pagination: Spring Data JPA Page vs Slice vs keyset Window scrolling, with the SQL and REST shapes
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b4bLvQVbZweAqqf54x
This commit is contained in:
@@ -0,0 +1 @@
|
||||
target/
|
||||
@@ -0,0 +1,48 @@
|
||||
# pagination
|
||||
|
||||
Companion code for **Spring Data JPA Pagination and Sorting: Page vs Slice vs Keyset Scrolling (Window API)** on [ankurm.com](https://ankurm.com).
|
||||
|
||||
| Component | Version |
|
||||
|---|---|
|
||||
| Spring Boot | 4.1.2 |
|
||||
| Spring Data (release train) / Spring Data JPA | 2026.0.2 / 4.1.2 |
|
||||
| Hibernate ORM | 7.4.13.Final (7.3.13.Final for the old-behaviour comparison) |
|
||||
| H2 (in-memory) | 2.4.240 |
|
||||
| JDK | Temurin 25.0.5 LTS |
|
||||
|
||||
No Docker needed. Every SQL statement is captured by `SqlRecorder`, a Hibernate `StatementInspector`, and every test writes what it saw to `output/` while asserting the same facts, so the numbers in the post fail the build if they stop being true.
|
||||
|
||||
```bash
|
||||
./scripts/run-all.sh # regenerates every file in output/
|
||||
./scripts/old-hibernate.sh # collection-fetch paging on Hibernate 7.3.13.Final
|
||||
./scripts/other-spring-data.sh # offset-then-keyset finding on Spring Data 2025.1.8 and 2026.0.0
|
||||
```
|
||||
|
||||
| Source file | What it shows |
|
||||
|---|---|
|
||||
| `ProductRepository` | every paging and scrolling form, numbered 1 to 10 |
|
||||
| `ProductController` | `Page` vs `PagedModel`, a keyset cursor endpoint, bad-sort handling |
|
||||
| `SqlRecorder` | the SQL capture mechanism |
|
||||
| `BrokenScrollRepository` (test) | a `String` `@Query` scroll method, which Spring Data rejects |
|
||||
|
||||
| Transcript | Shows |
|
||||
|---|---|
|
||||
| `output/01-page-slice-list.txt` | `Page` runs a count query, `Slice` and `List` do not |
|
||||
| `output/02-slice-boundary.txt` | how `Slice` knows there is a next page (n+1 rows) |
|
||||
| `output/03-count-query-skipped.txt` | when Spring Data skips the count query |
|
||||
| `output/04-sorting.txt` | sort handling and unsafe sort values |
|
||||
| `output/05-offset-vs-keyset-sql.txt` | offset SQL vs keyset SQL |
|
||||
| `output/06-deep-offset-scan-count.txt` | rows scanned: 10 / 90,010 / 11 |
|
||||
| `output/07-page-drift-insert.txt` | an insert repeats a row on the next page |
|
||||
| `output/08-page-drift-delete.txt` | a delete skips a row |
|
||||
| `output/09-keyset-ties-and-nulls.txt` | ties on the sort column, and the id tie-breaker |
|
||||
| `output/10-query-scroll-unsupported.txt` | String `@Query` scroll is rejected |
|
||||
| `output/11-offset-then-keyset-same-method-*.txt` | mixing offset and keyset scroll on one method |
|
||||
| `output/12-collection-fetch-paging-*.txt` | paging over a collection fetch, per Hibernate version |
|
||||
| `output/13-pageable-binding.txt` | what Spring MVC makes of `page`, `size`, `sort` |
|
||||
| `output/14-page-json-shapes.txt` | `Page` vs `PagedModel` JSON |
|
||||
| `output/15-rest-bad-sort.txt` | bad sort values over HTTP |
|
||||
| `output/16-keyset-cursor-walk.txt` | walking a dataset with an opaque cursor |
|
||||
| `output/17-page-json-via-dto.txt` | `serialization-mode=via-dto` |
|
||||
|
||||
There is no `docs/` folder: the explanations live in the post.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Page<Product> findByCategory("books", PageRequest.of(1, 10, Sort.by("id")))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
statements: 2, rows: 10, totalElements: 25, totalPages: 3, hasNext: true
|
||||
|
||||
# Slice<Product> findSliceByCategory("books", PageRequest.of(1, 10, Sort.by("id")))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only
|
||||
statements: 1, rows: 10, hasNext: true (no totalElements exists on a Slice)
|
||||
|
||||
# List<Product> findListByCategory("books", PageRequest.of(1, 10, Sort.by("id")))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only
|
||||
statements: 1, rows: 10 (no total, no hasNext)
|
||||
@@ -0,0 +1,4 @@
|
||||
# 5 toys in the table. Does the Slice know whether another slice follows?
|
||||
slice size 4 -> rows returned: 4, hasNext: true, statements: 1
|
||||
slice size 5 -> rows returned: 5, hasNext: false, statements: 1
|
||||
slice size 6 -> rows returned: 5, hasNext: false, statements: 1
|
||||
@@ -0,0 +1,7 @@
|
||||
# Page<Product>: how many statements does each request issue? (25 books, 5 toys)
|
||||
middle page page=1 size=10 rows=10 statements=2 totalElements=25
|
||||
first page, partial page=0 size=10 rows=5 statements=1 totalElements=5
|
||||
last page, partial page=2 size=10 rows=5 statements=1 totalElements=25
|
||||
full page in the middle page=1 size=5 rows=5 statements=2 totalElements=25
|
||||
last page, exactly full page=4 size=5 rows=5 statements=2 totalElements=25
|
||||
page past the end page=5 size=10 rows=0 statements=2 totalElements=25
|
||||
@@ -0,0 +1,21 @@
|
||||
# derived query, Sort.by("price").descending().and(Sort.by("id"))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.price desc,p1_0.id fetch first ? rows only
|
||||
|
||||
# derived query, unknown property
|
||||
org.springframework.data.core.PropertyReferenceException
|
||||
No property 'nope' found for type 'Product'; Did you mean 'name'
|
||||
|
||||
# @Query with Sort.by("price").descending() (alias p is applied for you)
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.price desc fetch first ? rows only
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
|
||||
# a value that is not a property reference at all: Sort.by("price) desc, (select 1")
|
||||
derived -> InvalidDataAccessApiUsageException
|
||||
Sort expression 'price) desc, (select 1: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…)
|
||||
@Query -> InvalidDataAccessApiUsageException
|
||||
Sort expression 'price) desc, (select 1: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…)
|
||||
|
||||
# @Query with a function-style sort value, three calls in one fresh context
|
||||
1. Sort.by("length(name)") -> rejected (InvalidDataAccessApiUsageException)
|
||||
2. JpaSort.unsafe("length(name)") -> accepted: order by character_length(p1_0.name) fetch first ? rows only
|
||||
3. Sort.by("length(name)") again -> accepted: order by character_length(p1_0.name) fetch first ? rows only
|
||||
@@ -0,0 +1,23 @@
|
||||
# offset paging, third page of 10: PageRequest.of(2, 10, Sort.by("id"))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only
|
||||
|
||||
# keyset scrolling, first window: ScrollPosition.keyset()
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
rows: 10, hasNext: true, position of the last row: KeysetScrollPosition [FORWARD, {id=10}]
|
||||
|
||||
# keyset scrolling, second window: the position from the first window
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.id>?) order by p1_0.id fetch first ? rows only
|
||||
rows: 10 (book-11 .. book-20), hasNext: true
|
||||
|
||||
# third window: 5 rows (book-21 .. book-25), hasNext: false
|
||||
|
||||
# WindowIterator over all 25 books, windows of 10
|
||||
rows visited: 25, statements: 3
|
||||
|
||||
# Query by Example: findBy(example, q -> q.sortBy(Sort.by("id")).limit(5).scroll(ScrollPosition.keyset()))
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
rows: 5, hasNext: true
|
||||
|
||||
# offset scrolling: ScrollPosition.offset(0) is a position, so the result starts AFTER it
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only
|
||||
first row returned: book-2
|
||||
@@ -0,0 +1,6 @@
|
||||
# 100,000 rows in category 'bulk'. H2 EXPLAIN ANALYZE reports scanCount: rows the database read.
|
||||
offset 0, fetch 10 scanCount: 10
|
||||
offset 90000, fetch 10 scanCount: 90,010
|
||||
keyset id > 90000, fetch 10 scanCount: 11
|
||||
|
||||
These are row counts, not timings. Timings depend on your database, indexes and cache.
|
||||
@@ -0,0 +1,7 @@
|
||||
# newest-first feed of 25 books, 5 per page; a new book is inserted between the two requests
|
||||
offset page 0: [book-25, book-24, book-23, book-22, book-21]
|
||||
keyset window 0: [book-25, book-24, book-23, book-22, book-21]
|
||||
-- book-NEW inserted --
|
||||
offset page 1: [book-21, book-20, book-19, book-18, book-17]
|
||||
keyset window 1: [book-20, book-19, book-18, book-17, book-16]
|
||||
offset: book-21 appears on both pages: true
|
||||
@@ -0,0 +1,7 @@
|
||||
# newest-first feed of 25 books, 5 per page; book-24 is deleted between the two requests
|
||||
offset page 0: [book-25, book-24, book-23, book-22, book-21]
|
||||
keyset window 0: [book-25, book-24, book-23, book-22, book-21]
|
||||
-- book-24 deleted --
|
||||
offset page 1: [book-19, book-18, book-17, book-16, book-15]
|
||||
keyset window 1: [book-20, book-19, book-18, book-17, book-16]
|
||||
offset skipped book-20: true
|
||||
@@ -0,0 +1,11 @@
|
||||
# sorting by price, which repeats: 25 books over 4 distinct prices, pages of 5
|
||||
rows visited: 25, distinct rows: 25, statements: 5
|
||||
second statement:
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.price>? or p1_0.price=? and p1_0.id>?) order by p1_0.price,p1_0.id fetch first ? rows only
|
||||
Method 6 names only the price in OrderBy; the id in the statement above was added by Spring Data.
|
||||
|
||||
# nullable sort column (H2 sorts NULLs first for ASC)
|
||||
full window: [b:null, d:null, a:1.00, c:2.00, e:3.00]
|
||||
after the first row (price null): [d, a, c, e]
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.price is not null or p1_0.price is null and p1_0.id>?) order by p1_0.price,p1_0.id fetch first ? rows only
|
||||
That predicate is right where NULLs sort first, as on H2. It was not run on a database that sorts them last.
|
||||
@@ -0,0 +1,4 @@
|
||||
# a String-based @Query method returning Window<Product>
|
||||
org.springframework.data.repository.query.QueryCreationException
|
||||
Scroll queries are not supported using String-based queries
|
||||
root cause: Scroll queries are not supported using String-based queries
|
||||
@@ -0,0 +1,11 @@
|
||||
Spring Data JPA: 4.0.8
|
||||
# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position)
|
||||
1. keyset, initial -> book-1..book-10
|
||||
2. keyset, after book-10 -> book-11..book-20
|
||||
3. offset, initial -> book-1..book-10
|
||||
4. offset, after row 10 -> book-11..book-20
|
||||
5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20)
|
||||
statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
|
||||
# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above
|
||||
keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16
|
||||
@@ -0,0 +1,11 @@
|
||||
Spring Data JPA: 4.1.0
|
||||
# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position)
|
||||
1. keyset, initial -> book-1..book-10
|
||||
2. keyset, after book-10 -> book-11..book-20
|
||||
3. offset, initial -> book-1..book-10
|
||||
4. offset, after row 10 -> book-11..book-20
|
||||
5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20)
|
||||
statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
|
||||
# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above
|
||||
keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16
|
||||
@@ -0,0 +1,11 @@
|
||||
Spring Data JPA: 4.1.2
|
||||
# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position)
|
||||
1. keyset, initial -> book-1..book-10
|
||||
2. keyset, after book-10 -> book-11..book-20
|
||||
3. offset, initial -> book-1..book-10
|
||||
4. offset, after row 10 -> book-11..book-20
|
||||
5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20)
|
||||
statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
|
||||
# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above
|
||||
keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16
|
||||
@@ -0,0 +1,25 @@
|
||||
Hibernate ORM: 7.3.13.Final
|
||||
# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews.
|
||||
|
||||
## method 8: @Query("select p from Product p join fetch p.reviews where ...")
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? order by p1_0.id
|
||||
select count(*) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=?
|
||||
products on the page: [book-1, book-2], reviews loaded on them: 4
|
||||
entities held by the persistence context: 18
|
||||
HHH90003004 (applying in memory) logged: true
|
||||
totalElements reported: 12 (products that actually have reviews: 6)
|
||||
|
||||
## method 8b: the same query with countQuery = "select count(distinct p) ..."
|
||||
select count(distinct p1_0.id) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=?
|
||||
totalElements reported: 6, totalPages: 3
|
||||
|
||||
## method 8c: @EntityGraph(attributePaths = "reviews") on a derived query, page of 4
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 left join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? order by p1_0.id
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
rows: 4, totalElements: 25, entities held by the persistence context: 37
|
||||
|
||||
## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids
|
||||
select p1_0.id from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
select distinct p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.id in (?,?) order by p1_0.id
|
||||
ids on the page: [1, 2], products fetched: 2, reviews fetched: 4
|
||||
@@ -0,0 +1,25 @@
|
||||
Hibernate ORM: 7.4.13.Final
|
||||
# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews.
|
||||
|
||||
## method 8: @Query("select p from Product p join fetch p.reviews where ...")
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from (select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? and exists(select 1 from review r1_0 where p1_0.id=r1_0.product_id) order by p1_0.id offset ? rows fetch first ? rows only) p1_0(id,category,name,price) join review r1_0 on p1_0.id=r1_0.product_id order by p1_0.id
|
||||
select count(*) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=?
|
||||
products on the page: [book-1, book-2], reviews loaded on them: 4
|
||||
entities held by the persistence context: 6
|
||||
HHH90003004 (applying in memory) logged: false
|
||||
totalElements reported: 12 (products that actually have reviews: 6)
|
||||
|
||||
## method 8b: the same query with countQuery = "select count(distinct p) ..."
|
||||
select count(distinct p1_0.id) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=?
|
||||
totalElements reported: 6, totalPages: 3
|
||||
|
||||
## method 8c: @EntityGraph(attributePaths = "reviews") on a derived query, page of 4
|
||||
select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from (select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only) p1_0(id,category,name,price) left join review r1_0 on p1_0.id=r1_0.product_id order by p1_0.id
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
rows: 4, totalElements: 25, entities held by the persistence context: 12
|
||||
|
||||
## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids
|
||||
select p1_0.id from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only
|
||||
select count(*) from product p1_0 where p1_0.category=?
|
||||
select distinct p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.id in (?,?) order by p1_0.id
|
||||
ids on the page: [1, 2], products fetched: 2, reviews fetched: 4
|
||||
@@ -0,0 +1,5 @@
|
||||
GET /products/echo -> 200 {"sort":"UNSORTED","size":20,"page":0}
|
||||
GET /products/echo?page=2&size=5&sort=price,desc&sort=id -> 200 {"sort":"price: DESC,id: ASC","size":5,"page":2}
|
||||
GET /products/echo?size=100000 -> 200 {"sort":"UNSORTED","size":2000,"page":0}
|
||||
GET /products/echo?page=-3&size=0 -> 200 {"sort":"UNSORTED","size":20,"page":0}
|
||||
GET /products/echo?page=abc -> 200 {"sort":"UNSORTED","size":20,"page":0}
|
||||
@@ -0,0 +1,5 @@
|
||||
GET /products/raw?size=2 (return type Page<ProductView>)
|
||||
{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":2,"pageable":{"offset":0,"pageNumber":0,"pageSize":2,"paged":true,"sort":{"empty":true,"sorted":false,"unsorted":true},"unpaged":false},"size":2,"sort":{"empty":true,"sorted":false,"unsorted":true},"totalElements":25,"totalPages":13}
|
||||
|
||||
GET /products/model?size=2 (return type PagedModel<ProductView>)
|
||||
{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"page":{"size":2,"number":0,"totalElements":25,"totalPages":13}}
|
||||
@@ -0,0 +1,8 @@
|
||||
GET /products/raw?size=1&sort=price,desc -> 200
|
||||
{"content":[{"id":3,"name":"book-3","price":13.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":1,"pageable":{"offset":0,"pageNumber":0,"pageSize":1,"paged":true,"sort":{"empty":false,"sorted":true,"unsorted":false},"unpaged":false},"size":1,"sort":{"empty":false,"sorted":true,"unsorted":false},"totalElements":25,"totalPages":25}
|
||||
GET /products/raw?size=1&sort=nope -> 400
|
||||
{"detail":"No property 'nope' found for type 'Product'; Did you mean 'name'","instance":"/products/raw","status":400,"title":"Bad Request"}
|
||||
GET /products/raw?size=1&sort=price;drop -> 400
|
||||
{"detail":"Sort expression 'price;drop: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…)","instance":"/products/raw","status":400,"title":"Bad Request"}
|
||||
GET /products/raw?size=1&sort=,desc -> 200
|
||||
{"content":[{"id":1,"name":"book-1","price":11.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":1,"pageable":{"offset":0,"pageNumber":0,"pageSize":1,"paged":true,"sort":{"empty":true,"sorted":false,"unsorted":true},"unpaged":false},"size":1,"sort":{"empty":true,"sorted":false,"unsorted":true},"totalElements":25,"totalPages":25}
|
||||
@@ -0,0 +1,4 @@
|
||||
GET /products/scroll -> 10 items, nextCursor="MTA"
|
||||
GET /products/scroll?after=MTA -> 10 items, nextCursor="MjA"
|
||||
GET /products/scroll?after=MjA -> 5 items, nextCursor=null
|
||||
calls: 3, items: 25, distinct: 25
|
||||
@@ -0,0 +1,3 @@
|
||||
spring.data.web.pageable.serialization-mode=via-dto
|
||||
GET /products/raw?size=2 (return type Page<ProductView>)
|
||||
{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"page":{"size":2,"number":0,"totalElements":25,"totalPages":13}}
|
||||
@@ -0,0 +1,57 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>4.1.2</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.ankurm</groupId>
|
||||
<artifactId>pagination-demo</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<java.version>25</java.version>
|
||||
<maven.compiler.release>25</maven.compiler.release>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-jpa</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-webmvc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-jpa-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-webmvc-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-runs the collection-fetch paging test on Hibernate 7.3.13.Final, the last 7.3 patch release, to reproduce
|
||||
# the HHH90003004 "applying in memory" behaviour that Hibernate 7.4 no longer has. Writes
|
||||
# output/12-collection-fetch-paging-7.3.13.Final.txt next to the 7.4 one written by the normal run.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mvn -B -q test -Dtest=CollectionFetchPagingTest -Dhibernate.version=7.3.13.Final
|
||||
cat output/12-collection-fetch-paging-7.3.13.Final.txt
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Replays the "offset then keyset on one method" finding on two other Spring Data release trains.
|
||||
# Each run writes output/11-offset-then-keyset-same-method-<Spring Data JPA version>.txt.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
for bom in 2025.1.8 2026.0.0; do
|
||||
mvn -B -q test -Dtest=KeysetScrollingTest#mixingOffsetAndKeysetOnOneMethod -Dspring-data-bom.version="$bom"
|
||||
done
|
||||
ls output/11-*
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerates every file in output/ by running the test suite (in-memory H2, no Docker needed).
|
||||
# The two "other version" scripts add the Hibernate 7.3.13 and extra Spring Data transcripts.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
rm -f output/*.txt
|
||||
mvn -B -q test
|
||||
./scripts/old-hibernate.sh >/dev/null
|
||||
./scripts/other-spring-data.sh >/dev/null
|
||||
echo "wrote:"; ls output
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class PaginationApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(PaginationApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.OneToMany;
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** A product in a catalogue. Prices repeat on purpose: ties are what break naive keyset scrolling. */
|
||||
@Entity
|
||||
public class Product {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
private String name;
|
||||
private String category;
|
||||
private BigDecimal price;
|
||||
|
||||
@OneToMany(mappedBy = "product")
|
||||
private List<Review> reviews = new ArrayList<>();
|
||||
|
||||
protected Product() {}
|
||||
|
||||
public Product(String name, String category, BigDecimal price) {
|
||||
this.name = name;
|
||||
this.category = category;
|
||||
this.price = price;
|
||||
}
|
||||
|
||||
public Long getId() { return id; }
|
||||
public String getName() { return name; }
|
||||
public String getCategory() { return category; }
|
||||
public BigDecimal getPrice() { return price; }
|
||||
public List<Review> getReviews() { return reviews; }
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.springframework.dao.InvalidDataAccessApiUsageException;
|
||||
import org.springframework.data.core.PropertyReferenceException;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.domain.ScrollPosition;
|
||||
import org.springframework.data.domain.Window;
|
||||
import org.springframework.data.web.PagedModel;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ProblemDetail;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/** The REST shapes the post compares. Entities are never returned directly: they carry lazy collections. */
|
||||
@RestController
|
||||
@RequestMapping("/products")
|
||||
public class ProductController {
|
||||
|
||||
/** What the API exposes about a product. */
|
||||
public record ProductView(Long id, String name, BigDecimal price) {
|
||||
static ProductView of(Product p) { return new ProductView(p.getId(), p.getName(), p.getPrice()); }
|
||||
}
|
||||
|
||||
/** Keyset response: the items plus an opaque cursor, and no total. */
|
||||
public record Scrolled(List<ProductView> items, String nextCursor) {}
|
||||
|
||||
private final ProductRepository repo;
|
||||
|
||||
public ProductController(ProductRepository repo) { this.repo = repo; }
|
||||
|
||||
/** 1. Returning a Page directly: Jackson serialises the PageImpl class itself. */
|
||||
@GetMapping("/raw")
|
||||
public Page<ProductView> raw(@RequestParam(defaultValue = "books") String category, Pageable pageable) {
|
||||
return repo.findByCategory(category, pageable).map(ProductView::of);
|
||||
}
|
||||
|
||||
/** 2. Wrapping the Page in PagedModel: a documented, stable JSON shape. */
|
||||
@GetMapping("/model")
|
||||
public PagedModel<ProductView> model(@RequestParam(defaultValue = "books") String category, Pageable pageable) {
|
||||
return new PagedModel<>(repo.findByCategory(category, pageable).map(ProductView::of));
|
||||
}
|
||||
|
||||
/** 3. Keyset over HTTP: the client sends back the cursor it was given, never a page number. */
|
||||
@GetMapping("/scroll")
|
||||
public Scrolled scroll(@RequestParam(defaultValue = "books") String category,
|
||||
@RequestParam(required = false) String after) {
|
||||
ScrollPosition position = after == null ? ScrollPosition.keyset() : decode(after);
|
||||
Window<Product> window = repo.findFirst10ByCategoryOrderByIdAsc(category, position);
|
||||
String next = window.hasNext() ? encode(window.positionAt(window.size() - 1)) : null;
|
||||
return new Scrolled(window.getContent().stream().map(ProductView::of).toList(), next);
|
||||
}
|
||||
|
||||
/** Shows what Spring MVC resolved from the query string, before any repository is involved. */
|
||||
@GetMapping("/echo")
|
||||
public Map<String, Object> echo(Pageable pageable) {
|
||||
return Map.of("page", pageable.getPageNumber(), "size", pageable.getPageSize(), "sort", pageable.getSort().toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* A bad sort value becomes a 400 instead of a 500. Two different exceptions can arrive here:
|
||||
* an unknown property (PropertyReferenceException) and a value that is not a property reference
|
||||
* at all (InvalidDataAccessApiUsageException). The second is broad, so a real application would
|
||||
* validate the sort against an allow-list before calling the repository.
|
||||
*/
|
||||
@ExceptionHandler({PropertyReferenceException.class, InvalidDataAccessApiUsageException.class})
|
||||
public ProblemDetail badSort(RuntimeException e) {
|
||||
return ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
|
||||
private static String encode(ScrollPosition position) {
|
||||
Object id = ((org.springframework.data.domain.KeysetScrollPosition) position).getKeys().get("id");
|
||||
return Base64.getUrlEncoder().withoutPadding().encodeToString(String.valueOf(id).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private static ScrollPosition decode(String cursor) {
|
||||
long id = Long.parseLong(new String(Base64.getUrlDecoder().decode(cursor), StandardCharsets.UTF_8));
|
||||
return ScrollPosition.forward(Map.of("id", id));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.domain.ScrollPosition;
|
||||
import org.springframework.data.domain.Slice;
|
||||
import org.springframework.data.domain.Window;
|
||||
import org.springframework.data.jpa.repository.EntityGraph;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
/** Every pagination form the post talks about, numbered so the post can point at them. */
|
||||
public interface ProductRepository extends JpaRepository<Product, Long> {
|
||||
|
||||
// 1. Page: the data query plus (usually) a count query.
|
||||
Page<Product> findByCategory(String category, Pageable pageable);
|
||||
|
||||
// 2. Slice: the data query only, fetching one extra row to learn whether a next slice exists.
|
||||
Slice<Product> findSliceByCategory(String category, Pageable pageable);
|
||||
|
||||
// 3. List with a Pageable: just the rows, no total, no "has next".
|
||||
List<Product> findListByCategory(String category, Pageable pageable);
|
||||
|
||||
// 4. Keyset (or offset) scrolling: the method name carries the page size, the argument carries the position.
|
||||
Window<Product> findFirst10ByCategoryOrderByIdAsc(String category, ScrollPosition position);
|
||||
|
||||
// 5. Keyset scrolling on a NON-unique column, WITH an id tie-breaker.
|
||||
Window<Product> findFirst5ByCategoryOrderByPriceAscIdAsc(String category, ScrollPosition position);
|
||||
|
||||
// 6. Keyset scrolling on a NON-unique column, WITHOUT a tie-breaker (the broken one).
|
||||
Window<Product> findFirst5ByCategoryOrderByPriceAsc(String category, ScrollPosition position);
|
||||
|
||||
// 7. @Query with a Pageable: sorting is applied to the alias you chose.
|
||||
@Query("select p from Product p where p.category = :category")
|
||||
Page<Product> queryByCategory(@Param("category") String category, Pageable pageable);
|
||||
|
||||
// 8. The trap: a collection join fetch combined with paging.
|
||||
@Query("select p from Product p join fetch p.reviews where p.category = :category")
|
||||
Page<Product> withReviewsInMemoryPaging(@Param("category") String category, Pageable pageable);
|
||||
|
||||
// 8b. Same query, but with an explicit count query that counts parents, not joined rows.
|
||||
@Query(value = "select p from Product p join fetch p.reviews where p.category = :category",
|
||||
countQuery = "select count(distinct p) from Product p join p.reviews where p.category = :category")
|
||||
Page<Product> withReviewsCorrectCount(@Param("category") String category, Pageable pageable);
|
||||
|
||||
// 8c. An entity graph does the same fetch without a hand-written query.
|
||||
@EntityGraph(attributePaths = "reviews")
|
||||
Page<Product> findGraphByCategory(String category, Pageable pageable);
|
||||
|
||||
// 9a/9b. The fix for old Hibernate versions: page the ids first, then fetch the collection for exactly those ids.
|
||||
@Query("select p.id from Product p where p.category = :category")
|
||||
Page<Long> idsByCategory(@Param("category") String category, Pageable pageable);
|
||||
|
||||
@Query("select distinct p from Product p join fetch p.reviews where p.id in :ids order by p.id")
|
||||
List<Product> fetchWithReviews(@Param("ids") Collection<Long> ids);
|
||||
|
||||
// 10. Newest-first feed, keyset style (used by the drift demo).
|
||||
Window<Product> findFirst5ByCategoryOrderByIdDesc(String category, ScrollPosition position);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
|
||||
/** A review of a product. Exists to demonstrate what a collection fetch does to paging. */
|
||||
@Entity
|
||||
public class Review {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private Product product;
|
||||
|
||||
private int rating;
|
||||
|
||||
protected Review() {}
|
||||
|
||||
public Review(Product product, int rating) {
|
||||
this.product = product;
|
||||
this.rating = rating;
|
||||
}
|
||||
|
||||
public Long getId() { return id; }
|
||||
public Product getProduct() { return product; }
|
||||
public int getRating() { return rating; }
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import org.hibernate.resource.jdbc.spi.StatementInspector;
|
||||
|
||||
/**
|
||||
* Records every SQL string Hibernate is about to send to the database.
|
||||
* Registered through hibernate.session_factory.statement_inspector in application.properties.
|
||||
* Hibernate instantiates it itself, so the captured list has to be static.
|
||||
*/
|
||||
public class SqlRecorder implements StatementInspector {
|
||||
private static final List<String> CAPTURED = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public String inspect(String sql) {
|
||||
CAPTURED.add(sql);
|
||||
return sql;
|
||||
}
|
||||
|
||||
public static void clear() { CAPTURED.clear(); }
|
||||
|
||||
public static List<String> captured() { return List.copyOf(CAPTURED); }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
spring.datasource.url=jdbc:h2:mem:pagination;DB_CLOSE_DELAY=-1
|
||||
spring.jpa.hibernate.ddl-auto=create-drop
|
||||
spring.jpa.open-in-view=false
|
||||
# Every SQL statement is captured by SqlRecorder (a Hibernate StatementInspector), not by show_sql.
|
||||
spring.jpa.properties.hibernate.session_factory.statement_inspector=com.ankurm.pagination.SqlRecorder
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.ankurm.brokenrepo;
|
||||
|
||||
import com.ankurm.pagination.Product;
|
||||
import org.springframework.data.domain.ScrollPosition;
|
||||
import org.springframework.data.domain.Window;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.Repository;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
/**
|
||||
* Deliberately broken: a String-based @Query that returns a Window.
|
||||
* It lives outside the com.ankurm.pagination package so Spring Boot does not try to register it at startup;
|
||||
* KeysetScrollingTest builds it by hand and records the real error.
|
||||
*/
|
||||
public interface BrokenScrollRepository extends Repository<Product, Long> {
|
||||
|
||||
@Query("select p from Product p where p.category = :category order by p.id")
|
||||
Window<Product> scrollByQuery(@Param("category") String category, ScrollPosition position);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import java.util.List;
|
||||
import org.hibernate.Session;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.boot.test.system.CapturedOutput;
|
||||
import org.springframework.boot.test.system.OutputCaptureExtension;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
|
||||
/**
|
||||
* Paging over a collection fetch. The transcript file name carries the Hibernate version that ran, because the
|
||||
* behaviour differs between 7.3 and 7.4: scripts/old-hibernate.sh re-runs this class on 7.3.13.Final.
|
||||
*/
|
||||
@DataJpaTest
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
|
||||
@ExtendWith(OutputCaptureExtension.class)
|
||||
class CollectionFetchPagingTest {
|
||||
|
||||
@Autowired ProductRepository repo;
|
||||
@Autowired EntityManager em;
|
||||
|
||||
@BeforeEach
|
||||
void seed() {
|
||||
TestData.catalogue(em);
|
||||
TestData.reviews(em);
|
||||
}
|
||||
|
||||
private long managed() { return em.unwrap(Session.class).getStatistics().getEntityCount(); }
|
||||
|
||||
private List<String> sqlThenReset() {
|
||||
List<String> sql = SqlRecorder.captured();
|
||||
SqlRecorder.clear();
|
||||
em.clear();
|
||||
return sql;
|
||||
}
|
||||
|
||||
@Test
|
||||
void pagingOverACollectionFetch(CapturedOutput out) {
|
||||
String hibernate = org.hibernate.Version.getVersionString();
|
||||
boolean pagedInSql = Integer.parseInt(hibernate.split("\\.")[1]) >= 4;
|
||||
Transcript t = new Transcript("12-collection-fetch-paging-" + hibernate + ".txt");
|
||||
t.line("Hibernate ORM: %s", hibernate);
|
||||
t.line("# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews.");
|
||||
|
||||
Page<Product> page = repo.withReviewsInMemoryPaging("books", PageRequest.of(0, 2, Sort.by("id")));
|
||||
long entities = managed();
|
||||
List<String> sql = sqlThenReset();
|
||||
boolean warned = out.getAll().contains("HHH90003004");
|
||||
t.line("");
|
||||
t.line("## method 8: @Query(\"select p from Product p join fetch p.reviews where ...\")");
|
||||
sql.forEach(t::line);
|
||||
t.line("products on the page: %s, reviews loaded on them: %d",
|
||||
page.getContent().stream().map(Product::getName).toList(),
|
||||
page.getContent().stream().mapToInt(p -> p.getReviews().size()).sum());
|
||||
t.line("entities held by the persistence context: %d", entities);
|
||||
t.line("HHH90003004 (applying in memory) logged: %s", warned);
|
||||
t.line("totalElements reported: %d (products that actually have reviews: 6)", page.getTotalElements());
|
||||
assertThat(page.getContent()).hasSize(2);
|
||||
assertThat(page.getTotalElements()).as("the derived count query counts joined rows").isEqualTo(12);
|
||||
if (pagedInSql) {
|
||||
assertThat(sql.get(0)).contains("offset ? rows fetch first ? rows only");
|
||||
assertThat(entities).isEqualTo(6);
|
||||
assertThat(warned).isFalse();
|
||||
} else {
|
||||
assertThat(sql.get(0)).doesNotContain("fetch first");
|
||||
assertThat(entities).isEqualTo(18);
|
||||
assertThat(warned).isTrue();
|
||||
}
|
||||
|
||||
Page<Product> counted = repo.withReviewsCorrectCount("books", PageRequest.of(0, 2, Sort.by("id")));
|
||||
List<String> countedSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("## method 8b: the same query with countQuery = \"select count(distinct p) ...\"");
|
||||
t.line(countedSql.get(1));
|
||||
t.line("totalElements reported: %d, totalPages: %d", counted.getTotalElements(), counted.getTotalPages());
|
||||
assertThat(counted.getTotalElements()).isEqualTo(6);
|
||||
|
||||
Page<Product> graph = repo.findGraphByCategory("books", PageRequest.of(0, 4, Sort.by("id")));
|
||||
long graphEntities = managed();
|
||||
List<String> graphSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("## method 8c: @EntityGraph(attributePaths = \"reviews\") on a derived query, page of 4");
|
||||
graphSql.forEach(t::line);
|
||||
t.line("rows: %d, totalElements: %d, entities held by the persistence context: %d",
|
||||
graph.getContent().size(), graph.getTotalElements(), graphEntities);
|
||||
assertThat(graph.getTotalElements()).as("the count query ignores the fetch").isEqualTo(25);
|
||||
|
||||
Page<Long> ids = repo.idsByCategory("books", PageRequest.of(0, 2, Sort.by("id")));
|
||||
List<Product> withReviews = repo.fetchWithReviews(ids.getContent());
|
||||
List<String> twoStepSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids");
|
||||
twoStepSql.forEach(t::line);
|
||||
t.line("ids on the page: %s, products fetched: %d, reviews fetched: %d", ids.getContent(), withReviews.size(),
|
||||
withReviews.stream().mapToInt(p -> p.getReviews().size()).sum());
|
||||
assertThat(withReviews).hasSize(2);
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import com.ankurm.brokenrepo.BrokenScrollRepository;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.data.domain.Example;
|
||||
import org.springframework.data.domain.ExampleMatcher;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.ScrollPosition;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.domain.Window;
|
||||
import org.springframework.data.jpa.repository.support.JpaRepositoryFactory;
|
||||
import org.springframework.data.support.WindowIterator;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
|
||||
/** Offset versus keyset scrolling. Every test writes output/NN-*.txt while asserting the same facts. */
|
||||
@DataJpaTest
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
|
||||
class KeysetScrollingTest {
|
||||
|
||||
@Autowired ProductRepository repo;
|
||||
@Autowired EntityManager em;
|
||||
|
||||
@BeforeEach
|
||||
void seed() { TestData.catalogue(em); }
|
||||
|
||||
private List<String> sqlThenReset() {
|
||||
List<String> sql = SqlRecorder.captured();
|
||||
SqlRecorder.clear();
|
||||
em.clear();
|
||||
return sql;
|
||||
}
|
||||
|
||||
private static List<String> names(Iterable<Product> products) {
|
||||
List<String> out = new ArrayList<>();
|
||||
products.forEach(p -> out.add(p.getName()));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- 05: the SQL of offset paging versus keyset scrolling ----------------------------
|
||||
|
||||
@Test
|
||||
void offsetVersusKeysetSql() {
|
||||
Transcript t = new Transcript("05-offset-vs-keyset-sql.txt");
|
||||
|
||||
repo.findListByCategory("books", PageRequest.of(2, 10, Sort.by("id")));
|
||||
t.line("# offset paging, third page of 10: PageRequest.of(2, 10, Sort.by(\"id\"))");
|
||||
List<String> offsetSql = sqlThenReset();
|
||||
offsetSql.forEach(t::line);
|
||||
assertThat(offsetSql.get(0)).contains("offset ? rows fetch first ? rows only");
|
||||
|
||||
Window<Product> w1 = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.keyset());
|
||||
List<String> sql1 = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# keyset scrolling, first window: ScrollPosition.keyset()");
|
||||
sql1.forEach(t::line);
|
||||
t.line("rows: %d, hasNext: %s, position of the last row: %s", w1.size(), w1.hasNext(), w1.positionAt(w1.size() - 1));
|
||||
assertThat(sql1.get(0)).doesNotContain("offset").doesNotContain("id>?");
|
||||
|
||||
Window<Product> w2 = repo.findFirst10ByCategoryOrderByIdAsc("books", w1.positionAt(w1.size() - 1));
|
||||
List<String> sql2 = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# keyset scrolling, second window: the position from the first window");
|
||||
sql2.forEach(t::line);
|
||||
t.line("rows: %d (%s .. %s), hasNext: %s", w2.size(), w2.getContent().get(0).getName(),
|
||||
w2.getContent().get(w2.size() - 1).getName(), w2.hasNext());
|
||||
assertThat(sql2.get(0)).contains("p1_0.id>?").doesNotContain("offset");
|
||||
assertThat(w2.getContent().get(0).getName()).isEqualTo("book-11");
|
||||
|
||||
Window<Product> w3 = repo.findFirst10ByCategoryOrderByIdAsc("books", w2.positionAt(w2.size() - 1));
|
||||
sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# third window: %d rows (%s .. %s), hasNext: %s", w3.size(), w3.getContent().get(0).getName(),
|
||||
w3.getContent().get(w3.size() - 1).getName(), w3.hasNext());
|
||||
assertThat(w3.hasNext()).isFalse();
|
||||
|
||||
WindowIterator<Product> it = WindowIterator
|
||||
.of(position -> repo.findFirst10ByCategoryOrderByIdAsc("books", position))
|
||||
.startingAt(ScrollPosition.keyset());
|
||||
int rows = 0;
|
||||
while (it.hasNext()) { it.next(); rows++; }
|
||||
List<String> itSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# WindowIterator over all 25 books, windows of 10");
|
||||
t.line("rows visited: %d, statements: %d", rows, itSql.size());
|
||||
assertThat(rows).isEqualTo(25);
|
||||
assertThat(itSql).hasSize(3);
|
||||
|
||||
Product probe = new Product(null, "books", null);
|
||||
Window<Product> fluent = repo.findBy(Example.of(probe, ExampleMatcher.matching().withIgnorePaths("price")),
|
||||
q -> q.sortBy(Sort.by("id")).limit(5).scroll(ScrollPosition.keyset()));
|
||||
List<String> fsql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# Query by Example: findBy(example, q -> q.sortBy(Sort.by(\"id\")).limit(5).scroll(ScrollPosition.keyset()))");
|
||||
fsql.forEach(t::line);
|
||||
t.line("rows: %d, hasNext: %s", fluent.size(), fluent.hasNext());
|
||||
assertThat(fluent.size()).isEqualTo(5);
|
||||
|
||||
// Last on purpose: after an offset call, keyset scrolling on the SAME method misbehaves (see test mixingOffsetAndKeysetOnOneMethod).
|
||||
Window<Product> o = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.offset(0));
|
||||
List<String> osql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# offset scrolling: ScrollPosition.offset(0) is a position, so the result starts AFTER it");
|
||||
osql.forEach(t::line);
|
||||
t.line("first row returned: %s", o.getContent().get(0).getName());
|
||||
assertThat(o.getContent().get(0).getName()).isEqualTo("book-2");
|
||||
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 06: what a deep offset costs ----------------------------------------------------
|
||||
|
||||
@Test
|
||||
void deepOffsetVersusKeysetScanCount() {
|
||||
em.createQuery("delete from Product").executeUpdate(); // only bulk rows, so scanCount is not padded by the seed
|
||||
em.createNativeQuery("insert into product(category, name, price) select 'bulk', 'p' || x, 10 from system_range(1, 100000)")
|
||||
.executeUpdate();
|
||||
em.flush();
|
||||
em.clear();
|
||||
|
||||
Transcript t = new Transcript("06-deep-offset-scan-count.txt");
|
||||
t.line("# 100,000 rows in category 'bulk'. H2 EXPLAIN ANALYZE reports scanCount: rows the database read.");
|
||||
record Probe(String label, String sql) {}
|
||||
List<Probe> probes = List.of(
|
||||
new Probe("offset 0, fetch 10",
|
||||
"explain analyze select id from product where category='bulk' order by id offset 0 rows fetch first 10 rows only"),
|
||||
new Probe("offset 90000, fetch 10",
|
||||
"explain analyze select id from product where category='bulk' order by id offset 90000 rows fetch first 10 rows only"),
|
||||
new Probe("keyset id > 90000, fetch 10",
|
||||
"explain analyze select id from product where category='bulk' and id>90000 order by id fetch first 10 rows only"));
|
||||
Pattern scan = Pattern.compile("scanCount: (\\d+)");
|
||||
int[] counts = new int[3];
|
||||
for (int i = 0; i < probes.size(); i++) {
|
||||
Object plan = em.createNativeQuery(probes.get(i).sql()).getSingleResult();
|
||||
Matcher m = scan.matcher(String.valueOf(plan));
|
||||
assertThat(m.find()).isTrue();
|
||||
counts[i] = Integer.parseInt(m.group(1));
|
||||
t.line("%-28s scanCount: %,d", probes.get(i).label(), counts[i]);
|
||||
}
|
||||
assertThat(counts[0]).isEqualTo(10);
|
||||
assertThat(counts[1]).isEqualTo(90_010);
|
||||
assertThat(counts[2]).isEqualTo(11);
|
||||
t.line("");
|
||||
t.line("These are row counts, not timings. Timings depend on your database, indexes and cache.");
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 07 / 08: page drift -------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void driftWhenARowIsInserted() {
|
||||
Transcript t = new Transcript("07-page-drift-insert.txt");
|
||||
t.line("# newest-first feed of 25 books, 5 per page; a new book is inserted between the two requests");
|
||||
|
||||
Page<Product> o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id")));
|
||||
Window<Product> k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
|
||||
t.line("offset page 0: %s", names(o0.getContent()));
|
||||
t.line("keyset window 0: %s", names(k0.getContent()));
|
||||
assertThat(names(o0.getContent())).isEqualTo(names(k0.getContent()));
|
||||
|
||||
em.persist(new Product("book-NEW", "books", BigDecimal.TEN));
|
||||
em.flush();
|
||||
em.clear();
|
||||
t.line("-- book-NEW inserted --");
|
||||
|
||||
Page<Product> o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id")));
|
||||
Window<Product> k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1));
|
||||
t.line("offset page 1: %s", names(o1.getContent()));
|
||||
t.line("keyset window 1: %s", names(k1.getContent()));
|
||||
List<String> seen = new ArrayList<>(names(o0.getContent()));
|
||||
seen.addAll(names(o1.getContent()));
|
||||
t.line("offset: book-21 appears on both pages: %s", seen.stream().filter("book-21"::equals).count() == 2);
|
||||
assertThat(names(o1.getContent()).get(0)).isEqualTo("book-21");
|
||||
assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20");
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void driftWhenARowIsDeleted() {
|
||||
Transcript t = new Transcript("08-page-drift-delete.txt");
|
||||
t.line("# newest-first feed of 25 books, 5 per page; book-24 is deleted between the two requests");
|
||||
|
||||
Page<Product> o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id")));
|
||||
Window<Product> k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
|
||||
t.line("offset page 0: %s", names(o0.getContent()));
|
||||
t.line("keyset window 0: %s", names(k0.getContent()));
|
||||
|
||||
em.createQuery("delete from Product p where p.name = 'book-24'").executeUpdate();
|
||||
em.clear();
|
||||
t.line("-- book-24 deleted --");
|
||||
|
||||
Page<Product> o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id")));
|
||||
Window<Product> k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1));
|
||||
t.line("offset page 1: %s", names(o1.getContent()));
|
||||
t.line("keyset window 1: %s", names(k1.getContent()));
|
||||
t.line("offset skipped book-20: %s", !names(o1.getContent()).contains("book-20"));
|
||||
assertThat(names(o1.getContent())).doesNotContain("book-20");
|
||||
assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20");
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 09: ties and nulls --------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void ties() {
|
||||
Transcript t = new Transcript("09-keyset-ties-and-nulls.txt");
|
||||
t.line("# sorting by price, which repeats: 25 books over 4 distinct prices, pages of 5");
|
||||
WindowIterator<Product> it = WindowIterator
|
||||
.of(position -> repo.findFirst5ByCategoryOrderByPriceAsc("books", position))
|
||||
.startingAt(ScrollPosition.keyset());
|
||||
List<String> all = new ArrayList<>();
|
||||
while (it.hasNext()) all.add(it.next().getName());
|
||||
List<String> sql = sqlThenReset();
|
||||
t.line("rows visited: %d, distinct rows: %d, statements: %d", all.size(), new HashSet<>(all).size(), sql.size());
|
||||
t.line("second statement:");
|
||||
t.line(sql.get(1));
|
||||
assertThat(all).hasSize(25);
|
||||
assertThat(new HashSet<>(all)).hasSize(25);
|
||||
assertThat(sql.get(1)).contains("p1_0.price>? or p1_0.price=? and p1_0.id>?");
|
||||
t.line("Method 6 names only the price in OrderBy; the id in the statement above was added by Spring Data.");
|
||||
|
||||
t.line("");
|
||||
t.line("# nullable sort column (H2 sorts NULLs first for ASC)");
|
||||
em.persist(new Product("a", "mixed", new BigDecimal("1")));
|
||||
em.persist(new Product("b", "mixed", null));
|
||||
em.persist(new Product("c", "mixed", new BigDecimal("2")));
|
||||
em.persist(new Product("d", "mixed", null));
|
||||
em.persist(new Product("e", "mixed", new BigDecimal("3")));
|
||||
em.flush();
|
||||
em.clear();
|
||||
SqlRecorder.clear();
|
||||
Window<Product> all5 = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", ScrollPosition.keyset());
|
||||
sqlThenReset();
|
||||
t.line("full window: %s", all5.getContent().stream().map(p -> p.getName() + ":" + p.getPrice()).toList());
|
||||
Window<Product> afterB = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", all5.positionAt(0));
|
||||
List<String> afterSql = sqlThenReset();
|
||||
t.line("after the first row (price null): %s", names(afterB.getContent()));
|
||||
t.line(afterSql.get(0));
|
||||
assertThat(names(afterB.getContent())).containsExactly("d", "a", "c", "e");
|
||||
assertThat(afterSql.get(0)).contains("p1_0.price is not null or p1_0.price is null and p1_0.id>?");
|
||||
t.line("That predicate is right where NULLs sort first, as on H2. It was not run on a database that sorts them last.");
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 10: @Query cannot scroll --------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void stringQueryCannotScroll() {
|
||||
Transcript t = new Transcript("10-query-scroll-unsupported.txt");
|
||||
t.line("# a String-based @Query method returning Window<Product>");
|
||||
try {
|
||||
new JpaRepositoryFactory(em).getRepository(BrokenScrollRepository.class);
|
||||
throw new AssertionError("expected the repository to be rejected");
|
||||
} catch (RuntimeException e) {
|
||||
t.line(e.getClass().getName());
|
||||
t.line(e.getMessage());
|
||||
Throwable root = e;
|
||||
while (root.getCause() != null) root = root.getCause();
|
||||
t.line("root cause: %s", root.getMessage());
|
||||
assertThat(root.getMessage()).contains("Scroll queries are not supported using String-based queries");
|
||||
}
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 11: mixing offset and keyset on one method --------------------------------------
|
||||
|
||||
/**
|
||||
* Observed on Spring Data JPA 4.1.2 (and on the 2025.1.8 and 2026.0.0 release trains, see scripts/other-spring-data.sh):
|
||||
* once a query method has been called with an OFFSET scroll position, later calls with a non-initial KEYSET
|
||||
* position on that same method are silently run without the keyset predicate. If this test starts failing after an
|
||||
* upgrade, the behaviour has been fixed and the post should be updated.
|
||||
*/
|
||||
@Test
|
||||
void mixingOffsetAndKeysetOnOneMethod() {
|
||||
String jpa = org.springframework.data.jpa.repository.JpaRepository.class.getPackage().getImplementationVersion();
|
||||
Transcript t = new Transcript("11-offset-then-keyset-same-method-" + jpa + ".txt");
|
||||
t.line("Spring Data JPA: %s", jpa);
|
||||
t.line("# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc(\"books\", position)");
|
||||
|
||||
Window<Product> k1 = window(ScrollPosition.keyset());
|
||||
t.line("1. keyset, initial -> %s", range(k1));
|
||||
Window<Product> k2 = window(k1.positionAt(9));
|
||||
t.line("2. keyset, after book-10 -> %s", range(k2));
|
||||
Window<Product> o1 = window(ScrollPosition.offset());
|
||||
t.line("3. offset, initial -> %s", range(o1));
|
||||
Window<Product> o2 = window(o1.positionAt(9));
|
||||
t.line("4. offset, after row 10 -> %s", range(o2));
|
||||
SqlRecorder.clear();
|
||||
Window<Product> k3 = window(k1.positionAt(9));
|
||||
List<String> sql = sqlThenReset();
|
||||
t.line("5. keyset, after book-10 again -> %s (step 2 returned %s)", range(k3), range(k2));
|
||||
t.line(" statement for step 5: %s", sql.get(0));
|
||||
|
||||
Window<Product> d1 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
|
||||
Window<Product> d2 = repo.findFirst5ByCategoryOrderByIdDesc("books", d1.positionAt(4));
|
||||
t.line("");
|
||||
t.line("# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above");
|
||||
t.line("keyset initial -> %s, keyset after book-21 -> %s", range(d1), range(d2));
|
||||
|
||||
assertThat(range(k2)).isEqualTo("book-11..book-20");
|
||||
assertThat(range(o2)).isEqualTo("book-11..book-20");
|
||||
assertThat(range(k3)).isEqualTo("book-1..book-10");
|
||||
assertThat(sql.get(0)).doesNotContain("p1_0.id>?");
|
||||
assertThat(range(d2)).isEqualTo("book-20..book-16");
|
||||
t.write();
|
||||
}
|
||||
|
||||
private Window<Product> window(ScrollPosition position) {
|
||||
Window<Product> w = repo.findFirst10ByCategoryOrderByIdAsc("books", position);
|
||||
em.clear();
|
||||
return w;
|
||||
}
|
||||
|
||||
private static String range(Window<Product> w) {
|
||||
return w.getContent().get(0).getName() + ".." + w.getContent().get(w.size() - 1).getName();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.dao.InvalidDataAccessApiUsageException;
|
||||
import org.springframework.data.core.PropertyReferenceException;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.domain.Slice;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.JpaSort;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
|
||||
/** Page, Slice, List, the count query, and sorting. Every test writes output/NN-*.txt while asserting the same facts. */
|
||||
@DataJpaTest
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
|
||||
class PaginationBasicsTest {
|
||||
|
||||
@Autowired ProductRepository repo;
|
||||
@Autowired EntityManager em;
|
||||
|
||||
@BeforeEach
|
||||
void seed() { TestData.catalogue(em); }
|
||||
|
||||
private List<String> sqlThenReset() {
|
||||
List<String> sql = SqlRecorder.captured();
|
||||
SqlRecorder.clear();
|
||||
em.clear();
|
||||
return sql;
|
||||
}
|
||||
|
||||
private String tryQuerySort(Sort sort) {
|
||||
try {
|
||||
repo.queryByCategory("books", PageRequest.of(0, 3, sort));
|
||||
String sql = SqlRecorder.captured().get(0);
|
||||
sqlThenReset();
|
||||
return "accepted: " + sql.substring(sql.indexOf(" order by "));
|
||||
} catch (InvalidDataAccessApiUsageException e) {
|
||||
sqlThenReset();
|
||||
return "rejected (" + e.getClass().getSimpleName() + ")";
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 01: the three return types ------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void pageSliceAndList() {
|
||||
Transcript t = new Transcript("01-page-slice-list.txt");
|
||||
Pageable second = PageRequest.of(1, 10, Sort.by("id"));
|
||||
|
||||
Page<Product> page = repo.findByCategory("books", second);
|
||||
List<String> pageSql = sqlThenReset();
|
||||
t.line("# Page<Product> findByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
|
||||
pageSql.forEach(t::line);
|
||||
t.line("statements: %d, rows: %d, totalElements: %d, totalPages: %d, hasNext: %s",
|
||||
pageSql.size(), page.getContent().size(), page.getTotalElements(), page.getTotalPages(), page.hasNext());
|
||||
assertThat(pageSql).hasSize(2);
|
||||
assertThat(pageSql.get(1)).startsWith("select count(*)");
|
||||
assertThat(page.getTotalElements()).isEqualTo(25);
|
||||
assertThat(page.getTotalPages()).isEqualTo(3);
|
||||
|
||||
Slice<Product> slice = repo.findSliceByCategory("books", second);
|
||||
List<String> sliceSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# Slice<Product> findSliceByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
|
||||
sliceSql.forEach(t::line);
|
||||
t.line("statements: %d, rows: %d, hasNext: %s (no totalElements exists on a Slice)",
|
||||
sliceSql.size(), slice.getContent().size(), slice.hasNext());
|
||||
assertThat(sliceSql).hasSize(1);
|
||||
assertThat(slice.hasNext()).isTrue();
|
||||
|
||||
List<Product> list = repo.findListByCategory("books", second);
|
||||
List<String> listSql = sqlThenReset();
|
||||
t.line("");
|
||||
t.line("# List<Product> findListByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
|
||||
listSql.forEach(t::line);
|
||||
t.line("statements: %d, rows: %d (no total, no hasNext)", listSql.size(), list.size());
|
||||
assertThat(listSql).hasSize(1);
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void sliceKnowsHasNextByFetchingOneExtraRow() {
|
||||
Transcript t = new Transcript("02-slice-boundary.txt");
|
||||
t.line("# 5 toys in the table. Does the Slice know whether another slice follows?");
|
||||
for (int size : new int[] {4, 5, 6}) {
|
||||
Slice<Product> s = repo.findSliceByCategory("toys", PageRequest.of(0, size, Sort.by("id")));
|
||||
t.line("slice size %d -> rows returned: %d, hasNext: %s, statements: %d",
|
||||
size, s.getContent().size(), s.hasNext(), SqlRecorder.captured().size());
|
||||
if (size == 4) { assertThat(s.hasNext()).isTrue(); assertThat(s.getContent()).hasSize(4); }
|
||||
else { assertThat(s.hasNext()).isFalse(); }
|
||||
sqlThenReset();
|
||||
}
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 03: when the count query is skipped ---------------------------------------------
|
||||
|
||||
@Test
|
||||
void countQueryIsSometimesSkipped() {
|
||||
Transcript t = new Transcript("03-count-query-skipped.txt");
|
||||
t.line("# Page<Product>: how many statements does each request issue? (25 books, 5 toys)");
|
||||
record Case(String label, String category, int page, int size, int expectedStatements) {}
|
||||
List<Case> cases = List.of(
|
||||
new Case("middle page ", "books", 1, 10, 2),
|
||||
new Case("first page, partial ", "toys", 0, 10, 1),
|
||||
new Case("last page, partial ", "books", 2, 10, 1),
|
||||
new Case("full page in the middle", "books", 1, 5, 2),
|
||||
new Case("last page, exactly full", "books", 4, 5, 2),
|
||||
new Case("page past the end ", "books", 5, 10, 2));
|
||||
for (Case c : cases) {
|
||||
Page<Product> p = repo.findByCategory(c.category(), PageRequest.of(c.page(), c.size(), Sort.by("id")));
|
||||
int statements = SqlRecorder.captured().size();
|
||||
t.line("%s page=%d size=%-2d rows=%-2d statements=%d totalElements=%d",
|
||||
c.label(), c.page(), c.size(), p.getContent().size(), statements, p.getTotalElements());
|
||||
assertThat(statements).as(c.label()).isEqualTo(c.expectedStatements());
|
||||
sqlThenReset();
|
||||
}
|
||||
t.write();
|
||||
}
|
||||
|
||||
// ---- 04: sorting ---------------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void sortingAndSortSafety() {
|
||||
Transcript t = new Transcript("04-sorting.txt");
|
||||
|
||||
repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending().and(Sort.by("id"))));
|
||||
t.line("# derived query, Sort.by(\"price\").descending().and(Sort.by(\"id\"))");
|
||||
sqlThenReset().stream().limit(1).forEach(t::line);
|
||||
|
||||
t.line("");
|
||||
t.line("# derived query, unknown property");
|
||||
try {
|
||||
repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("nope")));
|
||||
throw new AssertionError("expected an exception");
|
||||
} catch (PropertyReferenceException e) {
|
||||
t.line(e.getClass().getName());
|
||||
t.line(e.getMessage());
|
||||
}
|
||||
|
||||
repo.queryByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending()));
|
||||
t.line("");
|
||||
t.line("# @Query with Sort.by(\"price\").descending() (alias p is applied for you)");
|
||||
sqlThenReset().forEach(t::line);
|
||||
|
||||
t.line("");
|
||||
t.line("# a value that is not a property reference at all: Sort.by(\"price) desc, (select 1\")");
|
||||
for (String kind : new String[] {"derived", "@Query"}) {
|
||||
try {
|
||||
Pageable bad = PageRequest.of(0, 3, Sort.by("price) desc, (select 1"));
|
||||
if (kind.equals("derived")) repo.findByCategory("books", bad); else repo.queryByCategory("books", bad);
|
||||
throw new AssertionError("expected an exception");
|
||||
} catch (InvalidDataAccessApiUsageException e) {
|
||||
t.line("%s -> %s", kind, e.getClass().getSimpleName());
|
||||
t.line(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
t.line("");
|
||||
t.line("# @Query with a function-style sort value, three calls in one fresh context");
|
||||
t.line("1. Sort.by(\"length(name)\") -> %s", tryQuerySort(Sort.by("length(name)")));
|
||||
t.line("2. JpaSort.unsafe(\"length(name)\") -> %s", tryQuerySort(JpaSort.unsafe("length(name)")));
|
||||
t.line("3. Sort.by(\"length(name)\") again -> %s", tryQuerySort(Sort.by("length(name)")));
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/** What a client sees over HTTP: Pageable binding, size caps, JSON shapes, bad sorts, keyset cursors. */
|
||||
@SpringBootTest
|
||||
@AutoConfigureMockMvc
|
||||
@Transactional
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
|
||||
class RestPagingTest {
|
||||
|
||||
@Autowired MockMvc mvc;
|
||||
@Autowired EntityManager em;
|
||||
final JsonMapper json = JsonMapper.builder().build();
|
||||
|
||||
@BeforeEach
|
||||
void seed() {
|
||||
TestData.catalogue(em);
|
||||
em.flush();
|
||||
}
|
||||
|
||||
private String body(String url) throws Exception {
|
||||
return mvc.perform(get(url)).andReturn().getResponse().getContentAsString();
|
||||
}
|
||||
|
||||
private int status(String url) throws Exception {
|
||||
return mvc.perform(get(url)).andReturn().getResponse().getStatus();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageableBindingAndCaps() throws Exception {
|
||||
Transcript t = new Transcript("13-pageable-binding.txt");
|
||||
for (String q : List.of("", "?page=2&size=5&sort=price,desc&sort=id", "?size=100000", "?page=-3&size=0", "?page=abc")) {
|
||||
String url = "/products/echo" + q;
|
||||
t.line("GET %s -> %d %s", url, status(url), body(url));
|
||||
}
|
||||
assertThat(body("/products/echo")).contains("\"size\":20").contains("\"page\":0");
|
||||
assertThat(body("/products/echo?size=100000")).contains("\"size\":2000");
|
||||
assertThat(body("/products/echo?page=-3&size=0")).contains("\"size\":20").contains("\"page\":0");
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageJsonShapes() throws Exception {
|
||||
Transcript t = new Transcript("14-page-json-shapes.txt");
|
||||
String raw = body("/products/raw?size=2");
|
||||
String model = body("/products/model?size=2");
|
||||
t.line("GET /products/raw?size=2 (return type Page<ProductView>)");
|
||||
t.line(raw);
|
||||
t.line("");
|
||||
t.line("GET /products/model?size=2 (return type PagedModel<ProductView>)");
|
||||
t.line(model);
|
||||
JsonNode m = json.readTree(model);
|
||||
assertThat(m.has("content")).isTrue();
|
||||
assertThat(m.get("page").get("totalElements").asInt()).isEqualTo(25);
|
||||
assertThat(m.get("page").get("totalPages").asInt()).isEqualTo(13);
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void sortValidation() throws Exception {
|
||||
Transcript t = new Transcript("15-rest-bad-sort.txt");
|
||||
for (String q : List.of("sort=price,desc", "sort=nope", "sort=price;drop", "sort=,desc")) {
|
||||
String url = "/products/raw?size=1&" + q;
|
||||
t.line("GET %s -> %d", url, status(url));
|
||||
t.line(" %s", body(url).replaceAll("\\s+", " "));
|
||||
}
|
||||
assertThat(status("/products/raw?size=1&sort=nope")).isEqualTo(400);
|
||||
assertThat(status("/products/raw?size=1&sort=price,desc")).isEqualTo(200);
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void keysetCursorWalk() throws Exception {
|
||||
Transcript t = new Transcript("16-keyset-cursor-walk.txt");
|
||||
List<String> names = new ArrayList<>();
|
||||
String cursor = null;
|
||||
int calls = 0;
|
||||
do {
|
||||
String url = "/products/scroll" + (cursor == null ? "" : "?after=" + cursor);
|
||||
String b = body(url);
|
||||
calls++;
|
||||
JsonNode n = json.readTree(b);
|
||||
n.get("items").forEach(i -> names.add(i.get("name").asString()));
|
||||
t.line("GET %s -> %d items, nextCursor=%s", url, n.get("items").size(), n.get("nextCursor"));
|
||||
cursor = n.get("nextCursor").isNull() ? null : n.get("nextCursor").asString();
|
||||
} while (cursor != null);
|
||||
t.line("calls: %d, items: %d, distinct: %d", calls, names.size(), names.stream().distinct().count());
|
||||
assertThat(calls).isEqualTo(3);
|
||||
assertThat(names).hasSize(25).doesNotHaveDuplicates();
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/** Same endpoint as RestPagingTest, with spring.data.web.pageable.serialization-mode=via-dto. */
|
||||
@SpringBootTest(properties = "spring.data.web.pageable.serialization-mode=via-dto")
|
||||
@AutoConfigureMockMvc
|
||||
@Transactional
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
|
||||
class RestViaDtoTest {
|
||||
|
||||
@Autowired MockMvc mvc;
|
||||
@Autowired EntityManager em;
|
||||
|
||||
@Test
|
||||
void rawPageBecomesPagedModelShape() throws Exception {
|
||||
TestData.catalogue(em);
|
||||
em.flush();
|
||||
String body = mvc.perform(get("/products/raw?size=2")).andReturn().getResponse().getContentAsString();
|
||||
Transcript t = new Transcript("17-page-json-via-dto.txt");
|
||||
t.line("spring.data.web.pageable.serialization-mode=via-dto");
|
||||
t.line("GET /products/raw?size=2 (return type Page<ProductView>)");
|
||||
t.line(body);
|
||||
JsonNode n = JsonMapper.builder().build().readTree(body);
|
||||
assertThat(n.has("page")).isTrue();
|
||||
assertThat(n.get("page").get("totalElements").asInt()).isEqualTo(25);
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import java.math.BigDecimal;
|
||||
|
||||
/** Deterministic catalogue used by every test. Ids are 1..30 because each test gets a fresh context. */
|
||||
final class TestData {
|
||||
private TestData() {}
|
||||
|
||||
/** 25 books (ids 1-25, price 10 + i % 4, so prices tie) and 5 toys (ids 26-30). */
|
||||
static void catalogue(EntityManager em) {
|
||||
for (int i = 1; i <= 25; i++) em.persist(new Product("book-" + i, "books", BigDecimal.valueOf(10 + (i % 4))));
|
||||
for (int i = 1; i <= 5; i++) em.persist(new Product("toy-" + i, "toys", BigDecimal.valueOf(5 + i)));
|
||||
em.flush();
|
||||
em.clear();
|
||||
SqlRecorder.clear();
|
||||
}
|
||||
|
||||
/** Two reviews each for books 1-6, so 6 products have reviews and the join has 12 rows. */
|
||||
static void reviews(EntityManager em) {
|
||||
for (int i = 1; i <= 6; i++) {
|
||||
Product p = em.find(Product.class, (long) i);
|
||||
em.persist(new Review(p, 5));
|
||||
em.persist(new Review(p, 3));
|
||||
}
|
||||
em.flush();
|
||||
em.clear();
|
||||
SqlRecorder.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.ankurm.pagination;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** Writes output/NN-*.txt while a test asserts the same facts. */
|
||||
final class Transcript {
|
||||
private final List<String> lines = new ArrayList<>();
|
||||
private final Path file;
|
||||
|
||||
Transcript(String name) { this.file = Path.of("output/" + name); }
|
||||
|
||||
Transcript line(String fmt, Object... args) {
|
||||
String s = args.length == 0 ? fmt : fmt.formatted(args);
|
||||
lines.add(s);
|
||||
System.out.println(s);
|
||||
return this;
|
||||
}
|
||||
|
||||
void write() {
|
||||
try {
|
||||
Files.createDirectories(file.getParent());
|
||||
Files.writeString(file, String.join("\n", lines) + "\n");
|
||||
} catch (IOException e) { throw new RuntimeException(e); }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user