diff --git a/README.md b/README.md index c0dee27..6dfada5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Companion code for a series of posts on [ankurm.com](https://ankurm.com) coverin | [`distributed-cache/`](./distributed-cache) | Distributed Caching in Spring Boot: Hazelcast vs Infinispan vs Redis | Hazelcast 5.5.0 (embedded), Infinispan 16.1.4 (embedded), Redis 8.8.3 (Testcontainers) | | [`r2dbc/`](./r2dbc) | Reactive Data Access with Spring Data R2DBC: CRUD, Transactions and When to Choose It | Spring Data R2DBC 4.1.2, r2dbc-postgresql 1.1.3.RELEASE, PostgreSQL 17 (Testcontainers 2.0.5) | | [`projections/`](./projections) | Spring Data JPA Projections: Interface, Record and Dynamic Projections (With the SQL They Generate) | Spring Data JPA 4.1.2, Hibernate 7.4.13.Final, H2 2.4.240 | +| [`pagination/`](./pagination) | Spring Data JPA Pagination and Sorting: Page vs Slice vs Keyset Scrolling (Window API) | Spring Data JPA 4.1.2, Hibernate 7.4.13.Final, H2 2.4.240 | Each module has its own `pom.xml`, a `docker-compose.yml` for its backing services, and a `docs/output/` directory holding the real, re-runnable transcripts that every number quoted in its post traces back to. There are no narrative doc chapters here — the deeper material for each post lives in accordions inside the post itself. diff --git a/pagination/.gitignore b/pagination/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/pagination/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/pagination/README.md b/pagination/README.md new file mode 100644 index 0000000..4d359e5 --- /dev/null +++ b/pagination/README.md @@ -0,0 +1,48 @@ +# pagination + +Companion code for **Spring Data JPA Pagination and Sorting: Page vs Slice vs Keyset Scrolling (Window API)** on [ankurm.com](https://ankurm.com). + +| Component | Version | +|---|---| +| Spring Boot | 4.1.2 | +| Spring Data (release train) / Spring Data JPA | 2026.0.2 / 4.1.2 | +| Hibernate ORM | 7.4.13.Final (7.3.13.Final for the old-behaviour comparison) | +| H2 (in-memory) | 2.4.240 | +| JDK | Temurin 25.0.5 LTS | + +No Docker needed. Every SQL statement is captured by `SqlRecorder`, a Hibernate `StatementInspector`, and every test writes what it saw to `output/` while asserting the same facts, so the numbers in the post fail the build if they stop being true. + +```bash +./scripts/run-all.sh # regenerates every file in output/ +./scripts/old-hibernate.sh # collection-fetch paging on Hibernate 7.3.13.Final +./scripts/other-spring-data.sh # offset-then-keyset finding on Spring Data 2025.1.8 and 2026.0.0 +``` + +| Source file | What it shows | +|---|---| +| `ProductRepository` | every paging and scrolling form, numbered 1 to 10 | +| `ProductController` | `Page` vs `PagedModel`, a keyset cursor endpoint, bad-sort handling | +| `SqlRecorder` | the SQL capture mechanism | +| `BrokenScrollRepository` (test) | a `String` `@Query` scroll method, which Spring Data rejects | + +| Transcript | Shows | +|---|---| +| `output/01-page-slice-list.txt` | `Page` runs a count query, `Slice` and `List` do not | +| `output/02-slice-boundary.txt` | how `Slice` knows there is a next page (n+1 rows) | +| `output/03-count-query-skipped.txt` | when Spring Data skips the count query | +| `output/04-sorting.txt` | sort handling and unsafe sort values | +| `output/05-offset-vs-keyset-sql.txt` | offset SQL vs keyset SQL | +| `output/06-deep-offset-scan-count.txt` | rows scanned: 10 / 90,010 / 11 | +| `output/07-page-drift-insert.txt` | an insert repeats a row on the next page | +| `output/08-page-drift-delete.txt` | a delete skips a row | +| `output/09-keyset-ties-and-nulls.txt` | ties on the sort column, and the id tie-breaker | +| `output/10-query-scroll-unsupported.txt` | String `@Query` scroll is rejected | +| `output/11-offset-then-keyset-same-method-*.txt` | mixing offset and keyset scroll on one method | +| `output/12-collection-fetch-paging-*.txt` | paging over a collection fetch, per Hibernate version | +| `output/13-pageable-binding.txt` | what Spring MVC makes of `page`, `size`, `sort` | +| `output/14-page-json-shapes.txt` | `Page` vs `PagedModel` JSON | +| `output/15-rest-bad-sort.txt` | bad sort values over HTTP | +| `output/16-keyset-cursor-walk.txt` | walking a dataset with an opaque cursor | +| `output/17-page-json-via-dto.txt` | `serialization-mode=via-dto` | + +There is no `docs/` folder: the explanations live in the post. diff --git a/pagination/output/01-page-slice-list.txt b/pagination/output/01-page-slice-list.txt new file mode 100644 index 0000000..2f2809c --- /dev/null +++ b/pagination/output/01-page-slice-list.txt @@ -0,0 +1,12 @@ +# Page findByCategory("books", PageRequest.of(1, 10, Sort.by("id"))) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only +select count(*) from product p1_0 where p1_0.category=? +statements: 2, rows: 10, totalElements: 25, totalPages: 3, hasNext: true + +# Slice findSliceByCategory("books", PageRequest.of(1, 10, Sort.by("id"))) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only +statements: 1, rows: 10, hasNext: true (no totalElements exists on a Slice) + +# List findListByCategory("books", PageRequest.of(1, 10, Sort.by("id"))) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only +statements: 1, rows: 10 (no total, no hasNext) diff --git a/pagination/output/02-slice-boundary.txt b/pagination/output/02-slice-boundary.txt new file mode 100644 index 0000000..51a8124 --- /dev/null +++ b/pagination/output/02-slice-boundary.txt @@ -0,0 +1,4 @@ +# 5 toys in the table. Does the Slice know whether another slice follows? +slice size 4 -> rows returned: 4, hasNext: true, statements: 1 +slice size 5 -> rows returned: 5, hasNext: false, statements: 1 +slice size 6 -> rows returned: 5, hasNext: false, statements: 1 diff --git a/pagination/output/03-count-query-skipped.txt b/pagination/output/03-count-query-skipped.txt new file mode 100644 index 0000000..6b75f2c --- /dev/null +++ b/pagination/output/03-count-query-skipped.txt @@ -0,0 +1,7 @@ +# Page: how many statements does each request issue? (25 books, 5 toys) +middle page page=1 size=10 rows=10 statements=2 totalElements=25 +first page, partial page=0 size=10 rows=5 statements=1 totalElements=5 +last page, partial page=2 size=10 rows=5 statements=1 totalElements=25 +full page in the middle page=1 size=5 rows=5 statements=2 totalElements=25 +last page, exactly full page=4 size=5 rows=5 statements=2 totalElements=25 +page past the end page=5 size=10 rows=0 statements=2 totalElements=25 diff --git a/pagination/output/04-sorting.txt b/pagination/output/04-sorting.txt new file mode 100644 index 0000000..55b7300 --- /dev/null +++ b/pagination/output/04-sorting.txt @@ -0,0 +1,21 @@ +# derived query, Sort.by("price").descending().and(Sort.by("id")) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.price desc,p1_0.id fetch first ? rows only + +# derived query, unknown property +org.springframework.data.core.PropertyReferenceException +No property 'nope' found for type 'Product'; Did you mean 'name' + +# @Query with Sort.by("price").descending() (alias p is applied for you) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.price desc fetch first ? rows only +select count(*) from product p1_0 where p1_0.category=? + +# a value that is not a property reference at all: Sort.by("price) desc, (select 1") +derived -> InvalidDataAccessApiUsageException +Sort expression 'price) desc, (select 1: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…) +@Query -> InvalidDataAccessApiUsageException +Sort expression 'price) desc, (select 1: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…) + +# @Query with a function-style sort value, three calls in one fresh context +1. Sort.by("length(name)") -> rejected (InvalidDataAccessApiUsageException) +2. JpaSort.unsafe("length(name)") -> accepted: order by character_length(p1_0.name) fetch first ? rows only +3. Sort.by("length(name)") again -> accepted: order by character_length(p1_0.name) fetch first ? rows only diff --git a/pagination/output/05-offset-vs-keyset-sql.txt b/pagination/output/05-offset-vs-keyset-sql.txt new file mode 100644 index 0000000..573353b --- /dev/null +++ b/pagination/output/05-offset-vs-keyset-sql.txt @@ -0,0 +1,23 @@ +# offset paging, third page of 10: PageRequest.of(2, 10, Sort.by("id")) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only + +# keyset scrolling, first window: ScrollPosition.keyset() +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only +rows: 10, hasNext: true, position of the last row: KeysetScrollPosition [FORWARD, {id=10}] + +# keyset scrolling, second window: the position from the first window +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.id>?) order by p1_0.id fetch first ? rows only +rows: 10 (book-11 .. book-20), hasNext: true + +# third window: 5 rows (book-21 .. book-25), hasNext: false + +# WindowIterator over all 25 books, windows of 10 +rows visited: 25, statements: 3 + +# Query by Example: findBy(example, q -> q.sortBy(Sort.by("id")).limit(5).scroll(ScrollPosition.keyset())) +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only +rows: 5, hasNext: true + +# offset scrolling: ScrollPosition.offset(0) is a position, so the result starts AFTER it +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only +first row returned: book-2 diff --git a/pagination/output/06-deep-offset-scan-count.txt b/pagination/output/06-deep-offset-scan-count.txt new file mode 100644 index 0000000..37db20a --- /dev/null +++ b/pagination/output/06-deep-offset-scan-count.txt @@ -0,0 +1,6 @@ +# 100,000 rows in category 'bulk'. H2 EXPLAIN ANALYZE reports scanCount: rows the database read. +offset 0, fetch 10 scanCount: 10 +offset 90000, fetch 10 scanCount: 90,010 +keyset id > 90000, fetch 10 scanCount: 11 + +These are row counts, not timings. Timings depend on your database, indexes and cache. diff --git a/pagination/output/07-page-drift-insert.txt b/pagination/output/07-page-drift-insert.txt new file mode 100644 index 0000000..32135d9 --- /dev/null +++ b/pagination/output/07-page-drift-insert.txt @@ -0,0 +1,7 @@ +# newest-first feed of 25 books, 5 per page; a new book is inserted between the two requests +offset page 0: [book-25, book-24, book-23, book-22, book-21] +keyset window 0: [book-25, book-24, book-23, book-22, book-21] +-- book-NEW inserted -- +offset page 1: [book-21, book-20, book-19, book-18, book-17] +keyset window 1: [book-20, book-19, book-18, book-17, book-16] +offset: book-21 appears on both pages: true diff --git a/pagination/output/08-page-drift-delete.txt b/pagination/output/08-page-drift-delete.txt new file mode 100644 index 0000000..9d21ae4 --- /dev/null +++ b/pagination/output/08-page-drift-delete.txt @@ -0,0 +1,7 @@ +# newest-first feed of 25 books, 5 per page; book-24 is deleted between the two requests +offset page 0: [book-25, book-24, book-23, book-22, book-21] +keyset window 0: [book-25, book-24, book-23, book-22, book-21] +-- book-24 deleted -- +offset page 1: [book-19, book-18, book-17, book-16, book-15] +keyset window 1: [book-20, book-19, book-18, book-17, book-16] +offset skipped book-20: true diff --git a/pagination/output/09-keyset-ties-and-nulls.txt b/pagination/output/09-keyset-ties-and-nulls.txt new file mode 100644 index 0000000..78903e9 --- /dev/null +++ b/pagination/output/09-keyset-ties-and-nulls.txt @@ -0,0 +1,11 @@ +# sorting by price, which repeats: 25 books over 4 distinct prices, pages of 5 +rows visited: 25, distinct rows: 25, statements: 5 +second statement: +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.price>? or p1_0.price=? and p1_0.id>?) order by p1_0.price,p1_0.id fetch first ? rows only +Method 6 names only the price in OrderBy; the id in the statement above was added by Spring Data. + +# nullable sort column (H2 sorts NULLs first for ASC) +full window: [b:null, d:null, a:1.00, c:2.00, e:3.00] +after the first row (price null): [d, a, c, e] +select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where (p1_0.category=?) and (p1_0.price is not null or p1_0.price is null and p1_0.id>?) order by p1_0.price,p1_0.id fetch first ? rows only +That predicate is right where NULLs sort first, as on H2. It was not run on a database that sorts them last. diff --git a/pagination/output/10-query-scroll-unsupported.txt b/pagination/output/10-query-scroll-unsupported.txt new file mode 100644 index 0000000..53c9f8a --- /dev/null +++ b/pagination/output/10-query-scroll-unsupported.txt @@ -0,0 +1,4 @@ +# a String-based @Query method returning Window +org.springframework.data.repository.query.QueryCreationException +Scroll queries are not supported using String-based queries +root cause: Scroll queries are not supported using String-based queries diff --git a/pagination/output/11-offset-then-keyset-same-method-4.0.8.txt b/pagination/output/11-offset-then-keyset-same-method-4.0.8.txt new file mode 100644 index 0000000..cd5c046 --- /dev/null +++ b/pagination/output/11-offset-then-keyset-same-method-4.0.8.txt @@ -0,0 +1,11 @@ +Spring Data JPA: 4.0.8 +# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position) +1. keyset, initial -> book-1..book-10 +2. keyset, after book-10 -> book-11..book-20 +3. offset, initial -> book-1..book-10 +4. offset, after row 10 -> book-11..book-20 +5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20) + statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only + +# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above +keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16 diff --git a/pagination/output/11-offset-then-keyset-same-method-4.1.0.txt b/pagination/output/11-offset-then-keyset-same-method-4.1.0.txt new file mode 100644 index 0000000..224f7e2 --- /dev/null +++ b/pagination/output/11-offset-then-keyset-same-method-4.1.0.txt @@ -0,0 +1,11 @@ +Spring Data JPA: 4.1.0 +# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position) +1. keyset, initial -> book-1..book-10 +2. keyset, after book-10 -> book-11..book-20 +3. offset, initial -> book-1..book-10 +4. offset, after row 10 -> book-11..book-20 +5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20) + statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only + +# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above +keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16 diff --git a/pagination/output/11-offset-then-keyset-same-method-4.1.2.txt b/pagination/output/11-offset-then-keyset-same-method-4.1.2.txt new file mode 100644 index 0000000..3722337 --- /dev/null +++ b/pagination/output/11-offset-then-keyset-same-method-4.1.2.txt @@ -0,0 +1,11 @@ +Spring Data JPA: 4.1.2 +# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc("books", position) +1. keyset, initial -> book-1..book-10 +2. keyset, after book-10 -> book-11..book-20 +3. offset, initial -> book-1..book-10 +4. offset, after row 10 -> book-11..book-20 +5. keyset, after book-10 again -> book-1..book-10 (step 2 returned book-11..book-20) + statement for step 5: select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only + +# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above +keyset initial -> book-25..book-21, keyset after book-21 -> book-20..book-16 diff --git a/pagination/output/12-collection-fetch-paging-7.3.13.Final.txt b/pagination/output/12-collection-fetch-paging-7.3.13.Final.txt new file mode 100644 index 0000000..0ee8ce1 --- /dev/null +++ b/pagination/output/12-collection-fetch-paging-7.3.13.Final.txt @@ -0,0 +1,25 @@ +Hibernate ORM: 7.3.13.Final +# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews. + +## method 8: @Query("select p from Product p join fetch p.reviews where ...") +select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? order by p1_0.id +select count(*) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? +products on the page: [book-1, book-2], reviews loaded on them: 4 +entities held by the persistence context: 18 +HHH90003004 (applying in memory) logged: true +totalElements reported: 12 (products that actually have reviews: 6) + +## method 8b: the same query with countQuery = "select count(distinct p) ..." +select count(distinct p1_0.id) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? +totalElements reported: 6, totalPages: 3 + +## method 8c: @EntityGraph(attributePaths = "reviews") on a derived query, page of 4 +select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 left join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? order by p1_0.id +select count(*) from product p1_0 where p1_0.category=? +rows: 4, totalElements: 25, entities held by the persistence context: 37 + +## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids +select p1_0.id from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only +select count(*) from product p1_0 where p1_0.category=? +select distinct p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.id in (?,?) order by p1_0.id +ids on the page: [1, 2], products fetched: 2, reviews fetched: 4 diff --git a/pagination/output/12-collection-fetch-paging-7.4.13.Final.txt b/pagination/output/12-collection-fetch-paging-7.4.13.Final.txt new file mode 100644 index 0000000..545fdeb --- /dev/null +++ b/pagination/output/12-collection-fetch-paging-7.4.13.Final.txt @@ -0,0 +1,25 @@ +Hibernate ORM: 7.4.13.Final +# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews. + +## method 8: @Query("select p from Product p join fetch p.reviews where ...") +select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from (select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? and exists(select 1 from review r1_0 where p1_0.id=r1_0.product_id) order by p1_0.id offset ? rows fetch first ? rows only) p1_0(id,category,name,price) join review r1_0 on p1_0.id=r1_0.product_id order by p1_0.id +select count(*) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? +products on the page: [book-1, book-2], reviews loaded on them: 4 +entities held by the persistence context: 6 +HHH90003004 (applying in memory) logged: false +totalElements reported: 12 (products that actually have reviews: 6) + +## method 8b: the same query with countQuery = "select count(distinct p) ..." +select count(distinct p1_0.id) from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.category=? +totalElements reported: 6, totalPages: 3 + +## method 8c: @EntityGraph(attributePaths = "reviews") on a derived query, page of 4 +select p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from (select p1_0.id,p1_0.category,p1_0.name,p1_0.price from product p1_0 where p1_0.category=? order by p1_0.id offset ? rows fetch first ? rows only) p1_0(id,category,name,price) left join review r1_0 on p1_0.id=r1_0.product_id order by p1_0.id +select count(*) from product p1_0 where p1_0.category=? +rows: 4, totalElements: 25, entities held by the persistence context: 12 + +## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids +select p1_0.id from product p1_0 where p1_0.category=? order by p1_0.id fetch first ? rows only +select count(*) from product p1_0 where p1_0.category=? +select distinct p1_0.id,p1_0.category,p1_0.name,p1_0.price,r1_0.product_id,r1_0.id,r1_0.rating from product p1_0 join review r1_0 on p1_0.id=r1_0.product_id where p1_0.id in (?,?) order by p1_0.id +ids on the page: [1, 2], products fetched: 2, reviews fetched: 4 diff --git a/pagination/output/13-pageable-binding.txt b/pagination/output/13-pageable-binding.txt new file mode 100644 index 0000000..71947ec --- /dev/null +++ b/pagination/output/13-pageable-binding.txt @@ -0,0 +1,5 @@ +GET /products/echo -> 200 {"sort":"UNSORTED","size":20,"page":0} +GET /products/echo?page=2&size=5&sort=price,desc&sort=id -> 200 {"sort":"price: DESC,id: ASC","size":5,"page":2} +GET /products/echo?size=100000 -> 200 {"sort":"UNSORTED","size":2000,"page":0} +GET /products/echo?page=-3&size=0 -> 200 {"sort":"UNSORTED","size":20,"page":0} +GET /products/echo?page=abc -> 200 {"sort":"UNSORTED","size":20,"page":0} diff --git a/pagination/output/14-page-json-shapes.txt b/pagination/output/14-page-json-shapes.txt new file mode 100644 index 0000000..a9913c9 --- /dev/null +++ b/pagination/output/14-page-json-shapes.txt @@ -0,0 +1,5 @@ +GET /products/raw?size=2 (return type Page) +{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":2,"pageable":{"offset":0,"pageNumber":0,"pageSize":2,"paged":true,"sort":{"empty":true,"sorted":false,"unsorted":true},"unpaged":false},"size":2,"sort":{"empty":true,"sorted":false,"unsorted":true},"totalElements":25,"totalPages":13} + +GET /products/model?size=2 (return type PagedModel) +{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"page":{"size":2,"number":0,"totalElements":25,"totalPages":13}} diff --git a/pagination/output/15-rest-bad-sort.txt b/pagination/output/15-rest-bad-sort.txt new file mode 100644 index 0000000..1026f4b --- /dev/null +++ b/pagination/output/15-rest-bad-sort.txt @@ -0,0 +1,8 @@ +GET /products/raw?size=1&sort=price,desc -> 200 + {"content":[{"id":3,"name":"book-3","price":13.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":1,"pageable":{"offset":0,"pageNumber":0,"pageSize":1,"paged":true,"sort":{"empty":false,"sorted":true,"unsorted":false},"unpaged":false},"size":1,"sort":{"empty":false,"sorted":true,"unsorted":false},"totalElements":25,"totalPages":25} +GET /products/raw?size=1&sort=nope -> 400 + {"detail":"No property 'nope' found for type 'Product'; Did you mean 'name'","instance":"/products/raw","status":400,"title":"Bad Request"} +GET /products/raw?size=1&sort=price;drop -> 400 + {"detail":"Sort expression 'price;drop: ASC' must only contain property references or aliases used in the select clause; If you really want to use something other than that for sorting, please use JpaSort.unsafe(…)","instance":"/products/raw","status":400,"title":"Bad Request"} +GET /products/raw?size=1&sort=,desc -> 200 + {"content":[{"id":1,"name":"book-1","price":11.00}],"empty":false,"first":true,"last":false,"number":0,"numberOfElements":1,"pageable":{"offset":0,"pageNumber":0,"pageSize":1,"paged":true,"sort":{"empty":true,"sorted":false,"unsorted":true},"unpaged":false},"size":1,"sort":{"empty":true,"sorted":false,"unsorted":true},"totalElements":25,"totalPages":25} diff --git a/pagination/output/16-keyset-cursor-walk.txt b/pagination/output/16-keyset-cursor-walk.txt new file mode 100644 index 0000000..8e5c21d --- /dev/null +++ b/pagination/output/16-keyset-cursor-walk.txt @@ -0,0 +1,4 @@ +GET /products/scroll -> 10 items, nextCursor="MTA" +GET /products/scroll?after=MTA -> 10 items, nextCursor="MjA" +GET /products/scroll?after=MjA -> 5 items, nextCursor=null +calls: 3, items: 25, distinct: 25 diff --git a/pagination/output/17-page-json-via-dto.txt b/pagination/output/17-page-json-via-dto.txt new file mode 100644 index 0000000..b9fe90e --- /dev/null +++ b/pagination/output/17-page-json-via-dto.txt @@ -0,0 +1,3 @@ +spring.data.web.pageable.serialization-mode=via-dto +GET /products/raw?size=2 (return type Page) +{"content":[{"id":1,"name":"book-1","price":11.00},{"id":2,"name":"book-2","price":12.00}],"page":{"size":2,"number":0,"totalElements":25,"totalPages":13}} diff --git a/pagination/pom.xml b/pagination/pom.xml new file mode 100644 index 0000000..2074c09 --- /dev/null +++ b/pagination/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.2 + + + com.ankurm + pagination-demo + 1.0.0 + jar + + + 25 + 25 + + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-webmvc + + + com.h2database + h2 + runtime + + + org.springframework.boot + spring-boot-starter-data-jpa-test + test + + + org.springframework.boot + spring-boot-starter-webmvc-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/pagination/scripts/old-hibernate.sh b/pagination/scripts/old-hibernate.sh new file mode 100755 index 0000000..bb8cb2c --- /dev/null +++ b/pagination/scripts/old-hibernate.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Re-runs the collection-fetch paging test on Hibernate 7.3.13.Final, the last 7.3 patch release, to reproduce +# the HHH90003004 "applying in memory" behaviour that Hibernate 7.4 no longer has. Writes +# output/12-collection-fetch-paging-7.3.13.Final.txt next to the 7.4 one written by the normal run. +set -euo pipefail +cd "$(dirname "$0")/.." +mvn -B -q test -Dtest=CollectionFetchPagingTest -Dhibernate.version=7.3.13.Final +cat output/12-collection-fetch-paging-7.3.13.Final.txt diff --git a/pagination/scripts/other-spring-data.sh b/pagination/scripts/other-spring-data.sh new file mode 100755 index 0000000..31309fd --- /dev/null +++ b/pagination/scripts/other-spring-data.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Replays the "offset then keyset on one method" finding on two other Spring Data release trains. +# Each run writes output/11-offset-then-keyset-same-method-.txt. +set -euo pipefail +cd "$(dirname "$0")/.." +for bom in 2025.1.8 2026.0.0; do + mvn -B -q test -Dtest=KeysetScrollingTest#mixingOffsetAndKeysetOnOneMethod -Dspring-data-bom.version="$bom" +done +ls output/11-* diff --git a/pagination/scripts/run-all.sh b/pagination/scripts/run-all.sh new file mode 100755 index 0000000..36cb3c4 --- /dev/null +++ b/pagination/scripts/run-all.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Regenerates every file in output/ by running the test suite (in-memory H2, no Docker needed). +# The two "other version" scripts add the Hibernate 7.3.13 and extra Spring Data transcripts. +set -euo pipefail +cd "$(dirname "$0")/.." +rm -f output/*.txt +mvn -B -q test +./scripts/old-hibernate.sh >/dev/null +./scripts/other-spring-data.sh >/dev/null +echo "wrote:"; ls output diff --git a/pagination/src/main/java/com/ankurm/pagination/PaginationApplication.java b/pagination/src/main/java/com/ankurm/pagination/PaginationApplication.java new file mode 100644 index 0000000..78c495e --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/PaginationApplication.java @@ -0,0 +1,11 @@ +package com.ankurm.pagination; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class PaginationApplication { + public static void main(String[] args) { + SpringApplication.run(PaginationApplication.class, args); + } +} diff --git a/pagination/src/main/java/com/ankurm/pagination/Product.java b/pagination/src/main/java/com/ankurm/pagination/Product.java new file mode 100644 index 0000000..6d60d60 --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/Product.java @@ -0,0 +1,40 @@ +package com.ankurm.pagination; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.OneToMany; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** A product in a catalogue. Prices repeat on purpose: ties are what break naive keyset scrolling. */ +@Entity +public class Product { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String name; + private String category; + private BigDecimal price; + + @OneToMany(mappedBy = "product") + private List reviews = new ArrayList<>(); + + protected Product() {} + + public Product(String name, String category, BigDecimal price) { + this.name = name; + this.category = category; + this.price = price; + } + + public Long getId() { return id; } + public String getName() { return name; } + public String getCategory() { return category; } + public BigDecimal getPrice() { return price; } + public List getReviews() { return reviews; } +} diff --git a/pagination/src/main/java/com/ankurm/pagination/ProductController.java b/pagination/src/main/java/com/ankurm/pagination/ProductController.java new file mode 100644 index 0000000..8e8668e --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/ProductController.java @@ -0,0 +1,88 @@ +package com.ankurm.pagination; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.core.PropertyReferenceException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Window; +import org.springframework.data.web.PagedModel; +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** The REST shapes the post compares. Entities are never returned directly: they carry lazy collections. */ +@RestController +@RequestMapping("/products") +public class ProductController { + + /** What the API exposes about a product. */ + public record ProductView(Long id, String name, BigDecimal price) { + static ProductView of(Product p) { return new ProductView(p.getId(), p.getName(), p.getPrice()); } + } + + /** Keyset response: the items plus an opaque cursor, and no total. */ + public record Scrolled(List items, String nextCursor) {} + + private final ProductRepository repo; + + public ProductController(ProductRepository repo) { this.repo = repo; } + + /** 1. Returning a Page directly: Jackson serialises the PageImpl class itself. */ + @GetMapping("/raw") + public Page raw(@RequestParam(defaultValue = "books") String category, Pageable pageable) { + return repo.findByCategory(category, pageable).map(ProductView::of); + } + + /** 2. Wrapping the Page in PagedModel: a documented, stable JSON shape. */ + @GetMapping("/model") + public PagedModel model(@RequestParam(defaultValue = "books") String category, Pageable pageable) { + return new PagedModel<>(repo.findByCategory(category, pageable).map(ProductView::of)); + } + + /** 3. Keyset over HTTP: the client sends back the cursor it was given, never a page number. */ + @GetMapping("/scroll") + public Scrolled scroll(@RequestParam(defaultValue = "books") String category, + @RequestParam(required = false) String after) { + ScrollPosition position = after == null ? ScrollPosition.keyset() : decode(after); + Window window = repo.findFirst10ByCategoryOrderByIdAsc(category, position); + String next = window.hasNext() ? encode(window.positionAt(window.size() - 1)) : null; + return new Scrolled(window.getContent().stream().map(ProductView::of).toList(), next); + } + + /** Shows what Spring MVC resolved from the query string, before any repository is involved. */ + @GetMapping("/echo") + public Map echo(Pageable pageable) { + return Map.of("page", pageable.getPageNumber(), "size", pageable.getPageSize(), "sort", pageable.getSort().toString()); + } + + /** + * A bad sort value becomes a 400 instead of a 500. Two different exceptions can arrive here: + * an unknown property (PropertyReferenceException) and a value that is not a property reference + * at all (InvalidDataAccessApiUsageException). The second is broad, so a real application would + * validate the sort against an allow-list before calling the repository. + */ + @ExceptionHandler({PropertyReferenceException.class, InvalidDataAccessApiUsageException.class}) + public ProblemDetail badSort(RuntimeException e) { + return ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, e.getMessage()); + } + + private static String encode(ScrollPosition position) { + Object id = ((org.springframework.data.domain.KeysetScrollPosition) position).getKeys().get("id"); + return Base64.getUrlEncoder().withoutPadding().encodeToString(String.valueOf(id).getBytes(StandardCharsets.UTF_8)); + } + + private static ScrollPosition decode(String cursor) { + long id = Long.parseLong(new String(Base64.getUrlDecoder().decode(cursor), StandardCharsets.UTF_8)); + return ScrollPosition.forward(Map.of("id", id)); + } +} diff --git a/pagination/src/main/java/com/ankurm/pagination/ProductRepository.java b/pagination/src/main/java/com/ankurm/pagination/ProductRepository.java new file mode 100644 index 0000000..33c6b1e --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/ProductRepository.java @@ -0,0 +1,62 @@ +package com.ankurm.pagination; + +import java.util.Collection; +import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Slice; +import org.springframework.data.domain.Window; +import org.springframework.data.jpa.repository.EntityGraph; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +/** Every pagination form the post talks about, numbered so the post can point at them. */ +public interface ProductRepository extends JpaRepository { + + // 1. Page: the data query plus (usually) a count query. + Page findByCategory(String category, Pageable pageable); + + // 2. Slice: the data query only, fetching one extra row to learn whether a next slice exists. + Slice findSliceByCategory(String category, Pageable pageable); + + // 3. List with a Pageable: just the rows, no total, no "has next". + List findListByCategory(String category, Pageable pageable); + + // 4. Keyset (or offset) scrolling: the method name carries the page size, the argument carries the position. + Window findFirst10ByCategoryOrderByIdAsc(String category, ScrollPosition position); + + // 5. Keyset scrolling on a NON-unique column, WITH an id tie-breaker. + Window findFirst5ByCategoryOrderByPriceAscIdAsc(String category, ScrollPosition position); + + // 6. Keyset scrolling on a NON-unique column, WITHOUT a tie-breaker (the broken one). + Window findFirst5ByCategoryOrderByPriceAsc(String category, ScrollPosition position); + + // 7. @Query with a Pageable: sorting is applied to the alias you chose. + @Query("select p from Product p where p.category = :category") + Page queryByCategory(@Param("category") String category, Pageable pageable); + + // 8. The trap: a collection join fetch combined with paging. + @Query("select p from Product p join fetch p.reviews where p.category = :category") + Page withReviewsInMemoryPaging(@Param("category") String category, Pageable pageable); + + // 8b. Same query, but with an explicit count query that counts parents, not joined rows. + @Query(value = "select p from Product p join fetch p.reviews where p.category = :category", + countQuery = "select count(distinct p) from Product p join p.reviews where p.category = :category") + Page withReviewsCorrectCount(@Param("category") String category, Pageable pageable); + + // 8c. An entity graph does the same fetch without a hand-written query. + @EntityGraph(attributePaths = "reviews") + Page findGraphByCategory(String category, Pageable pageable); + + // 9a/9b. The fix for old Hibernate versions: page the ids first, then fetch the collection for exactly those ids. + @Query("select p.id from Product p where p.category = :category") + Page idsByCategory(@Param("category") String category, Pageable pageable); + + @Query("select distinct p from Product p join fetch p.reviews where p.id in :ids order by p.id") + List fetchWithReviews(@Param("ids") Collection ids); + + // 10. Newest-first feed, keyset style (used by the drift demo). + Window findFirst5ByCategoryOrderByIdDesc(String category, ScrollPosition position); +} diff --git a/pagination/src/main/java/com/ankurm/pagination/Review.java b/pagination/src/main/java/com/ankurm/pagination/Review.java new file mode 100644 index 0000000..ce7960f --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/Review.java @@ -0,0 +1,33 @@ +package com.ankurm.pagination; + +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; + +/** A review of a product. Exists to demonstrate what a collection fetch does to paging. */ +@Entity +public class Review { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + private Product product; + + private int rating; + + protected Review() {} + + public Review(Product product, int rating) { + this.product = product; + this.rating = rating; + } + + public Long getId() { return id; } + public Product getProduct() { return product; } + public int getRating() { return rating; } +} diff --git a/pagination/src/main/java/com/ankurm/pagination/SqlRecorder.java b/pagination/src/main/java/com/ankurm/pagination/SqlRecorder.java new file mode 100644 index 0000000..e50de31 --- /dev/null +++ b/pagination/src/main/java/com/ankurm/pagination/SqlRecorder.java @@ -0,0 +1,24 @@ +package com.ankurm.pagination; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.hibernate.resource.jdbc.spi.StatementInspector; + +/** + * Records every SQL string Hibernate is about to send to the database. + * Registered through hibernate.session_factory.statement_inspector in application.properties. + * Hibernate instantiates it itself, so the captured list has to be static. + */ +public class SqlRecorder implements StatementInspector { + private static final List CAPTURED = new CopyOnWriteArrayList<>(); + + @Override + public String inspect(String sql) { + CAPTURED.add(sql); + return sql; + } + + public static void clear() { CAPTURED.clear(); } + + public static List captured() { return List.copyOf(CAPTURED); } +} diff --git a/pagination/src/main/resources/application.properties b/pagination/src/main/resources/application.properties new file mode 100644 index 0000000..921c6e8 --- /dev/null +++ b/pagination/src/main/resources/application.properties @@ -0,0 +1,5 @@ +spring.datasource.url=jdbc:h2:mem:pagination;DB_CLOSE_DELAY=-1 +spring.jpa.hibernate.ddl-auto=create-drop +spring.jpa.open-in-view=false +# Every SQL statement is captured by SqlRecorder (a Hibernate StatementInspector), not by show_sql. +spring.jpa.properties.hibernate.session_factory.statement_inspector=com.ankurm.pagination.SqlRecorder diff --git a/pagination/src/test/java/com/ankurm/brokenrepo/BrokenScrollRepository.java b/pagination/src/test/java/com/ankurm/brokenrepo/BrokenScrollRepository.java new file mode 100644 index 0000000..eddd72d --- /dev/null +++ b/pagination/src/test/java/com/ankurm/brokenrepo/BrokenScrollRepository.java @@ -0,0 +1,19 @@ +package com.ankurm.brokenrepo; + +import com.ankurm.pagination.Product; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Window; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.Repository; +import org.springframework.data.repository.query.Param; + +/** + * Deliberately broken: a String-based @Query that returns a Window. + * It lives outside the com.ankurm.pagination package so Spring Boot does not try to register it at startup; + * KeysetScrollingTest builds it by hand and records the real error. + */ +public interface BrokenScrollRepository extends Repository { + + @Query("select p from Product p where p.category = :category order by p.id") + Window scrollByQuery(@Param("category") String category, ScrollPosition position); +} diff --git a/pagination/src/test/java/com/ankurm/pagination/CollectionFetchPagingTest.java b/pagination/src/test/java/com/ankurm/pagination/CollectionFetchPagingTest.java new file mode 100644 index 0000000..510c013 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/CollectionFetchPagingTest.java @@ -0,0 +1,109 @@ +package com.ankurm.pagination; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.persistence.EntityManager; +import java.util.List; +import org.hibernate.Session; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.test.annotation.DirtiesContext; + +/** + * Paging over a collection fetch. The transcript file name carries the Hibernate version that ran, because the + * behaviour differs between 7.3 and 7.4: scripts/old-hibernate.sh re-runs this class on 7.3.13.Final. + */ +@DataJpaTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +@ExtendWith(OutputCaptureExtension.class) +class CollectionFetchPagingTest { + + @Autowired ProductRepository repo; + @Autowired EntityManager em; + + @BeforeEach + void seed() { + TestData.catalogue(em); + TestData.reviews(em); + } + + private long managed() { return em.unwrap(Session.class).getStatistics().getEntityCount(); } + + private List sqlThenReset() { + List sql = SqlRecorder.captured(); + SqlRecorder.clear(); + em.clear(); + return sql; + } + + @Test + void pagingOverACollectionFetch(CapturedOutput out) { + String hibernate = org.hibernate.Version.getVersionString(); + boolean pagedInSql = Integer.parseInt(hibernate.split("\\.")[1]) >= 4; + Transcript t = new Transcript("12-collection-fetch-paging-" + hibernate + ".txt"); + t.line("Hibernate ORM: %s", hibernate); + t.line("# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews."); + + Page page = repo.withReviewsInMemoryPaging("books", PageRequest.of(0, 2, Sort.by("id"))); + long entities = managed(); + List sql = sqlThenReset(); + boolean warned = out.getAll().contains("HHH90003004"); + t.line(""); + t.line("## method 8: @Query(\"select p from Product p join fetch p.reviews where ...\")"); + sql.forEach(t::line); + t.line("products on the page: %s, reviews loaded on them: %d", + page.getContent().stream().map(Product::getName).toList(), + page.getContent().stream().mapToInt(p -> p.getReviews().size()).sum()); + t.line("entities held by the persistence context: %d", entities); + t.line("HHH90003004 (applying in memory) logged: %s", warned); + t.line("totalElements reported: %d (products that actually have reviews: 6)", page.getTotalElements()); + assertThat(page.getContent()).hasSize(2); + assertThat(page.getTotalElements()).as("the derived count query counts joined rows").isEqualTo(12); + if (pagedInSql) { + assertThat(sql.get(0)).contains("offset ? rows fetch first ? rows only"); + assertThat(entities).isEqualTo(6); + assertThat(warned).isFalse(); + } else { + assertThat(sql.get(0)).doesNotContain("fetch first"); + assertThat(entities).isEqualTo(18); + assertThat(warned).isTrue(); + } + + Page counted = repo.withReviewsCorrectCount("books", PageRequest.of(0, 2, Sort.by("id"))); + List countedSql = sqlThenReset(); + t.line(""); + t.line("## method 8b: the same query with countQuery = \"select count(distinct p) ...\""); + t.line(countedSql.get(1)); + t.line("totalElements reported: %d, totalPages: %d", counted.getTotalElements(), counted.getTotalPages()); + assertThat(counted.getTotalElements()).isEqualTo(6); + + Page graph = repo.findGraphByCategory("books", PageRequest.of(0, 4, Sort.by("id"))); + long graphEntities = managed(); + List graphSql = sqlThenReset(); + t.line(""); + t.line("## method 8c: @EntityGraph(attributePaths = \"reviews\") on a derived query, page of 4"); + graphSql.forEach(t::line); + t.line("rows: %d, totalElements: %d, entities held by the persistence context: %d", + graph.getContent().size(), graph.getTotalElements(), graphEntities); + assertThat(graph.getTotalElements()).as("the count query ignores the fetch").isEqualTo(25); + + Page ids = repo.idsByCategory("books", PageRequest.of(0, 2, Sort.by("id"))); + List withReviews = repo.fetchWithReviews(ids.getContent()); + List twoStepSql = sqlThenReset(); + t.line(""); + t.line("## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids"); + twoStepSql.forEach(t::line); + t.line("ids on the page: %s, products fetched: %d, reviews fetched: %d", ids.getContent(), withReviews.size(), + withReviews.stream().mapToInt(p -> p.getReviews().size()).sum()); + assertThat(withReviews).hasSize(2); + t.write(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/KeysetScrollingTest.java b/pagination/src/test/java/com/ankurm/pagination/KeysetScrollingTest.java new file mode 100644 index 0000000..316f772 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/KeysetScrollingTest.java @@ -0,0 +1,329 @@ +package com.ankurm.pagination; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.ankurm.brokenrepo.BrokenScrollRepository; +import jakarta.persistence.EntityManager; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; +import org.springframework.data.domain.Example; +import org.springframework.data.domain.ExampleMatcher; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Sort; +import org.springframework.data.domain.Window; +import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; +import org.springframework.data.support.WindowIterator; +import org.springframework.test.annotation.DirtiesContext; + +/** Offset versus keyset scrolling. Every test writes output/NN-*.txt while asserting the same facts. */ +@DataJpaTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +class KeysetScrollingTest { + + @Autowired ProductRepository repo; + @Autowired EntityManager em; + + @BeforeEach + void seed() { TestData.catalogue(em); } + + private List sqlThenReset() { + List sql = SqlRecorder.captured(); + SqlRecorder.clear(); + em.clear(); + return sql; + } + + private static List names(Iterable products) { + List out = new ArrayList<>(); + products.forEach(p -> out.add(p.getName())); + return out; + } + + // ---- 05: the SQL of offset paging versus keyset scrolling ---------------------------- + + @Test + void offsetVersusKeysetSql() { + Transcript t = new Transcript("05-offset-vs-keyset-sql.txt"); + + repo.findListByCategory("books", PageRequest.of(2, 10, Sort.by("id"))); + t.line("# offset paging, third page of 10: PageRequest.of(2, 10, Sort.by(\"id\"))"); + List offsetSql = sqlThenReset(); + offsetSql.forEach(t::line); + assertThat(offsetSql.get(0)).contains("offset ? rows fetch first ? rows only"); + + Window w1 = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.keyset()); + List sql1 = sqlThenReset(); + t.line(""); + t.line("# keyset scrolling, first window: ScrollPosition.keyset()"); + sql1.forEach(t::line); + t.line("rows: %d, hasNext: %s, position of the last row: %s", w1.size(), w1.hasNext(), w1.positionAt(w1.size() - 1)); + assertThat(sql1.get(0)).doesNotContain("offset").doesNotContain("id>?"); + + Window w2 = repo.findFirst10ByCategoryOrderByIdAsc("books", w1.positionAt(w1.size() - 1)); + List sql2 = sqlThenReset(); + t.line(""); + t.line("# keyset scrolling, second window: the position from the first window"); + sql2.forEach(t::line); + t.line("rows: %d (%s .. %s), hasNext: %s", w2.size(), w2.getContent().get(0).getName(), + w2.getContent().get(w2.size() - 1).getName(), w2.hasNext()); + assertThat(sql2.get(0)).contains("p1_0.id>?").doesNotContain("offset"); + assertThat(w2.getContent().get(0).getName()).isEqualTo("book-11"); + + Window w3 = repo.findFirst10ByCategoryOrderByIdAsc("books", w2.positionAt(w2.size() - 1)); + sqlThenReset(); + t.line(""); + t.line("# third window: %d rows (%s .. %s), hasNext: %s", w3.size(), w3.getContent().get(0).getName(), + w3.getContent().get(w3.size() - 1).getName(), w3.hasNext()); + assertThat(w3.hasNext()).isFalse(); + + WindowIterator it = WindowIterator + .of(position -> repo.findFirst10ByCategoryOrderByIdAsc("books", position)) + .startingAt(ScrollPosition.keyset()); + int rows = 0; + while (it.hasNext()) { it.next(); rows++; } + List itSql = sqlThenReset(); + t.line(""); + t.line("# WindowIterator over all 25 books, windows of 10"); + t.line("rows visited: %d, statements: %d", rows, itSql.size()); + assertThat(rows).isEqualTo(25); + assertThat(itSql).hasSize(3); + + Product probe = new Product(null, "books", null); + Window fluent = repo.findBy(Example.of(probe, ExampleMatcher.matching().withIgnorePaths("price")), + q -> q.sortBy(Sort.by("id")).limit(5).scroll(ScrollPosition.keyset())); + List fsql = sqlThenReset(); + t.line(""); + t.line("# Query by Example: findBy(example, q -> q.sortBy(Sort.by(\"id\")).limit(5).scroll(ScrollPosition.keyset()))"); + fsql.forEach(t::line); + t.line("rows: %d, hasNext: %s", fluent.size(), fluent.hasNext()); + assertThat(fluent.size()).isEqualTo(5); + + // Last on purpose: after an offset call, keyset scrolling on the SAME method misbehaves (see test mixingOffsetAndKeysetOnOneMethod). + Window o = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.offset(0)); + List osql = sqlThenReset(); + t.line(""); + t.line("# offset scrolling: ScrollPosition.offset(0) is a position, so the result starts AFTER it"); + osql.forEach(t::line); + t.line("first row returned: %s", o.getContent().get(0).getName()); + assertThat(o.getContent().get(0).getName()).isEqualTo("book-2"); + + t.write(); + } + + // ---- 06: what a deep offset costs ---------------------------------------------------- + + @Test + void deepOffsetVersusKeysetScanCount() { + em.createQuery("delete from Product").executeUpdate(); // only bulk rows, so scanCount is not padded by the seed + em.createNativeQuery("insert into product(category, name, price) select 'bulk', 'p' || x, 10 from system_range(1, 100000)") + .executeUpdate(); + em.flush(); + em.clear(); + + Transcript t = new Transcript("06-deep-offset-scan-count.txt"); + t.line("# 100,000 rows in category 'bulk'. H2 EXPLAIN ANALYZE reports scanCount: rows the database read."); + record Probe(String label, String sql) {} + List probes = List.of( + new Probe("offset 0, fetch 10", + "explain analyze select id from product where category='bulk' order by id offset 0 rows fetch first 10 rows only"), + new Probe("offset 90000, fetch 10", + "explain analyze select id from product where category='bulk' order by id offset 90000 rows fetch first 10 rows only"), + new Probe("keyset id > 90000, fetch 10", + "explain analyze select id from product where category='bulk' and id>90000 order by id fetch first 10 rows only")); + Pattern scan = Pattern.compile("scanCount: (\\d+)"); + int[] counts = new int[3]; + for (int i = 0; i < probes.size(); i++) { + Object plan = em.createNativeQuery(probes.get(i).sql()).getSingleResult(); + Matcher m = scan.matcher(String.valueOf(plan)); + assertThat(m.find()).isTrue(); + counts[i] = Integer.parseInt(m.group(1)); + t.line("%-28s scanCount: %,d", probes.get(i).label(), counts[i]); + } + assertThat(counts[0]).isEqualTo(10); + assertThat(counts[1]).isEqualTo(90_010); + assertThat(counts[2]).isEqualTo(11); + t.line(""); + t.line("These are row counts, not timings. Timings depend on your database, indexes and cache."); + t.write(); + } + + // ---- 07 / 08: page drift ------------------------------------------------------------- + + @Test + void driftWhenARowIsInserted() { + Transcript t = new Transcript("07-page-drift-insert.txt"); + t.line("# newest-first feed of 25 books, 5 per page; a new book is inserted between the two requests"); + + Page o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id"))); + Window k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset()); + t.line("offset page 0: %s", names(o0.getContent())); + t.line("keyset window 0: %s", names(k0.getContent())); + assertThat(names(o0.getContent())).isEqualTo(names(k0.getContent())); + + em.persist(new Product("book-NEW", "books", BigDecimal.TEN)); + em.flush(); + em.clear(); + t.line("-- book-NEW inserted --"); + + Page o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id"))); + Window k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1)); + t.line("offset page 1: %s", names(o1.getContent())); + t.line("keyset window 1: %s", names(k1.getContent())); + List seen = new ArrayList<>(names(o0.getContent())); + seen.addAll(names(o1.getContent())); + t.line("offset: book-21 appears on both pages: %s", seen.stream().filter("book-21"::equals).count() == 2); + assertThat(names(o1.getContent()).get(0)).isEqualTo("book-21"); + assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20"); + t.write(); + } + + @Test + void driftWhenARowIsDeleted() { + Transcript t = new Transcript("08-page-drift-delete.txt"); + t.line("# newest-first feed of 25 books, 5 per page; book-24 is deleted between the two requests"); + + Page o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id"))); + Window k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset()); + t.line("offset page 0: %s", names(o0.getContent())); + t.line("keyset window 0: %s", names(k0.getContent())); + + em.createQuery("delete from Product p where p.name = 'book-24'").executeUpdate(); + em.clear(); + t.line("-- book-24 deleted --"); + + Page o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id"))); + Window k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1)); + t.line("offset page 1: %s", names(o1.getContent())); + t.line("keyset window 1: %s", names(k1.getContent())); + t.line("offset skipped book-20: %s", !names(o1.getContent()).contains("book-20")); + assertThat(names(o1.getContent())).doesNotContain("book-20"); + assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20"); + t.write(); + } + + // ---- 09: ties and nulls -------------------------------------------------------------- + + @Test + void ties() { + Transcript t = new Transcript("09-keyset-ties-and-nulls.txt"); + t.line("# sorting by price, which repeats: 25 books over 4 distinct prices, pages of 5"); + WindowIterator it = WindowIterator + .of(position -> repo.findFirst5ByCategoryOrderByPriceAsc("books", position)) + .startingAt(ScrollPosition.keyset()); + List all = new ArrayList<>(); + while (it.hasNext()) all.add(it.next().getName()); + List sql = sqlThenReset(); + t.line("rows visited: %d, distinct rows: %d, statements: %d", all.size(), new HashSet<>(all).size(), sql.size()); + t.line("second statement:"); + t.line(sql.get(1)); + assertThat(all).hasSize(25); + assertThat(new HashSet<>(all)).hasSize(25); + assertThat(sql.get(1)).contains("p1_0.price>? or p1_0.price=? and p1_0.id>?"); + t.line("Method 6 names only the price in OrderBy; the id in the statement above was added by Spring Data."); + + t.line(""); + t.line("# nullable sort column (H2 sorts NULLs first for ASC)"); + em.persist(new Product("a", "mixed", new BigDecimal("1"))); + em.persist(new Product("b", "mixed", null)); + em.persist(new Product("c", "mixed", new BigDecimal("2"))); + em.persist(new Product("d", "mixed", null)); + em.persist(new Product("e", "mixed", new BigDecimal("3"))); + em.flush(); + em.clear(); + SqlRecorder.clear(); + Window all5 = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", ScrollPosition.keyset()); + sqlThenReset(); + t.line("full window: %s", all5.getContent().stream().map(p -> p.getName() + ":" + p.getPrice()).toList()); + Window afterB = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", all5.positionAt(0)); + List afterSql = sqlThenReset(); + t.line("after the first row (price null): %s", names(afterB.getContent())); + t.line(afterSql.get(0)); + assertThat(names(afterB.getContent())).containsExactly("d", "a", "c", "e"); + assertThat(afterSql.get(0)).contains("p1_0.price is not null or p1_0.price is null and p1_0.id>?"); + t.line("That predicate is right where NULLs sort first, as on H2. It was not run on a database that sorts them last."); + t.write(); + } + + // ---- 10: @Query cannot scroll -------------------------------------------------------- + + @Test + void stringQueryCannotScroll() { + Transcript t = new Transcript("10-query-scroll-unsupported.txt"); + t.line("# a String-based @Query method returning Window"); + try { + new JpaRepositoryFactory(em).getRepository(BrokenScrollRepository.class); + throw new AssertionError("expected the repository to be rejected"); + } catch (RuntimeException e) { + t.line(e.getClass().getName()); + t.line(e.getMessage()); + Throwable root = e; + while (root.getCause() != null) root = root.getCause(); + t.line("root cause: %s", root.getMessage()); + assertThat(root.getMessage()).contains("Scroll queries are not supported using String-based queries"); + } + t.write(); + } + + // ---- 11: mixing offset and keyset on one method -------------------------------------- + + /** + * Observed on Spring Data JPA 4.1.2 (and on the 2025.1.8 and 2026.0.0 release trains, see scripts/other-spring-data.sh): + * once a query method has been called with an OFFSET scroll position, later calls with a non-initial KEYSET + * position on that same method are silently run without the keyset predicate. If this test starts failing after an + * upgrade, the behaviour has been fixed and the post should be updated. + */ + @Test + void mixingOffsetAndKeysetOnOneMethod() { + String jpa = org.springframework.data.jpa.repository.JpaRepository.class.getPackage().getImplementationVersion(); + Transcript t = new Transcript("11-offset-then-keyset-same-method-" + jpa + ".txt"); + t.line("Spring Data JPA: %s", jpa); + t.line("# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc(\"books\", position)"); + + Window k1 = window(ScrollPosition.keyset()); + t.line("1. keyset, initial -> %s", range(k1)); + Window k2 = window(k1.positionAt(9)); + t.line("2. keyset, after book-10 -> %s", range(k2)); + Window o1 = window(ScrollPosition.offset()); + t.line("3. offset, initial -> %s", range(o1)); + Window o2 = window(o1.positionAt(9)); + t.line("4. offset, after row 10 -> %s", range(o2)); + SqlRecorder.clear(); + Window k3 = window(k1.positionAt(9)); + List sql = sqlThenReset(); + t.line("5. keyset, after book-10 again -> %s (step 2 returned %s)", range(k3), range(k2)); + t.line(" statement for step 5: %s", sql.get(0)); + + Window d1 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset()); + Window d2 = repo.findFirst5ByCategoryOrderByIdDesc("books", d1.positionAt(4)); + t.line(""); + t.line("# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above"); + t.line("keyset initial -> %s, keyset after book-21 -> %s", range(d1), range(d2)); + + assertThat(range(k2)).isEqualTo("book-11..book-20"); + assertThat(range(o2)).isEqualTo("book-11..book-20"); + assertThat(range(k3)).isEqualTo("book-1..book-10"); + assertThat(sql.get(0)).doesNotContain("p1_0.id>?"); + assertThat(range(d2)).isEqualTo("book-20..book-16"); + t.write(); + } + + private Window window(ScrollPosition position) { + Window w = repo.findFirst10ByCategoryOrderByIdAsc("books", position); + em.clear(); + return w; + } + + private static String range(Window w) { + return w.getContent().get(0).getName() + ".." + w.getContent().get(w.size() - 1).getName(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/PaginationBasicsTest.java b/pagination/src/test/java/com/ankurm/pagination/PaginationBasicsTest.java new file mode 100644 index 0000000..3b660a4 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/PaginationBasicsTest.java @@ -0,0 +1,174 @@ +package com.ankurm.pagination; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.persistence.EntityManager; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.core.PropertyReferenceException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.JpaSort; +import org.springframework.test.annotation.DirtiesContext; + +/** Page, Slice, List, the count query, and sorting. Every test writes output/NN-*.txt while asserting the same facts. */ +@DataJpaTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +class PaginationBasicsTest { + + @Autowired ProductRepository repo; + @Autowired EntityManager em; + + @BeforeEach + void seed() { TestData.catalogue(em); } + + private List sqlThenReset() { + List sql = SqlRecorder.captured(); + SqlRecorder.clear(); + em.clear(); + return sql; + } + + private String tryQuerySort(Sort sort) { + try { + repo.queryByCategory("books", PageRequest.of(0, 3, sort)); + String sql = SqlRecorder.captured().get(0); + sqlThenReset(); + return "accepted: " + sql.substring(sql.indexOf(" order by ")); + } catch (InvalidDataAccessApiUsageException e) { + sqlThenReset(); + return "rejected (" + e.getClass().getSimpleName() + ")"; + } + } + + // ---- 01: the three return types ------------------------------------------------------ + + @Test + void pageSliceAndList() { + Transcript t = new Transcript("01-page-slice-list.txt"); + Pageable second = PageRequest.of(1, 10, Sort.by("id")); + + Page page = repo.findByCategory("books", second); + List pageSql = sqlThenReset(); + t.line("# Page findByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))"); + pageSql.forEach(t::line); + t.line("statements: %d, rows: %d, totalElements: %d, totalPages: %d, hasNext: %s", + pageSql.size(), page.getContent().size(), page.getTotalElements(), page.getTotalPages(), page.hasNext()); + assertThat(pageSql).hasSize(2); + assertThat(pageSql.get(1)).startsWith("select count(*)"); + assertThat(page.getTotalElements()).isEqualTo(25); + assertThat(page.getTotalPages()).isEqualTo(3); + + Slice slice = repo.findSliceByCategory("books", second); + List sliceSql = sqlThenReset(); + t.line(""); + t.line("# Slice findSliceByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))"); + sliceSql.forEach(t::line); + t.line("statements: %d, rows: %d, hasNext: %s (no totalElements exists on a Slice)", + sliceSql.size(), slice.getContent().size(), slice.hasNext()); + assertThat(sliceSql).hasSize(1); + assertThat(slice.hasNext()).isTrue(); + + List list = repo.findListByCategory("books", second); + List listSql = sqlThenReset(); + t.line(""); + t.line("# List findListByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))"); + listSql.forEach(t::line); + t.line("statements: %d, rows: %d (no total, no hasNext)", listSql.size(), list.size()); + assertThat(listSql).hasSize(1); + t.write(); + } + + @Test + void sliceKnowsHasNextByFetchingOneExtraRow() { + Transcript t = new Transcript("02-slice-boundary.txt"); + t.line("# 5 toys in the table. Does the Slice know whether another slice follows?"); + for (int size : new int[] {4, 5, 6}) { + Slice s = repo.findSliceByCategory("toys", PageRequest.of(0, size, Sort.by("id"))); + t.line("slice size %d -> rows returned: %d, hasNext: %s, statements: %d", + size, s.getContent().size(), s.hasNext(), SqlRecorder.captured().size()); + if (size == 4) { assertThat(s.hasNext()).isTrue(); assertThat(s.getContent()).hasSize(4); } + else { assertThat(s.hasNext()).isFalse(); } + sqlThenReset(); + } + t.write(); + } + + // ---- 03: when the count query is skipped --------------------------------------------- + + @Test + void countQueryIsSometimesSkipped() { + Transcript t = new Transcript("03-count-query-skipped.txt"); + t.line("# Page: how many statements does each request issue? (25 books, 5 toys)"); + record Case(String label, String category, int page, int size, int expectedStatements) {} + List cases = List.of( + new Case("middle page ", "books", 1, 10, 2), + new Case("first page, partial ", "toys", 0, 10, 1), + new Case("last page, partial ", "books", 2, 10, 1), + new Case("full page in the middle", "books", 1, 5, 2), + new Case("last page, exactly full", "books", 4, 5, 2), + new Case("page past the end ", "books", 5, 10, 2)); + for (Case c : cases) { + Page p = repo.findByCategory(c.category(), PageRequest.of(c.page(), c.size(), Sort.by("id"))); + int statements = SqlRecorder.captured().size(); + t.line("%s page=%d size=%-2d rows=%-2d statements=%d totalElements=%d", + c.label(), c.page(), c.size(), p.getContent().size(), statements, p.getTotalElements()); + assertThat(statements).as(c.label()).isEqualTo(c.expectedStatements()); + sqlThenReset(); + } + t.write(); + } + + // ---- 04: sorting --------------------------------------------------------------------- + + @Test + void sortingAndSortSafety() { + Transcript t = new Transcript("04-sorting.txt"); + + repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending().and(Sort.by("id")))); + t.line("# derived query, Sort.by(\"price\").descending().and(Sort.by(\"id\"))"); + sqlThenReset().stream().limit(1).forEach(t::line); + + t.line(""); + t.line("# derived query, unknown property"); + try { + repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("nope"))); + throw new AssertionError("expected an exception"); + } catch (PropertyReferenceException e) { + t.line(e.getClass().getName()); + t.line(e.getMessage()); + } + + repo.queryByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending())); + t.line(""); + t.line("# @Query with Sort.by(\"price\").descending() (alias p is applied for you)"); + sqlThenReset().forEach(t::line); + + t.line(""); + t.line("# a value that is not a property reference at all: Sort.by(\"price) desc, (select 1\")"); + for (String kind : new String[] {"derived", "@Query"}) { + try { + Pageable bad = PageRequest.of(0, 3, Sort.by("price) desc, (select 1")); + if (kind.equals("derived")) repo.findByCategory("books", bad); else repo.queryByCategory("books", bad); + throw new AssertionError("expected an exception"); + } catch (InvalidDataAccessApiUsageException e) { + t.line("%s -> %s", kind, e.getClass().getSimpleName()); + t.line(e.getMessage()); + } + } + + t.line(""); + t.line("# @Query with a function-style sort value, three calls in one fresh context"); + t.line("1. Sort.by(\"length(name)\") -> %s", tryQuerySort(Sort.by("length(name)"))); + t.line("2. JpaSort.unsafe(\"length(name)\") -> %s", tryQuerySort(JpaSort.unsafe("length(name)"))); + t.line("3. Sort.by(\"length(name)\") again -> %s", tryQuerySort(Sort.by("length(name)"))); + t.write(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/RestPagingTest.java b/pagination/src/test/java/com/ankurm/pagination/RestPagingTest.java new file mode 100644 index 0000000..7834431 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/RestPagingTest.java @@ -0,0 +1,108 @@ +package com.ankurm.pagination; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; +import jakarta.persistence.EntityManager; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.annotation.Transactional; + +/** What a client sees over HTTP: Pageable binding, size caps, JSON shapes, bad sorts, keyset cursors. */ +@SpringBootTest +@AutoConfigureMockMvc +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +class RestPagingTest { + + @Autowired MockMvc mvc; + @Autowired EntityManager em; + final JsonMapper json = JsonMapper.builder().build(); + + @BeforeEach + void seed() { + TestData.catalogue(em); + em.flush(); + } + + private String body(String url) throws Exception { + return mvc.perform(get(url)).andReturn().getResponse().getContentAsString(); + } + + private int status(String url) throws Exception { + return mvc.perform(get(url)).andReturn().getResponse().getStatus(); + } + + @Test + void pageableBindingAndCaps() throws Exception { + Transcript t = new Transcript("13-pageable-binding.txt"); + for (String q : List.of("", "?page=2&size=5&sort=price,desc&sort=id", "?size=100000", "?page=-3&size=0", "?page=abc")) { + String url = "/products/echo" + q; + t.line("GET %s -> %d %s", url, status(url), body(url)); + } + assertThat(body("/products/echo")).contains("\"size\":20").contains("\"page\":0"); + assertThat(body("/products/echo?size=100000")).contains("\"size\":2000"); + assertThat(body("/products/echo?page=-3&size=0")).contains("\"size\":20").contains("\"page\":0"); + t.write(); + } + + @Test + void pageJsonShapes() throws Exception { + Transcript t = new Transcript("14-page-json-shapes.txt"); + String raw = body("/products/raw?size=2"); + String model = body("/products/model?size=2"); + t.line("GET /products/raw?size=2 (return type Page)"); + t.line(raw); + t.line(""); + t.line("GET /products/model?size=2 (return type PagedModel)"); + t.line(model); + JsonNode m = json.readTree(model); + assertThat(m.has("content")).isTrue(); + assertThat(m.get("page").get("totalElements").asInt()).isEqualTo(25); + assertThat(m.get("page").get("totalPages").asInt()).isEqualTo(13); + t.write(); + } + + @Test + void sortValidation() throws Exception { + Transcript t = new Transcript("15-rest-bad-sort.txt"); + for (String q : List.of("sort=price,desc", "sort=nope", "sort=price;drop", "sort=,desc")) { + String url = "/products/raw?size=1&" + q; + t.line("GET %s -> %d", url, status(url)); + t.line(" %s", body(url).replaceAll("\\s+", " ")); + } + assertThat(status("/products/raw?size=1&sort=nope")).isEqualTo(400); + assertThat(status("/products/raw?size=1&sort=price,desc")).isEqualTo(200); + t.write(); + } + + @Test + void keysetCursorWalk() throws Exception { + Transcript t = new Transcript("16-keyset-cursor-walk.txt"); + List names = new ArrayList<>(); + String cursor = null; + int calls = 0; + do { + String url = "/products/scroll" + (cursor == null ? "" : "?after=" + cursor); + String b = body(url); + calls++; + JsonNode n = json.readTree(b); + n.get("items").forEach(i -> names.add(i.get("name").asString())); + t.line("GET %s -> %d items, nextCursor=%s", url, n.get("items").size(), n.get("nextCursor")); + cursor = n.get("nextCursor").isNull() ? null : n.get("nextCursor").asString(); + } while (cursor != null); + t.line("calls: %d, items: %d, distinct: %d", calls, names.size(), names.stream().distinct().count()); + assertThat(calls).isEqualTo(3); + assertThat(names).hasSize(25).doesNotHaveDuplicates(); + t.write(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/RestViaDtoTest.java b/pagination/src/test/java/com/ankurm/pagination/RestViaDtoTest.java new file mode 100644 index 0000000..df1c7f6 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/RestViaDtoTest.java @@ -0,0 +1,41 @@ +package com.ankurm.pagination; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.annotation.Transactional; + +/** Same endpoint as RestPagingTest, with spring.data.web.pageable.serialization-mode=via-dto. */ +@SpringBootTest(properties = "spring.data.web.pageable.serialization-mode=via-dto") +@AutoConfigureMockMvc +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +class RestViaDtoTest { + + @Autowired MockMvc mvc; + @Autowired EntityManager em; + + @Test + void rawPageBecomesPagedModelShape() throws Exception { + TestData.catalogue(em); + em.flush(); + String body = mvc.perform(get("/products/raw?size=2")).andReturn().getResponse().getContentAsString(); + Transcript t = new Transcript("17-page-json-via-dto.txt"); + t.line("spring.data.web.pageable.serialization-mode=via-dto"); + t.line("GET /products/raw?size=2 (return type Page)"); + t.line(body); + JsonNode n = JsonMapper.builder().build().readTree(body); + assertThat(n.has("page")).isTrue(); + assertThat(n.get("page").get("totalElements").asInt()).isEqualTo(25); + t.write(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/TestData.java b/pagination/src/test/java/com/ankurm/pagination/TestData.java new file mode 100644 index 0000000..3fbee46 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/TestData.java @@ -0,0 +1,30 @@ +package com.ankurm.pagination; + +import jakarta.persistence.EntityManager; +import java.math.BigDecimal; + +/** Deterministic catalogue used by every test. Ids are 1..30 because each test gets a fresh context. */ +final class TestData { + private TestData() {} + + /** 25 books (ids 1-25, price 10 + i % 4, so prices tie) and 5 toys (ids 26-30). */ + static void catalogue(EntityManager em) { + for (int i = 1; i <= 25; i++) em.persist(new Product("book-" + i, "books", BigDecimal.valueOf(10 + (i % 4)))); + for (int i = 1; i <= 5; i++) em.persist(new Product("toy-" + i, "toys", BigDecimal.valueOf(5 + i))); + em.flush(); + em.clear(); + SqlRecorder.clear(); + } + + /** Two reviews each for books 1-6, so 6 products have reviews and the join has 12 rows. */ + static void reviews(EntityManager em) { + for (int i = 1; i <= 6; i++) { + Product p = em.find(Product.class, (long) i); + em.persist(new Review(p, 5)); + em.persist(new Review(p, 3)); + } + em.flush(); + em.clear(); + SqlRecorder.clear(); + } +} diff --git a/pagination/src/test/java/com/ankurm/pagination/Transcript.java b/pagination/src/test/java/com/ankurm/pagination/Transcript.java new file mode 100644 index 0000000..2747cf7 --- /dev/null +++ b/pagination/src/test/java/com/ankurm/pagination/Transcript.java @@ -0,0 +1,29 @@ +package com.ankurm.pagination; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** Writes output/NN-*.txt while a test asserts the same facts. */ +final class Transcript { + private final List lines = new ArrayList<>(); + private final Path file; + + Transcript(String name) { this.file = Path.of("output/" + name); } + + Transcript line(String fmt, Object... args) { + String s = args.length == 0 ? fmt : fmt.formatted(args); + lines.add(s); + System.out.println(s); + return this; + } + + void write() { + try { + Files.createDirectories(file.getParent()); + Files.writeString(file, String.join("\n", lines) + "\n"); + } catch (IOException e) { throw new RuntimeException(e); } + } +}