Add graphrag-neo4j module: structured-output entity and relation extraction into a typed Neo4j graph with a vector index, multi-hop retrieval by span linking and variable-length Cypher paths, compared with plain vector RAG over the same 35 chunks and 9 questions (complete evidence 7/9 vs 5/9 at k=8); Testcontainers on neo4j:2026.10.1 and 5.26. The extraction step is a rule-based stand-in, not a language model.
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01CifR2BE7PBvRy5hT8TS9oz
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,37 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
|
||||
/**
|
||||
* The last step of GraphRAG, and the only one that needs a real language model: put the selected paths and the
|
||||
* chunks that stated them in front of the model and ask. Nothing in this module's tests calls it with a real
|
||||
* model; {@link #prompt} is public so the tests can print exactly what would be sent.
|
||||
*/
|
||||
public class Answerer {
|
||||
|
||||
static final String SYSTEM = """
|
||||
Answer the question using only the facts and passages below. If they do not contain the answer,
|
||||
say so. Name the facts you used.
|
||||
""";
|
||||
|
||||
private final ChatClient client;
|
||||
|
||||
public Answerer(ChatModel model) {
|
||||
this.client = ChatClient.builder(model).defaultSystem(SYSTEM).build();
|
||||
}
|
||||
|
||||
public String answer(GraphRetriever.Retrieval retrieval, Map<String, String> chunkTexts) {
|
||||
return client.prompt().user(prompt(retrieval, chunkTexts)).call().content();
|
||||
}
|
||||
|
||||
public static String prompt(GraphRetriever.Retrieval r, Map<String, String> chunkTexts) {
|
||||
String facts = r.selected().stream().map(p -> "- " + p.path().text()).collect(Collectors.joining("\n"));
|
||||
String passages = r.chunkIds().stream().filter(chunkTexts::containsKey)
|
||||
.map(id -> "[" + id + "] " + chunkTexts.get(id)).collect(Collectors.joining("\n"));
|
||||
return "QUESTION: " + r.question() + "\n\nFACTS FROM THE GRAPH:\n" + facts + "\n\nSOURCE PASSAGES:\n" + passages;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
|
||||
|
||||
/** The 35 short documents under {@code src/main/resources/corpus}: file name = id, first line = title. */
|
||||
public final class Corpus {
|
||||
|
||||
public record Doc(String id, String title, String body) {
|
||||
/** Title and body as one chunk. These documents are short enough that one document is one chunk. */
|
||||
public String text() {
|
||||
return title + ". " + body;
|
||||
}
|
||||
}
|
||||
|
||||
private Corpus() {
|
||||
}
|
||||
|
||||
public static List<Doc> load() {
|
||||
try {
|
||||
List<Doc> docs = new ArrayList<>();
|
||||
for (Resource r : new PathMatchingResourcePatternResolver().getResources("classpath:corpus/*.txt")) {
|
||||
String id = r.getFilename().replace(".txt", "");
|
||||
String[] lines = new String(r.getInputStream().readAllBytes(), StandardCharsets.UTF_8).strip().split("\n", 2);
|
||||
docs.add(new Doc(id, lines[0].strip(), lines[1].strip()));
|
||||
}
|
||||
docs.sort(Comparator.comparing(Doc::id));
|
||||
return docs;
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
|
||||
/**
|
||||
* Step 2: decide which names are the same thing. A model that reads each chunk on its own writes "Card
|
||||
* Gateway" in one post-mortem and "Payments Gateway" in a service page; unless something merges them they
|
||||
* become two nodes and every path through one is invisible from the other.
|
||||
*
|
||||
* <p>Three layers, applied in order: names that differ only in case, punctuation or spacing are the same key;
|
||||
* a hand-kept alias table; and, optionally, cosine similarity of the name embeddings between entities of the
|
||||
* same type above a threshold.
|
||||
*/
|
||||
public final class EntityResolver {
|
||||
|
||||
/** Two distinct names of the same type and how close their embeddings are. */
|
||||
public record Pair(String a, String b, EntityType type, double cosine) {
|
||||
}
|
||||
|
||||
/** What every mention maps to, and what was merged on the way. */
|
||||
public record Resolution(Map<String, String> canonicalByName, Map<String, EntityType> typeByCanonical,
|
||||
List<Pair> merged) {
|
||||
}
|
||||
|
||||
private final EmbeddingModel embeddings;
|
||||
|
||||
public EntityResolver(EmbeddingModel embeddings) {
|
||||
this.embeddings = embeddings;
|
||||
}
|
||||
|
||||
public static String key(String name) {
|
||||
return name.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param mentions every entity mention from every chunk, repeats included (the most frequent spelling wins)
|
||||
* @param aliases lower-case-insensitive alias to canonical name
|
||||
* @param threshold merge same-type names whose cosine is at least this; pass a value above 1 to switch it off
|
||||
*/
|
||||
public Resolution resolve(List<Extraction.Entity> mentions, Map<String, String> aliases, double threshold) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
Map<String, EntityType> types = new LinkedHashMap<>();
|
||||
Map<String, String> spelling = new LinkedHashMap<>();
|
||||
for (Extraction.Entity e : mentions) {
|
||||
counts.merge(key(e.name()), 1, Integer::sum);
|
||||
types.putIfAbsent(key(e.name()), e.type());
|
||||
spelling.putIfAbsent(key(e.name()), e.name());
|
||||
}
|
||||
Map<String, String> aliasByKey = new HashMap<>();
|
||||
aliases.forEach((alias, canonical) -> aliasByKey.put(key(alias), canonical));
|
||||
|
||||
// key -> canonical key after the alias table
|
||||
Map<String, String> parent = new LinkedHashMap<>();
|
||||
for (String k : types.keySet()) {
|
||||
String canonical = aliasByKey.get(k);
|
||||
parent.put(k, canonical == null ? k : key(canonical));
|
||||
if (canonical != null) {
|
||||
types.putIfAbsent(key(canonical), types.get(k));
|
||||
spelling.putIfAbsent(key(canonical), canonical);
|
||||
}
|
||||
}
|
||||
|
||||
List<Pair> merged = new ArrayList<>();
|
||||
List<String> roots = parent.values().stream().distinct().toList();
|
||||
if (threshold <= 1.0) {
|
||||
for (Pair p : similarPairs(roots, types, spelling)) {
|
||||
if (p.cosine() >= threshold) {
|
||||
String ka = key(p.a());
|
||||
String kb = key(p.b());
|
||||
String winner = counts.getOrDefault(ka, 0) >= counts.getOrDefault(kb, 0) ? ka : kb;
|
||||
String loser = winner.equals(ka) ? kb : ka;
|
||||
final String w = winner;
|
||||
final String l = loser;
|
||||
parent.replaceAll((k, v) -> v.equals(l) ? w : v);
|
||||
merged.add(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, String> canonicalByName = new LinkedHashMap<>();
|
||||
Map<String, EntityType> typeByCanonical = new LinkedHashMap<>();
|
||||
for (Extraction.Entity e : mentions) {
|
||||
String root = parent.get(key(e.name()));
|
||||
String canonical = spelling.get(root);
|
||||
canonicalByName.put(e.name(), canonical);
|
||||
typeByCanonical.putIfAbsent(canonical, types.get(root));
|
||||
}
|
||||
return new Resolution(canonicalByName, typeByCanonical, merged);
|
||||
}
|
||||
|
||||
/** All pairs of distinct same-type names, most similar first. Used to merge and to print the evidence. */
|
||||
public List<Pair> similarPairs(List<String> keys, Map<String, EntityType> types, Map<String, String> spelling) {
|
||||
List<float[]> vectors = embeddings.embed(keys.stream().map(spelling::get).toList());
|
||||
List<Pair> out = new ArrayList<>();
|
||||
for (int i = 0; i < keys.size(); i++) {
|
||||
for (int j = i + 1; j < keys.size(); j++) {
|
||||
if (types.get(keys.get(i)) == types.get(keys.get(j))) {
|
||||
out.add(new Pair(spelling.get(keys.get(i)), spelling.get(keys.get(j)), types.get(keys.get(i)),
|
||||
cosine(vectors.get(i), vectors.get(j))));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort(Comparator.comparingDouble(Pair::cosine).reversed());
|
||||
return out;
|
||||
}
|
||||
|
||||
public static double cosine(float[] a, float[] b) {
|
||||
double dot = 0;
|
||||
double na = 0;
|
||||
double nb = 0;
|
||||
for (int i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
na += a[i] * a[i];
|
||||
nb += b[i] * b[i];
|
||||
}
|
||||
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
/** The node kinds the extraction schema allows. A closed set, so the model cannot invent "Concept" or "Thing". */
|
||||
public enum EntityType {
|
||||
SYSTEM, TEAM, PERSON, VENDOR, INCIDENT, PLACE
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* What the model must return for one chunk. {@code ChatClient.entity(Extraction.class)} turns this record into a
|
||||
* JSON schema, appends it to the prompt, and parses the reply back into the record.
|
||||
*/
|
||||
public record Extraction(List<Entity> entities, List<Relation> relations) {
|
||||
|
||||
public record Entity(String name, EntityType type) {
|
||||
}
|
||||
|
||||
public record Relation(String source, RelationType type, String target) {
|
||||
}
|
||||
|
||||
public static Extraction empty() {
|
||||
return new Extraction(List.of(), List.of());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
|
||||
/**
|
||||
* Step 1 of GraphRAG: one chunk of text in, entities and relations out, through
|
||||
* {@code ChatClient.entity(Extraction.class)}. The model sees the chunk and a JSON schema built from the
|
||||
* {@link Extraction} record; the enums in that record are what constrain it to a fixed vocabulary.
|
||||
*/
|
||||
public class Extractor {
|
||||
|
||||
static final String SYSTEM = """
|
||||
You turn short technical documents into a knowledge graph.
|
||||
List every named system, team, person, vendor, incident and place the document mentions as an entity,
|
||||
using the name exactly as the document writes it.
|
||||
List a relation only when the document states it. Never infer one.
|
||||
""";
|
||||
|
||||
static final String USER = """
|
||||
Extract the entities and relations from the document.
|
||||
|
||||
DOCUMENT_START
|
||||
{document}
|
||||
DOCUMENT_END
|
||||
""";
|
||||
|
||||
/** The cleaned extraction plus how many relations were dropped for naming an entity that was never listed. */
|
||||
public record Result(Extraction extraction, int droppedRelations) {
|
||||
}
|
||||
|
||||
private final ChatClient client;
|
||||
|
||||
public Extractor(ChatModel model) {
|
||||
this.client = ChatClient.builder(model).defaultSystem(SYSTEM).build();
|
||||
}
|
||||
|
||||
public Result extract(Corpus.Doc doc) {
|
||||
Extraction raw = client.prompt()
|
||||
.user(u -> u.text(USER).param("document", doc.text()))
|
||||
.call()
|
||||
.entity(Extraction.class);
|
||||
return clean(raw);
|
||||
}
|
||||
|
||||
/**
|
||||
* A relation is kept only if both endpoints appear in the entity list. A model that writes a relation to a
|
||||
* name it never listed would otherwise make the graph writer MATCH nothing and silently create no edge.
|
||||
*/
|
||||
static Result clean(Extraction raw) {
|
||||
if (raw == null || raw.entities() == null || raw.relations() == null) {
|
||||
return new Result(Extraction.empty(), 0);
|
||||
}
|
||||
Set<String> names = new HashSet<>();
|
||||
raw.entities().forEach(e -> names.add(e.name()));
|
||||
List<Extraction.Relation> kept = raw.relations().stream()
|
||||
.filter(r -> names.contains(r.source()) && names.contains(r.target()))
|
||||
.toList();
|
||||
return new Result(new Extraction(raw.entities(), kept), raw.relations().size() - kept.size());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
|
||||
/**
|
||||
* Multi-hop retrieval in three steps, with no language model anywhere in it:
|
||||
* <ol>
|
||||
* <li><b>Link</b>: cut the question into every 1 to 4 word span, embed them, ask the entity vector index for
|
||||
* the nearest entity of each, and keep the ones that clear {@link #linkThreshold}.</li>
|
||||
* <li><b>Walk</b>: from each seed entity, collect every simple path of up to {@code depth} relationships.</li>
|
||||
* <li><b>Rank</b>: render each path as a sentence, embed it, and keep the {@code topPaths} closest to the
|
||||
* question. The chunks that stated the relationships on those paths are the context.</li>
|
||||
* </ol>
|
||||
*/
|
||||
public class GraphRetriever {
|
||||
|
||||
private static final Set<String> STOP = Set.of("a", "an", "the", "of", "to", "in", "on", "is", "are", "who", "what",
|
||||
"which", "where", "that", "does", "do", "for", "by", "if", "and", "or", "it", "its", "this", "these", "those",
|
||||
"with", "from", "at", "as", "be", "goes", "down", "stop", "working", "whom", "behind");
|
||||
|
||||
public record Seed(String span, String name, double score) {
|
||||
}
|
||||
|
||||
public record RankedPath(GraphStore.GraphPath path, double score) {
|
||||
}
|
||||
|
||||
public record Retrieval(String question, List<Seed> seeds, int candidatePaths, List<RankedPath> selected,
|
||||
Set<String> chunkIds) {
|
||||
}
|
||||
|
||||
private final GraphStore graph;
|
||||
private final EmbeddingModel embeddings;
|
||||
private final double linkThreshold;
|
||||
|
||||
public GraphRetriever(GraphStore graph, EmbeddingModel embeddings, double linkThreshold) {
|
||||
this.graph = graph;
|
||||
this.embeddings = embeddings;
|
||||
this.linkThreshold = linkThreshold;
|
||||
}
|
||||
|
||||
public List<Seed> link(String question) {
|
||||
String[] words = question.replaceAll("[?,.]", "").split("\\s+");
|
||||
List<String> spans = new ArrayList<>();
|
||||
for (int i = 0; i < words.length; i++) {
|
||||
for (int n = 1; n <= 4 && i + n <= words.length; n++) {
|
||||
String first = words[i].toLowerCase();
|
||||
String last = words[i + n - 1].toLowerCase();
|
||||
if (STOP.contains(first) || STOP.contains(last)) {
|
||||
continue;
|
||||
}
|
||||
spans.add(String.join(" ", java.util.Arrays.copyOfRange(words, i, i + n)));
|
||||
}
|
||||
}
|
||||
List<float[]> vectors = embeddings.embed(spans);
|
||||
Map<String, Seed> best = new LinkedHashMap<>();
|
||||
for (int i = 0; i < spans.size(); i++) {
|
||||
for (GraphStore.EntityHit hit : graph.nearestEntities(vectors.get(i), 1)) {
|
||||
if (hit.score() >= linkThreshold) {
|
||||
Seed seed = new Seed(spans.get(i), hit.name(), hit.score());
|
||||
best.merge(hit.name(), seed, (a, b) -> a.score() >= b.score() ? a : b);
|
||||
}
|
||||
}
|
||||
}
|
||||
return best.values().stream().sorted(Comparator.comparingDouble(Seed::score).reversed()).toList();
|
||||
}
|
||||
|
||||
public Retrieval retrieve(String question, int depth, int topPaths) {
|
||||
List<Seed> seeds = link(question);
|
||||
// one walk per seed; a path found from two seeds in opposite directions counts once
|
||||
Map<Set<String>, GraphStore.GraphPath> unique = new LinkedHashMap<>();
|
||||
for (Seed s : seeds) {
|
||||
for (GraphStore.GraphPath p : graph.paths(EntityResolver.key(s.name()), depth, 5000)) {
|
||||
Set<String> edgeSet = p.edges().stream().map(GraphStore.Edge::sentence).collect(Collectors.toSet());
|
||||
unique.putIfAbsent(edgeSet, p);
|
||||
}
|
||||
}
|
||||
List<GraphStore.GraphPath> candidates = new ArrayList<>(unique.values());
|
||||
List<RankedPath> ranked = new ArrayList<>();
|
||||
if (!candidates.isEmpty()) {
|
||||
float[] q = embeddings.embed(question);
|
||||
List<float[]> vectors = embeddings.embed(candidates.stream().map(GraphStore.GraphPath::text).toList());
|
||||
for (int i = 0; i < candidates.size(); i++) {
|
||||
ranked.add(new RankedPath(candidates.get(i), EntityResolver.cosine(q, vectors.get(i))));
|
||||
}
|
||||
ranked.sort(Comparator.comparingDouble(RankedPath::score).reversed());
|
||||
}
|
||||
List<RankedPath> selected = ranked.subList(0, Math.min(topPaths, ranked.size()));
|
||||
Set<String> chunks = new LinkedHashSet<>();
|
||||
selected.forEach(rp -> rp.path().edges().forEach(e -> chunks.addAll(e.sources())));
|
||||
return new Retrieval(question, seeds, candidates.size(), List.copyOf(selected), chunks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.Record;
|
||||
import org.neo4j.driver.Value;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
|
||||
/**
|
||||
* Everything that talks Cypher. Entities are {@code (:Entity {key, name, type, embedding})}; each relation is a
|
||||
* real typed relationship (:DEPENDS_ON, :OWNED_BY, ...) carrying the ids of the chunks that stated it in a
|
||||
* {@code sources} list; chunks are {@code (:Chunk)} nodes written by Spring AI's Neo4jVectorStore, and a
|
||||
* {@code (:Chunk)-[:MENTIONS]->(:Entity)} edge records where every entity was seen.
|
||||
*/
|
||||
public class GraphStore {
|
||||
|
||||
public static final String ENTITY_INDEX = "entity_name_embedding";
|
||||
|
||||
/** One stated relation, in its real direction, with the chunks that stated it. */
|
||||
public record Edge(String from, RelationType type, String to, List<String> sources) {
|
||||
public String sentence() {
|
||||
return from + " " + type.phrase() + " " + to;
|
||||
}
|
||||
}
|
||||
|
||||
/** A walk through the graph from a seed entity. {@code edges} are in walking order, each in its true direction. */
|
||||
public record GraphPath(String seed, List<Edge> edges) {
|
||||
public String text() {
|
||||
return edges.stream().map(Edge::sentence).collect(Collectors.joining("; "));
|
||||
}
|
||||
}
|
||||
|
||||
public record EntityHit(String name, String key, EntityType type, double score) {
|
||||
}
|
||||
|
||||
private final Driver driver;
|
||||
private final EmbeddingModel embeddings;
|
||||
|
||||
public GraphStore(Driver driver, EmbeddingModel embeddings) {
|
||||
this.driver = driver;
|
||||
this.embeddings = embeddings;
|
||||
}
|
||||
|
||||
public void initSchema() {
|
||||
run("CREATE CONSTRAINT entity_key IF NOT EXISTS FOR (e:Entity) REQUIRE e.key IS UNIQUE", Map.of());
|
||||
run("CREATE VECTOR INDEX " + ENTITY_INDEX + " IF NOT EXISTS FOR (e:Entity) ON (e.embedding) "
|
||||
+ "OPTIONS {indexConfig: {`vector.dimensions`: " + embeddings.dimensions()
|
||||
+ ", `vector.similarity_function`: 'cosine'}}", Map.of());
|
||||
run("CALL db.awaitIndexes(60)", Map.of());
|
||||
}
|
||||
|
||||
public void clearEntities() {
|
||||
run("MATCH (e:Entity) DETACH DELETE e", Map.of());
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one resolved extraction per chunk. Relationship types are concatenated into the Cypher text because
|
||||
* Cypher has no parameter for them; {@link RelationType} is the allow-list that makes that safe.
|
||||
*/
|
||||
public void write(Map<String, Extraction> extractionByChunk, EntityResolver.Resolution resolution) {
|
||||
List<Map<String, Object>> entityRows = new ArrayList<>();
|
||||
for (Map.Entry<String, EntityType> e : resolution.typeByCanonical().entrySet()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("key", EntityResolver.key(e.getKey()));
|
||||
row.put("name", e.getKey());
|
||||
row.put("type", e.getValue().name());
|
||||
row.put("embedding", embeddings.embed(e.getKey()));
|
||||
entityRows.add(row);
|
||||
}
|
||||
run("""
|
||||
UNWIND $rows AS row
|
||||
MERGE (e:Entity {key: row.key})
|
||||
SET e.name = row.name, e.type = row.type
|
||||
WITH e, row
|
||||
CALL db.create.setNodeVectorProperty(e, 'embedding', row.embedding)
|
||||
""", Map.of("rows", entityRows));
|
||||
|
||||
List<Map<String, Object>> mentions = new ArrayList<>();
|
||||
Map<RelationType, List<Map<String, Object>>> byType = new LinkedHashMap<>();
|
||||
extractionByChunk.forEach((chunkId, extraction) -> {
|
||||
for (Extraction.Entity e : extraction.entities()) {
|
||||
mentions.add(Map.of("chunk", chunkId, "key", EntityResolver.key(resolution.canonicalByName().get(e.name()))));
|
||||
}
|
||||
for (Extraction.Relation r : extraction.relations()) {
|
||||
byType.computeIfAbsent(r.type(), t -> new ArrayList<>()).add(Map.of("chunk", chunkId,
|
||||
"from", EntityResolver.key(resolution.canonicalByName().get(r.source())),
|
||||
"to", EntityResolver.key(resolution.canonicalByName().get(r.target()))));
|
||||
}
|
||||
});
|
||||
run("""
|
||||
UNWIND $rows AS row
|
||||
MATCH (c:Chunk {id: row.chunk}), (e:Entity {key: row.key})
|
||||
MERGE (c)-[:MENTIONS]->(e)
|
||||
""", Map.of("rows", mentions));
|
||||
byType.forEach((type, rows) -> run("""
|
||||
UNWIND $rows AS row
|
||||
MATCH (a:Entity {key: row.from}), (b:Entity {key: row.to})
|
||||
MERGE (a)-[r:%s]->(b)
|
||||
SET r.sources = CASE WHEN row.chunk IN coalesce(r.sources, []) THEN r.sources
|
||||
ELSE coalesce(r.sources, []) + row.chunk END
|
||||
""".formatted(type.name()), Map.of("rows", rows)));
|
||||
}
|
||||
|
||||
/** The k entities whose name embedding is nearest to the vector, through the vector index. */
|
||||
public List<EntityHit> nearestEntities(float[] vector, int k) {
|
||||
return query("""
|
||||
CALL db.index.vector.queryNodes($index, $k, $vector) YIELD node, score
|
||||
RETURN node.name AS name, node.key AS key, node.type AS type, score
|
||||
ORDER BY score DESC
|
||||
""", Map.of("index", ENTITY_INDEX, "k", k, "vector", vector)).stream()
|
||||
.map(r -> new EntityHit(r.get("name").asString(), r.get("key").asString(),
|
||||
EntityType.valueOf(r.get("type").asString()), r.get("score").asDouble()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Every simple path of 1..maxHops relationships that starts at the seed, in either direction, over the
|
||||
* eight allowed relationship types. Chunk nodes are not walked through: the type list excludes MENTIONS.
|
||||
*/
|
||||
public List<GraphPath> paths(String seedKey, int maxHops, int limit) {
|
||||
String types = Arrays.stream(RelationType.values()).map(Enum::name).collect(Collectors.joining("|"));
|
||||
List<Record> rows = query("""
|
||||
MATCH p = (s:Entity {key: $key})-[:%s*1..%d]-(t:Entity)
|
||||
WHERE all(n IN nodes(p) WHERE single(m IN nodes(p) WHERE m = n))
|
||||
RETURN [r IN relationships(p) | {from: startNode(r).name, type: type(r), to: endNode(r).name,
|
||||
sources: r.sources}] AS edges
|
||||
LIMIT $limit
|
||||
""".formatted(types, maxHops), Map.of("key", seedKey, "limit", limit));
|
||||
List<GraphPath> out = new ArrayList<>();
|
||||
for (Record row : rows) {
|
||||
List<Edge> edges = new ArrayList<>();
|
||||
for (Value e : row.get("edges").values()) {
|
||||
edges.add(new Edge(e.get("from").asString(), RelationType.valueOf(e.get("type").asString()),
|
||||
e.get("to").asString(), e.get("sources").asList(Value::asString)));
|
||||
}
|
||||
out.add(new GraphPath(seedKey, edges));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
public Map<String, String> chunkTexts(Collection<String> ids) {
|
||||
Map<String, String> out = new LinkedHashMap<>();
|
||||
query("MATCH (c:Chunk) WHERE c.id IN $ids RETURN c.id AS id, c.text AS text ORDER BY id", Map.of("ids", List.copyOf(ids)))
|
||||
.forEach(r -> out.put(r.get("id").asString(), r.get("text").asString()));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- small helpers ---------------------------------------------------------------------------
|
||||
|
||||
public List<Record> query(String cypher, Map<String, Object> params) {
|
||||
return driver.executableQuery(cypher).withParameters(params).execute().records();
|
||||
}
|
||||
|
||||
public void run(String cypher, Map<String, Object> params) {
|
||||
driver.executableQuery(cypher).withParameters(params).execute();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.transformers.TransformersEmbeddingModel;
|
||||
|
||||
/**
|
||||
* all-MiniLM-L6-v2 (384 dimensions) running inside the JVM through ONNX Runtime. Both model files come from
|
||||
* Hugging Face and are cached under the user's home directory, so the first run downloads about 90 MB and
|
||||
* later runs are offline. Same approach as the rag-ingestion module in this repository.
|
||||
*/
|
||||
public final class MiniLm {
|
||||
|
||||
private static final String HF = "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/";
|
||||
private static EmbeddingModel shared;
|
||||
|
||||
private MiniLm() {
|
||||
}
|
||||
|
||||
public static synchronized EmbeddingModel get() {
|
||||
if (shared == null) {
|
||||
try {
|
||||
TransformersEmbeddingModel m = new TransformersEmbeddingModel(MetadataMode.NONE);
|
||||
m.setTokenizerResource(HF + "tokenizer.json");
|
||||
m.setModelResource(HF + "onnx/model.onnx");
|
||||
m.setResourceCacheDirectory(System.getProperty("user.home") + "/.cache/minilm");
|
||||
m.setTokenizerOptions(Map.of("padding", "true", "truncation", "true", "maxLength", "256"));
|
||||
m.afterPropertiesSet();
|
||||
shared = m;
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
return shared;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.neo4j.Neo4jVectorStore;
|
||||
|
||||
/**
|
||||
* The whole indexing side in one place: chunks into a Neo4j vector index (what plain vector RAG needs), then
|
||||
* extraction, entity resolution and graph writes (what GraphRAG adds on top).
|
||||
*/
|
||||
public class Pipeline {
|
||||
|
||||
/** What building the graph produced, for printing and for tests. */
|
||||
public record Built(Map<String, Extraction> extractionByChunk, EntityResolver.Resolution resolution,
|
||||
int droppedRelations, List<Extraction.Entity> allMentions) {
|
||||
}
|
||||
|
||||
private final GraphStore graph;
|
||||
private final EmbeddingModel embeddings;
|
||||
private final Extractor extractor;
|
||||
private final Neo4jVectorStore chunks;
|
||||
|
||||
public Pipeline(Driver driver, EmbeddingModel embeddings, ChatModel chatModel) {
|
||||
this.embeddings = embeddings;
|
||||
this.graph = new GraphStore(driver, embeddings);
|
||||
this.extractor = new Extractor(chatModel);
|
||||
this.chunks = Neo4jVectorStore.builder(driver, embeddings)
|
||||
.label("Chunk")
|
||||
.indexName("chunk_embedding")
|
||||
.embeddingDimension(embeddings.dimensions())
|
||||
.initializeSchema(true)
|
||||
.build();
|
||||
this.chunks.afterPropertiesSet();
|
||||
this.graph.initSchema();
|
||||
}
|
||||
|
||||
public GraphStore graph() {
|
||||
return graph;
|
||||
}
|
||||
|
||||
public Neo4jVectorStore chunks() {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
public void loadChunks(List<Corpus.Doc> docs) {
|
||||
chunks.add(docs.stream().map(d -> new Document(d.id(), d.text(), Map.of("title", d.title()))).toList());
|
||||
}
|
||||
|
||||
/** Plain vector RAG: the ids of the k chunks nearest to the question. */
|
||||
public List<String> vectorSearch(String question, int k) {
|
||||
return chunks.similaritySearch(SearchRequest.builder().query(question).topK(k).build())
|
||||
.stream().map(Document::getId).toList();
|
||||
}
|
||||
|
||||
/** Extract from every chunk, resolve names, and (re)write the entity graph. */
|
||||
public Built buildGraph(List<Corpus.Doc> docs, Map<String, String> aliases, double mergeThreshold) {
|
||||
Map<String, Extraction> byChunk = new LinkedHashMap<>();
|
||||
List<Extraction.Entity> mentions = new ArrayList<>();
|
||||
int dropped = 0;
|
||||
for (Corpus.Doc d : docs) {
|
||||
Extractor.Result r = extractor.extract(d);
|
||||
byChunk.put(d.id(), r.extraction());
|
||||
mentions.addAll(r.extraction().entities());
|
||||
dropped += r.droppedRelations();
|
||||
}
|
||||
EntityResolver.Resolution resolution = new EntityResolver(embeddings).resolve(mentions, aliases, mergeThreshold);
|
||||
graph.clearEntities();
|
||||
graph.write(byChunk, resolution);
|
||||
return new Built(byChunk, resolution, dropped, mentions);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* One evaluation question. {@code evidence} is the set of documents that together contain every fact needed to
|
||||
* answer it, worked out by hand from the corpus; retrieval is "complete" only when all of them are in the context.
|
||||
*/
|
||||
public record Question(String id, int hops, String text, Set<String> evidence, List<String> answers) {
|
||||
|
||||
public static List<Question> all() {
|
||||
return List.of(
|
||||
new Question("Q1", 1, "Who leads the Fulfilment Team?",
|
||||
Set.of("team-fulfilment"), List.of("Tomasz Nowak")),
|
||||
new Question("Q2", 2, "Who leads the team that owns the Payments Gateway?",
|
||||
Set.of("svc-payments-gateway", "team-payments"), List.of("Priya Raman")),
|
||||
new Question("Q3", 2, "Which team owns the root cause of INC-2041?",
|
||||
Set.of("inc-2041", "db-orders"), List.of("Core Data Team")),
|
||||
new Question("Q4", 3, "Who does the leader of the team that owns the Payments Gateway report to?",
|
||||
Set.of("svc-payments-gateway", "team-payments", "person-priya"), List.of("Dana Okafor")),
|
||||
new Question("Q5", 3, "Where is the company that hosts the database behind the Ledger Service located?",
|
||||
Set.of("svc-ledger", "db-orders", "vendor-borealis"), List.of("Frankfurt")),
|
||||
new Question("Q6", 3, "Who leads the teams that own the services the Checkout Service depends on?",
|
||||
Set.of("svc-checkout", "svc-payments-gateway", "svc-inventory", "team-payments", "team-fulfilment"),
|
||||
List.of("Priya Raman", "Tomasz Nowak")),
|
||||
new Question("Q7", 3, "Which services stop working if the Orders Database goes down?",
|
||||
Set.of("svc-ledger", "svc-payments-gateway", "svc-checkout"),
|
||||
List.of("Ledger Service", "Payments Gateway", "Checkout Service")),
|
||||
new Question("Q8", 2, "Which incidents affected services that the Checkout Service depends on directly?",
|
||||
Set.of("svc-checkout", "inc-2057", "inc-2060"), List.of("INC-2057", "INC-2060")),
|
||||
new Question("Q9", 3, "Who leads the team that owns the service INC-2057 affected?",
|
||||
Set.of("inc-2057", "svc-inventory", "team-fulfilment"), List.of("Tomasz Nowak")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.ankurm.graphrag;
|
||||
|
||||
/**
|
||||
* The relationship kinds the extraction schema allows. This enum is also the allow-list that makes it safe to
|
||||
* put a model-chosen relationship type into a Cypher string: Cypher cannot take a relationship type as a
|
||||
* parameter, so the type is concatenated into the query, and only these constants can ever get there.
|
||||
*/
|
||||
public enum RelationType {
|
||||
DEPENDS_ON("depends on"),
|
||||
OWNED_BY("is owned by"),
|
||||
LED_BY("is led by"),
|
||||
HOSTED_ON("is hosted on"),
|
||||
AFFECTED("affected"),
|
||||
CAUSED_BY("was caused by"),
|
||||
REPORTS_TO("reports to"),
|
||||
LOCATED_IN("is located in");
|
||||
|
||||
private final String phrase;
|
||||
|
||||
RelationType(String phrase) {
|
||||
this.phrase = phrase;
|
||||
}
|
||||
|
||||
/** The relation as an English phrase, used to render a path as a sentence. */
|
||||
public String phrase() {
|
||||
return phrase;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package com.ankurm.graphrag.stub;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.ankurm.graphrag.EntityType;
|
||||
import com.ankurm.graphrag.RelationType;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* A STAND-IN for a language model, not a language model. It reads the document out of the prompt the way
|
||||
* {@code Extractor} writes it, finds sentences built from eight fixed phrases ("depends on", "is owned by",
|
||||
* "is led by", ...) with regular expressions, and replies with the JSON a good model would produce for them.
|
||||
* It exists so that the whole pipeline (prompt, JSON schema, parsing, graph writes) runs offline and the same
|
||||
* way every time. It says nothing about how well a real model extracts entities from real prose, and no
|
||||
* number in this module claims to.
|
||||
*
|
||||
* <p>The {@link Mode} values reproduce replies that real models do produce: JSON wrapped in a markdown
|
||||
* fence, JSON cut off mid-way, a relation type outside the allowed list, a relation whose endpoint was never
|
||||
* listed as an entity.
|
||||
*/
|
||||
public final class RuleBasedExtractorModel implements org.springframework.ai.chat.model.ChatModel {
|
||||
|
||||
public enum Mode { CLEAN, FENCED, PROSE_THEN_FENCE, TRUNCATED, UNKNOWN_RELATION_TYPE, INJECTION_RELATION_TYPE, DANGLING_RELATION }
|
||||
|
||||
private static final String NAME = "[A-Z][\\w-]*(?: [A-Z][\\w-]*)*";
|
||||
private static final Pattern DEPENDS = Pattern.compile("(?:depends on|\\band on) (?:[Tt]he )?(" + NAME + ")");
|
||||
private static final Pattern FIRST_NAME = Pattern.compile("^\\s*(?:[Tt]he )?(" + NAME + ")");
|
||||
private static final JsonMapper JSON = JsonMapper.builder().build();
|
||||
|
||||
private final Mode mode;
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
private final AtomicLong inputChars = new AtomicLong();
|
||||
private volatile String lastUserText = "";
|
||||
|
||||
public RuleBasedExtractorModel() {
|
||||
this(Mode.CLEAN);
|
||||
}
|
||||
|
||||
public RuleBasedExtractorModel(Mode mode) {
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public int calls() {
|
||||
return calls.get();
|
||||
}
|
||||
|
||||
public long inputChars() {
|
||||
return inputChars.get();
|
||||
}
|
||||
|
||||
/** The full user message of the most recent call, including the JSON-schema instructions Spring AI appended. */
|
||||
public String lastUserText() {
|
||||
return lastUserText;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
calls.incrementAndGet();
|
||||
String user = prompt.getUserMessage().getText();
|
||||
lastUserText = user;
|
||||
inputChars.addAndGet(prompt.getContents().length());
|
||||
int start = user.lastIndexOf("DOCUMENT_START");
|
||||
int end = user.lastIndexOf("DOCUMENT_END");
|
||||
String document = user.substring(start + "DOCUMENT_START".length(), end).strip();
|
||||
String json = render(extract(document));
|
||||
return new ChatResponse(List.of(new Generation(new AssistantMessage(json))));
|
||||
}
|
||||
|
||||
// ---- the "model" ----------------------------------------------------------------------------------
|
||||
|
||||
private record Ent(String name, EntityType type) {
|
||||
}
|
||||
|
||||
private record Rel(String source, String type, String target) {
|
||||
}
|
||||
|
||||
private record Found(Set<Ent> entities, List<Rel> relations) {
|
||||
}
|
||||
|
||||
private Found extract(String document) {
|
||||
String[] sentences = document.split("(?<=\\.)\\s+");
|
||||
String title = sentences[0].replaceAll("\\.$", "");
|
||||
Set<Ent> entities = new LinkedHashSet<>();
|
||||
List<Rel> relations = new ArrayList<>();
|
||||
for (String s : sentences) {
|
||||
s = s.replaceAll("\\.$", "");
|
||||
for (RelationType type : RelationType.values()) {
|
||||
int at = s.indexOf(" " + type.phrase() + " ");
|
||||
if (at < 0) {
|
||||
continue;
|
||||
}
|
||||
String subject = subject(s.substring(0, at), title);
|
||||
if (subject == null) {
|
||||
continue;
|
||||
}
|
||||
List<String> objects = objects(type, s.substring(at));
|
||||
for (String object : objects) {
|
||||
entities.add(new Ent(subject, subjectType(type, subject)));
|
||||
entities.add(new Ent(object, objectType(type)));
|
||||
relations.add(new Rel(subject, type.name(), object));
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Found(entities, relations);
|
||||
}
|
||||
|
||||
private static String subject(String before, String title) {
|
||||
String b = before.strip();
|
||||
if (b.equals("It") || b.equals("She") || b.equals("He")) {
|
||||
return title;
|
||||
}
|
||||
Matcher m = FIRST_NAME.matcher(b);
|
||||
return m.matches() ? m.group(1) : null;
|
||||
}
|
||||
|
||||
private static List<String> objects(RelationType type, String fromVerb) {
|
||||
List<String> out = new ArrayList<>();
|
||||
if (type == RelationType.DEPENDS_ON) {
|
||||
Matcher m = DEPENDS.matcher(fromVerb);
|
||||
while (m.find()) {
|
||||
out.add(m.group(1));
|
||||
}
|
||||
} else {
|
||||
Matcher m = FIRST_NAME.matcher(fromVerb.substring((" " + type.phrase()).length()));
|
||||
if (m.find()) {
|
||||
out.add(m.group(1));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static EntityType subjectType(RelationType type, String subject) {
|
||||
return switch (type) {
|
||||
case DEPENDS_ON, OWNED_BY, HOSTED_ON -> EntityType.SYSTEM;
|
||||
case LED_BY -> EntityType.TEAM;
|
||||
case AFFECTED, CAUSED_BY -> EntityType.INCIDENT;
|
||||
case REPORTS_TO -> EntityType.PERSON;
|
||||
case LOCATED_IN -> subject.endsWith("Team") ? EntityType.TEAM : EntityType.VENDOR;
|
||||
};
|
||||
}
|
||||
|
||||
private static EntityType objectType(RelationType type) {
|
||||
return switch (type) {
|
||||
case DEPENDS_ON, AFFECTED, CAUSED_BY -> EntityType.SYSTEM;
|
||||
case OWNED_BY -> EntityType.TEAM;
|
||||
case LED_BY, REPORTS_TO -> EntityType.PERSON;
|
||||
case HOSTED_ON -> EntityType.VENDOR;
|
||||
case LOCATED_IN -> EntityType.PLACE;
|
||||
};
|
||||
}
|
||||
|
||||
private String render(Found found) {
|
||||
List<Map<String, String>> ents = new ArrayList<>();
|
||||
for (Ent e : found.entities()) {
|
||||
ents.add(Map.of("name", e.name(), "type", e.type().name()));
|
||||
}
|
||||
List<Map<String, String>> rels = new ArrayList<>();
|
||||
for (Rel r : found.relations()) {
|
||||
Map<String, String> m = new LinkedHashMap<>();
|
||||
m.put("source", r.source());
|
||||
m.put("type", r.type());
|
||||
m.put("target", r.target());
|
||||
rels.add(m);
|
||||
}
|
||||
switch (mode) {
|
||||
case UNKNOWN_RELATION_TYPE -> {
|
||||
if (!rels.isEmpty()) {
|
||||
rels.get(0).put("type", "IS_RESPONSIBLE_FOR");
|
||||
}
|
||||
}
|
||||
case INJECTION_RELATION_TYPE -> {
|
||||
if (!rels.isEmpty()) {
|
||||
rels.get(0).put("type", "OWNED_BY]->(x) DETACH DELETE x //");
|
||||
}
|
||||
}
|
||||
case DANGLING_RELATION -> rels.add(Map.of("source", rels.isEmpty() ? "Nobody" : rels.get(0).get("source"),
|
||||
"type", "OWNED_BY", "target", "Ghost Team"));
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
String json = JSON.writeValueAsString(Map.of("entities", ents, "relations", rels));
|
||||
return switch (mode) {
|
||||
case FENCED -> "```json\n" + json + "\n```";
|
||||
case PROSE_THEN_FENCE -> "Here is the extraction you asked for:\n```json\n" + json + "\n```";
|
||||
case TRUNCATED -> json.substring(0, Math.max(1, json.length() * 2 / 3));
|
||||
default -> json;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Reading the checkout latency dashboard
|
||||
The checkout latency dashboard shows p50, p95 and p99 per region. A red band on the p99 panel usually means the Payments Gateway is slow rather than the Checkout Service itself. Compare against the Inventory Service panel before paging anyone.
|
||||
@@ -0,0 +1,2 @@
|
||||
Orders Database
|
||||
The Orders Database is a PostgreSQL 16 cluster that stores orders and ledger entries. The Orders Database is owned by the Core Data Team. It is hosted on Borealis Hosting. Nightly backups are kept for 35 days.
|
||||
@@ -0,0 +1,2 @@
|
||||
FAQ: why does the stock count look wrong?
|
||||
Stock counts lag the warehouse by up to five minutes. Reserved stock is subtracted immediately at checkout. If a count is negative, file a ticket with the Fulfilment Team queue.
|
||||
@@ -0,0 +1,2 @@
|
||||
Incident process
|
||||
Incidents are numbered INC-NNNN and graded from severity one to four. A write-up is due within five working days. Write-ups are blameless and name systems, never individuals.
|
||||
@@ -0,0 +1,2 @@
|
||||
Month-end reconciliation guide
|
||||
Reconciliation compares settled amounts in the Ledger Service with the bank statements. Differences under one cent are rounded away. Anything larger is raised with the finance operations desk.
|
||||
@@ -0,0 +1,2 @@
|
||||
On-call rotation guide
|
||||
Every team keeps a weekly rotation with a primary and a secondary. Handover happens on Monday mornings. A page that is not acknowledged in ten minutes escalates to the secondary.
|
||||
@@ -0,0 +1,2 @@
|
||||
Warehouse scanner setup
|
||||
Scanners connect over the warehouse wifi and report counts in batches. Replace the battery every eighteen months. The Warehouse Sync job tolerates a scanner being offline for a day.
|
||||
@@ -0,0 +1,2 @@
|
||||
INC-2041
|
||||
INC-2041 was a 47-minute outage on 14 March. INC-2041 affected the Ledger Service. INC-2041 was caused by the Orders Database. A disk-full alert had been silenced two weeks earlier.
|
||||
@@ -0,0 +1,2 @@
|
||||
INC-2057
|
||||
INC-2057 was a stock-count drift that lasted a weekend. INC-2057 affected the Inventory Service. INC-2057 was caused by the Warehouse Sync job. Counts were corrected by hand on Monday.
|
||||
@@ -0,0 +1,2 @@
|
||||
INC-2060
|
||||
INC-2060 was a 12-minute slowdown on card payments. INC-2060 affected the Card Gateway. The Card Gateway is the on-call channel's name for the Payments Gateway. A retry storm after a network blip was the trigger. The write-up recommends capping retries.
|
||||
@@ -0,0 +1,2 @@
|
||||
Warehouse Sync
|
||||
The Warehouse Sync job pulls stock counts from every warehouse scanner every five minutes. The Warehouse Sync job is owned by the Fulfilment Team. It is hosted on Borealis Hosting.
|
||||
@@ -0,0 +1,2 @@
|
||||
Memo: cloud cost review
|
||||
Spend on Arcadia Cloud grew eleven per cent last quarter, mostly in the Notification Service and the Inventory Service. The Platform Team will propose a tagging scheme.
|
||||
@@ -0,0 +1,2 @@
|
||||
Memo: hosting contract renewal
|
||||
The colocation contract with Borealis Hosting renews in the autumn. Finance wants a quote for a second facility. Arcadia Cloud was considered for the databases and rejected for latency reasons.
|
||||
@@ -0,0 +1,2 @@
|
||||
Storefront redesign launch notes
|
||||
The new Checkout Service flow went to ten per cent of customers on Tuesday. The Storefront Team saw a small rise in abandoned carts on narrow phones and fixed the layout the same day.
|
||||
@@ -0,0 +1,2 @@
|
||||
Payments Team offsite notes
|
||||
The Payments Team offsite covered refunds, chargebacks and a proposal to move reporting off the Ledger Service. Nothing was decided. Notes from the whiteboard session are in the team wiki.
|
||||
@@ -0,0 +1,2 @@
|
||||
Engineering onboarding
|
||||
New engineers get a laptop, repository access and a buddy on day one. In the first month they shadow an on-call shift and ship one small change. Questions go to the engineering channel.
|
||||
@@ -0,0 +1,2 @@
|
||||
Elena Petrova
|
||||
Elena Petrova moved from design to engineering management in 2022. Elena Petrova reports to Kwame Mensah. She owns the storefront roadmap.
|
||||
@@ -0,0 +1,2 @@
|
||||
Marcus Webb
|
||||
Marcus Webb has worked on the company's databases since 2018. Marcus Webb reports to Dana Okafor. He chairs the schema review board.
|
||||
@@ -0,0 +1,2 @@
|
||||
Priya Raman
|
||||
Priya Raman joined the company in 2021 as a payments engineer. Priya Raman reports to Dana Okafor. She is the named contact for card network certificate renewals.
|
||||
@@ -0,0 +1,2 @@
|
||||
Tomasz Nowak
|
||||
Tomasz Nowak set up the first warehouse scanners. Tomasz Nowak reports to Kwame Mensah. He runs the fulfilment on-call rotation.
|
||||
@@ -0,0 +1,2 @@
|
||||
Card data handling policy
|
||||
Card numbers must never be written to logs. Only the Payments Gateway may see raw card data; the Checkout Service passes opaque tokens. Annual PCI reviews are scheduled in the autumn.
|
||||
@@ -0,0 +1,2 @@
|
||||
Runbook: restoring the Orders Database from backup
|
||||
Restore the latest nightly backup to a scratch cluster first and check row counts. Replay the write-ahead log to the minute you need. Promote the scratch cluster only after the Ledger Service owners have signed off.
|
||||
@@ -0,0 +1,2 @@
|
||||
Runbook: Payments Gateway timeouts
|
||||
When the Payments Gateway starts timing out, check the card network status pages first. Raise the read timeout only as a stopgap and tell the Payments Team channel. Never retry a charge without an idempotency key.
|
||||
@@ -0,0 +1,2 @@
|
||||
Checkout Service
|
||||
The Checkout Service takes a customer's cart and turns it into an order. It depends on the Payments Gateway to take card payments and on the Inventory Service to reserve stock. The Checkout Service is owned by the Storefront Team. Its latency dashboard lives in the shared Grafana folder.
|
||||
@@ -0,0 +1,2 @@
|
||||
Inventory Service
|
||||
The Inventory Service keeps stock counts per warehouse. It depends on the Warehouse Sync job to receive counts. The Inventory Service is owned by the Fulfilment Team. It is hosted on Arcadia Cloud.
|
||||
@@ -0,0 +1,2 @@
|
||||
Ledger Service
|
||||
The Ledger Service is the system of record for settled money movements. It depends on the Orders Database for storage. The Ledger Service is owned by the Core Data Team. Month-end reconciliation runs against it on the third working day.
|
||||
@@ -0,0 +1,2 @@
|
||||
Notification Service
|
||||
The Notification Service sends order emails and shipping texts. It depends on the Mailer Relay for outbound mail. The Notification Service is owned by the Platform Team. It is hosted on Arcadia Cloud.
|
||||
@@ -0,0 +1,2 @@
|
||||
Payments Gateway
|
||||
The Payments Gateway talks to the card networks on behalf of every other service. The Payments Gateway depends on the Ledger Service to record each authorised charge. The Payments Gateway is owned by the Payments Team. It is hosted on Arcadia Cloud.
|
||||
@@ -0,0 +1,2 @@
|
||||
Core Data Team
|
||||
The Core Data Team looks after the shared databases and the ledger. The Core Data Team is led by Marcus Webb. The Core Data Team is based in Austin. The team keeps a public changelog for schema changes.
|
||||
@@ -0,0 +1,2 @@
|
||||
Fulfilment Team
|
||||
The Fulfilment Team connects the shop to the warehouses. The Fulfilment Team is led by Tomasz Nowak. The Fulfilment Team is based in Gdansk.
|
||||
@@ -0,0 +1,2 @@
|
||||
Payments Team
|
||||
The Payments Team looks after card acceptance and refunds. The Payments Team is led by Priya Raman. The Payments Team is based in Pune. The team holds a weekly review on Thursdays.
|
||||
@@ -0,0 +1,2 @@
|
||||
Platform Team
|
||||
The Platform Team runs the build system and the shared tooling. The Platform Team is led by Aiko Tanaka. The Platform Team is based in Osaka.
|
||||
@@ -0,0 +1,2 @@
|
||||
Storefront Team
|
||||
The Storefront Team builds the web and mobile shopping experience. The Storefront Team is led by Elena Petrova. The Storefront Team is based in Lisbon.
|
||||
@@ -0,0 +1,2 @@
|
||||
Arcadia Cloud
|
||||
Arcadia Cloud is the public cloud used for stateless services. Arcadia Cloud is located in Dublin. Reserved capacity is renegotiated each spring.
|
||||
@@ -0,0 +1,2 @@
|
||||
Borealis Hosting
|
||||
Borealis Hosting runs the colocation facility used for databases and batch jobs. Borealis Hosting is located in Frankfurt. Its support line is staffed around the clock.
|
||||
Reference in New Issue
Block a user