The Map-Reduce paradigm allows you to process large inputs by splitting them into smaller chunks (the Map step) and then combining those results into a single output (the Reduce step).
To implement this, you typically create two specialized BaseNode subclasses:
Map Node:
prepAsync: Splits the large input (e.g., from sharedState) into an array of chunks.execAsync: Processes each individual chunk (e.g., via an LLM call).postAsync: Collects the results from all chunk executions and saves them into the sharedState for the next node.
Reduce Node:
prepAsync: Retrieves the array of processed chunks from the sharedState.execAsync: Performs a final operation (e.g., merging or summarizing) on the entire collection of results.postAsync: Saves the final consolidated result into the sharedState.
You connect them by calling mapNode.addSuccessor(reduceNode, DEFAULT_ACTION) and initializing a Flow with the map node.
import { BaseNode, Flow, DEFAULT_ACTION } from "../src/pocket";
// 1. Define the Map Node
export class MapSummaries extends BaseNode {
public async prepAsync(sharedState: any): Promise<string[]> {
// Logic to split sharedState.text into chunks
return chunks;
}
public async execAsync(chunk: string): Promise<string> {
// Logic to process a single chunk
return result;
}
public async postAsync(sharedState: any, prepResult: string[], execResultList: string[]): Promise<string> {
sharedState.summaries = execResultList;
return DEFAULT_ACTION;
}
}
// 2. Define the Reduce Node
export class ReduceSummaries extends BaseNode {
public async prepAsync(sharedState: any): Promise<string[]> {
return sharedState.summaries ?? [];
}
public async execAsync(summaries: string[]): Promise<string> {
// Logic to combine all summaries
return finalResult;
}
public async postAsync(sharedState: any, prepResult: string[], execResult: string): Promise<string> {
sharedState.final_summary = execResult;
return DEFAULT_ACTION;
}
}
// 3. Wire them together
const mapNode = new MapSummaries();
const reduceNode = new ReduceSummaries();
mapNode.addSuccessor(reduceNode, DEFAULT_ACTION);
const flow = new Flow(mapNode);
await flow.runAsync({ text: "large input..." });