LangGraph Crash Recovery with Automatic Checkpointing

Overview

When you use a checkpointer, LangGraph automatically saves state after every node execution, not just at interrupt points. This means if your program crashes at any time, you can resume from the last completed node.

The Key Concept

Automatic Checkpointing After Every Node - LangGraph continuously saves your workflow's state to SQLite (or another backend) after each node completes, enabling automatic crash recovery without any special code.

Complete Example: Crash Recovery


from langgraph.graph import StateGraph, START, END
from langgraph.checkpoint.sqlite import SqliteSaver
from typing import TypedDict
import time
import random

# Define state
class State(TypedDict):
    messages: list
    step_count: int
    result: str

# Nodes that do work
def step_1(state):
    print("Executing step_1...")
    time.sleep(1)
    return {
        "messages": state.get("messages", []) + ["Step 1 complete"],
        "step_count": 1
    }

def step_2(state):
    print("Executing step_2...")
    time.sleep(1)
    
    # Simulate random crash (20% chance)
    if random.random() < 0.2:
        print("💥 CRASH! Program terminated unexpectedly")
        raise SystemExit("Simulated crash")
    
    return {
        "messages": state["messages"] + ["Step 2 complete"],
        "step_count": 2
    }

def step_3(state):
    print("Executing step_3...")
    time.sleep(1)
    
    # Another potential crash point
    if random.random() < 0.2:
        print("💥 CRASH! Program terminated unexpectedly")
        raise SystemExit("Simulated crash")
    
    return {
        "messages": state["messages"] + ["Step 3 complete"],
        "step_count": 3,
        "result": "All steps completed successfully!"
    }

# Build graph
builder = StateGraph(State)
builder.add_node("step_1", step_1)
builder.add_node("step_2", step_2)
builder.add_node("step_3", step_3)

builder.add_edge(START, "step_1")
builder.add_edge("step_1", "step_2")
builder.add_edge("step_2", "step_3")
builder.add_edge("step_3", END)

# Compile with SQLite checkpointer
checkpointer = SqliteSaver.from_conn_string("checkpoints.db")
graph = builder.compile(checkpointer=checkpointer)

# === MAIN PROGRAM ===
def run_workflow(thread_id="workflow-1"):
    config = {"configurable": {"thread_id": thread_id}}
    
    # Check if we have existing state
    try:
        current_state = graph.get_state(config)
        
        if current_state.next:  # There are pending nodes
            print(f"\n🔄 RESUMING from checkpoint...")
            print(f"   Last completed: {current_state.values.get('messages', [])}")
            print(f"   Next nodes: {current_state.next}")
            print(f"   Step count: {current_state.values.get('step_count', 0)}")
            
            # Resume from where we left off (pass None as input)
            result = graph.invoke(None, config=config)
        else:
            print("\n▶️  STARTING new workflow...")
            # Start fresh workflow
            result = graph.invoke(
                {"messages": [], "step_count": 0},
                config=config
            )
        
        print("\n✅ WORKFLOW COMPLETED!")
        print(f"Final result: {result}")
        return result
        
    except SystemExit as e:
        print(f"\n💥 Program crashed: {e}")
        print("State has been saved. Restart the program to resume.")
        return None

# Run the workflow (keep trying until it completes)
if __name__ == "__main__":
    print("Starting workflow (may crash and need restart)...\n")
    
    result = None
    while result is None:
        result = run_workflow()
        
        if result is None:
            print("\n⏳ Restarting in 2 seconds...")
            time.sleep(2)

How It Works

First run (crashes at step 2):


▶️  STARTING new workflow...
Executing step_1...
Executing step_2...
💥 CRASH! Program terminated unexpectedly

💥 Program crashed: Simulated crash
State has been saved. Restart the program to resume.

At this point:

checkpoints.db contains state after step_1 completed
The graph knows step_2 is next

Second run (resumes from checkpoint):


🔄 RESUMING from checkpoint...
   Last completed: ['Step 1 complete']
   Next nodes: ('step_2',)
   Step count: 1
Executing step_2...
Executing step_3...

✅ WORKFLOW COMPLETED!
Final result: {'messages': ['Step 1 complete', 'Step 2 complete', 'Step 3 complete'], 
               'step_count': 3, 'result': 'All steps completed successfully!'}

Separate Script Approach (More Realistic)

workflow.py (the workflow definition):


from langgraph.graph import StateGraph, START, END
from langgraph.checkpoint.sqlite import SqliteSaver
from typing import TypedDict
import time

class State(TypedDict):
    messages: list
    data: dict

def fetch_data(state):
    print("Fetching data from API...")
    time.sleep(2)
    # Imagine this could fail due to network issues
    return {"data": {"user_count": 1000}}

def process_data(state):
    print("Processing data...")
    time.sleep(2)
    # Imagine this is long-running and could crash
    processed = state["data"]["user_count"] * 2
    return {"messages": ["Processed"], "data": {"processed": processed}}

def save_results(state):
    print("Saving results...")
    time.sleep(2)
    return {"messages": state["messages"] + ["Saved"]}

# Build and compile graph
def create_graph():
    builder = StateGraph(State)
    builder.add_node("fetch", fetch_data)
    builder.add_node("process", process_data)
    builder.add_node("save", save_results)
    
    builder.add_edge(START, "fetch")
    builder.add_edge("fetch", "process")
    builder.add_edge("process", "save")
    builder.add_edge("save", END)
    
    checkpointer = SqliteSaver.from_conn_string("workflow.db")
    return builder.compile(checkpointer=checkpointer)

run.py (the runner script):


from workflow import create_graph

def main():
    graph = create_graph()
    thread_id = "daily-job-2025-01-20"
    config = {"configurable": {"thread_id": thread_id}}
    
    # Check for existing state
    state = graph.get_state(config)
    
    if state.next:
        print("🔄 Found incomplete workflow. Resuming...")
        print(f"   Completed so far: {state.values.get('messages', [])}")
        print(f"   Resuming at: {state.next}")
        result = graph.invoke(None, config=config)
    else:
        print("▶️  Starting new workflow...")
        result = graph.invoke({"messages": [], "data": {}}, config=config)
    
    print("✅ Complete!", result)

if __name__ == "__main__":
    main()

Usage:


# First run - might crash during processing
python run.py

# If it crashed, just run again - it resumes automatically
python run.py

# Can run as many times as needed until completion
python run.py

Real-World Example: Long-Running Data Pipeline


from langgraph.graph import StateGraph, START, END
from langgraph.checkpoint.sqlite import SqliteSaver
from typing import TypedDict
import time

class PipelineState(TypedDict):
    files_processed: list
    total_records: int
    current_file: str
    errors: list

def list_files(state):
    """Get list of files to process"""
    print("Listing files...")
    files = ["file1.csv", "file2.csv", "file3.csv", "file4.csv"]
    return {"files_to_process": files, "files_processed": []}

def process_file(state):
    """Process one file at a time"""
    files_to_process = state.get("files_to_process", [])
    
    if not files_to_process:
        return state  # All done
    
    # Get next file
    current_file = files_to_process[0]
    print(f"Processing {current_file}...")
    
    # Simulate long processing
    time.sleep(3)
    
    # Simulate potential crash (power failure, OOM, etc.)
    # In real world, this would be actual crashes
    
    # Update state
    return {
        "files_to_process": files_to_process[1:],
        "files_processed": state.get("files_processed", []) + [current_file],
        "total_records": state.get("total_records", 0) + 1000
    }

def more_files(state):
    """Check if more files to process"""
    if state.get("files_to_process"):
        return "process_file"
    return END

# Build graph with loop
builder = StateGraph(PipelineState)
builder.add_node("list_files", list_files)
builder.add_node("process_file", process_file)

builder.add_edge(START, "list_files")
builder.add_edge("list_files", "process_file")
builder.add_conditional_edges("process_file", more_files, ["process_file", END])

# Persistent checkpointer
checkpointer = SqliteSaver.from_conn_string("pipeline.db")
graph = builder.compile(checkpointer=checkpointer)

def run_pipeline():
    thread_id = "pipeline-batch-001"
    config = {"configurable": {"thread_id": thread_id}}
    
    state = graph.get_state(config)
    
    if state.next:
        print(f"\n🔄 Resuming pipeline...")
        print(f"   Files completed: {state.values.get('files_processed', [])}")
        print(f"   Files remaining: {len(state.values.get('files_to_process', []))}")
        result = graph.invoke(None, config=config)
    else:
        print("\n▶️  Starting new pipeline...")
        result = graph.invoke({}, config=config)
    
    print(f"\n✅ Pipeline complete!")
    print(f"   Total files: {len(result.get('files_processed', []))}")
    print(f"   Total records: {result.get('total_records', 0)}")

if __name__ == "__main__":
    # Run this script multiple times if it crashes
    # It will always resume from the last completed file
    run_pipeline()

Key Points

Automatic checkpointing: LangGraph saves state after EVERY node completes, not just at interrupt points
No special code needed: You don't need to add checkpoint calls - it's automatic
Resume detection: Check if state.next exists to detect incomplete workflows
Thread ID is crucial: Same thread_id = same workflow instance
Idempotent nodes: Design nodes to be safely re-runnable (avoid duplicate side effects)
Only state data is saved, not LLMs: When your code restarts, it still needs to load and initialize any LLMs it uses. Checkpoints store serializable data (JSON-like structures). LLM objects, functions, and connections can't be serialized to JSON, so they must be recreated on restart.
What IS saved in checkpoints: ✅ State data (strings, numbers, lists, dicts, simple objects) ✅ Which node to execute next ✅ Conversation history / messages ✅ Variables and results from completed nodes What is NOT saved: ❌ LLM instances (model objects) ❌ Function definitions ❌ Tools ❌ Database connections ❌ File handles ❌ Any non-serializable objects

Inspecting Checkpoints Manually


from langgraph.checkpoint.sqlite import SqliteSaver

# Open the database
checkpointer = SqliteSaver.from_conn_string("checkpoints.db")

# Get all states for a thread
config = {"configurable": {"thread_id": "workflow-1"}}
state = checkpointer.get(config)

print(f"Current state: {state}")
print(f"Next nodes: {state.next}")
print(f"Values: {state.values}")

Production Patterns

Cron job that's crash-resistant:


xxxxxxxxxx
# This cron job will resume if it crashes mid-execution
0 2 * * * cd /app && python run_pipeline.py

Kubernetes job with restarts:


xxxxxxxxxx
apiVersion: batch/v1
kind: Job
spec:
  template:
    spec:
      restartPolicy: OnFailure  # Will restart on crash
      containers:
      - name: workflow
        image: myworkflow:latest
        command: ["python", "run.py"]

The workflow will automatically resume from the last checkpoint after each restart.

Bottom Line

With SQLite checkpointing, your LangGraph workflow is automatically crash-resistant:

State is saved after every node
On restart, check if there's existing state
If yes, resume with invoke(None, config)
No special interrupt code needed

This works for crashes from:

Power loss
OOM (Out of Memory) kills
Container restarts
Manual termination
Network failures
Any other unpredictable interruption

Checkpointer Options

MemorySaver (Development)


xxxxxxxxxx
from langgraph.checkpoint.memory import MemorySaver

checkpointer = MemorySaver()
# State is lost when program exits

SqliteSaver (Production - Single Machine)


xxxxxxxxxx
from langgraph.checkpoint.sqlite import SqliteSaver

checkpointer = SqliteSaver.from_conn_string("checkpoints.db")
# State persists across program restarts

PostgresSaver (Production - Distributed)


xxxxxxxxxx
from langgraph.checkpoint.postgres import PostgresSaver

checkpointer = PostgresSaver.from_conn_string(
    "postgresql://user:pass@localhost/db"
)
# State persists and can be shared across multiple workers