LlamaIndex — Data, Nodes & Indexing¶

from pathlib import Path

from llama_index.core import Document, SimpleDirectoryReader

documents = SimpleDirectoryReader(
    input_dir="data",
    recursive=True,                              # default False: only the top folder
    required_exts=[".md", ".txt"],
    exclude=["**/drafts/*"],
    filename_as_id=True,                         # doc_id = file path instead of a random UUID
    file_metadata=lambda path: {"file_name": Path(path).name, "source": "kb"},
).load_data()

for doc in documents:                            # portable ids: the default path is absolute
    doc.doc_id = doc.metadata["file_name"]

manual = Document(text="Refunds within 30 days.", doc_id="refund-policy", metadata={"topic": "billing"})

from llama_index.core.schema import MetadataMode

doc = Document(
    text="body",
    doc_id="x",
    metadata={"topic": "billing", "visibility": "public", "year": 2026},
    excluded_embed_metadata_keys=["year"],
    excluded_llm_metadata_keys=["year", "visibility"],
)
doc.get_content(metadata_mode=MetadataMode.EMBED)   # 'topic: billing\nvisibility: public\n\nbody'
doc.get_content(metadata_mode=MetadataMode.LLM)     # 'topic: billing\n\nbody'

from llama_index.core.node_parser import MarkdownNodeParser, SemanticSplitterNodeParser, SentenceSplitter

splitter = SentenceSplitter(
    chunk_size=512,                  # tokens; default 1024
    chunk_overlap=64,                # default 200, and it must be smaller than chunk_size
    id_func=lambda i, doc: f"{doc.doc_id}#{i}",     # deterministic node ids
)
nodes = splitter.get_nodes_from_documents(documents)

md_nodes = MarkdownNodeParser().get_nodes_from_documents(documents)     # metadata["header_path"] == "/Refund policy/"
semantic = SemanticSplitterNodeParser(
    buffer_size=1, breakpoint_percentile_threshold=95, embed_model=my_embed_model
)

from llama_index.core import Settings
from llama_index.core.node_parser import SentenceSplitter
from llama_index.llms.openai import OpenAI

Settings.llm = OpenAI(model="gpt-4o-mini", temperature=0)
Settings.embed_model = "local:BAAI/bge-small-en-v1.5"        # needs llama-index-embeddings-huggingface
Settings.node_parser = SentenceSplitter(chunk_size=512, chunk_overlap=64)

from llama_index.core import Document
from llama_index.core.ingestion import DocstoreStrategy, IngestionCache, IngestionPipeline
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core.storage.docstore import SimpleDocumentStore
from llama_index.core.vector_stores import SimpleVectorStore

store = SimpleVectorStore()
pipeline = IngestionPipeline(
    transformations=[SentenceSplitter(chunk_size=256, chunk_overlap=32), embed_model],   # last step embeds
    docstore=SimpleDocumentStore(),
    vector_store=store,
    cache=IngestionCache(),
    docstore_strategy=DocstoreStrategy.UPSERTS,
)

nodes = pipeline.run(documents=docs)            # new nodes; embedded and written to the vector store
nodes = pipeline.run(documents=docs)            # [] : nothing changed
pipeline.persist("./pipeline_storage")          # cache + docstore
pipeline.load("./pipeline_storage")             # later runs reuse them
nodes = await pipeline.arun(documents=docs, num_workers=4)   # async; num_workers sets processes

from llama_index.core import VectorStoreIndex

index = VectorStoreIndex.from_documents(docs, embed_model=embed_model, transformations=[splitter], show_progress=True)
index = VectorStoreIndex(nodes, embed_model=embed_model)            # from nodes you built
index = VectorStoreIndex.from_vector_store(vector_store, embed_model=embed_model)   # attach to existing data

import chromadb
from llama_index.core import StorageContext, VectorStoreIndex
from llama_index.vector_stores.chroma import ChromaVectorStore

client = chromadb.PersistentClient(path="chroma_db")
store = ChromaVectorStore(chroma_collection=client.get_or_create_collection("kb-docs"))   # name: 3+ chars
index = VectorStoreIndex.from_documents(
    docs, storage_context=StorageContext.from_defaults(vector_store=store), embed_model=embed_model
)

# later, in another process: no re-embedding
index = VectorStoreIndex.from_vector_store(
    ChromaVectorStore(chroma_collection=chromadb.PersistentClient(path="chroma_db").get_collection("kb-docs")),
    embed_model=embed_model,
)

from llama_index.vector_stores.postgres import PGVectorStore

store = PGVectorStore.from_params(
    host="localhost", port=5432, database="kb", user="app", password="<DB_PASSWORD>",
    table_name="kb_chunks",          # created as "data_kb_chunks"
    embed_dim=1536,                  # must equal the embedding model's dimension (default 1536)
    hybrid_search=True,              # adds a full-text column for vector_store_query_mode="hybrid"
    hnsw_kwargs={"hnsw_m": 16, "hnsw_ef_construction": 64, "hnsw_ef_search": 40, "hnsw_dist_method": "vector_cosine_ops"},
)

from llama_index.core import StorageContext, load_index_from_storage

index.storage_context.persist(persist_dir="storage")
# files: default__vector_store.json, docstore.json, index_store.json, graph_store.json, image__vector_store.json

storage = StorageContext.from_defaults(persist_dir="storage")
index = load_index_from_storage(storage, embed_model=embed_model)    # pass the same embed model for queries