API Reference - LiuEmbeddings & LiuVectorStore¶

from liuembeddings import LiuEmbeddings, LiuVectorStore

embedder = LiuEmbeddings(model_name="MiniLM")
store = LiuVectorStore(
    embedding_model=embedder,
    collection_name="my_documents",
    persist_path="./chroma_data"
)

texts = [
    "Python is a programming language",
    "JavaScript runs in browsers"
]

# Simple add
doc_ids = store.add_texts(texts)

# With metadata
store.add_texts(
    texts=texts,
    metadatas=[
        {"topic": "Programming", "lang": "Python"},
        {"topic": "Web", "lang": "JavaScript"}
    ],
    ids=["doc1", "doc2"]  # Optional custom IDs
)

docs = [f"Document {i}: Sample..." for i in range(1000)]
metas = [{"index": i, "source": "batch"} for i in range(1000)]

store.add_texts_batch(
    texts=docs,
    batch_size=100,
    metadatas=metas
)

query = "What is Python?"
raw, documents = store.query(query, n_results=2)

for doc in documents:
    print(doc)
# Output:
# "Python is a programming language"
# "Python is used for web development"

query = "Tell me about Python"
raw, results = store.similarity_search(
    query_text=query,
    n_results=2,
    with_score=0.4  # Filter results with score > 0.4
)

for result in results:
    print(f"ID: {result['id']}")
    print(f"Score: {result['similarity_score']:.2f}")
    print(f"Text: {result['document']}")
    print(f"Metadata: {result['metadata']}\n")

[
    {
        'id': 'doc_0_abc123',
        'document': 'Python is a high-level programming language',
        'metadata': {'topic': 'Programming', 'lang': 'Python'},
        'similarity_score': 0.89
    },
    {
        'id': 'doc_1_def456',
        'document': 'Python is used for machine learning',
        'metadata': {'topic': 'ML', 'lang': 'Python'},
        'similarity_score': 0.76
    }
]

doc = store.search_by_id("doc_0_abc123")

if doc:
    print(f"ID: {doc['id']}")
    print(f"Text: {doc['document']}")
    print(f"Metadata: {doc['metadata']}")
else:
    print("Document not found")

# Find all programming documents
results = store.search_by_metadata({"topic": "Programming"})

for doc in results:
    print(f"{doc['id']}: {doc['document']}")

all_docs = store.get_all()

print(f"Total documents: {len(all_docs)}")
for doc in all_docs:
    print(f"{doc['id']}: {doc['document'][:50]}...")

store.update_by_id(
    doc_id="doc_0_abc123",
    new_text="Python: A powerful programming language",
    new_metadata={"updated": True, "version": "2"}
)

try:
    store.delete_by_id("doc_0_abc123")
    print("Deleted successfully")
except:
    print("Document not found")

total = store.count_documents()
print(f"Collection has {total} documents")

info = store.info

print(f"Collection: {info['name']}")
print(f"Documents: {info['document_count']}")
print(f"Model: {info['embedding_model']}")

{
    'name': str,
    'document_count': int,
    'embedding_model': dict  # model_info
}

store.save("backup_collection.json")

long_doc = """
Machine learning is powerful. Feature engineering improves models. 
Deep learning uses neural networks.
"""

store.search(
    text_document=long_doc,
    chunk_size=150,
    chunk_overlap=30
)

raw, docs = store.search(
    query="What improves model performance?",
    n_results=2
)

raw, docs = store.search(
    query="machine learning",
    text_document=long_doc,
    chunk_size=150,
    chunk_overlap=30,
    n_results=1
)

from liuembeddings import LiuEmbeddings

# Default model (USE - 768 dimensions)
embedder = LiuEmbeddings()

# Custom model
embedder = LiuEmbeddings(model_name="MiniLM")

from liuembeddings import LiuEmbeddings, LiuVectorStore, LiuConfig

# Add a custom external embedding model
LiuConfig.AVAILABLE_MODELS['MPNetMini'] = {
    'id': "sentence-transformers/all-mpnet-base-v2",  # HuggingFace MPNet variant
    'dimension': 384,
    'full_name': 'MPNet Mini', # OPTIONAL
    'size': 90,  # OPTIONAL MB
    'description': 'Smaller MPNet variant, faster than full base', # OPTIONAL
    'accuracy': 0.80 # OPTIONAL
}

# Initialize the custom embedder
custom_embedder = LiuEmbeddings('MPNetMini')


custom_vector = LiuVectorStore(
    embedding_model=custom_embedder,
    collection_name="knowledge-NNLM"
)

# Multiple documents
documents = [
    "all boy's lovewin prize",
    "all boy's love money",
    "all boy's love protein",
    "lorem ipsum lorem ipsum",
    "loremipsum lorem iplorem"
]

custom_vector.add_texts(documents,)

raw,docs=custom_vector.search('what all the boys love')

print("answer:")
for i in docs:
    print(i)

>>> answer:
    all boy's love money
    all boy's love protein
    all boy's lovewin prize

print(raw)

info = embedder.model_info

print(info)
# {
#     'name': 'USE',
#     'id': 'intfloat/e5-base-v2',
#     'dimension': 768,
#     'size_mb': 300,
#     'description': 'High-quality semantic embeddings...',
#     'accuracy': 0.84
# }

{
    'name': str,           # Model name
    'id': str,             # HuggingFace model ID
    'dimension': int,      # Output dimension
    'size_mb': int,        # File size in MB
    'description': str,    # Model description
    'accuracy': float      # Accuracy score
}

from liuembeddings import split_text

text = "Machine learning is powerful. Feature engineering improves performance."

chunks = split_text(
    text=text,
    chunk_size=150,
    chunk_overlap=30,
    split_by_sentences=True,
    clean_before_split=True,
    lowercase=False
)

for i, chunk in enumerate(chunks):
    print(f"Chunk {i}: {chunk}")

from liuembeddings import clean_text

messy = "   Hello   WORLD!!! \n\n How are YOU?   "
cleaned = clean_text(messy, lowercase=True)
# Output: "hello world how are you"

raw, _ = store.similarity_search("query")
clean_results = clean(raw)

for result in clean_results:
    print(f"ID: {result['id']}")
    print(f"Document: {result['document']}")
    print(f"Metadata: {result['metadata']}")
    print(f"Distance: {result['distance']}\n")

from liuembeddings import validate_texts

texts = ["Hello", "World"]
is_valid = validate_texts(texts, min_length=1)
print(is_valid)  # True

from liuembeddings import batch_generator

items = list(range(100))
for batch in batch_generator(items, batch_size=10):
    print(f"Batch: {batch}")

from liuembeddings import LiuConfig

# View models
print(LiuConfig.AVAILABLE_MODELS)

# Change defaults
LiuConfig.DEFAULT_CHUNK_SIZE = 2000
LiuConfig.DEFAULT_BATCH_SIZE = 64

# Models
AVAILABLE_MODELS = {
    "MiniLM": {...},
    "MPNetBase": {...},
    "USE": {...},
    "USEL": {...}
}

# Chunking
DEFAULT_CHUNK_SIZE = 1000
DEFAULT_CHUNK_OVERLAP = 200

# Vector Store
DEFAULT_VECTOR_PATH = "./liu_db"
DEFAULT_COLLECTION_NAME = "default_collection"
DISTANCE_METRIC = "cosine"

# Search
DEFAULT_N_RESULTS = 3
MAX_N_RESULTS = 100

# Batch Processing
DEFAULT_BATCH_SIZE = 100

# Model Caching
ENABLE_MODEL_CACHE = True

# Logging
LOG_LEVEL = "INFO"
LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"

# Similarity
DEFAULT_SIMILARITY_SEARCH_SCORE_THRESHOLD = 0.4

fastquery(
        query: str=None,
        text_document: str,
        chunk_size: int, #deafult from config file
        chunk_overlap: int, #deafult from config file
        n_results: int, #deafult from config file
        with_score: float = None, #deafult from config file
        collection_name: str = fastquery.collection_name,
        model_name: str = fastquery.model_name 
) -> list:
        """
        One-line semantic search function.

        note -> use the same model for embedding and searching.
        by default its use "USE" model.

        Combines chunking, embedding, storage, and search in a single call.
        Perfect for quick prototyping and small applications.

        Args:
            text_document: Long text or document to search within
            query: Query string to search for
            chunk_size: Size of text chunks (default: from config)
            chunk_overlap: Overlap between chunks (default: from config)
            n_results: Number of results to return (default: from config)
            with_score: give smilarity search like answer
            collection_name: Name for the vector store collection (default: from config)
            model_name: Embedding model to use (default: "USE")

        Returns:
            List of most similar chunks from the document

        Raises:
            ValueError: If inputs are invalid
            RuntimeError: If operation failsalueError/RuntimeError: For invalid input or failures.
    """

from liuembeddings import fastquery

# Simple use: embed and search in 3 lines
text = "New York is the largest city in the United States. Washington D.C. is the capital. California is a state."

fastquery.collection_name="minimal_collection"

fastquery(text_document=text,)

raw,results = fastquery(
    query="Capital of USA?",
    n_results=2
)

for chunk in results:
    print(chunk)

from liuembeddings import fastquery
fastquery.collection_name='liu-collection'  
fastquery.model_name='USE'

# Custom chunk size and overlap

fastquery(
    text_document="Deep learning uses neural networks. Machine learning is a subset of AI.",
    chunk_size=80,
    chunk_overlap=15,
)


raw,results = fastquery(
    query="What is machine learning?",
    n_results=1
)
print("Best answer:", results[1][0])

from liuembeddings import fastquery

document = """
Luna loves exploring the night sky. Every weekend, she sets up her telescope on the rooftop to watch distant galaxies.
Her favorite constellation is Orion, and she can identify it even without a telescope.
Last month, she discovered a small comet passing near Jupiter and recorded its movement in her astronomy journal.
"""

# Add document to collection "story_collection"
fastquery(
    text_document=document,
    n_results=5,       
    collection_name="story_collection"
)

raw, ans = fastquery(
    query="What celestial object did Luna discover?",
    collection_name="story_collection"
)

for item in ans:
    print(f"Answer: {item}")

Answer: Last month, she discovered a small comet passing near Jupiter and recorded its movement in her astronomy journal.

raw, ans = fastquery(
    query="What celestial object did Luna discover?",
    with_score=0.5,
    collection_name="story_collection"
)

for item in ans:
    print(f"id: {item['id']}")
    print(f"Document: {item['document']}")
    print(f"Metadata: {item['metadata']}")
    print(f"Similarity score: {item['similarity_score']}")

id: doc_1_1761378265744
Document: Last month, she discovered a small comet passing near Jupiter and recorded its movement in her astronomy journal.
Metadata: {'source': 'story_collection'}
Similarity score: 0.41

for item in ans:
    if item['similarity_score'] < 0.5:
        print(f"Answer: {item['document']}")

Answer: Last month, she discovered a small comet passing near Jupiter and recorded its movement in her astronomy journal.

fastquery(
    text_document="...", 
    query="...", 
    model_name="USE",
    collection_name="my_new_collection"
)

from liuembeddings import fastquery

long_doc = """
The solar system includes the Sun and the objects that orbit it, such as planets,
asteroids, and comets. Planets like Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus,
and Neptune revolve around the Sun.
"""

#storing and query at same 
#use with CAUTION re ingesting same data leads to data dublication add ones and Query multiple time
results = fastquery(
    text_document=long_doc,
    query="Which planets orbit the sun?",
    n_results=3
)

for answer in results:
    print(answer)

embedder = LiuEmbeddings(model_name="USE")
embedding = embedder.embed_query("What is machine learning?")

print(len(embedding))  # 768 (for USE model)
print(embedding[:5])   # [-0.004, -0.072, -0.060, -0.007, -0.022]

documents = [
    "Python is a programming language",
    "Machine learning requires data",
    "Data science uses statistics"
]

embeddings = embedder.embed_documents(documents)

print(f"Generated {len(embeddings)} embeddings")
print(f"Each embedding has {len(embeddings[0])} dimensions")

from liuembeddings import (
    LiuEmbeddings,
    LiuVectorStore,
    split_text,
    LiuConfig
)

# 1. Initialize embedder
embedder = LiuEmbeddings(model_name="MiniLM")

# 2. Create vector store
store = LiuVectorStore(embedder, "knowledge_base")

# 3. Add documents
docs = [
    "Python is a programming language",
    "Machine learning requires data",
    "Data science combines stats and coding"
]
store.add_texts(docs, metadatas=[{"type": "intro"} for _ in docs])

# 4. Search
query = "What is Python?"
raw, results = store.similarity_search(query, n_results=2)

# 5. Process results
for result in results:
    print(f"Score: {result['similarity_score']:.2f}")
    print(f"Text: {result['document']}\n")

# 6. Update
store.update_by_id(results[0]['id'], "Python is a powerful language")

# 7. Get info
print(f"Total docs: {store.count_documents()}")
print(f"Collection: {store.info}")

from liuembeddings import LiuEmbeddings, LiuVectorStore

try:
    embedder = LiuEmbeddings(model_name="InvalidModel")
except ValueError as e:
    print(f"Model error: {e}")

try:
    store = LiuVectorStore(None, "test")  # Invalid embedder
except TypeError as e:
    print(f"Type error: {e}")

try:
    results = store.query("")  # Empty query
except ValueError as e:
    print(f"Query error: {e}")