e,
                )
                raise e
        from alibabacloud_ha3engine_vector import models
        id_list = [sha1(t.encode("utf-8")).hexdigest() for t in texts]
        embeddings = self.embedding.embed_documents(list(texts))
        metadatas = metadatas or [{} for _ in texts]
        field_name_map = self.config.field_name_mapping
        add_doc_list = []
        text_list = list(texts)
        for idx, doc_id in enumerate(id_list):
            embedding = embeddings[idx] if idx < len(embeddings) else None
            metadata = metadatas[idx] if idx < len(metadatas) else None
            text = text_list[idx] if idx < len(text_list) else None
            add_doc: Dict[str, Any] = dict()
            add_doc_fields: Dict[str, Any] = dict()
            add_doc_fields.__setitem__(field_name_map["id"], doc_id)
            add_doc_fields.__setitem__(field_name_map["document"], text)
            if embedding is not None:
                add_doc_fields.__setitem__(
                    field_name_map["embedding"],
                    self.config.embedding_field_separator.join(
                        str(unit) for unit in embedding
                    ),
                )
            if metadata is not None:
                for md_key, md_value in metadata.items():
                    add_doc_fields.__setitem__(
                        field_name_map[md_key].split(",")[0], md_value
                    )
            add_doc.__setitem__("fields", add_doc_fields)
            add_doc.__setitem__("cmd", "add")
            add_doc_list.append(add_doc)
        return _upsert(add_doc_list)
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,


self,
        query: str,
        k: int = 4,
        search_filter: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Perform similarity retrieval based on text.
        Args:
            query: Vectorize text for retrieval.，should not be empty.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        embedding = self.embedding.embed_query(query)
        return self.create_results(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        search_filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Perform similarity retrieval based on text with scores.
        Args:
            query: Vectorize text for retrieval.，should not be empty.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        embedding: List[float] = self.embedding.embed_query(query)
        return self.create_results_with_score(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        search_filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Perform retrieval directly using vectors.


) -> List[Document]:
        """Perform retrieval directly using vectors.
        Args:
            embedding: vectors.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        return self.create_results(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def inner_embedding_query(
        self,
        embedding: List[float],
        search_filter: Optional[Dict[str, Any]] = None,
        k: int = 4,
    ) -> Dict[str, Any]:
        def generate_filter_query() -> str:
            if search_filter is None:
                return ""
            filter_clause = " AND ".join(
                [
                    create_filter(md_key, md_value)
                    for md_key, md_value in search_filter.items()
                ]
            )
            return filter_clause
        def create_filter(md_key: str, md_value: Any) -> str:
            md_filter_expr = self.config.field_name_mapping[md_key]
            if md_filter_expr is None:
                return ""
            expr = md_filter_expr.split(",")
            if len(expr) != 2:
                logger.error(
                    f"filter {md_filter_expr} express is not correct, "
                    f"must contain mapping field and operator."
                )
                return ""
            md_filter_key = expr[0].strip()
            md_filter_operator = expr[1].strip()
            if isinstance(md_value, numbers.Number):
                return f"{md_filter_key} {md_filter_operator} {md_value}"
            return f'{md_filter_key}{md_filter_operator}"{md_value}"'


return f'{md_filter_key}{md_filter_operator}"{md_value}"'
        def search_data() -> Dict[str, Any]:
            request = QueryRequest(
                table_name=self.config.table_name,
                namespace=self.config.namespace,
                vector=embedding,
                include_vector=True,
                output_fields=self.config.output_fields,
                filter=generate_filter_query(),
                top_k=k,
            )
            query_result = self.ha3_engine_client.query(request)
            return json.loads(query_result.body)
        from alibabacloud_ha3engine_vector.models import QueryRequest
        try:
            json_response = search_data()
            if (
                "errorCode" in json_response
                and "errorMsg" in json_response
                and len(json_response["errorMsg"]) > 0
            ):
                logger.error(
                    f"query {self.config.endpoint} {self.config.instance_id} "
                    f"failed:{json_response['errorMsg']}."
                )
            else:
                return json_response
        except Exception as e:
            logger.error(
                f"query instance endpoint:{self.config.endpoint} "
                f"instance_id:{self.config.instance_id} failed.",
                e,
            )
        return {}
[docs]    def create_results(self, json_result: Dict[str, Any]) -> List[Document]:
        """Assemble documents."""
        items = json_result["result"]
        query_result_list: List[Document] = []
        for item in items:
            if (
                "fields" not in item
                or self.config.field_name_mapping["document"] not in item["fields"]
            ):
                query_result_list.append(Document())
            else:
                fields = item["fields"]
                query_result_list.append(
                    Document(


fields = item["fields"]
                query_result_list.append(
                    Document(
                        page_content=fields[self.config.field_name_mapping["document"]],
                        metadata=self.create_inverse_metadata(fields),
                    )
                )
        return query_result_list
[docs]    def create_inverse_metadata(self, fields: Dict[str, Any]) -> Dict[str, Any]:
        """Create metadata from fields.
        Args:
            fields: The fields of the document. The fields must be a dict.
        Returns:
            metadata: The metadata of the document. The metadata must be a dict.
        """
        metadata: Dict[str, Any] = {}
        for key, value in fields.items():
            if key == "id" or key == "document" or key == "embedding":
                continue
            metadata[self.config.inverse_field_name_mapping[key]] = value
        return metadata
[docs]    def create_results_with_score(
        self, json_result: Dict[str, Any]
    ) -> List[Tuple[Document, float]]:
        """Parsing the returned results with scores.
        Args:
            json_result: Results from OpenSearch query.
        Returns:
            query_result_list: Results with scores.
        """
        items = json_result["result"]
        query_result_list: List[Tuple[Document, float]] = []
        for item in items:
            fields = item["fields"]
            query_result_list.append(
                (
                    Document(
                        page_content=fields[self.config.field_name_mapping["document"]],
                        metadata=self.create_inverse_metadata(fields),
                    ),
                    float(item["score"]),
                )
            )
        return query_result_list
[docs]    def delete_documents_with_texts(self, texts: List[str]) -> bool:


[docs]    def delete_documents_with_texts(self, texts: List[str]) -> bool:
        """Delete documents based on their page content.
        Args:
            texts: List of document page content.
        Returns:
           Whether the deletion was successful or not.
        """
        id_list = [sha1(t.encode("utf-8")).hexdigest() for t in texts]
        return self.delete_documents_with_document_id(id_list)
[docs]    def delete_documents_with_document_id(self, id_list: List[str]) -> bool:
        """Delete documents based on their IDs.
        Args:
            id_list: List of document IDs.
        Returns:
            Whether the deletion was successful or not.
        """
        if id_list is None or len(id_list) == 0:
            return True
        from alibabacloud_ha3engine_vector import models
        delete_doc_list = []
        for doc_id in id_list:
            delete_doc_list.append(
                {
                    "fields": {self.config.field_name_mapping["id"]: doc_id},
                    "cmd": "delete",
                }
            )
        delete_request = models.PushDocumentsRequest(
            self.options_headers, delete_doc_list
        )
        try:
            delete_response = self.ha3_engine_client.push_documents(
                self.config.opt_table_name,
                self.config.field_name_mapping["id"],
                delete_request,
            )
            json_response = json.loads(delete_response.body)
            return json_response["status"] == "OK"
        except Exception as e:
            logger.error(
                f"delete doc from :{self.config.endpoint} "
                f"instance_id:{self.config.instance_id} failed.",
                e,
            )
            raise e
[docs]    @classmethod


e,
            )
            raise e
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        config: Optional[AlibabaCloudOpenSearchSettings] = None,
        **kwargs: Any,
    ) -> "AlibabaCloudOpenSearch":
        """Create alibaba cloud opensearch vector store instance.
        Args:
            texts: The text segments to be inserted into the vector storage,
             should not be empty.
            embedding: Embedding function, Embedding function.
            config: Alibaba OpenSearch instance configuration.
            metadatas: Metadata information.
        Returns:
            AlibabaCloudOpenSearch: Alibaba cloud opensearch vector store instance.
        """
        if texts is None or len(texts) == 0:
            raise Exception("the inserted text segments, should not be empty.")
        if embedding is None:
            raise Exception("the embeddings should not be empty.")
        if config is None:
            raise Exception("config should not be none.")
        ctx = cls(embedding, config, **kwargs)
        ctx.add_texts(texts=texts, metadatas=metadatas)
        return ctx
[docs]    @classmethod
    def from_documents(
        cls,
        documents: List[Document],
        embedding: Embeddings,
        config: Optional[AlibabaCloudOpenSearchSettings] = None,
        **kwargs: Any,
    ) -> "AlibabaCloudOpenSearch":
        """Create alibaba cloud opensearch vector store instance.
        Args:
            documents: Documents to be inserted into the vector storage,
             should not be empty.


documents: Documents to be inserted into the vector storage,
             should not be empty.
            embedding: Embedding function, Embedding function.
            config: Alibaba OpenSearch instance configuration.
            ids: Specify the ID for the inserted document. If left empty, the ID will be
             automatically generated based on the text content.
        Returns:
            AlibabaCloudOpenSearch: Alibaba cloud opensearch vector store instance.
        """
        if documents is None or len(documents) == 0:
            raise Exception("the inserted documents, should not be empty.")
        if embedding is None:
            raise Exception("the embeddings should not be empty.")
        if config is None:
            raise Exception("config can't be none")
        texts = [d.page_content for d in documents]
        metadatas = [d.metadata for d in documents]
        return cls.from_texts(
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            config=config,
            **kwargs,
        )


Source code for langchain.vectorstores.annoy
from __future__ import annotations
import os
import pickle
import uuid
from configparser import ConfigParser
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
import numpy as np
from langchain.docstore.base import Docstore
from langchain.docstore.document import Document
from langchain.docstore.in_memory import InMemoryDocstore
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
INDEX_METRICS = frozenset(["angular", "euclidean", "manhattan", "hamming", "dot"])
DEFAULT_METRIC = "angular"
[docs]def dependable_annoy_import() -> Any:
    """Import annoy if available, otherwise raise error."""
    try:
        import annoy
    except ImportError:
        raise ImportError(
            "Could not import annoy python package. "
            "Please install it with `pip install --user annoy` "
        )
    return annoy
[docs]class Annoy(VectorStore):
    """`Annoy` vector store.
    To use, you should have the ``annoy`` python package installed.
    Example:
        .. code-block:: python
            from langchain.vectorstores import Annoy
            db = Annoy(embedding_function, index, docstore, index_to_docstore_id)
    """
[docs]    def __init__(
        self,
        embedding_function: Callable,
        index: Any,
        metric: str,
        docstore: Docstore,
        index_to_docstore_id: Dict[int, str],
    ):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function
        self.index = index


self.embedding_function = embedding_function
        self.index = index
        self.metric = metric
        self.docstore = docstore
        self.index_to_docstore_id = index_to_docstore_id
    @property
    def embeddings(self) -> Optional[Embeddings]:
        # TODO: Accept embedding object directly
        return None
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        raise NotImplementedError(
            "Annoy does not allow to add new data once the index is build."
        )
[docs]    def process_index_results(
        self, idxs: List[int], dists: List[float]
    ) -> List[Tuple[Document, float]]:
        """Turns annoy results into a list of documents and scores.
        Args:
            idxs: List of indices of the documents in the index.
            dists: List of distances of the documents in the index.
        Returns:
            List of Documents and scores.
        """
        docs = []
        for idx, dist in zip(idxs, dists):
            _id = self.index_to_docstore_id[idx]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append((doc, dist))
        return docs
[docs]    def similarity_search_with_score_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:


"""Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_vector(
            embedding, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_item(
            docstore_index, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.


Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        embedding = self.embedding_function(query)
        docs = self.similarity_search_with_score_by_vector(embedding, k, search_k)
        return docs
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_vector(
            embedding, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to docstore_index.
        Args:
            docstore_index: Index of document in docstore
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:


to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_index(
            docstore_index, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search(
        self, query: str, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query.
        """
        docs_and_scores = self.similarity_search_with_score(query, k, search_k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            k: Number of Documents to return. Defaults to 4.
            lambda_mult: Number between 0 and 1 that determines the degree


lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        idxs = self.index.get_nns_by_vector(
            embedding, fetch_k, search_k=-1, include_distances=False
        )
        embeddings = [self.index.get_item_vector(i) for i in idxs]
        mmr_selected = maximal_marginal_relevance(
            np.array([embedding], dtype=np.float32),
            embeddings,
            k=k,
            lambda_mult=lambda_mult,
        )
        # ignore the -1's if not enough docs are returned/indexed
        selected_indices = [idxs[i] for i in mmr_selected if i != -1]
        docs = []
        for i in selected_indices:
            _id = self.index_to_docstore_id[i]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append(doc)
        return docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.


Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self.embedding_function(query)
        docs = self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult=lambda_mult
        )
        return docs
    @classmethod
    def __from(
        cls,
        texts: List[str],
        embeddings: List[List[float]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        if metric not in INDEX_METRICS:
            raise ValueError(
                (
                    f"Unsupported distance metric: {metric}. "
                    f"Expected one of {list(INDEX_METRICS)}"
                )
            )
        annoy = dependable_annoy_import()
        if not embeddings:
            raise ValueError("embeddings must be provided to build AnnoyIndex")
        f = len(embeddings[0])
        index = annoy.AnnoyIndex(f, metric=metric)
        for i, emb in enumerate(embeddings):
            index.add_item(i, emb)
        index.build(trees, n_jobs=n_jobs)
        documents = []


index.build(trees, n_jobs=n_jobs)
        documents = []
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas else {}
            documents.append(Document(page_content=text, metadata=metadata))
        index_to_id = {i: str(uuid.uuid4()) for i in range(len(documents))}
        docstore = InMemoryDocstore(
            {index_to_id[i]: doc for i, doc in enumerate(documents)}
        )
        return cls(embedding.embed_query, index, metric, docstore, index_to_id)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from raw documents.
        Args:
            texts: List of documents to index.
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Creates an in memory docstore
            3. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Annoy


.. code-block:: python
                from langchain.vectorstores import Annoy
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                index = Annoy.from_texts(texts, embeddings)
        """
        embeddings = embedding.embed_documents(texts)
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    @classmethod
    def from_embeddings(
        cls,
        text_embeddings: List[Tuple[str, List[float]]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from embeddings.
        Args:
            text_embeddings: List of tuples of (text, embedding)
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1
        This is a user friendly interface that:
            1. Creates an in memory docstore with provided embeddings
            2. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Annoy
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                text_embeddings = embeddings.embed_documents(texts)


embeddings = OpenAIEmbeddings()
                text_embeddings = embeddings.embed_documents(texts)
                text_embedding_pairs = list(zip(texts, text_embeddings))
                db = Annoy.from_embeddings(text_embedding_pairs, embeddings)
        """
        texts = [t[0] for t in text_embeddings]
        embeddings = [t[1] for t in text_embeddings]
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    def save_local(self, folder_path: str, prefault: bool = False) -> None:
        """Save Annoy index, docstore, and index_to_docstore_id to disk.
        Args:
            folder_path: folder path to save index, docstore,
                and index_to_docstore_id to.
            prefault: Whether to pre-load the index into memory.
        """
        path = Path(folder_path)
        os.makedirs(path, exist_ok=True)
        # save index, index config, docstore and index_to_docstore_id
        config_object = ConfigParser()
        config_object["ANNOY"] = {
            "f": self.index.f,
            "metric": self.metric,
        }
        self.index.save(str(path / "index.annoy"), prefault=prefault)
        with open(path / "index.pkl", "wb") as file:
            pickle.dump((self.docstore, self.index_to_docstore_id, config_object), file)
[docs]    @classmethod
    def load_local(
        cls,
        folder_path: str,
        embeddings: Embeddings,
    ) -> Annoy:
        """Load Annoy index, docstore, and index_to_docstore_id to disk.
        Args:


Args:
            folder_path: folder path to load index, docstore,
                and index_to_docstore_id from.
            embeddings: Embeddings to use when generating queries.
        """
        path = Path(folder_path)
        # load index separately since it is not picklable
        annoy = dependable_annoy_import()
        # load docstore and index_to_docstore_id
        with open(path / "index.pkl", "rb") as file:
            docstore, index_to_docstore_id, config_object = pickle.load(file)
        f = int(config_object["ANNOY"]["f"])
        metric = config_object["ANNOY"]["metric"]
        index = annoy.AnnoyIndex(f, metric=metric)
        index.load(str(path / "index.annoy"))
        return cls(
            embeddings.embed_query, index, metric, docstore, index_to_docstore_id
        )


Source code for langchain.vectorstores.docarray.base
from abc import ABC
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Type
import numpy as np
from langchain.pydantic_v1 import Field
from langchain.schema import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
if TYPE_CHECKING:
    from docarray import BaseDoc
    from docarray.index.abstract import BaseDocIndex
def _check_docarray_import() -> None:
    try:
        import docarray
        da_version = docarray.__version__.split(".")
        if int(da_version[0]) == 0 and int(da_version[1]) <= 31:
            raise ImportError(
                f"To use the DocArrayHnswSearch VectorStore the docarray "
                f"version >=0.32.0 is expected, received: {docarray.__version__}."
                f"To upgrade, please run: `pip install -U docarray`."
            )
    except ImportError:
        raise ImportError(
            "Could not import docarray python package. "
            'Please install it with `pip install "langchain[docarray]"`.'
        )
[docs]class DocArrayIndex(VectorStore, ABC):
    """Base class for `DocArray` based vector stores."""
[docs]    def __init__(
        self,
        doc_index: "BaseDocIndex",
        embedding: Embeddings,
    ):
        """Initialize a vector store from DocArray's DocIndex."""
        self.doc_index = doc_index
        self.embedding = embedding
    @staticmethod
    def _get_doc_cls(**embeddings_params: Any) -> Type["BaseDoc"]:


def _get_doc_cls(**embeddings_params: Any) -> Type["BaseDoc"]:
        """Get docarray Document class describing the schema of DocIndex."""
        from docarray import BaseDoc
        from docarray.typing import NdArray
        class DocArrayDoc(BaseDoc):
            text: Optional[str]
            embedding: Optional[NdArray] = Field(**embeddings_params)
            metadata: Optional[dict]
        return DocArrayDoc
    @property
    def doc_cls(self) -> Type["BaseDoc"]:
        if self.doc_index._schema is None:
            raise ValueError("doc_index expected to have non-null _schema attribute.")
        return self.doc_index._schema
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Embed texts and add to the vector store.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        ids: List[str] = []
        embeddings = self.embedding.embed_documents(list(texts))
        for i, (t, e) in enumerate(zip(texts, embeddings)):
            m = metadatas[i] if metadatas else {}
            doc = self.doc_cls(text=t, embedding=e, metadata=m)
            self.doc_index.index([doc])
            ids.append(str(doc.id))
        return ids
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, **kwargs: Any


self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of documents most similar to the query text and
            cosine distance in float for each.
            Lower score represents more similarity.
        """
        query_embedding = self.embedding.embed_query(query)
        query_doc = self.doc_cls(embedding=query_embedding)  # type: ignore
        docs, scores = self.doc_index.find(query_doc, search_field="embedding", limit=k)
        result = [
            (Document(page_content=doc.text, metadata=doc.metadata), score)
            for doc, score in zip(docs, scores)
        ]
        return result
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the query.
        """
        results = self.similarity_search_with_score(query, k=k, **kwargs)
        return [doc for doc, _ in results]
    def _similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return docs and relevance scores, normalized on a scale from 0 to 1.


"""Return docs and relevance scores, normalized on a scale from 0 to 1.
        0 is dissimilar, 1 is most similar.
        """
        raise NotImplementedError()
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the query vector.
        """
        query_doc = self.doc_cls(embedding=embedding)  # type: ignore
        docs = self.doc_index.find(
            query_doc, search_field="embedding", limit=k
        ).documents
        result = [
            Document(page_content=doc.text, metadata=doc.metadata) for doc in docs
        ]
        return result
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                of diversity among the results with 0 corresponding


of diversity among the results with 0 corresponding
                to maximum diversity and 1 to minimum diversity.
                Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        query_embedding = self.embedding.embed_query(query)
        query_doc = self.doc_cls(embedding=query_embedding)  # type: ignore
        docs = self.doc_index.find(
            query_doc, search_field="embedding", limit=fetch_k
        ).documents
        mmr_selected = maximal_marginal_relevance(
            np.array(query_embedding), docs.embedding, k=k
        )
        results = [
            Document(page_content=docs[idx].text, metadata=docs[idx].metadata)
            for idx in mmr_selected
        ]
        return results


Source code for langchain.vectorstores.docarray.hnsw
from __future__ import annotations
from typing import Any, List, Literal, Optional
from langchain.schema.embeddings import Embeddings
from langchain.vectorstores.docarray.base import (
    DocArrayIndex,
    _check_docarray_import,
)
[docs]class DocArrayHnswSearch(DocArrayIndex):
    """`HnswLib` storage using `DocArray` package.
    To use it, you should have the ``docarray`` package with version >=0.32.0 installed.
    You can install it with `pip install "langchain[docarray]"`.
    """
[docs]    @classmethod
    def from_params(
        cls,
        embedding: Embeddings,
        work_dir: str,
        n_dim: int,
        dist_metric: Literal["cosine", "ip", "l2"] = "cosine",
        max_elements: int = 1024,
        index: bool = True,
        ef_construction: int = 200,
        ef: int = 10,
        M: int = 16,
        allow_replace_deleted: bool = True,
        num_threads: int = 1,
        **kwargs: Any,
    ) -> DocArrayHnswSearch:
        """Initialize DocArrayHnswSearch store.
        Args:
            embedding (Embeddings): Embedding function.
            work_dir (str): path to the location where all the data will be stored.
            n_dim (int): dimension of an embedding.
            dist_metric (str): Distance metric for DocArrayHnswSearch can be one of:
                "cosine", "ip", and "l2". Defaults to "cosine".


"cosine", "ip", and "l2". Defaults to "cosine".
            max_elements (int): Maximum number of vectors that can be stored.
                Defaults to 1024.
            index (bool): Whether an index should be built for this field.
                Defaults to True.
            ef_construction (int): defines a construction time/accuracy trade-off.
                Defaults to 200.
            ef (int): parameter controlling query time/accuracy trade-off.
                Defaults to 10.
            M (int): parameter that defines the maximum number of outgoing
                connections in the graph. Defaults to 16.
            allow_replace_deleted (bool): Enables replacing of deleted elements
                with new added ones. Defaults to True.
            num_threads (int): Sets the number of cpu threads to use. Defaults to 1.
            **kwargs: Other keyword arguments to be passed to the get_doc_cls method.
        """
        _check_docarray_import()
        from docarray.index import HnswDocumentIndex
        doc_cls = cls._get_doc_cls(
            dim=n_dim,
            space=dist_metric,
            max_elements=max_elements,
            index=index,
            ef_construction=ef_construction,
            ef=ef,
            M=M,
            allow_replace_deleted=allow_replace_deleted,
            num_threads=num_threads,
            **kwargs,
        )
        doc_index = HnswDocumentIndex[doc_cls](work_dir=work_dir)  # type: ignore
        return cls(doc_index, embedding)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        work_dir: Optional[str] = None,


work_dir: Optional[str] = None,
        n_dim: Optional[int] = None,
        **kwargs: Any,
    ) -> DocArrayHnswSearch:
        """Create an DocArrayHnswSearch store and insert data.
        Args:
            texts (List[str]): Text data.
            embedding (Embeddings): Embedding function.
            metadatas (Optional[List[dict]]): Metadata for each text if it exists.
                Defaults to None.
            work_dir (str): path to the location where all the data will be stored.
            n_dim (int): dimension of an embedding.
            **kwargs: Other keyword arguments to be passed to the __init__ method.
        Returns:
            DocArrayHnswSearch Vector Store
        """
        if work_dir is None:
            raise ValueError("`work_dir` parameter has not been set.")
        if n_dim is None:
            raise ValueError("`n_dim` parameter has not been set.")
        store = cls.from_params(embedding, work_dir, n_dim, **kwargs)
        store.add_texts(texts=texts, metadatas=metadatas)
        return store


Source code for langchain.vectorstores.docarray.in_memory
"""Wrapper around in-memory storage."""
from __future__ import annotations
from typing import Any, Dict, List, Literal, Optional
from langchain.schema.embeddings import Embeddings
from langchain.vectorstores.docarray.base import (
    DocArrayIndex,
    _check_docarray_import,
)
[docs]class DocArrayInMemorySearch(DocArrayIndex):
    """In-memory `DocArray` storage for exact search.
    To use it, you should have the ``docarray`` package with version >=0.32.0 installed.
    You can install it with `pip install "langchain[docarray]"`.
    """
[docs]    @classmethod
    def from_params(
        cls,
        embedding: Embeddings,
        metric: Literal[
            "cosine_sim", "euclidian_dist", "sgeuclidean_dist"
        ] = "cosine_sim",
        **kwargs: Any,
    ) -> DocArrayInMemorySearch:
        """Initialize DocArrayInMemorySearch store.
        Args:
            embedding (Embeddings): Embedding function.
            metric (str): metric for exact nearest-neighbor search.
                Can be one of: "cosine_sim", "euclidean_dist" and "sqeuclidean_dist".
                Defaults to "cosine_sim".
            **kwargs: Other keyword arguments to be passed to the get_doc_cls method.
        """
        _check_docarray_import()
        from docarray.index import InMemoryExactNNIndex
        doc_cls = cls._get_doc_cls(space=metric, **kwargs)
        doc_index = InMemoryExactNNIndex[doc_cls]()  # type: ignore
        return cls(doc_index, embedding)
[docs]    @classmethod
    def from_texts(


[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[Dict[Any, Any]]] = None,
        **kwargs: Any,
    ) -> DocArrayInMemorySearch:
        """Create an DocArrayInMemorySearch store and insert data.
        Args:
            texts (List[str]): Text data.
            embedding (Embeddings): Embedding function.
            metadatas (Optional[List[Dict[Any, Any]]]): Metadata for each text
                if it exists. Defaults to None.
            metric (str): metric for exact nearest-neighbor search.
                Can be one of: "cosine_sim", "euclidean_dist" and "sqeuclidean_dist".
                Defaults to "cosine_sim".
        Returns:
            DocArrayInMemorySearch Vector Store
        """
        store = cls.from_params(embedding, **kwargs)
        store.add_texts(texts=texts, metadatas=metadatas)
        return store


Source code for langchain.vectorstores.redis.schema
from __future__ import annotations
import os
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
import numpy as np
import yaml
from typing_extensions import TYPE_CHECKING, Literal
from langchain.pydantic_v1 import BaseModel, Field, validator
from langchain.vectorstores.redis.constants import REDIS_VECTOR_DTYPE_MAP
if TYPE_CHECKING:
    from redis.commands.search.field import (  # type: ignore
        NumericField,
        TagField,
        TextField,
        VectorField,
    )
[docs]class RedisDistanceMetric(str, Enum):
    """Distance metrics for Redis vector fields."""
    l2 = "L2"
    cosine = "COSINE"
    ip = "IP"
[docs]class RedisField(BaseModel):
    """Base class for Redis fields."""
    name: str = Field(...)
[docs]class TextFieldSchema(RedisField):
    """Schema for text fields in Redis."""
    weight: float = 1
    no_stem: bool = False
    phonetic_matcher: Optional[str] = None
    withsuffixtrie: bool = False
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> TextField:
        from redis.commands.search.field import TextField  # type: ignore
        return TextField(
            self.name,
            weight=self.weight,
            no_stem=self.no_stem,
            phonetic_matcher=self.phonetic_matcher,  # type: ignore
            sortable=self.sortable,
            no_index=self.no_index,
        )
[docs]class TagFieldSchema(RedisField):
    """Schema for tag fields in Redis."""


"""Schema for tag fields in Redis."""
    separator: str = ","
    case_sensitive: bool = False
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> TagField:
        from redis.commands.search.field import TagField  # type: ignore
        return TagField(
            self.name,
            separator=self.separator,
            case_sensitive=self.case_sensitive,
            sortable=self.sortable,
            no_index=self.no_index,
        )
[docs]class NumericFieldSchema(RedisField):
    """Schema for numeric fields in Redis."""
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> NumericField:
        from redis.commands.search.field import NumericField  # type: ignore
        return NumericField(self.name, sortable=self.sortable, no_index=self.no_index)
[docs]class RedisVectorField(RedisField):
    """Base class for Redis vector fields."""
    dims: int = Field(...)
    algorithm: object = Field(...)
    datatype: str = Field(default="FLOAT32")
    distance_metric: RedisDistanceMetric = Field(default="COSINE")
    initial_cap: Optional[int] = None
    @validator("algorithm", "datatype", "distance_metric", pre=True, each_item=True)
    def uppercase_strings(cls, v: str) -> str:
        return v.upper()
    @validator("datatype", pre=True)
    def uppercase_and_check_dtype(cls, v: str) -> str:
        if v.upper() not in REDIS_VECTOR_DTYPE_MAP:
            raise ValueError(
                f"datatype must be one of {REDIS_VECTOR_DTYPE_MAP.keys()}. Got {v}"
            )
        return v.upper()


)
        return v.upper()
    def _fields(self) -> Dict[str, Any]:
        field_data = {
            "TYPE": self.datatype,
            "DIM": self.dims,
            "DISTANCE_METRIC": self.distance_metric,
        }
        if self.initial_cap is not None:  # Only include it if it's set
            field_data["INITIAL_CAP"] = self.initial_cap
        return field_data
[docs]class FlatVectorField(RedisVectorField):
    """Schema for flat vector fields in Redis."""
    algorithm: Literal["FLAT"] = "FLAT"
    block_size: Optional[int] = None
[docs]    def as_field(self) -> VectorField:
        from redis.commands.search.field import VectorField  # type: ignore
        field_data = super()._fields()
        if self.block_size is not None:
            field_data["BLOCK_SIZE"] = self.block_size
        return VectorField(self.name, self.algorithm, field_data)
[docs]class HNSWVectorField(RedisVectorField):
    """Schema for HNSW vector fields in Redis."""
    algorithm: Literal["HNSW"] = "HNSW"
    m: int = Field(default=16)
    ef_construction: int = Field(default=200)
    ef_runtime: int = Field(default=10)
    epsilon: float = Field(default=0.01)
[docs]    def as_field(self) -> VectorField:
        from redis.commands.search.field import VectorField  # type: ignore
        field_data = super()._fields()
        field_data.update(
            {
                "M": self.m,
                "EF_CONSTRUCTION": self.ef_construction,
                "EF_RUNTIME": self.ef_runtime,
                "EPSILON": self.epsilon,


"EF_RUNTIME": self.ef_runtime,
                "EPSILON": self.epsilon,
            }
        )
        return VectorField(self.name, self.algorithm, field_data)
[docs]class RedisModel(BaseModel):
    """Schema for Redis index."""
    # always have a content field for text
    text: List[TextFieldSchema] = [TextFieldSchema(name="content")]
    tag: Optional[List[TagFieldSchema]] = None
    numeric: Optional[List[NumericFieldSchema]] = None
    extra: Optional[List[RedisField]] = None
    # filled by default_vector_schema
    vector: Optional[List[Union[FlatVectorField, HNSWVectorField]]] = None
    content_key: str = "content"
    content_vector_key: str = "content_vector"
[docs]    def add_content_field(self) -> None:
        if self.text is None:
            self.text = []
        for field in self.text:
            if field.name == self.content_key:
                return
        self.text.append(TextFieldSchema(name=self.content_key))
[docs]    def add_vector_field(self, vector_field: Dict[str, Any]) -> None:
        # catch case where user inputted no vector field spec
        # in the index schema
        if self.vector is None:
            self.vector = []
        # ignore types as pydantic is handling type validation and conversion
        if vector_field["algorithm"] == "FLAT":
            self.vector.append(FlatVectorField(**vector_field))  # type: ignore
        elif vector_field["algorithm"] == "HNSW":
            self.vector.append(HNSWVectorField(**vector_field))  # type: ignore
        else:
            raise ValueError(
                f"algorithm must be either FLAT or HNSW. Got "


f"algorithm must be either FLAT or HNSW. Got "
                f"{vector_field['algorithm']}"
            )
[docs]    def as_dict(self) -> Dict[str, List[Any]]:
        schemas: Dict[str, List[Any]] = {"text": [], "tag": [], "numeric": []}
        # iter over all class attributes
        for attr, attr_value in self.__dict__.items():
            # only non-empty lists
            if isinstance(attr_value, list) and len(attr_value) > 0:
                field_values: List[Dict[str, Any]] = []
                # iterate over all fields in each category (tag, text, etc)
                for val in attr_value:
                    value: Dict[str, Any] = {}
                    # iterate over values within each field to extract
                    # settings for that field (i.e. name, weight, etc)
                    for field, field_value in val.__dict__.items():
                        # make enums into strings
                        if isinstance(field_value, Enum):
                            value[field] = field_value.value
                        # don't write null values
                        elif field_value is not None:
                            value[field] = field_value
                    field_values.append(value)
                schemas[attr] = field_values
        schema: Dict[str, List[Any]] = {}
        # only write non-empty lists from defaults
        for k, v in schemas.items():
            if len(v) > 0:
                schema[k] = v
        return schema
    @property
    def content_vector(self) -> Union[FlatVectorField, HNSWVectorField]:
        if not self.vector:
            raise ValueError("No vector fields found")
        for field in self.vector:
            if field.name == self.content_vector_key:
                return field


if field.name == self.content_vector_key:
                return field
        raise ValueError("No content_vector field found")
    @property
    def vector_dtype(self) -> np.dtype:
        # should only ever be called after pydantic has validated the schema
        return REDIS_VECTOR_DTYPE_MAP[self.content_vector.datatype]
    @property
    def is_empty(self) -> bool:
        return all(
            field is None for field in [self.tag, self.text, self.numeric, self.vector]
        )
[docs]    def get_fields(self) -> List["RedisField"]:
        redis_fields: List["RedisField"] = []
        if self.is_empty:
            return redis_fields
        for field_name in self.__fields__.keys():
            if field_name not in ["content_key", "content_vector_key", "extra"]:
                field_group = getattr(self, field_name)
                if field_group is not None:
                    for field in field_group:
                        redis_fields.append(field.as_field())
        return redis_fields
    @property
    def metadata_keys(self) -> List[str]:
        keys: List[str] = []
        if self.is_empty:
            return keys
        for field_name in self.__fields__.keys():
            field_group = getattr(self, field_name)
            if field_group is not None:
                for field in field_group:
                    # check if it's a metadata field. exclude vector and content key
                    if not isinstance(field, str) and field.name not in [
                        self.content_key,
                        self.content_vector_key,
                    ]:
                        keys.append(field.name)
        return keys
[docs]def read_schema(
    index_schema: Optional[Union[Dict[str, List[Any]], str, os.PathLike]]
) -> Dict[str, Any]:


) -> Dict[str, Any]:
    """Reads in the index schema from a dict or yaml file.
    Check if it is a dict and return RedisModel otherwise, check if it's a path and
    read in the file assuming it's a yaml file and return a RedisModel
    """
    if isinstance(index_schema, dict):
        return index_schema
    elif isinstance(index_schema, Path):
        with open(index_schema, "rb") as f:
            return yaml.safe_load(f)
    elif isinstance(index_schema, str):
        if Path(index_schema).resolve().is_file():
            with open(index_schema, "rb") as f:
                return yaml.safe_load(f)
        else:
            raise FileNotFoundError(f"index_schema file {index_schema} does not exist")
    else:
        raise TypeError(
            f"index_schema must be a dict, or path to a yaml file "
            f"Got {type(index_schema)}"
        )


Source code for langchain.vectorstores.redis.base
"""Wrapper around Redis vector database."""
from __future__ import annotations
import logging
import os
import uuid
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Mapping,
    Optional,
    Tuple,
    Type,
    Union,
    cast,
)
import numpy as np
import yaml
from langchain._api import deprecated
from langchain.callbacks.manager import CallbackManagerForRetrieverRun
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore, VectorStoreRetriever
from langchain.utilities.redis import (
    _array_to_buffer,
    _buffer_to_array,
    check_redis_module_exist,
    get_client,
)
from langchain.utils import get_from_dict_or_env
from langchain.vectorstores.redis.constants import (
    REDIS_REQUIRED_MODULES,
    REDIS_TAG_SEPARATOR,
)
from langchain.vectorstores.utils import maximal_marginal_relevance
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
    from redis.client import Redis as RedisType
    from redis.commands.search.query import Query
    from langchain.vectorstores.redis.filters import RedisFilterExpression
    from langchain.vectorstores.redis.schema import RedisModel
def _default_relevance_score(val: float) -> float:
    return 1 - val
[docs]def check_index_exists(client: RedisType, index_name: str) -> bool:
    """Check if Redis index exists."""
    try:
        client.ft(index_name).info()
    except:  # noqa: E722
        logger.info("Index does not exist")
        return False
    logger.info("Index already exists")


return False
    logger.info("Index already exists")
    return True
[docs]class Redis(VectorStore):
    """Redis vector database.
    To use, you should have the ``redis`` python package installed
    and have a running Redis Enterprise or Redis-Stack server
    For production use cases, it is recommended to use Redis Enterprise
    as the scaling, performance, stability and availability is much
    better than Redis-Stack.
    For testing and prototyping, however, this is not required.
    Redis-Stack is available as a docker container the full vector
    search API available.
    .. code-block:: bash
        # to run redis stack in docker locally
        docker run -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
    Once running, you can connect to the redis server with the following url schemas:
    - redis://<host>:<port> # simple connection
    - redis://<username>:<password>@<host>:<port> # connection with authentication
    - rediss://<host>:<port> # connection with SSL
    - rediss://<username>:<password>@<host>:<port> # connection with SSL and auth
    Examples:
    The following examples show various ways to use the Redis VectorStore with
    LangChain.
    For all the following examples assume we have the following imports:
    .. code-block:: python
        from langchain.vectorstores import Redis
        from langchain.embeddings import OpenAIEmbeddings
    Initialize, create index, and load Documents
        .. code-block:: python
            from langchain.vectorstores import Redis
            from langchain.embeddings import OpenAIEmbeddings
            rds = Redis.from_documents(
                documents, # a list of Document objects from loaders or created


documents, # a list of Document objects from loaders or created
                embeddings, # an Embeddings object
                redis_url="redis://localhost:6379",
            )
    Initialize, create index, and load Documents with metadata
        .. code-block:: python
            rds = Redis.from_texts(
                texts, # a list of strings
                metadata, # a list of metadata dicts
                embeddings, # an Embeddings object
                redis_url="redis://localhost:6379",
            )
    Initialize, create index, and load Documents with metadata and return keys
        .. code-block:: python
            rds, keys = Redis.from_texts_return_keys(
                texts, # a list of strings
                metadata, # a list of metadata dicts
                embeddings, # an Embeddings object
                redis_url="redis://localhost:6379",
            )
    For use cases where the index needs to stay alive, you can initialize
    with an index name such that it's easier to reference later
        .. code-block:: python
            rds = Redis.from_texts(
                texts, # a list of strings
                metadata, # a list of metadata dicts
                embeddings, # an Embeddings object
                index_name="my-index",
                redis_url="redis://localhost:6379",
            )
    Initialize and connect to an existing index (from above)
        .. code-block:: python
            rds = Redis.from_existing_index(
                embeddings, # an Embeddings object
                index_name="my-index",
                redis_url="redis://localhost:6379",
            )
    Advanced examples:
    Custom vector schema can be supplied to change the way that
    Redis creates the underlying vector schema. This is useful
    for production use cases where you want to optimize the


for production use cases where you want to optimize the
    vector schema for your use case. ex. using HNSW instead of
    FLAT (knn) which is the default
        .. code-block:: python
            vector_schema = {
                "algorithm": "HNSW"
            }
            rds = Redis.from_texts(
                texts, # a list of strings
                metadata, # a list of metadata dicts
                embeddings, # an Embeddings object
                vector_schema=vector_schema,
                redis_url="redis://localhost:6379",
            )
    Custom index schema can be supplied to change the way that the
    metadata is indexed. This is useful for you would like to use the
    hybrid querying (filtering) capability of Redis.
    By default, this implementation will automatically generate the index
    schema according to the following rules:
        - All strings are indexed as text fields
        - All numbers are indexed as numeric fields
        - All lists of strings are indexed as tag fields (joined by
            langchain.vectorstores.redis.constants.REDIS_TAG_SEPARATOR)
        - All None values are not indexed but still stored in Redis these are
            not retrievable through the interface here, but the raw Redis client
            can be used to retrieve them.
        - All other types are not indexed
    To override these rules, you can pass in a custom index schema like the following
        .. code-block:: yaml
            tag:
                - name: credit_score
            text:
                - name: user
                - name: job
    Typically, the ``credit_score`` field would be a text field since it's a string,
    however, we can override this behavior by specifying the field type as shown with
    the yaml config (can also be a dictionary) above and the code below.


the yaml config (can also be a dictionary) above and the code below.
        .. code-block:: python
            rds = Redis.from_texts(
                texts, # a list of strings
                metadata, # a list of metadata dicts
                embeddings, # an Embeddings object
                index_schema="path/to/index_schema.yaml", # can also be a dictionary
                redis_url="redis://localhost:6379",
            )
    When connecting to an existing index where a custom schema has been applied, it's
    important to pass in the same schema to the ``from_existing_index`` method.
    Otherwise, the schema for newly added samples will be incorrect and metadata
    will not be returned.
    """
    DEFAULT_VECTOR_SCHEMA = {
        "name": "content_vector",
        "algorithm": "FLAT",
        "dims": 1536,
        "distance_metric": "COSINE",
        "datatype": "FLOAT32",
    }
[docs]    def __init__(
        self,
        redis_url: str,
        index_name: str,
        embedding: Embeddings,
        index_schema: Optional[Union[Dict[str, str], str, os.PathLike]] = None,
        vector_schema: Optional[Dict[str, Union[str, int]]] = None,
        relevance_score_fn: Optional[Callable[[float], float]] = None,
        key_prefix: Optional[str] = None,
        **kwargs: Any,
    ):
        """Initialize with necessary components."""
        self._check_deprecated_kwargs(kwargs)
        try:
            # TODO use importlib to check if redis is installed
            import redis  # noqa: F401
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "


raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        self.index_name = index_name
        self._embeddings = embedding
        try:
            redis_client = get_client(redis_url=redis_url, **kwargs)
            # check if redis has redisearch module installed
            check_redis_module_exist(redis_client, REDIS_REQUIRED_MODULES)
        except ValueError as e:
            raise ValueError(f"Redis failed to connect: {e}")
        self.client = redis_client
        self.relevance_score_fn = relevance_score_fn
        self._schema = self._get_schema_with_defaults(index_schema, vector_schema)
        self.key_prefix = key_prefix if key_prefix is not None else f"doc:{index_name}"
    @property
    def embeddings(self) -> Optional[Embeddings]:
        """Access the query embedding object if available."""
        return self._embeddings
[docs]    @classmethod
    def from_texts_return_keys(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        index_name: Optional[str] = None,
        index_schema: Optional[Union[Dict[str, str], str, os.PathLike]] = None,
        vector_schema: Optional[Dict[str, Union[str, int]]] = None,
        **kwargs: Any,
    ) -> Tuple[Redis, List[str]]:
        """Create a Redis vectorstore from raw documents.
        This is a user-friendly interface that:
            1. Embeds documents.
            2. Creates a new Redis index if it doesn't already exist
            3. Adds the documents to the newly created Redis index.


3. Adds the documents to the newly created Redis index.
            4. Returns the keys of the newly created documents once stored.
        This method will generate schema based on the metadata passed in
        if the `index_schema` is not defined. If the `index_schema` is defined,
        it will compare against the generated schema and warn if there are
        differences. If you are purposefully defining the schema for the
        metadata, then you can ignore that warning.
        To examine the schema options, initialize an instance of this class
        and print out the schema using the `Redis.schema`` property. This
        will include the content and content_vector classes which are
        always present in the langchain schema.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Redis
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                redis, keys = Redis.from_texts_return_keys(
                    texts,
                    embeddings,
                    redis_url="redis://localhost:6379"
                )
        Args:
            texts (List[str]): List of texts to add to the vectorstore.
            embedding (Embeddings): Embeddings to use for the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadata
                dicts to add to the vectorstore. Defaults to None.
            index_name (Optional[str], optional): Optional name of the index to
                create or add to. Defaults to None.
            index_schema (Optional[Union[Dict[str, str], str, os.PathLike]], optional):
                Optional fields to index within the metadata. Overrides generated
                schema. Defaults to None.
            vector_schema (Optional[Dict[str, Union[str, int]]], optional): Optional
                vector schema to use. Defaults to None.


vector schema to use. Defaults to None.
            **kwargs (Any): Additional keyword arguments to pass to the Redis client.
        Returns:
            Tuple[Redis, List[str]]: Tuple of the Redis instance and the keys of
                the newly created documents.
        Raises:
            ValueError: If the number of metadatas does not match the number of texts.
        """
        try:
            # TODO use importlib to check if redis is installed
            import redis  # noqa: F401
            from langchain.vectorstores.redis.schema import read_schema
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        if "redis_url" in kwargs:
            kwargs.pop("redis_url")
        # flag to use generated schema
        if "generate" in kwargs:
            kwargs.pop("generate")
        # see if the user specified keys
        keys = None
        if "keys" in kwargs:
            keys = kwargs.pop("keys")
        # Name of the search index if not given
        if not index_name:
            index_name = uuid.uuid4().hex
        # type check for metadata
        if metadatas:
            if isinstance(metadatas, list) and len(metadatas) != len(texts):  # type: ignore  # noqa: E501
                raise ValueError("Number of metadatas must match number of texts")
            if not (isinstance(metadatas, list) and isinstance(metadatas[0], dict)):
                raise ValueError("Metadatas must be a list of dicts")


raise ValueError("Metadatas must be a list of dicts")
            generated_schema = _generate_field_schema(metadatas[0])
            if index_schema:
                # read in the schema solely to compare to the generated schema
                user_schema = read_schema(index_schema)  # type: ignore
                # the very rare case where a super user decides to pass the index
                # schema and a document loader is used that has metadata which
                # we need to map into fields.
                if user_schema != generated_schema:
                    logger.warning(
                        "`index_schema` does not match generated metadata schema.\n"
                        + "If you meant to manually override the schema, please "
                        + "ignore this message.\n"
                        + f"index_schema: {user_schema}\n"
                        + f"generated_schema: {generated_schema}\n"
                    )
            else:
                # use the generated schema
                index_schema = generated_schema
        # Create instance
        instance = cls(
            redis_url,
            index_name,
            embedding,
            index_schema=index_schema,
            vector_schema=vector_schema,
            **kwargs,
        )
        # Add data to Redis
        keys = instance.add_texts(texts, metadatas, keys=keys)
        return instance, keys
[docs]    @classmethod
    def from_texts(
        cls: Type[Redis],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        index_name: Optional[str] = None,
        index_schema: Optional[Union[Dict[str, str], str, os.PathLike]] = None,
        vector_schema: Optional[Dict[str, Union[str, int]]] = None,
        **kwargs: Any,


**kwargs: Any,
    ) -> Redis:
        """Create a Redis vectorstore from a list of texts.
        This is a user-friendly interface that:
            1. Embeds documents.
            2. Creates a new Redis index if it doesn't already exist
            3. Adds the documents to the newly created Redis index.
        This method will generate schema based on the metadata passed in
        if the `index_schema` is not defined. If the `index_schema` is defined,
        it will compare against the generated schema and warn if there are
        differences. If you are purposefully defining the schema for the
        metadata, then you can ignore that warning.
        To examine the schema options, initialize an instance of this class
        and print out the schema using the `Redis.schema`` property. This
        will include the content and content_vector classes which are
        always present in the langchain schema.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Redis
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                redisearch = RediSearch.from_texts(
                    texts,
                    embeddings,
                    redis_url="redis://username:password@localhost:6379"
                )
        Args:
            texts (List[str]): List of texts to add to the vectorstore.
            embedding (Embeddings): Embedding model class (i.e. OpenAIEmbeddings)
                for embedding queries.
            metadatas (Optional[List[dict]], optional): Optional list of metadata dicts
                to add to the vectorstore. Defaults to None.
            index_name (Optional[str], optional): Optional name of the index to create
                or add to. Defaults to None.


or add to. Defaults to None.
            index_schema (Optional[Union[Dict[str, str], str, os.PathLike]], optional):
                Optional fields to index within the metadata. Overrides generated
                schema. Defaults to None.
            vector_schema (Optional[Dict[str, Union[str, int]]], optional): Optional
                vector schema to use. Defaults to None.
            **kwargs (Any): Additional keyword arguments to pass to the Redis client.
        Returns:
            Redis: Redis VectorStore instance.
        Raises:
            ValueError: If the number of metadatas does not match the number of texts.
            ImportError: If the redis python package is not installed.
        """
        instance, _ = cls.from_texts_return_keys(
            texts,
            embedding,
            metadatas=metadatas,
            index_name=index_name,
            index_schema=index_schema,
            vector_schema=vector_schema,
            **kwargs,
        )
        return instance
[docs]    @classmethod
    def from_existing_index(
        cls,
        embedding: Embeddings,
        index_name: str,
        schema: Union[Dict[str, str], str, os.PathLike],
        **kwargs: Any,
    ) -> Redis:
        """Connect to an existing Redis index.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Redis
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                redisearch = Redis.from_existing_index(
                    embeddings,
                    index_name="my-index",
                    redis_url="redis://username:password@localhost:6379"
                )
        Args:
            embedding (Embeddings): Embedding model class (i.e. OpenAIEmbeddings)
                for embedding queries.


for embedding queries.
            index_name (str): Name of the index to connect to.
            schema (Union[Dict[str, str], str, os.PathLike]): Schema of the index
                and the vector schema. Can be a dict, or path to yaml file
            **kwargs (Any): Additional keyword arguments to pass to the Redis client.
        Returns:
            Redis: Redis VectorStore instance.
        Raises:
            ValueError: If the index does not exist.
            ImportError: If the redis python package is not installed.
        """
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        try:
            # We need to first remove redis_url from kwargs,
            # otherwise passing it to Redis will result in an error.
            if "redis_url" in kwargs:
                kwargs.pop("redis_url")
            client = get_client(redis_url=redis_url, **kwargs)
            # check if redis has redisearch module installed
            check_redis_module_exist(client, REDIS_REQUIRED_MODULES)
            # ensure that the index already exists
            assert check_index_exists(
                client, index_name
            ), f"Index {index_name} does not exist"
        except Exception as e:
            raise ValueError(f"Redis failed to connect: {e}")
        return cls(
            redis_url,
            index_name,
            embedding,
            index_schema=schema,
            **kwargs,
        )
    @property
    def schema(self) -> Dict[str, List[Any]]:
        """Return the schema of the index."""
        return self._schema.as_dict()
[docs]    def write_schema(self, path: Union[str, os.PathLike]) -> None:
        """Write the schema to a yaml file."""


"""Write the schema to a yaml file."""
        with open(path, "w+") as f:
            yaml.dump(self.schema, f)
[docs]    @staticmethod
    def delete(
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> bool:
        """
        Delete a Redis entry.
        Args:
            ids: List of ids (keys in redis) to delete.
            redis_url: Redis connection url. This should be passed in the kwargs
                or set as an environment variable: REDIS_URL.
        Returns:
            bool: Whether or not the deletions were successful.
        Raises:
            ValueError: If the redis python package is not installed.
            ValueError: If the ids (keys in redis) are not provided
        """
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        if ids is None:
            raise ValueError("'ids' (keys)() were not provided.")
        try:
            import redis  # noqa: F401
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        try:
            # We need to first remove redis_url from kwargs,
            # otherwise passing it to Redis will result in an error.
            if "redis_url" in kwargs:
                kwargs.pop("redis_url")
            client = get_client(redis_url=redis_url, **kwargs)
        except ValueError as e:
            raise ValueError(f"Your redis connected error: {e}")
        # Check if index exists
        try:
            client.delete(*ids)
            logger.info("Entries deleted")
            return True
        except:  # noqa: E722


return True
        except:  # noqa: E722
            # ids does not exist
            return False
[docs]    @staticmethod
    def drop_index(
        index_name: str,
        delete_documents: bool,
        **kwargs: Any,
    ) -> bool:
        """
        Drop a Redis search index.
        Args:
            index_name (str): Name of the index to drop.
            delete_documents (bool): Whether to drop the associated documents.
        Returns:
            bool: Whether or not the drop was successful.
        """
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        try:
            import redis  # noqa: F401
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        try:
            # We need to first remove redis_url from kwargs,
            # otherwise passing it to Redis will result in an error.
            if "redis_url" in kwargs:
                kwargs.pop("redis_url")
            client = get_client(redis_url=redis_url, **kwargs)
        except ValueError as e:
            raise ValueError(f"Your redis connected error: {e}")
        # Check if index exists
        try:
            client.ft(index_name).dropindex(delete_documents)
            logger.info("Drop index")
            return True
        except:  # noqa: E722
            # Index not exist
            return False
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        embeddings: Optional[List[List[float]]] = None,
        batch_size: int = 1000,


batch_size: int = 1000,
        clean_metadata: bool = True,
        **kwargs: Any,
    ) -> List[str]:
        """Add more texts to the vectorstore.
        Args:
            texts (Iterable[str]): Iterable of strings/text to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.
                Defaults to None.
            embeddings (Optional[List[List[float]]], optional): Optional pre-generated
                embeddings. Defaults to None.
            keys (List[str]) or ids (List[str]): Identifiers of entries.
                Defaults to None.
            batch_size (int, optional): Batch size to use for writes. Defaults to 1000.
        Returns:
            List[str]: List of ids added to the vectorstore
        """
        ids = []
        # Get keys or ids from kwargs
        # Other vectorstores use ids
        keys_or_ids = kwargs.get("keys", kwargs.get("ids"))
        # type check for metadata
        if metadatas:
            if isinstance(metadatas, list) and len(metadatas) != len(texts):  # type: ignore  # noqa: E501
                raise ValueError("Number of metadatas must match number of texts")
            if not (isinstance(metadatas, list) and isinstance(metadatas[0], dict)):
                raise ValueError("Metadatas must be a list of dicts")
        embeddings = embeddings or self._embeddings.embed_documents(list(texts))
        self._create_index_if_not_exist(dim=len(embeddings[0]))
        # Write data to redis
        pipeline = self.client.pipeline(transaction=False)
        for i, text in enumerate(texts):
            # Use provided values by default or fallback


# Use provided values by default or fallback
            key = keys_or_ids[i] if keys_or_ids else str(uuid.uuid4().hex)
            if not key.startswith(self.key_prefix + ":"):
                key = self.key_prefix + ":" + key
            metadata = metadatas[i] if metadatas else {}
            metadata = _prepare_metadata(metadata) if clean_metadata else metadata
            pipeline.hset(
                key,
                mapping={
                    self._schema.content_key: text,
                    self._schema.content_vector_key: _array_to_buffer(
                        embeddings[i], self._schema.vector_dtype
                    ),
                    **metadata,
                },
            )
            ids.append(key)
            # Write batch
            if i % batch_size == 0:
                pipeline.execute()
        # Cleanup final batch
        pipeline.execute()
        return ids
[docs]    def as_retriever(self, **kwargs: Any) -> RedisVectorStoreRetriever:
        tags = kwargs.pop("tags", None) or []
        tags.extend(self._get_retriever_tags())
        return RedisVectorStoreRetriever(vectorstore=self, **kwargs, tags=tags)
[docs]    @deprecated("0.0.272", alternative="similarity_search(distance_threshold=0.1)")
    def similarity_search_limit_score(
        self, query: str, k: int = 4, score_threshold: float = 0.2, **kwargs: Any
    ) -> List[Document]:
        """
        Returns the most similar indexed documents to the query text within the
        score_threshold range.
        Deprecated: Use similarity_search with distance_threshold instead.
        Args:
            query (str): The query text for which to find similar documents.


Args:
            query (str): The query text for which to find similar documents.
            k (int): The number of documents to return. Default is 4.
            score_threshold (float): The minimum matching *distance* required
                for a document to be considered a match. Defaults to 0.2.
        Returns:
            List[Document]: A list of documents that are most similar to the query text
                including the match score for each document.
        Note:
            If there are no documents that satisfy the score_threshold value,
            an empty list is returned.
        """
        return self.similarity_search(
            query, k=k, distance_threshold=score_threshold, **kwargs
        )
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        filter: Optional[RedisFilterExpression] = None,
        return_metadata: bool = True,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Run similarity search with **vector distance**.
        The "scores" returned from this function are the raw vector
        distances from the query vector. For similarity scores, use
        ``similarity_search_with_relevance_scores``.
        Args:
            query (str): The query text for which to find similar documents.
            k (int): The number of documents to return. Default is 4.
            filter (RedisFilterExpression, optional): Optional metadata filter.
                Defaults to None.
            return_metadata (bool, optional): Whether to return metadata.
                Defaults to True.
        Returns:
            List[Tuple[Document, float]]: A list of documents that are
                most similar to the query with the distance for each document.
        """
        try:


"""
        try:
            import redis
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        if "score_threshold" in kwargs:
            logger.warning(
                "score_threshold is deprecated. Use distance_threshold instead."
                + "score_threshold should only be used in "
                + "similarity_search_with_relevance_scores."
                + "score_threshold will be removed in a future release.",
            )
        query_embedding = self._embeddings.embed_query(query)
        redis_query, params_dict = self._prepare_query(
            query_embedding,
            k=k,
            filter=filter,
            with_metadata=return_metadata,
            with_distance=True,
            **kwargs,
        )
        # Perform vector search
        # ignore type because redis-py is wrong about bytes
        try:
            results = self.client.ft(self.index_name).search(redis_query, params_dict)  # type: ignore  # noqa: E501
        except redis.exceptions.ResponseError as e:
            # split error message and see if it starts with "Syntax"
            if str(e).split(" ")[0] == "Syntax":
                raise ValueError(
                    "Query failed with syntax error. "
                    + "This is likely due to malformation of "
                    + "filter, vector, or query argument"
                ) from e
            raise e
        # Prepare document results
        docs_with_scores: List[Tuple[Document, float]] = []
        for result in results.docs:
            metadata = {}
            if return_metadata:
                metadata = {"id": result.id}
                metadata.update(self._collect_metadata(result))


metadata = {"id": result.id}
                metadata.update(self._collect_metadata(result))
            doc = Document(page_content=result.content, metadata=metadata)
            distance = self._calculate_fp_distance(result.distance)
            docs_with_scores.append((doc, distance))
        return docs_with_scores
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[RedisFilterExpression] = None,
        return_metadata: bool = True,
        distance_threshold: Optional[float] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Run similarity search
        Args:
            query (str): The query text for which to find similar documents.
            k (int): The number of documents to return. Default is 4.
            filter (RedisFilterExpression, optional): Optional metadata filter.
                Defaults to None.
            return_metadata (bool, optional): Whether to return metadata.
                Defaults to True.
            distance_threshold (Optional[float], optional): Maximum vector distance
                between selected documents and the query vector. Defaults to None.
        Returns:
            List[Document]: A list of documents that are most similar to the query
                text.
        """
        query_embedding = self._embeddings.embed_query(query)
        return self.similarity_search_by_vector(
            query_embedding,
            k=k,
            filter=filter,
            return_metadata=return_metadata,
            distance_threshold=distance_threshold,
            **kwargs,
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[RedisFilterExpression] = None,
        return_metadata: bool = True,


return_metadata: bool = True,
        distance_threshold: Optional[float] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Run similarity search between a query vector and the indexed vectors.
        Args:
            embedding (List[float]): The query vector for which to find similar
                documents.
            k (int): The number of documents to return. Default is 4.
            filter (RedisFilterExpression, optional): Optional metadata filter.
                Defaults to None.
            return_metadata (bool, optional): Whether to return metadata.
                Defaults to True.
            distance_threshold (Optional[float], optional): Maximum vector distance
                between selected documents and the query vector. Defaults to None.
        Returns:
            List[Document]: A list of documents that are most similar to the query
                text.
        """
        try:
            import redis
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        if "score_threshold" in kwargs:
            logger.warning(
                "score_threshold is deprecated. Use distance_threshold instead."
                + "score_threshold should only be used in "
                + "similarity_search_with_relevance_scores."
                + "score_threshold will be removed in a future release.",
            )
        redis_query, params_dict = self._prepare_query(
            embedding,
            k=k,
            filter=filter,
            distance_threshold=distance_threshold,
            with_metadata=return_metadata,
            with_distance=False,
        )
        # Perform vector search
        # ignore type because redis-py is wrong about bytes
        try:


# ignore type because redis-py is wrong about bytes
        try:
            results = self.client.ft(self.index_name).search(redis_query, params_dict)  # type: ignore  # noqa: E501
        except redis.exceptions.ResponseError as e:
            # split error message and see if it starts with "Syntax"
            if str(e).split(" ")[0] == "Syntax":
                raise ValueError(
                    "Query failed with syntax error. "
                    + "This is likely due to malformation of "
                    + "filter, vector, or query argument"
                ) from e
            raise e
        # Prepare document results
        docs = []
        for result in results.docs:
            metadata = {}
            if return_metadata:
                metadata = {"id": result.id}
                metadata.update(self._collect_metadata(result))
            content_key = self._schema.content_key
            docs.append(
                Document(page_content=getattr(result, content_key), metadata=metadata)
            )
        return docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[RedisFilterExpression] = None,
        return_metadata: bool = True,
        distance_threshold: Optional[float] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
            among selected documents.
        Args:
            query (str): Text to look up documents similar to.
            k (int): Number of Documents to return. Defaults to 4.


k (int): Number of Documents to return. Defaults to 4.
            fetch_k (int): Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult (float): Number between 0 and 1 that determines the degree
                of diversity among the results with 0 corresponding
                to maximum diversity and 1 to minimum diversity.
                Defaults to 0.5.
            filter (RedisFilterExpression, optional): Optional metadata filter.
                Defaults to None.
            return_metadata (bool, optional): Whether to return metadata.
                Defaults to True.
            distance_threshold (Optional[float], optional): Maximum vector distance
                between selected documents and the query vector. Defaults to None.
        Returns:
            List[Document]: A list of Documents selected by maximal marginal relevance.
        """
        # Embed the query
        query_embedding = self._embeddings.embed_query(query)
        # Fetch the initial documents
        prefetch_docs = self.similarity_search_by_vector(
            query_embedding,
            k=fetch_k,
            filter=filter,
            return_metadata=return_metadata,
            distance_threshold=distance_threshold,
            **kwargs,
        )
        prefetch_ids = [doc.metadata["id"] for doc in prefetch_docs]
        # Get the embeddings for the fetched documents
        prefetch_embeddings = [
            _buffer_to_array(
                cast(
                    bytes,
                    self.client.hget(prefetch_id, self._schema.content_vector_key),
                ),
                dtype=self._schema.vector_dtype,
            )
            for prefetch_id in prefetch_ids
        ]
        # Select documents using maximal marginal relevance
        selected_indices = maximal_marginal_relevance(
            np.array(query_embedding), prefetch_embeddings, lambda_mult=lambda_mult, k=k
        )


np.array(query_embedding), prefetch_embeddings, lambda_mult=lambda_mult, k=k
        )
        selected_docs = [prefetch_docs[i] for i in selected_indices]
        return selected_docs
    def _collect_metadata(self, result: "Document") -> Dict[str, Any]:
        """Collect metadata from Redis.
        Method ensures that there isn't a mismatch between the metadata
        and the index schema passed to this class by the user or generated
        by this class.
        Args:
            result (Document): redis.commands.search.Document object returned
                from Redis.
        Returns:
            Dict[str, Any]: Collected metadata.
        """
        # new metadata dict as modified by this method
        meta = {}
        for key in self._schema.metadata_keys:
            try:
                meta[key] = getattr(result, key)
            except AttributeError:
                # warning about attribute missing
                logger.warning(
                    f"Metadata key {key} not found in metadata. "
                    + "Setting to None. \n"
                    + "Metadata fields defined for this instance: "
                    + f"{self._schema.metadata_keys}"
                )
                meta[key] = None
        return meta
    def _prepare_query(
        self,
        query_embedding: List[float],
        k: int = 4,
        filter: Optional[RedisFilterExpression] = None,
        distance_threshold: Optional[float] = None,
        with_metadata: bool = True,
        with_distance: bool = False,
    ) -> Tuple["Query", Dict[str, Any]]:
        # Creates Redis query
        params_dict: Dict[str, Union[str, bytes, float]] = {
            "vector": _array_to_buffer(query_embedding, self._schema.vector_dtype),
        }
        # prepare return fields including score


}
        # prepare return fields including score
        return_fields = [self._schema.content_key]
        if with_distance:
            return_fields.append("distance")
        if with_metadata:
            return_fields.extend(self._schema.metadata_keys)
        if distance_threshold:
            params_dict["distance_threshold"] = distance_threshold
            return (
                self._prepare_range_query(
                    k, filter=filter, return_fields=return_fields
                ),
                params_dict,
            )
        return (
            self._prepare_vector_query(k, filter=filter, return_fields=return_fields),
            params_dict,
        )
    def _prepare_range_query(
        self,
        k: int,
        filter: Optional[RedisFilterExpression] = None,
        return_fields: Optional[List[str]] = None,
    ) -> "Query":
        try:
            from redis.commands.search.query import Query
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        return_fields = return_fields or []
        vector_key = self._schema.content_vector_key
        base_query = f"@{vector_key}:[VECTOR_RANGE $distance_threshold $vector]"
        if filter:
            base_query = "(" + base_query + " " + str(filter) + ")"
        query_string = base_query + "=>{$yield_distance_as: distance}"
        return (
            Query(query_string)
            .return_fields(*return_fields)
            .sort_by("distance")
            .paging(0, k)
            .dialect(2)
        )
    def _prepare_vector_query(
        self,
        k: int,
        filter: Optional[RedisFilterExpression] = None,


k: int,
        filter: Optional[RedisFilterExpression] = None,
        return_fields: Optional[List[str]] = None,
    ) -> "Query":
        """Prepare query for vector search.
        Args:
            k: Number of results to return.
            filter: Optional metadata filter.
        Returns:
            query: Query object.
        """
        try:
            from redis.commands.search.query import Query
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        return_fields = return_fields or []
        query_prefix = "*"
        if filter:
            query_prefix = f"{str(filter)}"
        vector_key = self._schema.content_vector_key
        base_query = f"({query_prefix})=>[KNN {k} @{vector_key} $vector AS distance]"
        query = (
            Query(base_query)
            .return_fields(*return_fields)
            .sort_by("distance")
            .paging(0, k)
            .dialect(2)
        )
        return query
    def _get_schema_with_defaults(
        self,
        index_schema: Optional[Union[Dict[str, str], str, os.PathLike]] = None,
        vector_schema: Optional[Dict[str, Union[str, int]]] = None,
    ) -> "RedisModel":
        # should only be called after init of Redis (so Import handled)
        from langchain.vectorstores.redis.schema import RedisModel, read_schema
        schema = RedisModel()
        # read in schema (yaml file or dict) and
        # pass to the Pydantic validators
        if index_schema: