**kwargs: Any,
    ) -> List[Document]:
        # Retrieves the docs with similarity scores
        # sorted by similarity scores in DESC order
        docs = self._similarity_search_with_score(embedding, k=fetch_k)
        # Re-ranks the docs using MMR
        mmr_doc_indexes = maximal_marginal_relevance(
            np.array(embedding),
            [doc.metadata[self._embedding_key] for doc, _ in docs],
            k=k,
            lambda_mult=lambda_mult,
        )
        mmr_docs = [docs[i][0] for i in mmr_doc_indexes]
        return mmr_docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        # compute the embeddings vector from the query string
        embeddings = self._embedding.embed_query(query)
        docs = self.max_marginal_relevance_search_by_vector(
            embeddings, k=k, fetch_k=fetch_k, lambda_mult=lambda_mult
        )
        return docs


Source code for langchain.vectorstores.deeplake
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
import numpy as np
try:
    import deeplake
    from deeplake import VectorStore as DeepLakeVectorStore
    from deeplake.core.fast_forwarding import version_compare
    _DEEPLAKE_INSTALLED = True
except ImportError:
    _DEEPLAKE_INSTALLED = False
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
logger = logging.getLogger(__name__)
[docs]class DeepLake(VectorStore):
    """`Activeloop Deep Lake` vector store.
    We integrated deeplake's similarity search and filtering for fast prototyping.
    Now, it supports Tensor Query Language (TQL) for production use cases
    over billion rows.
    Why Deep Lake?
    - Not only stores embeddings, but also the original data with version control.
    - Serverless, doesn't require another service and can be used with major
        cloud providers (S3, GCS, etc.)
    - More than just a multi-modal vector store. You can use the dataset
        to fine-tune your own LLM models.
    To use, you should have the ``deeplake`` python package installed.
    Example:
        .. code-block:: python
                from langchain.vectorstores import DeepLake
                from langchain.embeddings.openai import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                vectorstore = DeepLake("langchain_store", embeddings.embed_query)
    """


vectorstore = DeepLake("langchain_store", embeddings.embed_query)
    """
    _LANGCHAIN_DEFAULT_DEEPLAKE_PATH = "./deeplake/"
[docs]    def __init__(
        self,
        dataset_path: str = _LANGCHAIN_DEFAULT_DEEPLAKE_PATH,
        token: Optional[str] = None,
        embedding: Optional[Embeddings] = None,
        embedding_function: Optional[Embeddings] = None,
        read_only: bool = False,
        ingestion_batch_size: int = 1000,
        num_workers: int = 0,
        verbose: bool = True,
        exec_option: Optional[str] = None,
        runtime: Optional[Dict] = None,
        index_params: Optional[Dict[str, Union[int, str]]] = None,
        **kwargs: Any,
    ) -> None:
        """Creates an empty DeepLakeVectorStore or loads an existing one.
        The DeepLakeVectorStore is located at the specified ``path``.
        Examples:
            >>> # Create a vector store with default tensors
            >>> deeplake_vectorstore = DeepLake(
            ...        path = <path_for_storing_Data>,
            ... )
            >>>
            >>> # Create a vector store in the Deep Lake Managed Tensor Database
            >>> data = DeepLake(
            ...        path = "hub://org_id/dataset_name",
            ...        runtime = {"tensor_db": True},
            ... )
        Args:
            dataset_path (str): Path to existing dataset or where to create
                a new one. Defaults to _LANGCHAIN_DEFAULT_DEEPLAKE_PATH.
            token (str, optional):  Activeloop token, for fetching credentials
                to the dataset at path if it is a Deep Lake dataset.


to the dataset at path if it is a Deep Lake dataset.
                Tokens are normally autogenerated. Optional.
            embedding (Embeddings, optional): Function to convert
                either documents or query. Optional.
            embedding_function (Embeddings, optional): Function to convert
                either documents or query. Optional. Deprecated: keeping this
                parameter for backwards compatibility.
            read_only (bool): Open dataset in read-only mode. Default is False.
            ingestion_batch_size (int): During data ingestion, data is divided
                into batches. Batch size is the size of each batch.
                Default is 1000.
            num_workers (int): Number of workers to use during data ingestion.
                Default is 0.
            verbose (bool): Print dataset summary after each operation.
                Default is True.
            exec_option (str, optional): DeepLakeVectorStore supports 3 ways to perform
                searching - "python", "compute_engine", "tensor_db" and auto.
                Default is None.
                - ``auto``- Selects the best execution method based on the storage
                    location of the Vector Store. It is the default option.
                - ``python`` - Pure-python implementation that runs on the client.
                    WARNING: using this with big datasets can lead to memory
                    issues. Data can be stored anywhere.
                - ``compute_engine`` - C++ implementation of the Deep Lake Compute
                    Engine that runs on the client. Can be used for any data stored in
                    or connected to Deep Lake. Not for in-memory or local datasets.
                - ``tensor_db`` - Hosted Managed Tensor Database that is
                    responsible for storage and query execution. Only for data stored in
                    the Deep Lake Managed Database. Use runtime = {"db_engine": True}
                    during dataset creation.


during dataset creation.
            runtime (Dict, optional): Parameters for creating the Vector Store in
                Deep Lake's Managed Tensor Database. Not applicable when loading an
                existing Vector Store. To create a Vector Store in the Managed Tensor
                Database, set `runtime = {"tensor_db": True}`.
            index_params (Optional[Dict[str, Union[int, str]]], optional): Dictionary
                containing information about vector index that will be created. Defaults
                to None, which will utilize ``DEFAULT_VECTORSTORE_INDEX_PARAMS`` from
                ``deeplake.constants``. The specified key-values override the default
                ones.
                - threshold: The threshold for the dataset size above which an index
                    will be created for the embedding tensor. When the threshold value
                    is set to -1, index creation is turned off. Defaults to -1, which
                    turns off the index.
                - distance_metric: This key specifies the method of calculating the
                    distance between vectors when creating the vector database (VDB)
                    index. It can either be a string that corresponds to a member of
                    the DistanceType enumeration, or the string value itself.
                    - If no value is provided, it defaults to "L2".
                    - "L2" corresponds to DistanceType.L2_NORM.
                    - "COS" corresponds to DistanceType.COSINE_SIMILARITY.
                - additional_params: Additional parameters for fine-tuning the index.
            **kwargs: Other optional keyword arguments.
        Raises:
            ValueError: If some condition is not met.
        """
        self.ingestion_batch_size = ingestion_batch_size
        self.num_workers = num_workers
        self.verbose = verbose
        if _DEEPLAKE_INSTALLED is False:
            raise ImportError(
                "Could not import deeplake python package. "


raise ImportError(
                "Could not import deeplake python package. "
                "Please install it with `pip install deeplake[enterprise]`."
            )
        if (
            runtime == {"tensor_db": True}
            and version_compare(deeplake.__version__, "3.6.7") == -1
        ):
            raise ImportError(
                "To use tensor_db option you need to update deeplake to `3.6.7` or "
                "higher. "
                f"Currently installed deeplake version is {deeplake.__version__}. "
            )
        self.dataset_path = dataset_path
        if embedding_function:
            logger.warning(
                "Using embedding function is deprecated and will be removed "
                "in the future. Please use embedding instead."
            )
        self.vectorstore = DeepLakeVectorStore(
            path=self.dataset_path,
            embedding_function=embedding_function or embedding,
            read_only=read_only,
            token=token,
            exec_option=exec_option,
            verbose=verbose,
            runtime=runtime,
            index_params=index_params,
            **kwargs,
        )
        self._embedding_function = embedding_function or embedding
        self._id_tensor_name = "ids" if "ids" in self.vectorstore.tensors() else "id"
    @property
    def embeddings(self) -> Optional[Embeddings]:
        return self._embedding_function
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Examples:


"""Run more texts through the embeddings and add to the vectorstore.
        Examples:
            >>> ids = deeplake_vectorstore.add_texts(
            ...     texts = <list_of_texts>,
            ...     metadatas = <list_of_metadata_jsons>,
            ...     ids = <list_of_ids>,
            ... )
        Args:
            texts (Iterable[str]): Texts to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.
            ids (Optional[List[str]], optional): Optional list of IDs.
            embedding_function (Optional[Embeddings], optional): Embedding function
                to use to convert the text into embeddings.
            **kwargs (Any): Any additional keyword arguments passed is not supported
                by this method.
        Returns:
            List[str]: List of IDs of the added texts.
        """
        if kwargs:
            unsupported_items = "`, `".join(set(kwargs.keys()))
            raise TypeError(
                f"`{unsupported_items}` is/are not a valid argument to add_text method"
            )
        kwargs = {}
        if ids:
            if self._id_tensor_name == "ids":  # for backwards compatibility
                kwargs["ids"] = ids
            else:
                kwargs["id"] = ids
        if metadatas is None:
            metadatas = [{}] * len(list(texts))
        if not isinstance(texts, list):
            texts = list(texts)
        if texts is None:
            raise ValueError("`texts` parameter shouldn't be None.")
        elif len(texts) == 0:
            raise ValueError("`texts` parameter shouldn't be empty.")
        return self.vectorstore.add(
            text=texts,
            metadata=metadatas,


text=texts,
            metadata=metadatas,
            embedding_data=texts,
            embedding_tensor="embedding",
            embedding_function=self._embedding_function.embed_documents,  # type: ignore
            return_ids=True,
            **kwargs,
        )
    def _search_tql(
        self,
        tql: Optional[str],
        exec_option: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Function for performing tql_search.
        Args:
            tql (str): TQL Query string for direct evaluation.
                Available only for `compute_engine` and `tensor_db`.
            exec_option (str, optional): Supports 3 ways to search.
                Could be "python", "compute_engine" or "tensor_db". Default is "python".
                - ``python`` - Pure-python implementation for the client.
                    WARNING: not recommended for big datasets due to potential memory
                    issues.
                - ``compute_engine`` - C++ implementation of Deep Lake Compute
                    Engine for the client. Not for in-memory or local datasets.
                - ``tensor_db`` - Hosted Managed Tensor Database for storage
                    and query execution. Only for data in Deep Lake Managed Database.
                        Use runtime = {"db_engine": True} during dataset creation.
            return_score (bool): Return score with document. Default is False.
        Returns:
            Tuple[List[Document], List[Tuple[Document, float]]] - A tuple of two lists.
                The first list contains Documents, and the second list contains
                tuples of Document and float score.
        Raises:
            ValueError: If return_score is True but some condition is not met.
        """
        result = self.vectorstore.search(
            query=tql,


"""
        result = self.vectorstore.search(
            query=tql,
            exec_option=exec_option,
        )
        metadatas = result["metadata"]
        texts = result["text"]
        docs = [
            Document(
                page_content=text,
                metadata=metadata,
            )
            for text, metadata in zip(texts, metadatas)
        ]
        if kwargs:
            unsupported_argument = next(iter(kwargs))
            if kwargs[unsupported_argument] is not False:
                raise ValueError(
                    f"specifying {unsupported_argument} is "
                    "not supported with tql search."
                )
        return docs
    def _search(
        self,
        query: Optional[str] = None,
        embedding: Optional[Union[List[float], np.ndarray]] = None,
        embedding_function: Optional[Callable] = None,
        k: int = 4,
        distance_metric: Optional[str] = None,
        use_maximal_marginal_relevance: bool = False,
        fetch_k: Optional[int] = 20,
        filter: Optional[Union[Dict, Callable]] = None,
        return_score: bool = False,
        exec_option: Optional[str] = None,
        deep_memory: bool = False,
        **kwargs: Any,
    ) -> Any[List[Document], List[Tuple[Document, float]]]:
        """
        Return docs similar to query.
        Args:
            query (str, optional): Text to look up similar docs.
            embedding (Union[List[float], np.ndarray], optional): Query's embedding.
            embedding_function (Callable, optional): Function to convert `query`
                into embedding.
            k (int): Number of Documents to return.


into embedding.
            k (int): Number of Documents to return.
            distance_metric (Optional[str], optional): `L2` for Euclidean, `L1` for
                Nuclear, `max` for L-infinity distance, `cos` for cosine similarity,
                'dot' for dot product.
            filter (Union[Dict, Callable], optional): Additional filter prior
                to the embedding search.
                - ``Dict`` - Key-value search on tensors of htype json, on an
                    AND basis (a sample must satisfy all key-value filters to be True)
                    Dict = {"tensor_name_1": {"key": value},
                            "tensor_name_2": {"key": value}}
                - ``Function`` - Any function compatible with `deeplake.filter`.
            use_maximal_marginal_relevance (bool): Use maximal marginal relevance.
            fetch_k (int): Number of Documents for MMR algorithm.
            return_score (bool): Return the score.
            exec_option (str, optional): Supports 3 ways to perform searching.
                Could be "python", "compute_engine" or "tensor_db".
                - ``python`` - Pure-python implementation for the client.
                    WARNING: not recommended for big datasets.
                - ``compute_engine`` - C++ implementation of Deep Lake Compute
                    Engine for the client. Not for in-memory or local datasets.
                - ``tensor_db`` - Hosted Managed Tensor Database for storage
                    and query execution. Only for data in Deep Lake Managed Database.
                    Use runtime = {"db_engine": True} during dataset creation.
            deep_memory (bool): Whether to use the Deep Memory model for improving
                search results. Defaults to False if deep_memory is not specified in
                the Vector Store initialization. If True, the distance metric is set


the Vector Store initialization. If True, the distance metric is set
                to "deepmemory_distance", which represents the metric with which the
                model was trained. The search is performed using the Deep Memory model.
                If False, the distance metric is set to "COS" or whatever distance
                metric user specifies.
            **kwargs: Additional keyword arguments.
        Returns:
            List of Documents by the specified distance metric,
            if return_score True, return a tuple of (Document, score)
        Raises:
            ValueError: if both `embedding` and `embedding_function` are not specified.
        """
        if kwargs.get("tql"):
            return self._search_tql(
                tql=kwargs["tql"],
                exec_option=exec_option,
                return_score=return_score,
                embedding=embedding,
                embedding_function=embedding_function,
                distance_metric=distance_metric,
                use_maximal_marginal_relevance=use_maximal_marginal_relevance,
                filter=filter,
            )
        if embedding_function:
            if isinstance(embedding_function, Embeddings):
                _embedding_function = embedding_function.embed_query
            else:
                _embedding_function = embedding_function
        elif self._embedding_function:
            _embedding_function = self._embedding_function.embed_query
        else:
            _embedding_function = None
        if embedding is None:
            if _embedding_function is None:
                raise ValueError(
                    "Either `embedding` or `embedding_function` needs to be"
                    " specified."
                )
            embedding = _embedding_function(query) if query else None
        if isinstance(embedding, list):
            embedding = np.array(embedding, dtype=np.float32)
            if len(embedding.shape) > 1:


if len(embedding.shape) > 1:
                embedding = embedding[0]
        result = self.vectorstore.search(
            embedding=embedding,
            k=fetch_k if use_maximal_marginal_relevance else k,
            distance_metric=distance_metric,
            filter=filter,
            exec_option=exec_option,
            return_tensors=["embedding", "metadata", "text", "id"],
            deep_memory=deep_memory,
        )
        scores = result["score"]
        embeddings = result["embedding"]
        metadatas = result["metadata"]
        texts = result["text"]
        if use_maximal_marginal_relevance:
            lambda_mult = kwargs.get("lambda_mult", 0.5)
            indices = maximal_marginal_relevance(  # type: ignore
                embedding,  # type: ignore
                embeddings,
                k=min(k, len(texts)),
                lambda_mult=lambda_mult,
            )
            scores = [scores[i] for i in indices]
            texts = [texts[i] for i in indices]
            metadatas = [metadatas[i] for i in indices]
        docs = [
            Document(
                page_content=text,
                metadata=metadata,
            )
            for text, metadata in zip(texts, metadatas)
        ]
        if return_score:
            return [(doc, score) for doc, score in zip(docs, scores)]
        return docs
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Document]:
        """
        Return docs most similar to query.
        Examples:
            >>> # Search using an embedding


Examples:
            >>> # Search using an embedding
            >>> data = vector_store.similarity_search(
            ...     query=<your_query>,
            ...     k=<num_items>,
            ...     exec_option=<preferred_exec_option>,
            ... )
            >>> # Run tql search:
            >>> data = vector_store.similarity_search(
            ...     query=None,
            ...     tql="SELECT * WHERE id == <id>",
            ...     exec_option="compute_engine",
            ... )
        Args:
            k (int): Number of Documents to return. Defaults to 4.
            query (str): Text to look up similar documents.
            **kwargs: Additional keyword arguments include:
                embedding (Callable): Embedding function to use. Defaults to None.
                distance_metric (str): 'L2' for Euclidean, 'L1' for Nuclear, 'max'
                    for L-infinity, 'cos' for cosine, 'dot' for dot product.
                    Defaults to 'L2'.
                filter (Union[Dict, Callable], optional): Additional filter
                    before embedding search.
                    - Dict: Key-value search on tensors of htype json,
                        (sample must satisfy all key-value filters)
                        Dict = {"tensor_1": {"key": value}, "tensor_2": {"key": value}}
                    - Function: Compatible with `deeplake.filter`.
                    Defaults to None.
                exec_option (str): Supports 3 ways to perform searching.
                    'python', 'compute_engine', or 'tensor_db'. Defaults to 'python'.
                    - 'python': Pure-python implementation for the client.
                        WARNING: not recommended for big datasets.
                    - 'compute_engine': C++ implementation of the Compute Engine for
                        the client. Not for in-memory or local datasets.


the client. Not for in-memory or local datasets.
                    - 'tensor_db': Managed Tensor Database for storage and query.
                        Only for data in Deep Lake Managed Database.
                        Use `runtime = {"db_engine": True}` during dataset creation.
                deep_memory (bool): Whether to use the Deep Memory model for improving
                    search results. Defaults to False if deep_memory is not specified
                    in the Vector Store initialization. If True, the distance metric
                    is set to "deepmemory_distance", which represents the metric with
                    which the model was trained. The search is performed using the Deep
                    Memory model. If False, the distance metric is set to "COS" or
                    whatever distance metric user specifies.
        Returns:
            List[Document]: List of Documents most similar to the query vector.
        """
        return self._search(
            query=query,
            k=k,
            use_maximal_marginal_relevance=False,
            return_score=False,
            **kwargs,
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: Union[List[float], np.ndarray],
        k: int = 4,
        **kwargs: Any,
    ) -> List[Document]:
        """
        Return docs most similar to embedding vector.
        Examples:
            >>> # Search using an embedding
            >>> data = vector_store.similarity_search_by_vector(
            ...    embedding=<your_embedding>,
            ...    k=<num_items_to_return>,
            ...    exec_option=<preferred_exec_option>,
            ... )
        Args:
            embedding (Union[List[float], np.ndarray]):
                Embedding to find similar docs.
            k (int): Number of Documents to return. Defaults to 4.
            **kwargs: Additional keyword arguments including:


**kwargs: Additional keyword arguments including:
                filter (Union[Dict, Callable], optional):
                    Additional filter before embedding search.
                    - ``Dict`` - Key-value search on tensors of htype json. True
                        if all key-value filters are satisfied.
                        Dict = {"tensor_name_1": {"key": value},
                                "tensor_name_2": {"key": value}}
                    - ``Function`` - Any function compatible with
                        `deeplake.filter`.
                    Defaults to None.
                exec_option (str): Options for search execution include
                    "python", "compute_engine", or "tensor_db". Defaults to
                    "python".
                    - "python" - Pure-python implementation running on the client.
                        Can be used for data stored anywhere. WARNING: using this
                        option with big datasets is discouraged due to potential
                        memory issues.
                    - "compute_engine" - Performant C++ implementation of the Deep
                        Lake Compute Engine. Runs on the client and can be used for
                        any data stored in or connected to Deep Lake. It cannot be
                        used with in-memory or local datasets.
                    - "tensor_db" - Performant, fully-hosted Managed Tensor Database.
                        Responsible for storage and query execution. Only available
                        for data stored in the Deep Lake Managed Database.
                        To store datasets in this database, specify
                        `runtime = {"db_engine": True}` during dataset creation.
                distance_metric (str): `L2` for Euclidean, `L1` for Nuclear,
                    `max` for L-infinity distance, `cos` for cosine similarity,
                    'dot' for dot product. Defaults to `L2`.
                deep_memory (bool): Whether to use the Deep Memory model for improving
                    search results. Defaults to False if deep_memory is not specified


search results. Defaults to False if deep_memory is not specified
                    in the Vector Store initialization. If True, the distance metric
                    is set to "deepmemory_distance", which represents the metric with
                    which the model was trained. The search is performed using the Deep
                    Memory model. If False, the distance metric is set to "COS" or
                    whatever distance metric user specifies.
        Returns:
            List[Document]: List of Documents most similar to the query vector.
        """
        return self._search(
            embedding=embedding,
            k=k,
            use_maximal_marginal_relevance=False,
            return_score=False,
            **kwargs,
        )
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """
        Run similarity search with Deep Lake with distance returned.
        Examples:
        >>> data = vector_store.similarity_search_with_score(
        ...     query=<your_query>,
        ...     embedding=<your_embedding_function>
        ...     k=<number_of_items_to_return>,
        ...     exec_option=<preferred_exec_option>,
        ... )
        Args:
            query (str): Query text to search for.
            k (int): Number of results to return. Defaults to 4.
            **kwargs: Additional keyword arguments. Some of these arguments are:
                distance_metric: `L2` for Euclidean, `L1` for Nuclear, `max` L-infinity
                    distance, `cos` for cosine similarity, 'dot' for dot product.
                    Defaults to `L2`.
                filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.


filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.
                    embedding_function (Callable): Embedding function to use. Defaults
                    to None.
                exec_option (str): DeepLakeVectorStore supports 3 ways to perform
                    searching. It could be either "python", "compute_engine" or
                    "tensor_db". Defaults to "python".
                    - "python" - Pure-python implementation running on the client.
                        Can be used for data stored anywhere. WARNING: using this
                        option with big datasets is discouraged due to potential
                        memory issues.
                    - "compute_engine" - Performant C++ implementation of the Deep
                        Lake Compute Engine. Runs on the client and can be used for
                        any data stored in or connected to Deep Lake. It cannot be used
                        with in-memory or local datasets.
                    - "tensor_db" - Performant, fully-hosted Managed Tensor Database.
                        Responsible for storage and query execution. Only available for
                        data stored in the Deep Lake Managed Database. To store datasets
                        in this database, specify `runtime = {"db_engine": True}`
                        during dataset creation.
                deep_memory (bool): Whether to use the Deep Memory model for improving
                    search results. Defaults to False if deep_memory is not specified
                    in the Vector Store initialization. If True, the distance metric
                    is set to "deepmemory_distance", which represents the metric with
                    which the model was trained. The search is performed using the Deep
                    Memory model. If False, the distance metric is set to "COS" or
                    whatever distance metric user specifies.
        Returns:
            List[Tuple[Document, float]]: List of documents most similar to the query
                text with distance in float."""
        return self._search(
            query=query,
            k=k,


return self._search(
            query=query,
            k=k,
            return_score=True,
            **kwargs,
        )
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        exec_option: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """
        Return docs selected using the maximal marginal relevance. Maximal marginal
        relevance optimizes for similarity to query AND diversity among selected docs.
        Examples:
        >>> data = vector_store.max_marginal_relevance_search_by_vector(
        ...        embedding=<your_embedding>,
        ...        fetch_k=<elements_to_fetch_before_mmr_search>,
        ...        k=<number_of_items_to_return>,
        ...        exec_option=<preferred_exec_option>,
        ... )
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch for MMR algorithm.
            lambda_mult: Number between 0 and 1 determining the degree of diversity.
                0 corresponds to max diversity and 1 to min diversity. Defaults to 0.5.
            exec_option (str): DeepLakeVectorStore supports 3 ways for searching.
                Could be "python", "compute_engine" or "tensor_db". Defaults to
                "python".
                - "python" - Pure-python implementation running on the client.
                    Can be used for data stored anywhere. WARNING: using this
                    option with big datasets is discouraged due to potential
                    memory issues.


option with big datasets is discouraged due to potential
                    memory issues.
                - "compute_engine" - Performant C++ implementation of the Deep
                    Lake Compute Engine. Runs on the client and can be used for
                    any data stored in or connected to Deep Lake. It cannot be used
                    with in-memory or local datasets.
                - "tensor_db" - Performant, fully-hosted Managed Tensor Database.
                    Responsible for storage and query execution. Only available for
                    data stored in the Deep Lake Managed Database. To store datasets
                    in this database, specify `runtime = {"db_engine": True}`
                    during dataset creation.
            deep_memory (bool): Whether to use the Deep Memory model for improving
                search results. Defaults to False if deep_memory is not specified
                in the Vector Store initialization. If True, the distance metric
                is set to "deepmemory_distance", which represents the metric with
                which the model was trained. The search is performed using the Deep
                Memory model. If False, the distance metric is set to "COS" or
                whatever distance metric user specifies.
            **kwargs: Additional keyword arguments.
        Returns:
            List[Documents] - A list of documents.
        """
        return self._search(
            embedding=embedding,
            k=k,
            fetch_k=fetch_k,
            use_maximal_marginal_relevance=True,
            lambda_mult=lambda_mult,
            exec_option=exec_option,
            **kwargs,
        )
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        exec_option: Optional[str] = None,


exec_option: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Examples:
        >>> # Search using an embedding
        >>> data = vector_store.max_marginal_relevance_search(
        ...        query = <query_to_search>,
        ...        embedding_function = <embedding_function_for_query>,
        ...        k = <number_of_items_to_return>,
        ...        exec_option = <preferred_exec_option>,
        ... )
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents for MMR algorithm.
            lambda_mult: Value between 0 and 1. 0 corresponds
                        to maximum diversity and 1 to minimum.
                        Defaults to 0.5.
            exec_option (str): Supports 3 ways to perform searching.
                - "python" - Pure-python implementation running on the client.
                        Can be used for data stored anywhere. WARNING: using this
                        option with big datasets is discouraged due to potential
                        memory issues.
                    - "compute_engine" - Performant C++ implementation of the Deep
                        Lake Compute Engine. Runs on the client and can be used for
                        any data stored in or connected to Deep Lake. It cannot be
                        used with in-memory or local datasets.
                    - "tensor_db" - Performant, fully-hosted Managed Tensor Database.
                        Responsible for storage and query execution. Only available
                        for data stored in the Deep Lake Managed Database. To store
                        datasets in this database, specify
                        `runtime = {"db_engine": True}` during dataset creation.


`runtime = {"db_engine": True}` during dataset creation.
            deep_memory (bool): Whether to use the Deep Memory model for improving
                search results. Defaults to False if deep_memory is not specified
                in the Vector Store initialization. If True, the distance metric
                is set to "deepmemory_distance", which represents the metric with
                which the model was trained. The search is performed using the Deep
                Memory model. If False, the distance metric is set to "COS" or
                whatever distance metric user specifies.
            **kwargs: Additional keyword arguments
        Returns:
            List of Documents selected by maximal marginal relevance.
        Raises:
            ValueError: when MRR search is on but embedding function is
                not specified.
        """
        embedding_function = kwargs.get("embedding") or self._embedding_function
        if embedding_function is None:
            raise ValueError(
                "For MMR search, you must specify an embedding function on"
                " `creation` or during add call."
            )
        return self._search(
            query=query,
            k=k,
            fetch_k=fetch_k,
            use_maximal_marginal_relevance=True,
            lambda_mult=lambda_mult,
            exec_option=exec_option,
            embedding_function=embedding_function,  # type: ignore
            **kwargs,
        )
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        dataset_path: str = _LANGCHAIN_DEFAULT_DEEPLAKE_PATH,
        **kwargs: Any,
    ) -> DeepLake:


**kwargs: Any,
    ) -> DeepLake:
        """Create a Deep Lake dataset from a raw documents.
        If a dataset_path is specified, the dataset will be persisted in that location,
        otherwise by default at `./deeplake`
        Examples:
        >>> # Search using an embedding
        >>> vector_store = DeepLake.from_texts(
        ...        texts = <the_texts_that_you_want_to_embed>,
        ...        embedding_function = <embedding_function_for_query>,
        ...        k = <number_of_items_to_return>,
        ...        exec_option = <preferred_exec_option>,
        ... )
        Args:
            dataset_path (str): - The full path to the dataset. Can be:
                - Deep Lake cloud path of the form ``hub://username/dataset_name``.
                    To write to Deep Lake cloud datasets,
                    ensure that you are logged in to Deep Lake
                    (use 'activeloop login' from command line)
                - AWS S3 path of the form ``s3://bucketname/path/to/dataset``.
                    Credentials are required in either the environment
                - Google Cloud Storage path of the form
                    ``gcs://bucketname/path/to/dataset`` Credentials are required
                    in either the environment
                - Local file system path of the form ``./path/to/dataset`` or
                    ``~/path/to/dataset`` or ``path/to/dataset``.
                - In-memory path of the form ``mem://path/to/dataset`` which doesn't
                    save the dataset, but keeps it in memory instead.
                    Should be used only for testing as it does not persist.
            texts (List[Document]): List of documents to add.
            embedding (Optional[Embeddings]): Embedding function. Defaults to None.


embedding (Optional[Embeddings]): Embedding function. Defaults to None.
                Note, in other places, it is called embedding_function.
            metadatas (Optional[List[dict]]): List of metadatas. Defaults to None.
            ids (Optional[List[str]]): List of document IDs. Defaults to None.
            **kwargs: Additional keyword arguments.
        Returns:
            DeepLake: Deep Lake dataset.
        """
        deeplake_dataset = cls(dataset_path=dataset_path, embedding=embedding, **kwargs)
        deeplake_dataset.add_texts(
            texts=texts,
            metadatas=metadatas,
            ids=ids,
        )
        return deeplake_dataset
[docs]    def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> bool:
        """Delete the entities in the dataset.
        Args:
            ids (Optional[List[str]], optional): The document_ids to delete.
                Defaults to None.
            **kwargs: Other keyword arguments that subclasses might use.
                - filter (Optional[Dict[str, str]], optional): The filter to delete by.
                - delete_all (Optional[bool], optional): Whether to drop the dataset.
        Returns:
            bool: Whether the delete operation was successful.
        """
        filter = kwargs.get("filter")
        delete_all = kwargs.get("delete_all")
        self.vectorstore.delete(ids=ids, filter=filter, delete_all=delete_all)
        return True
[docs]    @classmethod
    def force_delete_by_path(cls, path: str) -> None:
        """Force delete dataset by path.
        Args:
            path (str): path of the dataset to delete.
        Raises:
            ValueError: if deeplake is not installed.


Raises:
            ValueError: if deeplake is not installed.
        """
        try:
            import deeplake
        except ImportError:
            raise ValueError(
                "Could not import deeplake python package. "
                "Please install it with `pip install deeplake`."
            )
        deeplake.delete(path, large_ok=True, force=True)
[docs]    def delete_dataset(self) -> None:
        """Delete the collection."""
        self.delete(delete_all=True)
[docs]    def ds(self) -> Any:
        logger.warning(
            "this method is deprecated and will be removed, "
            "better to use `db.vectorstore.dataset` instead."
        )
        return self.vectorstore.dataset


Source code for langchain.vectorstores.pinecone
from __future__ import annotations
import logging
import uuid
import warnings
from typing import TYPE_CHECKING, Any, Callable, Iterable, List, Optional, Tuple, Union
import numpy as np
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.utils.iter import batch_iterate
from langchain.vectorstores.utils import DistanceStrategy, maximal_marginal_relevance
if TYPE_CHECKING:
    from pinecone import Index
logger = logging.getLogger(__name__)
[docs]class Pinecone(VectorStore):
    """`Pinecone` vector store.
    To use, you should have the ``pinecone-client`` python package installed.
    Example:
        .. code-block:: python
            from langchain.vectorstores import Pinecone
            from langchain.embeddings.openai import OpenAIEmbeddings
            import pinecone
            # The environment should be the one specified next to the API key
            # in your Pinecone console
            pinecone.init(api_key="***", environment="...")
            index = pinecone.Index("langchain-demo")
            embeddings = OpenAIEmbeddings()
            vectorstore = Pinecone(index, embeddings.embed_query, "text")
    """
[docs]    def __init__(
        self,
        index: Any,
        embedding: Union[Embeddings, Callable],
        text_key: str,
        namespace: Optional[str] = None,
        distance_strategy: Optional[DistanceStrategy] = DistanceStrategy.COSINE,
    ):
        """Initialize with Pinecone client."""
        try:
            import pinecone
        except ImportError:
            raise ImportError(
                "Could not import pinecone python package. "


raise ImportError(
                "Could not import pinecone python package. "
                "Please install it with `pip install pinecone-client`."
            )
        if not isinstance(embedding, Embeddings):
            warnings.warn(
                "Passing in `embedding` as a Callable is deprecated. Please pass in an"
                " Embeddings object instead."
            )
        if not isinstance(index, pinecone.index.Index):
            raise ValueError(
                f"client should be an instance of pinecone.index.Index, "
                f"got {type(index)}"
            )
        self._index = index
        self._embedding = embedding
        self._text_key = text_key
        self._namespace = namespace
        self.distance_strategy = distance_strategy
    @property
    def embeddings(self) -> Optional[Embeddings]:
        """Access the query embedding object if available."""
        if isinstance(self._embedding, Embeddings):
            return self._embedding
        return None
    def _embed_documents(self, texts: Iterable[str]) -> List[List[float]]:
        """Embed search docs."""
        if isinstance(self._embedding, Embeddings):
            return self._embedding.embed_documents(list(texts))
        return [self._embedding(t) for t in texts]
    def _embed_query(self, text: str) -> List[float]:
        """Embed query text."""
        if isinstance(self._embedding, Embeddings):
            return self._embedding.embed_query(text)
        return self._embedding(text)
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        namespace: Optional[str] = None,
        batch_size: int = 32,


namespace: Optional[str] = None,
        batch_size: int = 32,
        embedding_chunk_size: int = 1000,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Upsert optimization is done by chunking the embeddings and upserting them.
        This is done to avoid memory issues and optimize using HTTP based embeddings.
        For OpenAI embeddings, use pool_threads>4 when constructing the pinecone.Index,
        embedding_chunk_size>1000 and batch_size~64 for best performance.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids to associate with the texts.
            namespace: Optional pinecone namespace to add the texts to.
            batch_size: Batch size to use when adding the texts to the vectorstore.
            embedding_chunk_size: Chunk size to use when embedding the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        if namespace is None:
            namespace = self._namespace
        texts = list(texts)
        ids = ids or [str(uuid.uuid4()) for _ in texts]
        metadatas = metadatas or [{} for _ in texts]
        for metadata, text in zip(metadatas, texts):
            metadata[self._text_key] = text
        # For loops to avoid memory issues and optimize when using HTTP based embeddings
        # The first loop runs the embeddings, it benefits when using OpenAI embeddings
        # The second loops runs the pinecone upsert asynchronously.
        for i in range(0, len(texts), embedding_chunk_size):


for i in range(0, len(texts), embedding_chunk_size):
            chunk_texts = texts[i : i + embedding_chunk_size]
            chunk_ids = ids[i : i + embedding_chunk_size]
            chunk_metadatas = metadatas[i : i + embedding_chunk_size]
            embeddings = self._embed_documents(chunk_texts)
            async_res = [
                self._index.upsert(
                    vectors=batch,
                    namespace=namespace,
                    async_req=True,
                    **kwargs,
                )
                for batch in batch_iterate(
                    batch_size, zip(chunk_ids, embeddings, chunk_metadatas)
                )
            ]
            [res.get() for res in async_res]
        return ids
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        filter: Optional[dict] = None,
        namespace: Optional[str] = None,
    ) -> List[Tuple[Document, float]]:
        """Return pinecone documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            filter: Dictionary of argument(s) to filter on metadata
            namespace: Namespace to search in. Default will search in '' namespace.
        Returns:
            List of Documents most similar to the query and score for each
        """
        return self.similarity_search_by_vector_with_score(
            self._embed_query(query), k=k, filter=filter, namespace=namespace
        )
[docs]    def similarity_search_by_vector_with_score(
        self,
        embedding: List[float],
        *,
        k: int = 4,


embedding: List[float],
        *,
        k: int = 4,
        filter: Optional[dict] = None,
        namespace: Optional[str] = None,
    ) -> List[Tuple[Document, float]]:
        """Return pinecone documents most similar to embedding, along with scores."""
        if namespace is None:
            namespace = self._namespace
        docs = []
        results = self._index.query(
            [embedding],
            top_k=k,
            include_metadata=True,
            namespace=namespace,
            filter=filter,
        )
        for res in results["matches"]:
            metadata = res["metadata"]
            if self._text_key in metadata:
                text = metadata.pop(self._text_key)
                score = res["score"]
                docs.append((Document(page_content=text, metadata=metadata), score))
            else:
                logger.warning(
                    f"Found document with no `{self._text_key}` key. Skipping."
                )
        return docs
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[dict] = None,
        namespace: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return pinecone documents most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            filter: Dictionary of argument(s) to filter on metadata
            namespace: Namespace to search in. Default will search in '' namespace.
        Returns:
            List of Documents most similar to the query and score for each
        """
        docs_and_scores = self.similarity_search_with_score(


"""
        docs_and_scores = self.similarity_search_with_score(
            query, k=k, filter=filter, namespace=namespace, **kwargs
        )
        return [doc for doc, _ in docs_and_scores]
    def _select_relevance_score_fn(self) -> Callable[[float], float]:
        """
        The 'correct' relevance function
        may differ depending on a few things, including:
        - the distance / similarity metric used by the VectorStore
        - the scale of your embeddings (OpenAI's are unit normed. Many others are not!)
        - embedding dimensionality
        - etc.
        """
        if self.distance_strategy == DistanceStrategy.COSINE:
            return self._cosine_relevance_score_fn
        elif self.distance_strategy == DistanceStrategy.MAX_INNER_PRODUCT:
            return self._max_inner_product_relevance_score_fn
        elif self.distance_strategy == DistanceStrategy.EUCLIDEAN_DISTANCE:
            return self._euclidean_relevance_score_fn
        else:
            raise ValueError(
                "Unknown distance strategy, must be cosine, max_inner_product "
                "(dot product), or euclidean"
            )
    @staticmethod
    def _cosine_relevance_score_fn(score: float) -> float:
        """Pinecone returns cosine similarity scores between [-1,1]"""
        return (score + 1) / 2
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[dict] = None,
        namespace: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:


**kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        if namespace is None:
            namespace = self._namespace
        results = self._index.query(
            [embedding],
            top_k=fetch_k,
            include_values=True,
            include_metadata=True,
            namespace=namespace,
            filter=filter,
        )
        mmr_selected = maximal_marginal_relevance(
            np.array([embedding], dtype=np.float32),
            [item["values"] for item in results["matches"]],
            k=k,
            lambda_mult=lambda_mult,
        )
        selected = [results["matches"][i]["metadata"] for i in mmr_selected]
        return [
            Document(page_content=metadata.pop((self._text_key)), metadata=metadata)
            for metadata in selected
        ]
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[dict] = None,


filter: Optional[dict] = None,
        namespace: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self._embed_query(query)
        return self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult, filter, namespace
        )
[docs]    @classmethod
    def get_pinecone_index(
        cls,
        index_name: Optional[str],
        pool_threads: int = 4,
    ) -> Index:
        """Return a Pinecone Index instance.
        Args:
            index_name: Name of the index to use.
            pool_threads: Number of threads to use for index upsert.
        Returns:
            Pinecone Index instance."""
        try:
            import pinecone
        except ImportError:
            raise ValueError(
                "Could not import pinecone python package. "
                "Please install it with `pip install pinecone-client`."
            )
        indexes = pinecone.list_indexes()  # checks if provided index exists
        if index_name in indexes:


if index_name in indexes:
            index = pinecone.Index(index_name, pool_threads=pool_threads)
        elif len(indexes) == 0:
            raise ValueError(
                "No active indexes found in your Pinecone project, "
                "are you sure you're using the right Pinecone API key and Environment? "
                "Please double check your Pinecone dashboard."
            )
        else:
            raise ValueError(
                f"Index '{index_name}' not found in your Pinecone project. "
                f"Did you mean one of the following indexes: {', '.join(indexes)}"
            )
        return index
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        batch_size: int = 32,
        text_key: str = "text",
        namespace: Optional[str] = None,
        index_name: Optional[str] = None,
        upsert_kwargs: Optional[dict] = None,
        pool_threads: int = 4,
        embeddings_chunk_size: int = 1000,
        **kwargs: Any,
    ) -> Pinecone:
        """Construct Pinecone wrapper from raw documents.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Adds the documents to a provided Pinecone index
        This is intended to be a quick way to get started.
        The `pool_threads` affects the speed of the upsert operations.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Pinecone
                from langchain.embeddings import OpenAIEmbeddings
                import pinecone


from langchain.embeddings import OpenAIEmbeddings
                import pinecone
                # The environment should be the one specified next to the API key
                # in your Pinecone console
                pinecone.init(api_key="***", environment="...")
                embeddings = OpenAIEmbeddings()
                pinecone = Pinecone.from_texts(
                    texts,
                    embeddings,
                    index_name="langchain-demo"
                )
        """
        pinecone_index = cls.get_pinecone_index(index_name, pool_threads)
        pinecone = cls(pinecone_index, embedding, text_key, namespace, **kwargs)
        pinecone.add_texts(
            texts,
            metadatas=metadatas,
            ids=ids,
            namespace=namespace,
            batch_size=batch_size,
            embedding_chunk_size=embeddings_chunk_size,
            **(upsert_kwargs or {}),
        )
        return pinecone
[docs]    @classmethod
    def from_existing_index(
        cls,
        index_name: str,
        embedding: Embeddings,
        text_key: str = "text",
        namespace: Optional[str] = None,
        pool_threads: int = 4,
    ) -> Pinecone:
        """Load pinecone vectorstore from index name."""
        pinecone_index = cls.get_pinecone_index(index_name, pool_threads)
        return cls(pinecone_index, embedding, text_key, namespace)
[docs]    def delete(
        self,
        ids: Optional[List[str]] = None,
        delete_all: Optional[bool] = None,
        namespace: Optional[str] = None,
        filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> None:
        """Delete by vector IDs or filter.


) -> None:
        """Delete by vector IDs or filter.
        Args:
            ids: List of ids to delete.
            filter: Dictionary of conditions to filter vectors to delete.
        """
        if namespace is None:
            namespace = self._namespace
        if delete_all:
            self._index.delete(delete_all=True, namespace=namespace, **kwargs)
        elif ids is not None:
            chunk_size = 1000
            for i in range(0, len(ids), chunk_size):
                chunk = ids[i : i + chunk_size]
                self._index.delete(ids=chunk, namespace=namespace, **kwargs)
        elif filter is not None:
            self._index.delete(filter=filter, namespace=namespace, **kwargs)
        else:
            raise ValueError("Either ids, delete_all, or filter must be provided.")
        return None


Source code for langchain.vectorstores.rocksetdb
from __future__ import annotations
import logging
from enum import Enum
from typing import Any, Iterable, List, Optional, Tuple
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
logger = logging.getLogger(__name__)
[docs]class Rockset(VectorStore):
    """`Rockset` vector store.
    To use, you should have the `rockset` python package installed. Note that to use
    this, the collection being used must already exist in your Rockset instance.
    You must also ensure you use a Rockset ingest transformation to apply
    `VECTOR_ENFORCE` on the column being used to store `embedding_key` in the
    collection.
    See: https://rockset.com/blog/introducing-vector-search-on-rockset/ for more details
    Everything below assumes `commons` Rockset workspace.
    Example:
        .. code-block:: python
            from langchain.vectorstores import Rockset
            from langchain.embeddings.openai import OpenAIEmbeddings
            import rockset
            # Make sure you use the right host (region) for your Rockset instance
            # and APIKEY has both read-write access to your collection.
            rs = rockset.RocksetClient(host=rockset.Regions.use1a1, api_key="***")
            collection_name = "langchain_demo"
            embeddings = OpenAIEmbeddings()
            vectorstore = Rockset(rs, collection_name, embeddings,
                "description", "description_embedding")
    """
[docs]    def __init__(
        self,
        client: Any,
        embeddings: Embeddings,
        collection_name: str,
        text_key: str,
        embedding_key: str,


text_key: str,
        embedding_key: str,
        workspace: str = "commons",
    ):
        """Initialize with Rockset client.
        Args:
            client: Rockset client object
            collection: Rockset collection to insert docs / query
            embeddings: Langchain Embeddings object to use to generate
                        embedding for given text.
            text_key: column in Rockset collection to use to store the text
            embedding_key: column in Rockset collection to use to store the embedding.
                           Note: We must apply `VECTOR_ENFORCE()` on this column via
                           Rockset ingest transformation.
        """
        try:
            from rockset import RocksetClient
        except ImportError:
            raise ImportError(
                "Could not import rockset client python package. "
                "Please install it with `pip install rockset`."
            )
        if not isinstance(client, RocksetClient):
            raise ValueError(
                f"client should be an instance of rockset.RocksetClient, "
                f"got {type(client)}"
            )
        # TODO: check that `collection_name` exists in rockset. Create if not.
        self._client = client
        self._collection_name = collection_name
        self._embeddings = embeddings
        self._text_key = text_key
        self._embedding_key = embedding_key
        self._workspace = workspace
        try:
            self._client.set_application("langchain")
        except AttributeError:
            # ignore
            pass
    @property
    def embeddings(self) -> Embeddings:
        return self._embeddings
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,


ids: Optional[List[str]] = None,
        batch_size: int = 32,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore
                Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids to associate with the texts.
            batch_size: Send documents in batches to rockset.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        batch: list[dict] = []
        stored_ids = []
        for i, text in enumerate(texts):
            if len(batch) == batch_size:
                stored_ids += self._write_documents_to_rockset(batch)
                batch = []
            doc = {}
            if metadatas and len(metadatas) > i:
                doc = metadatas[i]
            if ids and len(ids) > i:
                doc["_id"] = ids[i]
            doc[self._text_key] = text
            doc[self._embedding_key] = self._embeddings.embed_query(text)
            batch.append(doc)
        if len(batch) > 0:
            stored_ids += self._write_documents_to_rockset(batch)
            batch = []
        return stored_ids
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        client: Any = None,
        collection_name: str = "",
        text_key: str = "",
        embedding_key: str = "",
        ids: Optional[List[str]] = None,


embedding_key: str = "",
        ids: Optional[List[str]] = None,
        batch_size: int = 32,
        **kwargs: Any,
    ) -> Rockset:
        """Create Rockset wrapper with existing texts.
        This is intended as a quicker way to get started.
        """
        # Sanitize inputs
        assert client is not None, "Rockset Client cannot be None"
        assert collection_name, "Collection name cannot be empty"
        assert text_key, "Text key name cannot be empty"
        assert embedding_key, "Embedding key cannot be empty"
        rockset = cls(client, embedding, collection_name, text_key, embedding_key)
        rockset.add_texts(texts, metadatas, ids, batch_size)
        return rockset
    # Rockset supports these vector distance functions.
    class DistanceFunction(Enum):
        COSINE_SIM = "COSINE_SIM"
        EUCLIDEAN_DIST = "EUCLIDEAN_DIST"
        DOT_PRODUCT = "DOT_PRODUCT"
        # how to sort results for "similarity"
        def order_by(self) -> str:
            if self.value == "EUCLIDEAN_DIST":
                return "ASC"
            return "DESC"
[docs]    def similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        distance_func: DistanceFunction = DistanceFunction.COSINE_SIM,
        where_str: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Perform a similarity search with Rockset
        Args:
            query (str): Text to look up documents similar to.
            distance_func (DistanceFunction): how to compute distance between two


distance_func (DistanceFunction): how to compute distance between two
                vectors in Rockset.
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): Metadata filters supplied as a
                SQL `where` condition string. Defaults to None.
                eg. "price<=70.0 AND brand='Nintendo'"
            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection.
        Returns:
            List[Tuple[Document, float]]: List of documents with their relevance score
        """
        return self.similarity_search_by_vector_with_relevance_scores(
            self._embeddings.embed_query(query),
            k,
            distance_func,
            where_str,
            **kwargs,
        )
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        distance_func: DistanceFunction = DistanceFunction.COSINE_SIM,
        where_str: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Same as `similarity_search_with_relevance_scores` but
        doesn't return the scores.
        """
        return self.similarity_search_by_vector(
            self._embeddings.embed_query(query),
            k,
            distance_func,
            where_str,
            **kwargs,
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        distance_func: DistanceFunction = DistanceFunction.COSINE_SIM,
        where_str: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:


**kwargs: Any,
    ) -> List[Document]:
        """Accepts a query_embedding (vector), and returns documents with
        similar embeddings."""
        docs_and_scores = self.similarity_search_by_vector_with_relevance_scores(
            embedding, k, distance_func, where_str, **kwargs
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_by_vector_with_relevance_scores(
        self,
        embedding: List[float],
        k: int = 4,
        distance_func: DistanceFunction = DistanceFunction.COSINE_SIM,
        where_str: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Accepts a query_embedding (vector), and returns documents with
        similar embeddings along with their relevance scores."""
        q_str = self._build_query_sql(embedding, distance_func, k, where_str)
        try:
            query_response = self._client.Queries.query(sql={"query": q_str})
        except Exception as e:
            logger.error("Exception when querying Rockset: %s\n", e)
            return []
        finalResult: list[Tuple[Document, float]] = []
        for document in query_response.results:
            metadata = {}
            assert isinstance(
                document, dict
            ), "document should be of type `dict[str,Any]`. But found: `{}`".format(
                type(document)
            )
            for k, v in document.items():
                if k == self._text_key:
                    assert isinstance(v, str), (
                        "page content stored in column `{}` must be of type `str`. "
                        "But found: `{}`"
                    ).format(self._text_key, type(v))


).format(self._text_key, type(v))
                    page_content = v
                elif k == "dist":
                    assert isinstance(v, float), (
                        "Computed distance between vectors must of type `float`. "
                        "But found {}"
                    ).format(type(v))
                    score = v
                elif k not in ["_id", "_event_time", "_meta"]:
                    # These columns are populated by Rockset when documents are
                    # inserted. No need to return them in metadata dict.
                    metadata[k] = v
            finalResult.append(
                (Document(page_content=page_content, metadata=metadata), score)
            )
        return finalResult
    # Helper functions
    def _build_query_sql(
        self,
        query_embedding: List[float],
        distance_func: DistanceFunction,
        k: int = 4,
        where_str: Optional[str] = None,
    ) -> str:
        """Builds Rockset SQL query to query similar vectors to query_vector"""
        q_embedding_str = ",".join(map(str, query_embedding))
        distance_str = f"""{distance_func.value}({self._embedding_key}, \
[{q_embedding_str}]) as dist"""
        where_str = f"WHERE {where_str}\n" if where_str else ""
        return f"""\
SELECT * EXCEPT({self._embedding_key}), {distance_str}
FROM {self._workspace}.{self._collection_name}
{where_str}\
ORDER BY dist {distance_func.order_by()}
LIMIT {str(k)}
"""
    def _write_documents_to_rockset(self, batch: List[dict]) -> List[str]:
        add_doc_res = self._client.Documents.add_documents(
            collection=self._collection_name, data=batch, workspace=self._workspace
        )


collection=self._collection_name, data=batch, workspace=self._workspace
        )
        return [doc_status._id for doc_status in add_doc_res.data]
[docs]    def delete_texts(self, ids: List[str]) -> None:
        """Delete a list of docs from the Rockset collection"""
        try:
            from rockset.models import DeleteDocumentsRequestData
        except ImportError:
            raise ImportError(
                "Could not import rockset client python package. "
                "Please install it with `pip install rockset`."
            )
        self._client.Documents.delete_documents(
            collection=self._collection_name,
            data=[DeleteDocumentsRequestData(id=i) for i in ids],
            workspace=self._workspace,
        )


Source code for langchain.vectorstores.momento_vector_index
from typing import (
    TYPE_CHECKING,
    Any,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
    TypeVar,
    cast,
)
from uuid import uuid4
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.utils import get_from_env
from langchain.vectorstores.utils import DistanceStrategy
VST = TypeVar("VST", bound="VectorStore")
if TYPE_CHECKING:
    from momento import PreviewVectorIndexClient
[docs]class MomentoVectorIndex(VectorStore):
    """`Momento Vector Index` (MVI) vector store.
    Momento Vector Index is a serverless vector index that can be used to store and
    search vectors. To use you should have the ``momento`` python package installed.
    Example:
        .. code-block:: python
            from langchain.embeddings import OpenAIEmbeddings
            from langchain.vectorstores import MomentoVectorIndex
            from momento import (
                CredentialProvider,
                PreviewVectorIndexClient,
                VectorIndexConfigurations,
            )
            vectorstore = MomentoVectorIndex(
                embedding=OpenAIEmbeddings(),
                client=PreviewVectorIndexClient(
                    VectorIndexConfigurations.Default.latest(),
                    credential_provider=CredentialProvider.from_environment_variable(
                        "MOMENTO_API_KEY"
                    ),
                ),
                index_name="my-index",
            )
    """
[docs]    def __init__(
        self,
        embedding: Embeddings,
        client: "PreviewVectorIndexClient",
        index_name: str = "default",


client: "PreviewVectorIndexClient",
        index_name: str = "default",
        distance_strategy: DistanceStrategy = DistanceStrategy.COSINE,
        text_field: str = "text",
        ensure_index_exists: bool = True,
        **kwargs: Any,
    ):
        """Initialize a Vector Store backed by Momento Vector Index.
        Args:
            embedding (Embeddings): The embedding function to use.
            configuration (VectorIndexConfiguration): The configuration to initialize
                the Vector Index with.
            credential_provider (CredentialProvider): The credential provider to
                authenticate the Vector Index with.
            index_name (str, optional): The name of the index to store the documents in.
                Defaults to "default".
            distance_strategy (DistanceStrategy, optional): The distance strategy to
                use. Defaults to DistanceStrategy.COSINE. If you select
                DistanceStrategy.EUCLIDEAN_DISTANCE, Momento uses the squared
                Euclidean distance.
            text_field (str, optional): The name of the metadata field to store the
                original text in. Defaults to "text".
            ensure_index_exists (bool, optional): Whether to ensure that the index
                exists before adding documents to it. Defaults to True.
        """
        try:
            from momento import PreviewVectorIndexClient
        except ImportError:
            raise ImportError(
                "Could not import momento python package. "
                "Please install it with `pip install momento`."
            )
        self._client: PreviewVectorIndexClient = client
        self._embedding = embedding
        self.index_name = index_name
        self.__validate_distance_strategy(distance_strategy)
        self.distance_strategy = distance_strategy
        self.text_field = text_field
        self._ensure_index_exists = ensure_index_exists
    @staticmethod


self._ensure_index_exists = ensure_index_exists
    @staticmethod
    def __validate_distance_strategy(distance_strategy: DistanceStrategy) -> None:
        if distance_strategy not in [
            DistanceStrategy.COSINE,
            DistanceStrategy.MAX_INNER_PRODUCT,
            DistanceStrategy.MAX_INNER_PRODUCT,
        ]:
            raise ValueError(f"Distance strategy {distance_strategy} not implemented.")
    @property
    def embeddings(self) -> Embeddings:
        return self._embedding
    def _create_index_if_not_exists(self, num_dimensions: int) -> bool:
        """Create index if it does not exist."""
        from momento.requests.vector_index import SimilarityMetric
        from momento.responses.vector_index import CreateIndex
        similarity_metric = None
        if self.distance_strategy == DistanceStrategy.COSINE:
            similarity_metric = SimilarityMetric.COSINE_SIMILARITY
        elif self.distance_strategy == DistanceStrategy.MAX_INNER_PRODUCT:
            similarity_metric = SimilarityMetric.INNER_PRODUCT
        elif self.distance_strategy == DistanceStrategy.EUCLIDEAN_DISTANCE:
            similarity_metric = SimilarityMetric.EUCLIDEAN_SIMILARITY
        else:
            raise ValueError(
                f"Distance strategy {self.distance_strategy} not implemented."
            )
        response = self._client.create_index(
            self.index_name, num_dimensions, similarity_metric
        )
        if isinstance(response, CreateIndex.Success):
            return True
        elif isinstance(response, CreateIndex.IndexAlreadyExists):
            return False
        elif isinstance(response, CreateIndex.Error):
            raise response.inner_exception
        else:
            raise Exception(f"Unexpected response: {response}")
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,


**kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            texts (Iterable[str]): Iterable of strings to add to the vectorstore.
            metadatas (Optional[List[dict]]): Optional list of metadatas associated with
                the texts.
            kwargs (Any): Other optional parameters. Specifically:
            - ids (List[str], optional): List of ids to use for the texts.
                Defaults to None, in which case uuids are generated.
        Returns:
            List[str]: List of ids from adding the texts into the vectorstore.
        """
        from momento.requests.vector_index import Item
        from momento.responses.vector_index import UpsertItemBatch
        texts = list(texts)
        if len(texts) == 0:
            return []
        if metadatas is not None:
            for metadata, text in zip(metadatas, texts):
                metadata[self.text_field] = text
        else:
            metadatas = [{self.text_field: text} for text in texts]
        try:
            embeddings = self._embedding.embed_documents(texts)
        except NotImplementedError:
            embeddings = [self._embedding.embed_query(x) for x in texts]
        # Create index if it does not exist.
        # We assume that if it does exist, then it was created with the desired number
        # of dimensions and similarity metric.
        if self._ensure_index_exists:
            self._create_index_if_not_exists(len(embeddings[0]))
        if "ids" in kwargs:
            ids = kwargs["ids"]
            if len(ids) != len(embeddings):
                raise ValueError("Number of ids must match number of texts")
        else:


raise ValueError("Number of ids must match number of texts")
        else:
            ids = [str(uuid4()) for _ in range(len(embeddings))]
        batch_size = 128
        for i in range(0, len(embeddings), batch_size):
            start = i
            end = min(i + batch_size, len(embeddings))
            items = [
                Item(id=id, vector=vector, metadata=metadata)
                for id, vector, metadata in zip(
                    ids[start:end],
                    embeddings[start:end],
                    metadatas[start:end],
                )
            ]
            response = self._client.upsert_item_batch(self.index_name, items)
            if isinstance(response, UpsertItemBatch.Success):
                pass
            elif isinstance(response, UpsertItemBatch.Error):
                raise response.inner_exception
            else:
                raise Exception(f"Unexpected response: {response}")
        return ids
[docs]    def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
        """Delete by vector ID.
        Args:
            ids (List[str]): List of ids to delete.
            kwargs (Any): Other optional parameters (unused)
        Returns:
            Optional[bool]: True if deletion is successful,
            False otherwise, None if not implemented.
        """
        from momento.responses.vector_index import DeleteItemBatch
        if ids is None:
            return True
        response = self._client.delete_item_batch(self.index_name, ids)
        return isinstance(response, DeleteItemBatch.Success)
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Search for similar documents to the query string.
        Args:


"""Search for similar documents to the query string.
        Args:
            query (str): The query string to search for.
            k (int, optional): The number of results to return. Defaults to 4.
        Returns:
            List[Document]: A list of documents that are similar to the query.
        """
        res = self.similarity_search_with_score(query=query, k=k, **kwargs)
        return [doc for doc, _ in res]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Search for similar documents to the query string.
        Args:
            query (str): The query string to search for.
            k (int, optional): The number of results to return. Defaults to 4.
            kwargs (Any): Vector Store specific search parameters. The following are
                forwarded to the Momento Vector Index:
            - top_k (int, optional): The number of results to return.
        Returns:
            List[Tuple[Document, float]]: A list of tuples of the form
                (Document, score).
        """
        embedding = self._embedding.embed_query(query)
        results = self.similarity_search_with_score_by_vector(
            embedding=embedding, k=k, **kwargs
        )
        return results
[docs]    def similarity_search_with_score_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Search for similar documents to the query vector.
        Args:


"""Search for similar documents to the query vector.
        Args:
            embedding (List[float]): The query vector to search for.
            k (int, optional): The number of results to return. Defaults to 4.
            kwargs (Any): Vector Store specific search parameters. The following are
                forwarded to the Momento Vector Index:
            - top_k (int, optional): The number of results to return.
        Returns:
            List[Tuple[Document, float]]: A list of tuples of the form
                (Document, score).
        """
        from momento.requests.vector_index import ALL_METADATA
        from momento.responses.vector_index import Search
        if "top_k" in kwargs:
            k = kwargs["k"]
        response = self._client.search(
            self.index_name, embedding, top_k=k, metadata_fields=ALL_METADATA
        )
        if not isinstance(response, Search.Success):
            return []
        results = []
        for hit in response.hits:
            text = cast(str, hit.metadata.pop(self.text_field))
            doc = Document(page_content=text, metadata=hit.metadata)
            pair = (doc, hit.score)
            results.append(pair)
        return results
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Search for similar documents to the query vector.
        Args:
            embedding (List[float]): The query vector to search for.
            k (int, optional): The number of results to return. Defaults to 4.
        Returns:
            List[Document]: A list of documents that are similar to the query.
        """
        results = self.similarity_search_with_score_by_vector(


"""
        results = self.similarity_search_with_score_by_vector(
            embedding=embedding, k=k, **kwargs
        )
        return [doc for doc, _ in results]
[docs]    @classmethod
    def from_texts(
        cls: Type[VST],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> VST:
        """Return the Vector Store initialized from texts and embeddings.
        Args:
            cls (Type[VST]): The Vector Store class to use to initialize
                the Vector Store.
            texts (List[str]): The texts to initialize the Vector Store with.
            embedding (Embeddings): The embedding function to use.
            metadatas (Optional[List[dict]], optional): The metadata associated with
                the texts. Defaults to None.
            kwargs (Any): Vector Store specific parameters. The following are forwarded
                to the Vector Store constructor and required:
            - index_name (str, optional): The name of the index to store the documents
                in. Defaults to "default".
            - text_field (str, optional): The name of the metadata field to store the
                original text in. Defaults to "text".
            - distance_strategy (DistanceStrategy, optional): The distance strategy to
                use. Defaults to DistanceStrategy.COSINE. If you select
                DistanceStrategy.EUCLIDEAN_DISTANCE, Momento uses the squared
                Euclidean distance.
            - ensure_index_exists (bool, optional): Whether to ensure that the index
                exists before adding documents to it. Defaults to True.
            Additionally you can either pass in a client or an API key
            - client (PreviewVectorIndexClient): The Momento Vector Index client to use.


- client (PreviewVectorIndexClient): The Momento Vector Index client to use.
            - api_key (Optional[str]): The configuration to use to initialize
                the Vector Index with. Defaults to None. If None, the configuration
                is initialized from the environment variable `MOMENTO_API_KEY`.
        Returns:
            VST: Momento Vector Index vector store initialized from texts and
                embeddings.
        """
        from momento import (
            CredentialProvider,
            PreviewVectorIndexClient,
            VectorIndexConfigurations,
        )
        if "client" in kwargs:
            client = kwargs.pop("client")
        else:
            supplied_api_key = kwargs.pop("api_key", None)
            api_key = supplied_api_key or get_from_env("api_key", "MOMENTO_API_KEY")
            client = PreviewVectorIndexClient(
                configuration=VectorIndexConfigurations.Default.latest(),
                credential_provider=CredentialProvider.from_string(api_key),
            )
        vector_db = cls(embedding=embedding, client=client, **kwargs)  # type: ignore
        vector_db.add_texts(texts=texts, metadatas=metadatas, **kwargs)
        return vector_db


Source code for langchain.vectorstores.elasticsearch
import logging
import uuid
from abc import ABC, abstractmethod
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Literal,
    Optional,
    Tuple,
    Union,
)
import numpy as np
from langchain.docstore.document import Document
from langchain.schema.embeddings import Embeddings
from langchain.schema.vectorstore import VectorStore
from langchain.vectorstores.utils import DistanceStrategy, maximal_marginal_relevance
if TYPE_CHECKING:
    from elasticsearch import Elasticsearch
logger = logging.getLogger(__name__)
[docs]class BaseRetrievalStrategy(ABC):
    """Base class for `Elasticsearch` retrieval strategies."""
[docs]    @abstractmethod
    def query(
        self,
        query_vector: Union[List[float], None],
        query: Union[str, None],
        *,
        k: int,
        fetch_k: int,
        vector_query_field: str,
        text_field: str,
        filter: List[dict],
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        """
        Executes when a search is performed on the store.
        Args:
            query_vector: The query vector,
                          or None if not using vector-based query.
            query: The text query, or None if not using text-based query.
            k: The total number of results to retrieve.
            fetch_k: The number of results to fetch initially.
            vector_query_field: The field containing the vector
                                representations in the index.
            text_field: The field containing the text data in the index.
            filter: List of filter clauses to apply to the query.


filter: List of filter clauses to apply to the query.
            similarity: The similarity strategy to use, or None if not using one.
        Returns:
            Dict: The Elasticsearch query body.
        """
[docs]    @abstractmethod
    def index(
        self,
        dims_length: Union[int, None],
        vector_query_field: str,
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        """
        Executes when the index is created.
        Args:
            dims_length: Numeric length of the embedding vectors,
                        or None if not using vector-based query.
            vector_query_field: The field containing the vector
                                representations in the index.
            similarity: The similarity strategy to use,
                        or None if not using one.
        Returns:
            Dict: The Elasticsearch settings and mappings for the strategy.
        """
[docs]    def before_index_setup(
        self, client: "Elasticsearch", text_field: str, vector_query_field: str
    ) -> None:
        """
        Executes before the index is created. Used for setting up
        any required Elasticsearch resources like a pipeline.
        Args:
            client: The Elasticsearch client.
            text_field: The field containing the text data in the index.
            vector_query_field: The field containing the vector
                                representations in the index.
        """
[docs]    def require_inference(self) -> bool:
        """
        Returns whether or not the strategy requires inference
        to be performed on the text before it is added to the index.
        Returns:
            bool: Whether or not the strategy requires inference
            to be performed on the text before it is added to the index.
        """
        return True


"""
        return True
[docs]class ApproxRetrievalStrategy(BaseRetrievalStrategy):
    """Approximate retrieval strategy using the `HNSW` algorithm."""
[docs]    def __init__(
        self,
        query_model_id: Optional[str] = None,
        hybrid: Optional[bool] = False,
        rrf: Optional[Union[dict, bool]] = True,
    ):
        self.query_model_id = query_model_id
        self.hybrid = hybrid
        # RRF has two optional parameters
        # 'rank_constant', 'window_size'
        # https://www.elastic.co/guide/en/elasticsearch/reference/current/rrf.html
        self.rrf = rrf
[docs]    def query(
        self,
        query_vector: Union[List[float], None],
        query: Union[str, None],
        k: int,
        fetch_k: int,
        vector_query_field: str,
        text_field: str,
        filter: List[dict],
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        knn = {
            "filter": filter,
            "field": vector_query_field,
            "k": k,
            "num_candidates": fetch_k,
        }
        # Embedding provided via the embedding function
        if query_vector and not self.query_model_id:
            knn["query_vector"] = query_vector
        # Case 2: Used when model has been deployed to
        # Elasticsearch and can infer the query vector from the query text
        elif query and self.query_model_id:
            knn["query_vector_builder"] = {
                "text_embedding": {
                    "model_id": self.query_model_id,  # use 'model_id' argument


"model_id": self.query_model_id,  # use 'model_id' argument
                    "model_text": query,  # use 'query' argument
                }
            }
        else:
            raise ValueError(
                "You must provide an embedding function or a"
                " query_model_id to perform a similarity search."
            )
        # If hybrid, add a query to the knn query
        # RRF is used to even the score from the knn query and text query
        # RRF has two optional parameters: {'rank_constant':int, 'window_size':int}
        # https://www.elastic.co/guide/en/elasticsearch/reference/current/rrf.html
        if self.hybrid:
            query_body = {
                "knn": knn,
                "query": {
                    "bool": {
                        "must": [
                            {
                                "match": {
                                    text_field: {
                                        "query": query,
                                    }
                                }
                            }
                        ],
                        "filter": filter,
                    }
                },
            }
            if isinstance(self.rrf, dict):
                query_body["rank"] = {"rrf": self.rrf}
            elif isinstance(self.rrf, bool) and self.rrf is True:
                query_body["rank"] = {"rrf": {}}
            return query_body
        else:
            return {"knn": knn}
[docs]    def index(
        self,
        dims_length: Union[int, None],
        vector_query_field: str,
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        """Create the mapping for the Elasticsearch index."""
        if similarity is DistanceStrategy.COSINE:
            similarityAlgo = "cosine"


similarityAlgo = "cosine"
        elif similarity is DistanceStrategy.EUCLIDEAN_DISTANCE:
            similarityAlgo = "l2_norm"
        elif similarity is DistanceStrategy.DOT_PRODUCT:
            similarityAlgo = "dot_product"
        else:
            raise ValueError(f"Similarity {similarity} not supported.")
        return {
            "mappings": {
                "properties": {
                    vector_query_field: {
                        "type": "dense_vector",
                        "dims": dims_length,
                        "index": True,
                        "similarity": similarityAlgo,
                    },
                }
            }
        }
[docs]class ExactRetrievalStrategy(BaseRetrievalStrategy):
    """Exact retrieval strategy using the `script_score` query."""
[docs]    def query(
        self,
        query_vector: Union[List[float], None],
        query: Union[str, None],
        k: int,
        fetch_k: int,
        vector_query_field: str,
        text_field: str,
        filter: Union[List[dict], None],
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        if similarity is DistanceStrategy.COSINE:
            similarityAlgo = (
                f"cosineSimilarity(params.query_vector, '{vector_query_field}') + 1.0"
            )
        elif similarity is DistanceStrategy.EUCLIDEAN_DISTANCE:
            similarityAlgo = (
                f"1 / (1 + l2norm(params.query_vector, '{vector_query_field}'))"
            )
        elif similarity is DistanceStrategy.DOT_PRODUCT:
            similarityAlgo = f"""
            double value = dotProduct(params.query_vector, '{vector_query_field}');


double value = dotProduct(params.query_vector, '{vector_query_field}');
            return sigmoid(1, Math.E, -value);
            """
        else:
            raise ValueError(f"Similarity {similarity} not supported.")
        queryBool: Dict = {"match_all": {}}
        if filter:
            queryBool = {"bool": {"filter": filter}}
        return {
            "query": {
                "script_score": {
                    "query": queryBool,
                    "script": {
                        "source": similarityAlgo,
                        "params": {"query_vector": query_vector},
                    },
                },
            }
        }
[docs]    def index(
        self,
        dims_length: Union[int, None],
        vector_query_field: str,
        similarity: Union[DistanceStrategy, None],
    ) -> Dict:
        """Create the mapping for the Elasticsearch index."""
        return {
            "mappings": {
                "properties": {
                    vector_query_field: {
                        "type": "dense_vector",
                        "dims": dims_length,
                        "index": False,
                    },
                }
            }
        }
[docs]class SparseRetrievalStrategy(BaseRetrievalStrategy):
    """Sparse retrieval strategy using the `text_expansion` processor."""
[docs]    def __init__(self, model_id: Optional[str] = None):
        self.model_id = model_id or ".elser_model_1"
[docs]    def query(
        self,
        query_vector: Union[List[float], None],
        query: Union[str, None],
        k: int,
        fetch_k: int,
        vector_query_field: str,
        text_field: str,