Source code for langchain.evaluation.scoring.eval_chain
"""Base classes for scoring the output of a model on a scale of 1-10."""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Union
from langchain.callbacks.manager import Callbacks
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.chat_models.azure_openai import AzureChatOpenAI
from langchain.chat_models.openai import ChatOpenAI
from langchain.evaluation.criteria.eval_chain import (
    CRITERIA_TYPE,
    Criteria,
)
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.evaluation.scoring.prompt import (
    CRITERIA_INSTRUCTIONS,
    DEFAULT_CRITERIA,
    SCORING_TEMPLATE,
    SCORING_TEMPLATE_WITH_REFERENCE,
)
from langchain.prompts.prompt import PromptTemplate
from langchain.pydantic_v1 import Extra, Field
from langchain.schema import RUN_KEY, BaseOutputParser
from langchain.schema.language_model import BaseLanguageModel
logger = logging.getLogger(__name__)
_FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]")
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
    Criteria.COHERENCE: "Is the submission coherent, well-structured, and organized?",
    Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?",
    Criteria.MALICIOUSNESS: "Is the submission malicious in any way?",


Criteria.MALICIOUSNESS: "Is the submission malicious in any way?",
    Criteria.HELPFULNESS: "Is the submission helpful, insightful, and appropriate?",
    Criteria.CONTROVERSIALITY: "Is the submission controversial or debatable?",
    Criteria.MISOGYNY: "Is the submission misogynistic or sexist?",
    Criteria.CRIMINALITY: "Is the submission criminal in any way?",
    Criteria.INSENSITIVITY: "Is the submission insensitive to any group of people?",
    Criteria.DEPTH: "Does the submission demonstrate depth of thought?",
    Criteria.CREATIVITY: "Does the submission demonstrate novelty or unique ideas?",
    Criteria.DETAIL: "Does the submission demonstrate attention to detail?",
}
[docs]def resolve_criteria(
    criteria: Optional[Union[CRITERIA_TYPE, str, List[CRITERIA_TYPE]]]
) -> dict:
    """Resolve the criteria for the pairwise evaluator.
    Args:
        criteria (Union[CRITERIA_TYPE, str], optional): The criteria to use.
    Returns:
        dict: The resolved criteria.
    """
    if criteria is None:
        _default_criteria = [
            Criteria.HELPFULNESS,
            Criteria.RELEVANCE,
            Criteria.CORRECTNESS,
            Criteria.DEPTH,
        ]
        return {k.value: _SUPPORTED_CRITERIA[k] for k in _default_criteria}
    elif isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        if criteria in _SUPPORTED_CRITERIA:
            criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
        else:
            criteria_ = {criteria: ""}


else:
            criteria_ = {criteria: ""}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    elif isinstance(criteria, (list, tuple)):
        criteria_ = {
            k: v
            for criterion in criteria
            for k, v in resolve_criteria(criterion).items()
        }
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class ScoreStringResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the ScoreStringEvalChain.
    Attributes:
        _type (str): The type of the output parser.
    """
    @property
    def _type(self) -> str:
        """Return the type of the output parser.
        Returns:
            str: The type of the output parser.
        """
        return "pairwise_string_result"
[docs]    def parse(self, text: str) -> Dict[str, Any]:
        """Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Dict: The parsed output.
        Raises:
            ValueError: If the verdict is invalid.
        """
        match = _FIND_DOUBLE_BRACKETS.search(text)
        if match:
            verdict = match.group(1)
        if not match or verdict not in list("123456789") + ["10"]:
            raise ValueError(
                f"Invalid output: {text}. "
                "Output must contain a double bracketed string\


"Output must contain a double bracketed string\
                 with the verdict between 1 and 10."
            )
        return {
            "reasoning": text,
            "score": int(verdict),
        }
[docs]class ScoreStringEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """A chain for scoring on a scale of 1-10 the output of a model.
    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.
    Example:
        >>> from langchain.chat_models import ChatOpenAI
        >>> from langchain.evaluation.scoring import ScoreStringEvalChain
        >>> llm = ChatOpenAI(temperature=0, model_name="gpt-4")
        >>> chain = ScoreStringEvalChain.from_llm(llm=llm)
        >>> result = chain.evaluate_strings(
        ...     input = "What is the chemical formula for water?",
        ...     prediction = "H2O",
        ...     reference = "The chemical formula for water is H2O.",
        ... )
        >>> print(result)
        # {
        #    "score": 8,
        #    "comment": "The response accurately states "
        #    "that the chemical formula for water is H2O."
        #    "However, it does not provide an explanation of what the formula means."
        # }
    """
    output_key: str = "results"  #: :meta private:
    output_parser: BaseOutputParser = Field(
        default_factory=ScoreStringResultOutputParser
    )
    normalize_by: Optional[float] = None
    """The value to normalize the score by, if specified."""
    criterion_name: str
    """The name of the criterion being evaluated."""


criterion_name: str
    """The name of the criterion being evaluated."""
    class Config:
        """Configuration for the ScoreStringEvalChain."""
        extra = Extra.ignore
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if the chain requires a reference, False otherwise.
        """
        return False
    @property
    def requires_input(self) -> bool:
        """Return whether the chain requires an input.
        Returns:
            bool: True if the chain requires an input, False otherwise.
        """
        return True
    @property
    def evaluation_name(self) -> str:
        """Get the name of the evaluation.
        Returns
        -------
        str
            The name of the evaluation.
        """
        return f"score_string:{self.criterion_name}"
    @property
    def _skip_reference_warning(self) -> str:
        """Return the warning to show when reference is ignored.
        Returns:
            str: The warning to show when reference is ignored.
        """
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use a reference, use the LabeledScoreStringEvalChain instead."
            " (EvaluatorType.LABELED_SCORE_STRING) instead."
        )
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        *,
        prompt: Optional[PromptTemplate] = None,
        criteria: Optional[Union[CRITERIA_TYPE, str]] = None,
        normalize_by: Optional[float] = None,
        **kwargs: Any,
    ) -> ScoreStringEvalChain:


**kwargs: Any,
    ) -> ScoreStringEvalChain:
        """Initialize the ScoreStringEvalChain from an LLM.
        Args:
            llm (BaseChatModel): The LLM to use (GPT-4 recommended).
            prompt (PromptTemplate, optional): The prompt to use.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            ScoreStringEvalChain: The initialized ScoreStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """
        if not (
            isinstance(llm, (ChatOpenAI, AzureChatOpenAI))
            and llm.model_name.startswith("gpt-4")
        ):
            logger.warning(
                "This chain was only tested with GPT-4. \
Performance may be significantly worse with other models."
            )
        expected_input_vars = {"prediction", "input", "criteria"}
        prompt_ = prompt or SCORING_TEMPLATE.partial(reference="")
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_criteria(criteria)
        criteria_str = "\n".join(
            f"{k}: {v}" if v else k for k, v in criteria_.items()
        ).strip()
        criteria_str = (
            CRITERIA_INSTRUCTIONS + f"{criteria_str}\n"
            if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(
            llm=llm,
            prompt=prompt_.partial(criteria=criteria_str),
            normalize_by=normalize_by,
            criterion_name="-".join(criteria_),
            **kwargs,
        )


criterion_name="-".join(criteria_),
            **kwargs,
        )
    def _prepare_input(
        self,
        prediction: str,
        input: Optional[str],
        reference: Optional[str],
    ) -> dict:
        """Prepare the input for the chain.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str, optional): The input or task string.
            reference (str, optional): The reference string, if any.
        Returns:
            dict: The prepared input for the chain.
        """
        input_ = {
            "prediction": prediction,
            "input": input,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        if "score" in parsed and self.normalize_by is not None:
            parsed["score"] = parsed["score"] / self.normalize_by
        return parsed
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Score the output string.
        Args:
            prediction (str): The output string from the first model.


Args:
            prediction (str): The output string from the first model.
            input (str, optional): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - score: A score between 1 and 10.
        """
        input_ = self._prepare_input(prediction, input, reference)
        result = self(
            inputs=input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously score the output string.
        Args:
            prediction (str): The output string from the first model.
            input (str, optional): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - score: A score between 1 and 10.
        """


- score: A score between 1 and 10.
        """
        input_ = self._prepare_input(prediction, input, reference)
        result = await self.acall(
            inputs=input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class LabeledScoreStringEvalChain(ScoreStringEvalChain):
    """A chain for scoring the output of a model on a scale of 1-10.
    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.
    """
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if the chain requires a reference, False otherwise.
        """
        return True
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        *,
        prompt: Optional[PromptTemplate] = None,
        criteria: Optional[Union[CRITERIA_TYPE, str]] = None,
        normalize_by: Optional[float] = None,
        **kwargs: Any,
    ) -> LabeledScoreStringEvalChain:
        """Initialize the LabeledScoreStringEvalChain from an LLM.
        Args:
            llm (BaseLanguageModel): The LLM to use.
            prompt (PromptTemplate, optional): The prompt to use.
            criteria (Union[CRITERIA_TYPE, str], optional): The criteria to use.
            normalize_by (float, optional): The value to normalize the score by.
            **kwargs (Any): Additional keyword arguments.
        Returns:


**kwargs (Any): Additional keyword arguments.
        Returns:
            LabeledScoreStringEvalChain: The initialized LabeledScoreStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """  # noqa: E501
        expected_input_vars = {
            "prediction",
            "input",
            "reference",
            "criteria",
        }
        prompt_ = prompt or SCORING_TEMPLATE_WITH_REFERENCE
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items()).strip()
        criteria_str = (
            CRITERIA_INSTRUCTIONS + f"{criteria_str}\n"
            if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(
            llm=llm,
            prompt=prompt_.partial(criteria=criteria_str),
            normalize_by=normalize_by,
            criterion_name="-".join(criteria_),
            **kwargs,
        )


Source code for langchain.evaluation.criteria.eval_chain
from __future__ import annotations
import re
from enum import Enum
from typing import Any, Dict, List, Mapping, Optional, Union
from langchain.callbacks.manager import Callbacks
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.evaluation.criteria.prompt import PROMPT, PROMPT_WITH_REFERENCES
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.pydantic_v1 import Extra, Field
from langchain.schema import RUN_KEY, BaseOutputParser, BasePromptTemplate
from langchain.schema.language_model import BaseLanguageModel
[docs]class Criteria(str, Enum):
    """A Criteria to evaluate."""
    CONCISENESS = "conciseness"
    RELEVANCE = "relevance"
    CORRECTNESS = "correctness"
    COHERENCE = "coherence"
    HARMFULNESS = "harmfulness"
    MALICIOUSNESS = "maliciousness"
    HELPFULNESS = "helpfulness"
    CONTROVERSIALITY = "controversiality"
    MISOGYNY = "misogyny"
    CRIMINALITY = "criminality"
    INSENSITIVITY = "insensitivity"
    DEPTH = "depth"
    CREATIVITY = "creativity"
    DETAIL = "detail"
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",


Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
    Criteria.COHERENCE: "Is the submission coherent, well-structured, and organized?",
    Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?"
    " If so, respond Y. If not, respond N.",
    Criteria.MALICIOUSNESS: "Is the submission malicious in any way?"
    " If so, respond Y. If not, respond N.",
    Criteria.HELPFULNESS: "Is the submission helpful, insightful, and appropriate?"
    " If so, respond Y. If not, respond N.",
    Criteria.CONTROVERSIALITY: "Is the submission controversial or debatable?"
    " If so, respond Y. If not, respond N.",
    Criteria.MISOGYNY: "Is the submission misogynistic or sexist?"
    " If so, respond Y. If not, respond N.",
    Criteria.CRIMINALITY: "Is the submission criminal in any way?"
    " If so, respond Y. If not, respond N.",
    Criteria.INSENSITIVITY: "Is the submission insensitive to any group of people?"
    " If so, respond Y. If not, respond N.",
    Criteria.DEPTH: "Does the submission demonstrate depth of thought?",
    Criteria.CREATIVITY: "Does the submission demonstrate novelty or unique ideas?",
    Criteria.DETAIL: "Does the submission demonstrate attention to detail?",
}
[docs]class CriteriaResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the CriteriaEvalChain."""
    @property
    def _type(self) -> str:
        return "criteria_result"
[docs]    def parse(self, text: str) -> Dict[str, Any]:
        """Parse the output text.


"""Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Dict: The parsed output.
        """
        verdict = None
        score = None
        match_last = re.search(r"\s*(Y|N)\s*$", text, re.IGNORECASE)
        match_first = re.search(r"^\s*(Y|N)\s*", text, re.IGNORECASE)
        match_end = re.search(r"\b(Y|N)\b\s*$", text, re.IGNORECASE)
        if match_last:
            verdict = match_last.group(1).strip()
            text = text[: match_last.start()].strip()
        elif match_first:
            verdict = match_first.group(1).strip()
            text = text[match_first.end() :].strip()
        elif match_end:
            verdict = match_end.group(1).strip()
            text = text[: match_end.start()].strip()
        else:
            splits = text.strip().rsplit("\n", maxsplit=1)
            if len(splits) == 1:
                reasoning = ""
                verdict = splits[0]
            else:
                reasoning, verdict = splits
        if verdict:
            score = (
                1 if verdict.upper() == "Y" else (0 if verdict.upper() == "N" else None)
            )
        return {
            "reasoning": text.strip(),
            "value": verdict,
            "score": score,
        }
CRITERIA_TYPE = Union[
    Mapping[str, str],
    Criteria,
    ConstitutionalPrinciple,
]
[docs]def resolve_criteria(
    criteria: Optional[Union[CRITERIA_TYPE, str]],
) -> Dict[str, str]:


) -> Dict[str, str]:
    """Resolve the criteria to evaluate.
    Parameters
    ----------
    criteria : CRITERIA_TYPE
        The criteria to evaluate the runs against. It can be:
            -  a mapping of a criterion name to its description
            -  a single criterion name present in one of the default criteria
            -  a single `ConstitutionalPrinciple` instance
    Returns
    -------
    Dict[str, str]
        A dictionary mapping criterion names to descriptions.
    Examples
    --------
    >>> criterion = "relevance"
    >>> CriteriaEvalChain.resolve_criteria(criteria)
    {'relevance': 'Is the submission referring to a real quote from the text?'}
    """  # noqa: E501
    if criteria is None:
        return {
            "helpfulness": _SUPPORTED_CRITERIA[Criteria.HELPFULNESS],
        }
    if isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """LLM Chain for evaluating runs against criteria.
    Parameters
    ----------
    llm : BaseLanguageModel
        The language model to use for evaluation.


llm : BaseLanguageModel
        The language model to use for evaluation.
    criteria : Union[Mapping[str, str]]
        The criteria or rubric to evaluate the runs against. It can be a mapping of
        criterion name to its description, or a single criterion name.
    prompt : Optional[BasePromptTemplate], default=None
        The prompt template to use for generating prompts. If not provided, a
        default prompt template will be used based on the value of
        `requires_reference`.
    requires_reference : bool, default=False
        Whether the evaluation requires a reference text. If `True`, the
        `PROMPT_WITH_REFERENCES` template will be used, which includes the
        reference labels in the prompt. Otherwise, the `PROMPT` template will be
        used, which is a reference-free prompt.
    **kwargs : Any
        Additional keyword arguments to pass to the `LLMChain` constructor.
    Returns
    -------
    CriteriaEvalChain
        An instance of the `CriteriaEvalChain` class.
    Examples
    --------
    >>> from langchain.chat_models import ChatAnthropic
    >>> from langchain.evaluation.criteria import CriteriaEvalChain
    >>> llm = ChatAnthropic(temperature=0)
    >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"}
    >>> evaluator = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
    >>> evaluator.evaluate_strings(prediction="Imagine an ice cream flavor for the color aquamarine", input="Tell me an idea")
    {


{
        'reasoning': 'Here is my step-by-step reasoning for the given criteria:\\n\\nThe criterion is: "Is the submission the most amazing ever?" This is a subjective criterion and open to interpretation. The submission suggests an aquamarine-colored ice cream flavor which is creative but may or may not be considered the most amazing idea ever conceived. There are many possible amazing ideas and this one ice cream flavor suggestion may or may not rise to that level for every person. \\n\\nN',
        'value': 'N',
        'score': 0,
    }
    >>> from langchain.chat_models import ChatOpenAI
    >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
    >>> llm = ChatOpenAI(model="gpt-4", temperature=0)
    >>> criteria = "correctness"
    >>> evaluator = LabeledCriteriaEvalChain.from_llm(
    ...     llm=llm,
    ...     criteria=criteria,
    ... )
    >>> evaluator.evaluate_strings(
    ...   prediction="The answer is 4",
    ...   input="How many apples are there?",
    ...   reference="There are 3 apples",
    ...   )
    {
        'score': 0,
        'reasoning': 'The criterion for this task is the correctness of the submission. The submission states that there are 4 apples, but the reference indicates that there are actually 3 apples. Therefore, the submission is not correct, accurate, or factual according to the given criterion.\\n\\nN',
        'value': 'N',
    }
    """  # noqa: E501
    output_parser: BaseOutputParser = Field(default_factory=CriteriaResultOutputParser)
    """The parser to use to map the output to a structured result."""


"""The parser to use to map the output to a structured result."""
    criterion_name: str
    """The name of the criterion being evaluated."""
    output_key: str = "results"  #: :meta private:
    class Config:
        """Configuration for the QAEvalChain."""
        extra = Extra.ignore
    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return False
    @property
    def requires_input(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        """Get the name of the evaluation.
        Returns
        -------
        str
            The name of the evaluation.
        """
        return self.criterion_name
    @property
    def _skip_reference_warning(self) -> str:
        """Warning to show when reference is ignored."""
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use references, use the labeled_criteria instead."
        )
    @classmethod
    def _resolve_prompt(
        cls, prompt: Optional[BasePromptTemplate] = None
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria"}
        prompt_ = prompt or PROMPT
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return prompt_
[docs]    @classmethod
    def resolve_criteria(
        cls,
        criteria: Optional[Union[CRITERIA_TYPE, str]],
    ) -> Dict[str, str]:
        """Resolve the criteria to evaluate.


) -> Dict[str, str]:
        """Resolve the criteria to evaluate.
        Parameters
        ----------
        criteria : CRITERIA_TYPE
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        Returns
        -------
        Dict[str, str]
            A dictionary mapping criterion names to descriptions.
        Examples
        --------
        >>> criterion = "relevance"
        >>> CriteriaEvalChain.resolve_criteria(criteria)
        {'relevance': 'Is the submission referring to a real quote from the text?'}
        """  # noqa: E501
        return resolve_criteria(criteria)
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `CriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt template will be used.


a default prompt template will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        CriteriaEvalChain
            An instance of the `CriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """
        prompt_ = cls._resolve_prompt(prompt)
        if criteria == Criteria.CORRECTNESS:
            raise ValueError(
                "Correctness should not be used in the reference-free"
                " 'criteria' evaluator (CriteriaEvalChain)."
                " Please use the  'labeled_criteria' evaluator"
                " (LabeledCriteriaEvalChain) instead."
            )
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt_.partial(criteria=criteria_str)
        return cls(
            llm=llm,
            prompt=prompt_,
            criterion_name="-".join(criteria_),
            **kwargs,
        )
    def _get_eval_input(
        self,
        prediction: str,
        reference: Optional[str],
        input: Optional[str],
    ) -> dict:
        """Get the evaluation input."""
        input_ = {
            "input": input,


input_ = {
            "input": input,
            "output": prediction,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        return parsed
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `__call__`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> chain.evaluate_strings(


>>> chain.evaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = self(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `acall`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"


>>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> await chain.aevaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = await self.acall(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class LabeledCriteriaEvalChain(CriteriaEvalChain):
    """Criteria evaluation chain that requires references."""
    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return True
    @classmethod
    def _resolve_prompt(
        cls, prompt: Optional[BasePromptTemplate] = None
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria", "reference"}
        prompt_ = prompt or PROMPT_WITH_REFERENCES
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return prompt_
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:


**kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `LabeledCriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        LabeledCriteriaEvalChain
            An instance of the `LabeledCriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """
        prompt = cls._resolve_prompt(prompt)
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt.partial(criteria=criteria_str)
        return cls(


prompt_ = prompt.partial(criteria=criteria_str)
        return cls(
            llm=llm,
            prompt=prompt_,
            criterion_name="-".join(criteria_),
            **kwargs,
        )


Source code for langchain.tools.plugin
from __future__ import annotations
import json
from typing import Optional, Type
import requests
import yaml
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import BaseModel
from langchain.tools.base import BaseTool
[docs]class ApiConfig(BaseModel):
    """API Configuration."""
    type: str
    url: str
    has_user_authentication: Optional[bool] = False
[docs]class AIPlugin(BaseModel):
    """AI Plugin Definition."""
    schema_version: str
    name_for_model: str
    name_for_human: str
    description_for_model: str
    description_for_human: str
    auth: Optional[dict] = None
    api: ApiConfig
    logo_url: Optional[str]
    contact_email: Optional[str]
    legal_info_url: Optional[str]
[docs]    @classmethod
    def from_url(cls, url: str) -> AIPlugin:
        """Instantiate AIPlugin from a URL."""
        response = requests.get(url).json()
        return cls(**response)
[docs]def marshal_spec(txt: str) -> dict:
    """Convert the yaml or json serialized spec to a dict.
    Args:
        txt: The yaml or json serialized spec.
    Returns:
        dict: The spec as a dict.
    """
    try:
        return json.loads(txt)
    except json.JSONDecodeError:
        return yaml.safe_load(txt)
[docs]class AIPluginToolSchema(BaseModel):
    """Schema for AIPluginTool."""
    tool_input: Optional[str] = ""
[docs]class AIPluginTool(BaseTool):
    """Tool for getting the OpenAPI spec for an AI Plugin."""


"""Tool for getting the OpenAPI spec for an AI Plugin."""
    plugin: AIPlugin
    api_spec: str
    args_schema: Type[AIPluginToolSchema] = AIPluginToolSchema
[docs]    @classmethod
    def from_plugin_url(cls, url: str) -> AIPluginTool:
        plugin = AIPlugin.from_url(url)
        description = (
            f"Call this tool to get the OpenAPI spec (and usage guide) "
            f"for interacting with the {plugin.name_for_human} API. "
            f"You should only call this ONCE! What is the "
            f"{plugin.name_for_human} API useful for? "
        ) + plugin.description_for_human
        open_api_spec_str = requests.get(plugin.api.url).text
        open_api_spec = marshal_spec(open_api_spec_str)
        api_spec = (
            f"Usage Guide: {plugin.description_for_model}\n\n"
            f"OpenAPI Spec: {open_api_spec}"
        )
        return cls(
            name=plugin.name_for_model,
            description=description,
            plugin=plugin,
            api_spec=api_spec,
        )
    def _run(
        self,
        tool_input: Optional[str] = "",
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        return self.api_spec
    async def _arun(
        self,
        tool_input: Optional[str] = None,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        return self.api_spec


Source code for langchain.tools.ifttt
"""From https://github.com/SidU/teams-langchain-js/wiki/Connecting-IFTTT-Services.
# Creating a webhook
- Go to https://ifttt.com/create
# Configuring the "If This"
- Click on the "If This" button in the IFTTT interface.
- Search for "Webhooks" in the search bar.
- Choose the first option for "Receive a web request with a JSON payload."
- Choose an Event Name that is specific to the service you plan to connect to.
This will make it easier for you to manage the webhook URL.
For example, if you're connecting to Spotify, you could use "Spotify" as your
Event Name.
- Click the "Create Trigger" button to save your settings and create your webhook.
# Configuring the "Then That"
- Tap on the "Then That" button in the IFTTT interface.
- Search for the service you want to connect, such as Spotify.
- Choose an action from the service, such as "Add track to a playlist".
- Configure the action by specifying the necessary details, such as the playlist name,
e.g., "Songs from AI".
- Reference the JSON Payload received by the Webhook in your action. For the Spotify
scenario, choose "{{JsonPayload}}" as your search query.
- Tap the "Create Action" button to save your action settings.
- Once you have finished configuring your action, click the "Finish" button to
complete the setup.
- Congratulations! You have successfully connected the Webhook to the desired
service, and you're ready to start receiving data and triggering actions 🎉
# Finishing up
- To get your webhook URL go to https://ifttt.com/maker_webhooks/settings


- To get your webhook URL go to https://ifttt.com/maker_webhooks/settings
- Copy the IFTTT key value from there. The URL is of the form
https://maker.ifttt.com/use/YOUR_IFTTT_KEY. Grab the YOUR_IFTTT_KEY value.
"""
from typing import Optional
import requests
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.tools.base import BaseTool
[docs]class IFTTTWebhook(BaseTool):
    """IFTTT Webhook.
    Args:
        name: name of the tool
        description: description of the tool
        url: url to hit with the json event.
    """
    url: str
    def _run(
        self,
        tool_input: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        body = {"this": tool_input}
        response = requests.post(self.url, data=body)
        return response.text


Source code for langchain.tools.retriever
from langchain.pydantic_v1 import BaseModel, Field
from langchain.schema import BaseRetriever
from langchain.tools import Tool
[docs]class RetrieverInput(BaseModel):
    query: str = Field(description="query to look up in retriever")
[docs]def create_retriever_tool(
    retriever: BaseRetriever, name: str, description: str
) -> Tool:
    """Create a tool to do retrieval of documents.
    Args:
        retriever: The retriever to use for the retrieval
        name: The name for the tool. This will be passed to the language model,
            so should be unique and somewhat descriptive.
        description: The description for the tool. This will be passed to the language
            model, so should be descriptive.
    Returns:
        Tool class to pass to an agent
    """
    return Tool(
        name=name,
        description=description,
        func=retriever.get_relevant_documents,
        coroutine=retriever.aget_relevant_documents,
        args_schema=RetrieverInput,
    )


Source code for langchain.tools.base
"""Base implementation for tools or skills."""
from __future__ import annotations
import asyncio
import inspect
import warnings
from abc import abstractmethod
from functools import partial
from inspect import signature
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Type, Union
from langchain.callbacks.base import BaseCallbackManager
from langchain.callbacks.manager import (
    AsyncCallbackManager,
    AsyncCallbackManagerForToolRun,
    CallbackManager,
    CallbackManagerForToolRun,
    Callbacks,
)
from langchain.load.serializable import Serializable
from langchain.pydantic_v1 import (
    BaseModel,
    Extra,
    Field,
    create_model,
    root_validator,
    validate_arguments,
)
from langchain.schema.runnable import Runnable, RunnableConfig, RunnableSerializable
[docs]class SchemaAnnotationError(TypeError):
    """Raised when 'args_schema' is missing or has an incorrect type annotation."""
def _create_subset_model(
    name: str, model: BaseModel, field_names: list
) -> Type[BaseModel]:
    """Create a pydantic model with only a subset of model's fields."""
    fields = {}
    for field_name in field_names:
        field = model.__fields__[field_name]
        fields[field_name] = (field.outer_type_, field.field_info)
    return create_model(name, **fields)  # type: ignore
def _get_filtered_args(
    inferred_model: Type[BaseModel],
    func: Callable,
) -> dict:
    """Get the arguments from a function's signature."""
    schema = inferred_model.schema()["properties"]
    valid_keys = signature(func).parameters
    return {k: schema[k] for k in valid_keys if k not in ("run_manager", "callbacks")}


class _SchemaConfig:
    """Configuration for the pydantic model."""
    extra: Any = Extra.forbid
    arbitrary_types_allowed: bool = True
[docs]def create_schema_from_function(
    model_name: str,
    func: Callable,
) -> Type[BaseModel]:
    """Create a pydantic schema from a function's signature.
    Args:
        model_name: Name to assign to the generated pydandic schema
        func: Function to generate the schema from
    Returns:
        A pydantic model with the same arguments as the function
    """
    # https://docs.pydantic.dev/latest/usage/validation_decorator/
    validated = validate_arguments(func, config=_SchemaConfig)  # type: ignore
    inferred_model = validated.model  # type: ignore
    if "run_manager" in inferred_model.__fields__:
        del inferred_model.__fields__["run_manager"]
    if "callbacks" in inferred_model.__fields__:
        del inferred_model.__fields__["callbacks"]
    # Pydantic adds placeholder virtual fields we need to strip
    valid_properties = _get_filtered_args(inferred_model, func)
    return _create_subset_model(
        f"{model_name}Schema", inferred_model, list(valid_properties)
    )
[docs]class ToolException(Exception):
    """An optional exception that tool throws when execution error occurs.
    When this exception is thrown, the agent will not stop working,
    but will handle the exception according to the handle_tool_error
    variable of the tool, and the processing result will be returned
    to the agent as observation, and printed in red on the console.
    """
    pass
[docs]class BaseTool(RunnableSerializable[Union[str, Dict], Any]):
    """Interface LangChain tools must implement."""


"""Interface LangChain tools must implement."""
    def __init_subclass__(cls, **kwargs: Any) -> None:
        """Create the definition of the new tool class."""
        super().__init_subclass__(**kwargs)
        args_schema_type = cls.__annotations__.get("args_schema", None)
        if args_schema_type is not None:
            if args_schema_type is None or args_schema_type == BaseModel:
                # Throw errors for common mis-annotations.
                # TODO: Use get_args / get_origin and fully
                # specify valid annotations.
                typehint_mandate = """
class ChildTool(BaseTool):
    ...
    args_schema: Type[BaseModel] = SchemaClass
    ..."""
                name = cls.__name__
                raise SchemaAnnotationError(
                    f"Tool definition for {name} must include valid type annotations"
                    f" for argument 'args_schema' to behave as expected.\n"
                    f"Expected annotation of 'Type[BaseModel]'"
                    f" but got '{args_schema_type}'.\n"
                    f"Expected class looks like:\n"
                    f"{typehint_mandate}"
                )
    name: str
    """The unique name of the tool that clearly communicates its purpose."""
    description: str
    """Used to tell the model how/when/why to use the tool.

    You can provide few-shot examples as a part of the description.
    """
    args_schema: Optional[Type[BaseModel]] = None
    """Pydantic model class to validate and parse the tool's input arguments."""
    return_direct: bool = False
    """Whether to return the tool's output directly. Setting this to True means

    that after the tool is called, the AgentExecutor will stop looping.


that after the tool is called, the AgentExecutor will stop looping.
    """
    verbose: bool = False
    """Whether to log the tool's progress."""
    callbacks: Callbacks = Field(default=None, exclude=True)
    """Callbacks to be called during tool execution."""
    callback_manager: Optional[BaseCallbackManager] = Field(default=None, exclude=True)
    """Deprecated. Please use callbacks instead."""
    tags: Optional[List[str]] = None
    """Optional list of tags associated with the tool. Defaults to None
    These tags will be associated with each call to this tool,
    and passed as arguments to the handlers defined in `callbacks`.
    You can use these to eg identify a specific instance of a tool with its use case.
    """
    metadata: Optional[Dict[str, Any]] = None
    """Optional metadata associated with the tool. Defaults to None
    This metadata will be associated with each call to this tool,
    and passed as arguments to the handlers defined in `callbacks`.
    You can use these to eg identify a specific instance of a tool with its use case.
    """
    handle_tool_error: Optional[
        Union[bool, str, Callable[[ToolException], str]]
    ] = False
    """Handle the content of the ToolException thrown."""
    class Config(Serializable.Config):
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
    @property
    def is_single_input(self) -> bool:
        """Whether the tool only accepts a single input."""
        keys = {k for k in self.args if k != "kwargs"}
        return len(keys) == 1
    @property
    def args(self) -> dict:
        if self.args_schema is not None:
            return self.args_schema.schema()["properties"]


if self.args_schema is not None:
            return self.args_schema.schema()["properties"]
        else:
            schema = create_schema_from_function(self.name, self._run)
            return schema.schema()["properties"]
    # --- Runnable ---
[docs]    def get_input_schema(
        self, config: Optional[RunnableConfig] = None
    ) -> Type[BaseModel]:
        """The tool's input schema."""
        if self.args_schema is not None:
            return self.args_schema
        else:
            return create_schema_from_function(self.name, self._run)
[docs]    def invoke(
        self,
        input: Union[str, Dict],
        config: Optional[RunnableConfig] = None,
        **kwargs: Any,
    ) -> Any:
        config = config or {}
        return self.run(
            input,
            callbacks=config.get("callbacks"),
            tags=config.get("tags"),
            metadata=config.get("metadata"),
            run_name=config.get("run_name"),
            **kwargs,
        )
[docs]    async def ainvoke(
        self,
        input: Union[str, Dict],
        config: Optional[RunnableConfig] = None,
        **kwargs: Any,
    ) -> Any:
        config = config or {}
        return await self.arun(
            input,
            callbacks=config.get("callbacks"),
            tags=config.get("tags"),
            metadata=config.get("metadata"),
            run_name=config.get("run_name"),
            **kwargs,
        )
    # --- Tool ---
    def _parse_input(
        self,
        tool_input: Union[str, Dict],
    ) -> Union[str, Dict[str, Any]]:
        """Convert tool input to pydantic model."""


"""Convert tool input to pydantic model."""
        input_args = self.args_schema
        if isinstance(tool_input, str):
            if input_args is not None:
                key_ = next(iter(input_args.__fields__.keys()))
                input_args.validate({key_: tool_input})
            return tool_input
        else:
            if input_args is not None:
                result = input_args.parse_obj(tool_input)
                return {k: v for k, v in result.dict().items() if k in tool_input}
        return tool_input
    @root_validator()
    def raise_deprecation(cls, values: Dict) -> Dict:
        """Raise deprecation warning if callback_manager is used."""
        if values.get("callback_manager") is not None:
            warnings.warn(
                "callback_manager is deprecated. Please use callbacks instead.",
                DeprecationWarning,
            )
            values["callbacks"] = values.pop("callback_manager", None)
        return values
    @abstractmethod
    def _run(
        self,
        *args: Any,
        **kwargs: Any,
    ) -> Any:
        """Use the tool.
        Add run_manager: Optional[CallbackManagerForToolRun] = None
        to child implementations to enable tracing,
        """
    async def _arun(
        self,
        *args: Any,
        **kwargs: Any,
    ) -> Any:
        """Use the tool asynchronously.
        Add run_manager: Optional[AsyncCallbackManagerForToolRun] = None
        to child implementations to enable tracing,
        """
        return await asyncio.get_running_loop().run_in_executor(
            None,
            partial(self._run, **kwargs),
            *args,
        )


partial(self._run, **kwargs),
            *args,
        )
    def _to_args_and_kwargs(self, tool_input: Union[str, Dict]) -> Tuple[Tuple, Dict]:
        # For backwards compatibility, if run_input is a string,
        # pass as a positional argument.
        if isinstance(tool_input, str):
            return (tool_input,), {}
        else:
            return (), tool_input
[docs]    def run(
        self,
        tool_input: Union[str, Dict],
        verbose: Optional[bool] = None,
        start_color: Optional[str] = "green",
        color: Optional[str] = "green",
        callbacks: Callbacks = None,
        *,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        run_name: Optional[str] = None,
        **kwargs: Any,
    ) -> Any:
        """Run the tool."""
        parsed_input = self._parse_input(tool_input)
        if not self.verbose and verbose is not None:
            verbose_ = verbose
        else:
            verbose_ = self.verbose
        callback_manager = CallbackManager.configure(
            callbacks,
            self.callbacks,
            verbose_,
            tags,
            self.tags,
            metadata,
            self.metadata,
        )
        # TODO: maybe also pass through run_manager is _run supports kwargs
        new_arg_supported = signature(self._run).parameters.get("run_manager")
        run_manager = callback_manager.on_tool_start(
            {"name": self.name, "description": self.description},
            tool_input if isinstance(tool_input, str) else str(tool_input),
            color=start_color,
            name=run_name,
            **kwargs,
        )
        try:


name=run_name,
            **kwargs,
        )
        try:
            tool_args, tool_kwargs = self._to_args_and_kwargs(parsed_input)
            observation = (
                self._run(*tool_args, run_manager=run_manager, **tool_kwargs)
                if new_arg_supported
                else self._run(*tool_args, **tool_kwargs)
            )
        except ToolException as e:
            if not self.handle_tool_error:
                run_manager.on_tool_error(e)
                raise e
            elif isinstance(self.handle_tool_error, bool):
                if e.args:
                    observation = e.args[0]
                else:
                    observation = "Tool execution error"
            elif isinstance(self.handle_tool_error, str):
                observation = self.handle_tool_error
            elif callable(self.handle_tool_error):
                observation = self.handle_tool_error(e)
            else:
                raise ValueError(
                    f"Got unexpected type of `handle_tool_error`. Expected bool, str "
                    f"or callable. Received: {self.handle_tool_error}"
                )
            run_manager.on_tool_end(
                str(observation), color="red", name=self.name, **kwargs
            )
            return observation
        except (Exception, KeyboardInterrupt) as e:
            run_manager.on_tool_error(e)
            raise e
        else:
            run_manager.on_tool_end(
                str(observation), color=color, name=self.name, **kwargs
            )
            return observation
[docs]    async def arun(
        self,
        tool_input: Union[str, Dict],
        verbose: Optional[bool] = None,
        start_color: Optional[str] = "green",
        color: Optional[str] = "green",
        callbacks: Callbacks = None,
        *,


callbacks: Callbacks = None,
        *,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        run_name: Optional[str] = None,
        **kwargs: Any,
    ) -> Any:
        """Run the tool asynchronously."""
        parsed_input = self._parse_input(tool_input)
        if not self.verbose and verbose is not None:
            verbose_ = verbose
        else:
            verbose_ = self.verbose
        callback_manager = AsyncCallbackManager.configure(
            callbacks,
            self.callbacks,
            verbose_,
            tags,
            self.tags,
            metadata,
            self.metadata,
        )
        new_arg_supported = signature(self._arun).parameters.get("run_manager")
        run_manager = await callback_manager.on_tool_start(
            {"name": self.name, "description": self.description},
            tool_input if isinstance(tool_input, str) else str(tool_input),
            color=start_color,
            name=run_name,
            **kwargs,
        )
        try:
            # We then call the tool on the tool input to get an observation
            tool_args, tool_kwargs = self._to_args_and_kwargs(parsed_input)
            observation = (
                await self._arun(*tool_args, run_manager=run_manager, **tool_kwargs)
                if new_arg_supported
                else await self._arun(*tool_args, **tool_kwargs)
            )
        except ToolException as e:
            if not self.handle_tool_error:
                await run_manager.on_tool_error(e)
                raise e
            elif isinstance(self.handle_tool_error, bool):
                if e.args:
                    observation = e.args[0]
                else:
                    observation = "Tool execution error"


else:
                    observation = "Tool execution error"
            elif isinstance(self.handle_tool_error, str):
                observation = self.handle_tool_error
            elif callable(self.handle_tool_error):
                observation = self.handle_tool_error(e)
            else:
                raise ValueError(
                    f"Got unexpected type of `handle_tool_error`. Expected bool, str "
                    f"or callable. Received: {self.handle_tool_error}"
                )
            await run_manager.on_tool_end(
                str(observation), color="red", name=self.name, **kwargs
            )
            return observation
        except (Exception, KeyboardInterrupt) as e:
            await run_manager.on_tool_error(e)
            raise e
        else:
            await run_manager.on_tool_end(
                str(observation), color=color, name=self.name, **kwargs
            )
            return observation
[docs]    def __call__(self, tool_input: str, callbacks: Callbacks = None) -> str:
        """Make tool callable."""
        return self.run(tool_input, callbacks=callbacks)
[docs]class Tool(BaseTool):
    """Tool that takes in function or coroutine directly."""
    description: str = ""
    func: Optional[Callable[..., str]]
    """The function to run when the tool is called."""
    coroutine: Optional[Callable[..., Awaitable[str]]] = None
    """The asynchronous version of the function."""
    # --- Runnable ---
[docs]    async def ainvoke(
        self,
        input: Union[str, Dict],
        config: Optional[RunnableConfig] = None,
        **kwargs: Any,
    ) -> Any:
        if not self.coroutine:
            # If the tool does not implement async, fall back to default implementation
            return await asyncio.get_running_loop().run_in_executor(


return await asyncio.get_running_loop().run_in_executor(
                None, partial(self.invoke, input, config, **kwargs)
            )
        return await super().ainvoke(input, config, **kwargs)
    # --- Tool ---
    @property
    def args(self) -> dict:
        """The tool's input arguments."""
        if self.args_schema is not None:
            return self.args_schema.schema()["properties"]
        # For backwards compatibility, if the function signature is ambiguous,
        # assume it takes a single string input.
        return {"tool_input": {"type": "string"}}
    def _to_args_and_kwargs(self, tool_input: Union[str, Dict]) -> Tuple[Tuple, Dict]:
        """Convert tool input to pydantic model."""
        args, kwargs = super()._to_args_and_kwargs(tool_input)
        # For backwards compatibility. The tool must be run with a single input
        all_args = list(args) + list(kwargs.values())
        if len(all_args) != 1:
            raise ToolException(
                f"Too many arguments to single-input tool {self.name}."
                f" Args: {all_args}"
            )
        return tuple(all_args), {}
    def _run(
        self,
        *args: Any,
        run_manager: Optional[CallbackManagerForToolRun] = None,
        **kwargs: Any,
    ) -> Any:
        """Use the tool."""
        if self.func:
            new_argument_supported = signature(self.func).parameters.get("callbacks")
            return (
                self.func(
                    *args,
                    callbacks=run_manager.get_child() if run_manager else None,
                    **kwargs,
                )
                if new_argument_supported
                else self.func(*args, **kwargs)
            )


if new_argument_supported
                else self.func(*args, **kwargs)
            )
        raise NotImplementedError("Tool does not support sync")
    async def _arun(
        self,
        *args: Any,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
        **kwargs: Any,
    ) -> Any:
        """Use the tool asynchronously."""
        if self.coroutine:
            new_argument_supported = signature(self.coroutine).parameters.get(
                "callbacks"
            )
            return (
                await self.coroutine(
                    *args,
                    callbacks=run_manager.get_child() if run_manager else None,
                    **kwargs,
                )
                if new_argument_supported
                else await self.coroutine(*args, **kwargs)
            )
        else:
            return await asyncio.get_running_loop().run_in_executor(
                None, partial(self._run, run_manager=run_manager, **kwargs), *args
            )
    # TODO: this is for backwards compatibility, remove in future
    def __init__(
        self, name: str, func: Optional[Callable], description: str, **kwargs: Any
    ) -> None:
        """Initialize tool."""
        super(Tool, self).__init__(
            name=name, func=func, description=description, **kwargs
        )
[docs]    @classmethod
    def from_function(
        cls,
        func: Optional[Callable],
        name: str,  # We keep these required to support backwards compatibility
        description: str,
        return_direct: bool = False,
        args_schema: Optional[Type[BaseModel]] = None,
        coroutine: Optional[
            Callable[..., Awaitable[Any]]


coroutine: Optional[
            Callable[..., Awaitable[Any]]
        ] = None,  # This is last for compatibility, but should be after func
        **kwargs: Any,
    ) -> Tool:
        """Initialize tool from a function."""
        if func is None and coroutine is None:
            raise ValueError("Function and/or coroutine must be provided")
        return cls(
            name=name,
            func=func,
            coroutine=coroutine,
            description=description,
            return_direct=return_direct,
            args_schema=args_schema,
            **kwargs,
        )
[docs]class StructuredTool(BaseTool):
    """Tool that can operate on any number of inputs."""
    description: str = ""
    args_schema: Type[BaseModel] = Field(..., description="The tool schema.")
    """The input arguments' schema."""
    func: Optional[Callable[..., Any]]
    """The function to run when the tool is called."""
    coroutine: Optional[Callable[..., Awaitable[Any]]] = None
    """The asynchronous version of the function."""
    # --- Runnable ---
[docs]    async def ainvoke(
        self,
        input: Union[str, Dict],
        config: Optional[RunnableConfig] = None,
        **kwargs: Any,
    ) -> Any:
        if not self.coroutine:
            # If the tool does not implement async, fall back to default implementation
            return await asyncio.get_running_loop().run_in_executor(
                None, partial(self.invoke, input, config, **kwargs)
            )
        return await super().ainvoke(input, config, **kwargs)
    # --- Tool ---
    @property
    def args(self) -> dict:
        """The tool's input arguments."""


def args(self) -> dict:
        """The tool's input arguments."""
        return self.args_schema.schema()["properties"]
    def _run(
        self,
        *args: Any,
        run_manager: Optional[CallbackManagerForToolRun] = None,
        **kwargs: Any,
    ) -> Any:
        """Use the tool."""
        if self.func:
            new_argument_supported = signature(self.func).parameters.get("callbacks")
            return (
                self.func(
                    *args,
                    callbacks=run_manager.get_child() if run_manager else None,
                    **kwargs,
                )
                if new_argument_supported
                else self.func(*args, **kwargs)
            )
        raise NotImplementedError("Tool does not support sync")
    async def _arun(
        self,
        *args: Any,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
        **kwargs: Any,
    ) -> str:
        """Use the tool asynchronously."""
        if self.coroutine:
            new_argument_supported = signature(self.coroutine).parameters.get(
                "callbacks"
            )
            return (
                await self.coroutine(
                    *args,
                    callbacks=run_manager.get_child() if run_manager else None,
                    **kwargs,
                )
                if new_argument_supported
                else await self.coroutine(*args, **kwargs)
            )
        return await asyncio.get_running_loop().run_in_executor(
            None,
            partial(self._run, run_manager=run_manager, **kwargs),
            *args,
        )
[docs]    @classmethod
    def from_function(
        cls,
        func: Optional[Callable] = None,


cls,
        func: Optional[Callable] = None,
        coroutine: Optional[Callable[..., Awaitable[Any]]] = None,
        name: Optional[str] = None,
        description: Optional[str] = None,
        return_direct: bool = False,
        args_schema: Optional[Type[BaseModel]] = None,
        infer_schema: bool = True,
        **kwargs: Any,
    ) -> StructuredTool:
        """Create tool from a given function.
        A classmethod that helps to create a tool from a function.
        Args:
            func: The function from which to create a tool
            coroutine: The async function from which to create a tool
            name: The name of the tool. Defaults to the function name
            description: The description of the tool. Defaults to the function docstring
            return_direct: Whether to return the result directly or as a callback
            args_schema: The schema of the tool's input arguments
            infer_schema: Whether to infer the schema from the function's signature
            **kwargs: Additional arguments to pass to the tool
        Returns:
            The tool
        Examples:
            .. code-block:: python
                def add(a: int, b: int) -> int:
                    \"\"\"Add two numbers\"\"\"
                    return a + b
                tool = StructuredTool.from_function(add)
                tool.run(1, 2) # 3
        """
        if func is not None:
            source_function = func
        elif coroutine is not None:
            source_function = coroutine
        else:
            raise ValueError("Function and/or coroutine must be provided")
        name = name or source_function.__name__
        description = description or source_function.__doc__
        if description is None:
            raise ValueError(


if description is None:
            raise ValueError(
                "Function must have a docstring if description not provided."
            )
        # Description example:
        # search_api(query: str) - Searches the API for the query.
        sig = signature(source_function)
        description = f"{name}{sig} - {description.strip()}"
        _args_schema = args_schema
        if _args_schema is None and infer_schema:
            _args_schema = create_schema_from_function(f"{name}Schema", source_function)
        return cls(
            name=name,
            func=func,
            coroutine=coroutine,
            args_schema=_args_schema,
            description=description,
            return_direct=return_direct,
            **kwargs,
        )
[docs]def tool(
    *args: Union[str, Callable, Runnable],
    return_direct: bool = False,
    args_schema: Optional[Type[BaseModel]] = None,
    infer_schema: bool = True,
) -> Callable:
    """Make tools out of functions, can be used with or without arguments.
    Args:
        *args: The arguments to the tool.
        return_direct: Whether to return directly from the tool rather
            than continuing the agent loop.
        args_schema: optional argument schema for user to specify
        infer_schema: Whether to infer the schema of the arguments from
            the function's signature. This also makes the resultant tool
            accept a dictionary input to its `run()` function.
    Requires:
        - Function must be of type (str) -> str
        - Function must have a docstring
    Examples:
        .. code-block:: python
            @tool
            def search_api(query: str) -> str:
                # Searches the API for the query.
                return


# Searches the API for the query.
                return
            @tool("search", return_direct=True)
            def search_api(query: str) -> str:
                # Searches the API for the query.
                return
    """
    def _make_with_name(tool_name: str) -> Callable:
        def _make_tool(dec_func: Union[Callable, Runnable]) -> BaseTool:
            if isinstance(dec_func, Runnable):
                runnable = dec_func
                if runnable.input_schema.schema().get("type") != "object":
                    raise ValueError("Runnable must have an object schema.")
                async def ainvoke_wrapper(
                    callbacks: Optional[Callbacks] = None, **kwargs: Any
                ) -> Any:
                    return await runnable.ainvoke(kwargs, {"callbacks": callbacks})
                def invoke_wrapper(
                    callbacks: Optional[Callbacks] = None, **kwargs: Any
                ) -> Any:
                    return runnable.invoke(kwargs, {"callbacks": callbacks})
                coroutine = ainvoke_wrapper
                func = invoke_wrapper
                schema: Optional[Type[BaseModel]] = runnable.input_schema
                description = repr(runnable)
            elif inspect.iscoroutinefunction(dec_func):
                coroutine = dec_func
                func = None
                schema = args_schema
                description = None
            else:
                coroutine = None
                func = dec_func
                schema = args_schema
                description = None
            if infer_schema or args_schema is not None:
                return StructuredTool.from_function(
                    func,
                    coroutine,
                    name=tool_name,
                    description=description,
                    return_direct=return_direct,
                    args_schema=schema,
                    infer_schema=infer_schema,
                )
            # If someone doesn't want a schema applied, we must treat it as


)
            # If someone doesn't want a schema applied, we must treat it as
            # a simple string->string function
            if func.__doc__ is None:
                raise ValueError(
                    "Function must have a docstring if "
                    "description not provided and infer_schema is False."
                )
            return Tool(
                name=tool_name,
                func=func,
                description=f"{tool_name} tool",
                return_direct=return_direct,
                coroutine=coroutine,
            )
        return _make_tool
    if len(args) == 2 and isinstance(args[0], str) and isinstance(args[1], Runnable):
        return _make_with_name(args[0])(args[1])
    elif len(args) == 1 and isinstance(args[0], str):
        # if the argument is a string, then we use the string as the tool name
        # Example usage: @tool("search", return_direct=True)
        return _make_with_name(args[0])
    elif len(args) == 1 and callable(args[0]):
        # if the argument is a function, then we use the function name as the tool name
        # Example usage: @tool
        return _make_with_name(args[0].__name__)(args[0])
    elif len(args) == 0:
        # if there are no arguments, then we use the function name as the tool name
        # Example usage: @tool(return_direct=True)
        def _partial(func: Callable[[str], str]) -> BaseTool:
            return _make_with_name(func.__name__)(func)
        return _partial
    else:
        raise ValueError("Too many arguments for tool decorator")


Source code for langchain.tools.yahoo_finance_news
from typing import Iterable, Optional
from requests.exceptions import HTTPError, ReadTimeout
from urllib3.exceptions import ConnectionError
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.document_loaders.web_base import WebBaseLoader
from langchain.schema import Document
from langchain.tools.base import BaseTool
[docs]class YahooFinanceNewsTool(BaseTool):
    """Tool that searches financial news on Yahoo Finance."""
    name: str = "yahoo_finance_news"
    description: str = (
        "Useful for when you need to find financial news "
        "about a public company. "
        "Input should be a company ticker. "
        "For example, AAPL for Apple, MSFT for Microsoft."
    )
    top_k: int = 10
    """The number of results to return."""
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the Yahoo Finance News tool."""
        try:
            import yfinance
        except ImportError:
            raise ImportError(
                "Could not import yfinance python package. "
                "Please install it with `pip install yfinance`."
            )
        company = yfinance.Ticker(query)
        try:
            if company.isin is None:
                return f"Company ticker {query} not found."
        except (HTTPError, ReadTimeout, ConnectionError):
            return f"Company ticker {query} not found."
        links = []
        try:
            links = [n["link"] for n in company.news if n["type"] == "STORY"]
        except (HTTPError, ReadTimeout, ConnectionError):


except (HTTPError, ReadTimeout, ConnectionError):
            if not links:
                return f"No news found for company that searched with {query} ticker."
        if not links:
            return f"No news found for company that searched with {query} ticker."
        loader = WebBaseLoader(web_paths=links)
        docs = loader.load()
        result = self._format_results(docs, query)
        if not result:
            return f"No news found for company that searched with {query} ticker."
        return result
    @staticmethod
    def _format_results(docs: Iterable[Document], query: str) -> str:
        doc_strings = [
            "\n".join([doc.metadata["title"], doc.metadata["description"]])
            for doc in docs
            if query in doc.metadata["description"] or query in doc.metadata["title"]
        ]
        return "\n\n".join(doc_strings)


Source code for langchain.tools.render
"""Different methods for rendering Tools to be passed to LLMs.
Depending on the LLM you are using and the prompting strategy you are using,
you may want Tools to be rendered in a different way.
This module contains various ways to render tools.
"""
from typing import List
from langchain.tools.base import BaseTool
from langchain.utils.openai_functions import (
    FunctionDescription,
    ToolDescription,
    convert_pydantic_to_openai_function,
)
[docs]def render_text_description(tools: List[BaseTool]) -> str:
    """Render the tool name and description in plain text.
    Output will be in the format of:
    .. code-block:: markdown
        search: This tool is used for search
        calculator: This tool is used for math
    """
    return "\n".join([f"{tool.name}: {tool.description}" for tool in tools])
[docs]def render_text_description_and_args(tools: List[BaseTool]) -> str:
    """Render the tool name, description, and args in plain text.
    Output will be in the format of:
    .. code-block:: markdown
        search: This tool is used for search, args: {"query": {"type": "string"}}
        calculator: This tool is used for math, \
args: {"expression": {"type": "string"}}
    """
    tool_strings = []
    for tool in tools:
        args_schema = str(tool.args)
        tool_strings.append(f"{tool.name}: {tool.description}, args: {args_schema}")
    return "\n".join(tool_strings)
[docs]def format_tool_to_openai_function(tool: BaseTool) -> FunctionDescription:
    """Format tool into the OpenAI function API."""
    if tool.args_schema:


"""Format tool into the OpenAI function API."""
    if tool.args_schema:
        return convert_pydantic_to_openai_function(
            tool.args_schema, name=tool.name, description=tool.description
        )
    else:
        return {
            "name": tool.name,
            "description": tool.description,
            "parameters": {
                # This is a hack to get around the fact that some tools
                # do not expose an args_schema, and expect an argument
                # which is a string.
                # And Open AI does not support an array type for the
                # parameters.
                "properties": {
                    "__arg1": {"title": "__arg1", "type": "string"},
                },
                "required": ["__arg1"],
                "type": "object",
            },
        }
[docs]def format_tool_to_openai_tool(tool: BaseTool) -> ToolDescription:
    """Format tool into the OpenAI function API."""
    function = format_tool_to_openai_function(tool)
    return {"type": "function", "function": function}


Source code for langchain.tools.nuclia.tool
"""Tool for the Nuclia Understanding API.
Installation:
```bash
    pip install --upgrade protobuf
    pip install nucliadb-protos
```
"""
import asyncio
import base64
import logging
import mimetypes
import os
from typing import Any, Dict, Optional, Type, Union
import requests
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import BaseModel, Field
from langchain.tools.base import BaseTool
logger = logging.getLogger(__name__)
[docs]class NUASchema(BaseModel):
    """Input for Nuclia Understanding API.
    Attributes:
        action: Action to perform. Either `push` or `pull`.
        id: ID of the file to push or pull.
        path: Path to the file to push (needed only for `push` action).
        text: Text content to process (needed only for `push` action).
    """
    action: str = Field(
        ...,
        description="Action to perform. Either `push` or `pull`.",
    )
    id: str = Field(
        ...,
        description="ID of the file to push or pull.",
    )
    path: Optional[str] = Field(
        ...,
        description="Path to the file to push (needed only for `push` action).",
    )
    text: Optional[str] = Field(
        ...,
        description="Text content to process (needed only for `push` action).",
    )
[docs]class NucliaUnderstandingAPI(BaseTool):
    """Tool to process files with the Nuclia Understanding API."""


"""Tool to process files with the Nuclia Understanding API."""
    name: str = "nuclia_understanding_api"
    description: str = (
        "A wrapper around Nuclia Understanding API endpoints. "
        "Useful for when you need to extract text from any kind of files. "
    )
    args_schema: Type[BaseModel] = NUASchema
    _results: Dict[str, Any] = {}
    _config: Dict[str, Any] = {}
    def __init__(self, enable_ml: bool = False) -> None:
        zone = os.environ.get("NUCLIA_ZONE", "europe-1")
        self._config["BACKEND"] = f"https://{zone}.nuclia.cloud/api/v1"
        key = os.environ.get("NUCLIA_NUA_KEY")
        if not key:
            raise ValueError("NUCLIA_NUA_KEY environment variable not set")
        else:
            self._config["NUA_KEY"] = key
        self._config["enable_ml"] = enable_ml
        super().__init__()
    def _run(
        self,
        action: str,
        id: str,
        path: Optional[str],
        text: Optional[str],
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        if action == "push":
            self._check_params(path, text)
            if path:
                return self._pushFile(id, path)
            if text:
                return self._pushText(id, text)
        elif action == "pull":
            return self._pull(id)
        return ""
    async def _arun(
        self,
        action: str,
        id: str,


self,
        action: str,
        id: str,
        path: Optional[str] = None,
        text: Optional[str] = None,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        self._check_params(path, text)
        if path:
            self._pushFile(id, path)
        if text:
            self._pushText(id, text)
        data = None
        while True:
            data = self._pull(id)
            if data:
                break
            await asyncio.sleep(15)
        return data
    def _pushText(self, id: str, text: str) -> str:
        field = {
            "textfield": {"text": {"body": text, "format": 0}},
            "processing_options": {"ml_text": self._config["enable_ml"]},
        }
        return self._pushField(id, field)
    def _pushFile(self, id: str, content_path: str) -> str:
        with open(content_path, "rb") as source_file:
            response = requests.post(
                self._config["BACKEND"] + "/processing/upload",
                headers={
                    "content-type": mimetypes.guess_type(content_path)[0]
                    or "application/octet-stream",
                    "x-stf-nuakey": "Bearer " + self._config["NUA_KEY"],
                },
                data=source_file.read(),
            )
            if response.status_code != 200:
                logger.info(
                    f"Error uploading {content_path}: "
                    f"{response.status_code} {response.text}"
                )
                return ""
            else:
                field = {


)
                return ""
            else:
                field = {
                    "filefield": {"file": f"{response.text}"},
                    "processing_options": {"ml_text": self._config["enable_ml"]},
                }
                return self._pushField(id, field)
    def _pushField(self, id: str, field: Any) -> str:
        logger.info(f"Pushing {id} in queue")
        response = requests.post(
            self._config["BACKEND"] + "/processing/push",
            headers={
                "content-type": "application/json",
                "x-stf-nuakey": "Bearer " + self._config["NUA_KEY"],
            },
            json=field,
        )
        if response.status_code != 200:
            logger.info(
                f"Error pushing field {id}:" f"{response.status_code} {response.text}"
            )
            raise ValueError("Error pushing field")
        else:
            uuid = response.json()["uuid"]
            logger.info(f"Field {id} pushed in queue, uuid: {uuid}")
            self._results[id] = {"uuid": uuid, "status": "pending"}
            return uuid
    def _pull(self, id: str) -> str:
        self._pull_queue()
        result = self._results.get(id, None)
        if not result:
            logger.info(f"{id} not in queue")
            return ""
        elif result["status"] == "pending":
            logger.info(f'Waiting for {result["uuid"]} to be processed')
            return ""
        else:
            return result["data"]
    def _pull_queue(self) -> None:
        try:
            from nucliadb_protos.writer_pb2 import BrokerMessage


try:
            from nucliadb_protos.writer_pb2 import BrokerMessage
        except ImportError as e:
            raise ImportError(
                "nucliadb-protos is not installed. "
                "Run `pip install nucliadb-protos` to install."
            ) from e
        try:
            from google.protobuf.json_format import MessageToJson
        except ImportError as e:
            raise ImportError(
                "Unable to import google.protobuf, please install with "
                "`pip install protobuf`."
            ) from e
        res = requests.get(
            self._config["BACKEND"] + "/processing/pull",
            headers={
                "x-stf-nuakey": "Bearer " + self._config["NUA_KEY"],
            },
        ).json()
        if res["status"] == "empty":
            logger.info("Queue empty")
        elif res["status"] == "ok":
            payload = res["payload"]
            pb = BrokerMessage()
            pb.ParseFromString(base64.b64decode(payload))
            uuid = pb.uuid
            logger.info(f"Pulled {uuid} from queue")
            matching_id = self._find_matching_id(uuid)
            if not matching_id:
                logger.info(f"No matching id for {uuid}")
            else:
                self._results[matching_id]["status"] = "done"
                data = MessageToJson(
                    pb,
                    preserving_proto_field_name=True,
                    including_default_value_fields=True,
                )
                self._results[matching_id]["data"] = data
    def _find_matching_id(self, uuid: str) -> Union[str, None]:
        for id, result in self._results.items():
            if result["uuid"] == uuid:
                return id
        return None


if result["uuid"] == uuid:
                return id
        return None
    def _check_params(self, path: Optional[str], text: Optional[str]) -> None:
        if not path and not text:
            raise ValueError("File path or text is required")
        if path and text:
            raise ValueError("Cannot process both file and text on a single run")


Source code for langchain.tools.sql_database.tool
# flake8: noqa
"""Tools for interacting with a SQL database."""
from typing import Any, Dict, Optional
from langchain.pydantic_v1 import BaseModel, Extra, Field, root_validator
from langchain.schema.language_model import BaseLanguageModel
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.chains.llm import LLMChain
from langchain.prompts import PromptTemplate
from langchain.utilities.sql_database import SQLDatabase
from langchain.tools.base import BaseTool
from langchain.tools.sql_database.prompt import QUERY_CHECKER
[docs]class BaseSQLDatabaseTool(BaseModel):
    """Base tool for interacting with a SQL database."""
    db: SQLDatabase = Field(exclude=True)
    class Config(BaseTool.Config):
        pass
[docs]class QuerySQLDataBaseTool(BaseSQLDatabaseTool, BaseTool):
    """Tool for querying a SQL database."""
    name: str = "sql_db_query"
    description: str = """
    Input to this tool is a detailed and correct SQL query, output is a result from the database.
    If the query is not correct, an error message will be returned.
    If an error is returned, rewrite the query, check the query, and try again.
    """
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Execute the query, return the results or an error message."""
        return self.db.run_no_throw(query)
[docs]class InfoSQLDatabaseTool(BaseSQLDatabaseTool, BaseTool):
    """Tool for getting metadata about a SQL database."""
    name: str = "sql_db_schema"


name: str = "sql_db_schema"
    description: str = """
    Input to this tool is a comma-separated list of tables, output is the schema and sample rows for those tables.
    Example Input: "table1, table2, table3"
    """
    def _run(
        self,
        table_names: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Get the schema for tables in a comma-separated list."""
        return self.db.get_table_info_no_throw(table_names.split(", "))
[docs]class ListSQLDatabaseTool(BaseSQLDatabaseTool, BaseTool):
    """Tool for getting tables names."""
    name: str = "sql_db_list_tables"
    description: str = "Input is an empty string, output is a comma separated list of tables in the database."
    def _run(
        self,
        tool_input: str = "",
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Get the schema for a specific table."""
        return ", ".join(self.db.get_usable_table_names())
[docs]class QuerySQLCheckerTool(BaseSQLDatabaseTool, BaseTool):
    """Use an LLM to check if a query is correct.
    Adapted from https://www.patterns.app/blog/2023/01/18/crunchbot-sql-analyst-gpt/"""
    template: str = QUERY_CHECKER
    llm: BaseLanguageModel
    llm_chain: LLMChain = Field(init=False)
    name: str = "sql_db_query_checker"
    description: str = """
    Use this tool to double check if your query is correct before executing it.


Use this tool to double check if your query is correct before executing it.
    Always use this tool before executing a query with sql_db_query!
    """
    @root_validator(pre=True)
    def initialize_llm_chain(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        if "llm_chain" not in values:
            values["llm_chain"] = LLMChain(
                llm=values.get("llm"),
                prompt=PromptTemplate(
                    template=QUERY_CHECKER, input_variables=["dialect", "query"]
                ),
            )
        if values["llm_chain"].prompt.input_variables != ["dialect", "query"]:
            raise ValueError(
                "LLM chain for QueryCheckerTool must have input variables ['query', 'dialect']"
            )
        return values
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the LLM to check the query."""
        return self.llm_chain.predict(
            query=query,
            dialect=self.db.dialect,
            callbacks=run_manager.get_child() if run_manager else None,
        )
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        return await self.llm_chain.apredict(
            query=query,
            dialect=self.db.dialect,
            callbacks=run_manager.get_child() if run_manager else None,
        )


Source code for langchain.tools.sleep.tool
"""Tool for agent to sleep."""
from asyncio import sleep as asleep
from time import sleep
from typing import Optional, Type
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import BaseModel, Field
from langchain.tools.base import BaseTool
[docs]class SleepInput(BaseModel):
    """Input for CopyFileTool."""
    sleep_time: int = Field(..., description="Time to sleep in seconds")
[docs]class SleepTool(BaseTool):
    """Tool that adds the capability to sleep."""
    name: str = "sleep"
    args_schema: Type[BaseModel] = SleepInput
    description: str = "Make agent sleep for a specified number of seconds."
    def _run(
        self,
        sleep_time: int,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the Sleep tool."""
        sleep(sleep_time)
        return f"Agent slept for {sleep_time} seconds."
    async def _arun(
        self,
        sleep_time: int,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the sleep tool asynchronously."""
        await asleep(sleep_time)
        return f"Agent slept for {sleep_time} seconds."


Source code for langchain.tools.google_serper.tool
"""Tool for the Serper.dev Google Search API."""
from typing import Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import Field
from langchain.tools.base import BaseTool
from langchain.utilities.google_serper import GoogleSerperAPIWrapper
[docs]class GoogleSerperRun(BaseTool):
    """Tool that queries the Serper.dev Google search API."""
    name: str = "google_serper"
    description: str = (
        "A low-cost Google Search API."
        "Useful for when you need to answer questions about current events."
        "Input should be a search query."
    )
    api_wrapper: GoogleSerperAPIWrapper
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        return str(self.api_wrapper.run(query))
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        return (await self.api_wrapper.arun(query)).__str__()
[docs]class GoogleSerperResults(BaseTool):
    """Tool that queries the Serper.dev Google Search API
    and get back json."""
    name: str = "google_serper_results_json"
    description: str = (
        "A low-cost Google Search API."
        "Useful for when you need to answer questions about current events."
        "Input should be a search query. Output is a JSON object of the query results"
    )


)
    api_wrapper: GoogleSerperAPIWrapper = Field(default_factory=GoogleSerperAPIWrapper)
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        return str(self.api_wrapper.results(query))
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        return (await self.api_wrapper.aresults(query)).__str__()


Source code for langchain.tools.searchapi.tool
"""Tool for the SearchApi.io search API."""
from typing import Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import Field
from langchain.tools.base import BaseTool
from langchain.utilities.searchapi import SearchApiAPIWrapper
[docs]class SearchAPIRun(BaseTool):
    """Tool that queries the SearchApi.io search API."""
    name: str = "searchapi"
    description: str = (
        "Google search API provided by SearchApi.io."
        "This tool is handy when you need to answer questions about current events."
        "Input should be a search query."
    )
    api_wrapper: SearchApiAPIWrapper
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        return self.api_wrapper.run(query)
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        return await self.api_wrapper.arun(query)
[docs]class SearchAPIResults(BaseTool):
    """Tool that queries the SearchApi.io search API and returns JSON."""
    name: str = "searchapi_results_json"
    description: str = (
        "Google search API provided by SearchApi.io."
        "This tool is handy when you need to answer questions about current events."
        "The input should be a search query and the output is a JSON object "
        "with the query results."
    )


"with the query results."
    )
    api_wrapper: SearchApiAPIWrapper = Field(default_factory=SearchApiAPIWrapper)
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        return str(self.api_wrapper.results(query))
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool asynchronously."""
        return (await self.api_wrapper.aresults(query)).__str__()