callbacks=callbacks,
                streaming=True
            )
            print(llm("What is Deep Learning?"))
    """
    max_new_tokens: int = 512
    """Maximum number of generated tokens"""
    top_k: Optional[int] = None
    """The number of highest probability vocabulary tokens to keep for
    top-k-filtering."""
    top_p: Optional[float] = 0.95
    """If set to < 1, only the smallest set of most probable tokens with probabilities
    that add up to `top_p` or higher are kept for generation."""
    typical_p: Optional[float] = 0.95
    """Typical Decoding mass. See [Typical Decoding for Natural Language
    Generation](https://arxiv.org/abs/2202.00666) for more information."""
    temperature: Optional[float] = 0.8
    """The value used to module the logits distribution."""
    repetition_penalty: Optional[float] = None
    """The parameter for repetition penalty. 1.0 means no penalty.
    See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details."""
    return_full_text: bool = False
    """Whether to prepend the prompt to the generated text"""
    truncate: Optional[int] = None
    """Truncate inputs tokens to the given size"""
    stop_sequences: List[str] = Field(default_factory=list)
    """Stop generating tokens if a member of `stop_sequences` is generated"""
    seed: Optional[int] = None
    """Random sampling seed"""
    inference_server_url: str = ""
    """text-generation-inference instance base url"""
    timeout: int = 120
    """Timeout in seconds"""
    streaming: bool = False
    """Whether to generate a stream of tokens asynchronously"""


streaming: bool = False
    """Whether to generate a stream of tokens asynchronously"""
    do_sample: bool = False
    """Activate logits sampling"""
    watermark: bool = False
    """Watermarking with [A Watermark for Large Language Models]
    (https://arxiv.org/abs/2301.10226)"""
    server_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any text-generation-inference server parameters not explicitly specified"""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `call` not explicitly specified"""
    client: Any
    async_client: Any
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = get_pydantic_field_names(cls)
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name in extra:
                raise ValueError(f"Found {field_name} supplied twice.")
            if field_name not in all_required_field_names:
                logger.warning(
                    f"""WARNING! {field_name} is not default parameter.
                    {field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        invalid_model_kwargs = all_required_field_names.intersection(extra.keys())
        if invalid_model_kwargs:
            raise ValueError(
                f"Parameters {invalid_model_kwargs} should be specified explicitly. "


f"Parameters {invalid_model_kwargs} should be specified explicitly. "
                f"Instead they were passed in as part of `model_kwargs` parameter."
            )
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that python package exists in environment."""
        try:
            import text_generation
            values["client"] = text_generation.Client(
                values["inference_server_url"],
                timeout=values["timeout"],
                **values["server_kwargs"],
            )
            values["async_client"] = text_generation.AsyncClient(
                values["inference_server_url"],
                timeout=values["timeout"],
                **values["server_kwargs"],
            )
        except ImportError:
            raise ImportError(
                "Could not import text_generation python package. "
                "Please install it with `pip install text_generation`."
            )
        return values
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "huggingface_textgen_inference"
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling text generation inference API."""
        return {
            "max_new_tokens": self.max_new_tokens,
            "top_k": self.top_k,
            "top_p": self.top_p,
            "typical_p": self.typical_p,
            "temperature": self.temperature,
            "repetition_penalty": self.repetition_penalty,
            "return_full_text": self.return_full_text,
            "truncate": self.truncate,
            "stop_sequences": self.stop_sequences,
            "seed": self.seed,
            "do_sample": self.do_sample,


"seed": self.seed,
            "do_sample": self.do_sample,
            "watermark": self.watermark,
            **self.model_kwargs,
        }
    def _invocation_params(
        self, runtime_stop: Optional[List[str]], **kwargs: Any
    ) -> Dict[str, Any]:
        params = {**self._default_params, **kwargs}
        params["stop_sequences"] = params["stop_sequences"] + (runtime_stop or [])
        return params
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        if self.streaming:
            completion = ""
            for chunk in self._stream(prompt, stop, run_manager, **kwargs):
                completion += chunk.text
            return completion
        invocation_params = self._invocation_params(stop, **kwargs)
        res = self.client.generate(prompt, **invocation_params)
        # remove stop sequences from the end of the generated text
        for stop_seq in invocation_params["stop_sequences"]:
            if stop_seq in res.generated_text:
                res.generated_text = res.generated_text[
                    : res.generated_text.index(stop_seq)
                ]
        return res.generated_text
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        if self.streaming:
            completion = ""
            async for chunk in self._astream(prompt, stop, run_manager, **kwargs):
                completion += chunk.text
            return completion


completion += chunk.text
            return completion
        invocation_params = self._invocation_params(stop, **kwargs)
        res = await self.async_client.generate(prompt, **invocation_params)
        # remove stop sequences from the end of the generated text
        for stop_seq in invocation_params["stop_sequences"]:
            if stop_seq in res.generated_text:
                res.generated_text = res.generated_text[
                    : res.generated_text.index(stop_seq)
                ]
        return res.generated_text
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        invocation_params = self._invocation_params(stop, **kwargs)
        for res in self.client.generate_stream(prompt, **invocation_params):
            # identify stop sequence in generated text, if any
            stop_seq_found: Optional[str] = None
            for stop_seq in invocation_params["stop_sequences"]:
                if stop_seq in res.token.text:
                    stop_seq_found = stop_seq
            # identify text to yield
            text: Optional[str] = None
            if res.token.special:
                text = None
            elif stop_seq_found:
                text = res.token.text[: res.token.text.index(stop_seq_found)]
            else:
                text = res.token.text
            # yield text, if any
            if text:
                chunk = GenerationChunk(text=text)
                yield chunk
                if run_manager:
                    run_manager.on_llm_new_token(chunk.text)
            # break if stop sequence found
            if stop_seq_found:
                break
    async def _astream(
        self,
        prompt: str,


async def _astream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> AsyncIterator[GenerationChunk]:
        invocation_params = self._invocation_params(stop, **kwargs)
        async for res in self.async_client.generate_stream(prompt, **invocation_params):
            # identify stop sequence in generated text, if any
            stop_seq_found: Optional[str] = None
            for stop_seq in invocation_params["stop_sequences"]:
                if stop_seq in res.token.text:
                    stop_seq_found = stop_seq
            # identify text to yield
            text: Optional[str] = None
            if res.token.special:
                text = None
            elif stop_seq_found:
                text = res.token.text[: res.token.text.index(stop_seq_found)]
            else:
                text = res.token.text
            # yield text, if any
            if text:
                chunk = GenerationChunk(text=text)
                yield chunk
                if run_manager:
                    await run_manager.on_llm_new_token(chunk.text)
            # break if stop sequence found
            if stop_seq_found:
                break


Source code for langchain.llms.arcee
from typing import Any, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utilities.arcee import ArceeWrapper, DALMFilter
from langchain.utils import get_from_dict_or_env
[docs]class Arcee(LLM):
    """Arcee's Domain Adapted Language Models (DALMs).
    To use, set the ``ARCEE_API_KEY`` environment variable with your Arcee API key,
    or pass ``arcee_api_key`` as a named parameter.
    Example:
        .. code-block:: python
            from langchain.llms import Arcee
            arcee = Arcee(
                model="DALM-PubMed",
                arcee_api_key="ARCEE-API-KEY"
            )
            response = arcee("AI-driven music therapy")
    """
    _client: Optional[ArceeWrapper] = None  #: :meta private:
    """Arcee _client."""
    arcee_api_key: str = ""
    """Arcee API Key"""
    model: str
    """Arcee DALM name"""
    arcee_api_url: str = "https://api.arcee.ai"
    """Arcee API URL"""
    arcee_api_version: str = "v2"
    """Arcee API Version"""
    arcee_app_url: str = "https://app.arcee.ai"
    """Arcee App URL"""
    model_id: str = ""
    """Arcee Model ID"""
    model_kwargs: Optional[Dict[str, Any]] = None
    """Keyword arguments to pass to the model."""
    class Config:


"""Keyword arguments to pass to the model."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        underscore_attrs_are_private = True
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "arcee"
    def __init__(self, **data: Any) -> None:
        """Initializes private fields."""
        super().__init__(**data)
        self._client = None
        self._client = ArceeWrapper(
            arcee_api_key=self.arcee_api_key,
            arcee_api_url=self.arcee_api_url,
            arcee_api_version=self.arcee_api_version,
            model_kwargs=self.model_kwargs,
            model_name=self.model,
        )
        self._client.validate_model_training_status()
    @root_validator()
    def validate_environments(cls, values: Dict) -> Dict:
        """Validate Arcee environment variables."""
        # validate env vars
        values["arcee_api_key"] = get_from_dict_or_env(
            values,
            "arcee_api_key",
            "ARCEE_API_KEY",
        )
        values["arcee_api_url"] = get_from_dict_or_env(
            values,
            "arcee_api_url",
            "ARCEE_API_URL",
        )
        values["arcee_app_url"] = get_from_dict_or_env(
            values,
            "arcee_app_url",
            "ARCEE_APP_URL",
        )
        values["arcee_api_version"] = get_from_dict_or_env(
            values,
            "arcee_api_version",
            "ARCEE_API_VERSION",
        )
        # validate model kwargs
        if values["model_kwargs"]:


)
        # validate model kwargs
        if values["model_kwargs"]:
            kw = values["model_kwargs"]
            # validate size
            if kw.get("size") is not None:
                if not kw.get("size") >= 0:
                    raise ValueError("`size` must be positive")
            # validate filters
            if kw.get("filters") is not None:
                if not isinstance(kw.get("filters"), List):
                    raise ValueError("`filters` must be a list")
                for f in kw.get("filters"):
                    DALMFilter(**f)
        return values
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Generate text from Arcee DALM.
        Args:
            prompt: Prompt to generate text from.
            size: The max number of context results to retrieve.
            Defaults to 3. (Can be less if filters are provided).
            filters: Filters to apply to the context dataset.
        """
        try:
            if not self._client:
                raise ValueError("Client is not initialized.")
            return self._client.generate(prompt=prompt, **kwargs)
        except Exception as e:
            raise Exception(f"Failed to generate text: {e}") from e


Source code for langchain.llms.utils
"""Common utility functions for LLM APIs."""
import re
from typing import List
[docs]def enforce_stop_tokens(text: str, stop: List[str]) -> str:
    """Cut off the text as soon as any stop words occur."""
    return re.split("|".join(stop), text, maxsplit=1)[0]


Source code for langchain.llms.predibase
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field
[docs]class Predibase(LLM):
    """Use your Predibase models with Langchain.
    To use, you should have the ``predibase`` python package installed,
    and have your Predibase API key.
    """
    model: str
    predibase_api_key: str
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    @property
    def _llm_type(self) -> str:
        return "predibase"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        try:
            from predibase import PredibaseClient
            pc = PredibaseClient(token=self.predibase_api_key)
        except ImportError as e:
            raise ImportError(
                "Could not import Predibase Python package. "
                "Please install it with `pip install predibase`."
            ) from e
        except ValueError as e:
            raise ValueError("Your API key is not correct. Please try again") from e
        # load model and version
        results = pc.prompt(prompt, model_name=self.model)
        return results[0].response
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_kwargs": self.model_kwargs},
        }


Source code for langchain.llms.forefrontai
from typing import Any, Dict, List, Mapping, Optional
import requests
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utils import get_from_dict_or_env
[docs]class ForefrontAI(LLM):
    """ForefrontAI large language models.
    To use, you should have the environment variable ``FOREFRONTAI_API_KEY``
    set with your API key.
    Example:
        .. code-block:: python
            from langchain.llms import ForefrontAI
            forefrontai = ForefrontAI(endpoint_url="")
    """
    endpoint_url: str = ""
    """Model name to use."""
    temperature: float = 0.7
    """What sampling temperature to use."""
    length: int = 256
    """The maximum number of tokens to generate in the completion."""
    top_p: float = 1.0
    """Total probability mass of tokens to consider at each step."""
    top_k: int = 40
    """The number of highest probability vocabulary tokens to
    keep for top-k-filtering."""
    repetition_penalty: int = 1
    """Penalizes repeated tokens according to frequency."""
    forefrontai_api_key: Optional[str] = None
    base_url: Optional[str] = None
    """Base url to use, if None decides based on model name."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key exists in environment."""


"""Validate that api key exists in environment."""
        forefrontai_api_key = get_from_dict_or_env(
            values, "forefrontai_api_key", "FOREFRONTAI_API_KEY"
        )
        values["forefrontai_api_key"] = forefrontai_api_key
        return values
    @property
    def _default_params(self) -> Mapping[str, Any]:
        """Get the default parameters for calling ForefrontAI API."""
        return {
            "temperature": self.temperature,
            "length": self.length,
            "top_p": self.top_p,
            "top_k": self.top_k,
            "repetition_penalty": self.repetition_penalty,
        }
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{"endpoint_url": self.endpoint_url}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "forefrontai"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to ForefrontAI's complete endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = ForefrontAI("Tell me a joke.")
        """
        response = requests.post(
            url=self.endpoint_url,
            headers={


response = requests.post(
            url=self.endpoint_url,
            headers={
                "Authorization": f"Bearer {self.forefrontai_api_key}",
                "Content-Type": "application/json",
            },
            json={"text": prompt, **self._default_params, **kwargs},
        )
        response_json = response.json()
        text = response_json["result"][0]["completion"]
        if stop is not None:
            # I believe this is required since the stop tokens
            # are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.aviary
import dataclasses
import os
from typing import Any, Dict, List, Mapping, Optional, Union, cast
import requests
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utils import get_from_dict_or_env
TIMEOUT = 60
[docs]@dataclasses.dataclass
class AviaryBackend:
    """Aviary backend.
    Attributes:
        backend_url: The URL for the Aviary backend.
        bearer: The bearer token for the Aviary backend.
    """
    backend_url: str
    bearer: str
    def __post_init__(self) -> None:
        self.header = {"Authorization": self.bearer}
[docs]    @classmethod
    def from_env(cls) -> "AviaryBackend":
        aviary_url = os.getenv("AVIARY_URL")
        assert aviary_url, "AVIARY_URL must be set"
        aviary_token = os.getenv("AVIARY_TOKEN", "")
        bearer = f"Bearer {aviary_token}" if aviary_token else ""
        aviary_url += "/" if not aviary_url.endswith("/") else ""
        return cls(aviary_url, bearer)
[docs]def get_models() -> List[str]:
    """List available models"""
    backend = AviaryBackend.from_env()
    request_url = backend.backend_url + "-/routes"
    response = requests.get(request_url, headers=backend.header, timeout=TIMEOUT)
    try:
        result = response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(


except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {request_url}. Text response: {response.text}"
        ) from e
    result = sorted(
        [k.lstrip("/").replace("--", "/") for k in result.keys() if "--" in k]
    )
    return result
[docs]def get_completions(
    model: str,
    prompt: str,
    use_prompt_format: bool = True,
    version: str = "",
) -> Dict[str, Union[str, float, int]]:
    """Get completions from Aviary models."""
    backend = AviaryBackend.from_env()
    url = backend.backend_url + model.replace("/", "--") + "/" + version + "query"
    response = requests.post(
        url,
        headers=backend.header,
        json={"prompt": prompt, "use_prompt_format": use_prompt_format},
        timeout=TIMEOUT,
    )
    try:
        return response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {url}. Text response: {response.text}"
        ) from e
[docs]class Aviary(LLM):
    """Aviary hosted models.
    Aviary is a backend for hosted models. You can
    find out more about aviary at
    http://github.com/ray-project/aviary
    To get a list of the models supported on an
    aviary, follow the instructions on the website to
    install the aviary CLI and then use:
    `aviary models`
    AVIARY_URL and AVIARY_TOKEN environment variables must be set.
    Attributes:


Attributes:
        model: The name of the model to use. Defaults to "amazon/LightGPT".
        aviary_url: The URL for the Aviary backend. Defaults to None.
        aviary_token: The bearer token for the Aviary backend. Defaults to None.
        use_prompt_format: If True, the prompt template for the model will be ignored.
            Defaults to True.
        version: API version to use for Aviary. Defaults to None.
    Example:
        .. code-block:: python
            from langchain.llms import Aviary
            os.environ["AVIARY_URL"] = "<URL>"
            os.environ["AVIARY_TOKEN"] = "<TOKEN>"
            light = Aviary(model='amazon/LightGPT')
            output = light('How do you make fried rice?')
    """
    model: str = "amazon/LightGPT"
    aviary_url: Optional[str] = None
    aviary_token: Optional[str] = None
    # If True the prompt template for the model will be ignored.
    use_prompt_format: bool = True
    # API version to use for Aviary
    version: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        aviary_url = get_from_dict_or_env(values, "aviary_url", "AVIARY_URL")
        aviary_token = get_from_dict_or_env(values, "aviary_token", "AVIARY_TOKEN")
        # Set env viarables for aviary sdk
        os.environ["AVIARY_URL"] = aviary_url


os.environ["AVIARY_URL"] = aviary_url
        os.environ["AVIARY_TOKEN"] = aviary_token
        try:
            aviary_models = get_models()
        except requests.exceptions.RequestException as e:
            raise ValueError(e)
        model = values.get("model")
        if model and model not in aviary_models:
            raise ValueError(f"{aviary_url} does not support model {values['model']}.")
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model,
            "aviary_url": self.aviary_url,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return f"aviary-{self.model.replace('/', '-')}"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Aviary
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = aviary("Tell me a joke.")
        """
        kwargs = {"use_prompt_format": self.use_prompt_format}
        if self.version:
            kwargs["version"] = self.version
        output = get_completions(
            model=self.model,
            prompt=prompt,
            **kwargs,
        )
        text = cast(str, output["generated_text"])
        if stop:


)
        text = cast(str, output["generated_text"])
        if stop:
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.loading
"""Base interface for loading large language model APIs."""
import json
from pathlib import Path
from typing import Union
import yaml
from langchain.llms import get_type_to_cls_dict
from langchain.llms.base import BaseLLM
[docs]def load_llm_from_config(config: dict) -> BaseLLM:
    """Load LLM from Config Dict."""
    if "_type" not in config:
        raise ValueError("Must specify an LLM Type in config")
    config_type = config.pop("_type")
    type_to_cls_dict = get_type_to_cls_dict()
    if config_type not in type_to_cls_dict:
        raise ValueError(f"Loading {config_type} LLM not supported")
    llm_cls = type_to_cls_dict[config_type]()
    return llm_cls(**config)
[docs]def load_llm(file: Union[str, Path]) -> BaseLLM:
    """Load LLM from file."""
    # Convert file to Path object.
    if isinstance(file, str):
        file_path = Path(file)
    else:
        file_path = file
    # Load from either json or yaml.
    if file_path.suffix == ".json":
        with open(file_path) as f:
            config = json.load(f)
    elif file_path.suffix == ".yaml":
        with open(file_path, "r") as f:
            config = yaml.safe_load(f)
    else:
        raise ValueError("File type must be json or yaml")
    # Load the LLM from the config now.
    return load_llm_from_config(config)


Source code for langchain.llms.aleph_alpha
from typing import Any, Dict, List, Optional, Sequence
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
[docs]class AlephAlpha(LLM):
    """Aleph Alpha large language models.
    To use, you should have the ``aleph_alpha_client`` python package installed, and the
    environment variable ``ALEPH_ALPHA_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Parameters are explained more in depth here:
    https://github.com/Aleph-Alpha/aleph-alpha-client/blob/c14b7dd2b4325c7da0d6a119f6e76385800e097b/aleph_alpha_client/completion.py#L10
    Example:
        .. code-block:: python
            from langchain.llms import AlephAlpha
            aleph_alpha = AlephAlpha(aleph_alpha_api_key="my-api-key")
    """
    client: Any  #: :meta private:
    model: Optional[str] = "luminous-base"
    """Model name to use."""
    maximum_tokens: int = 64
    """The maximum number of tokens to be generated."""
    temperature: float = 0.0
    """A non-negative float that tunes the degree of randomness in generation."""
    top_k: int = 0
    """Number of most likely tokens to consider at each step."""
    top_p: float = 0.0
    """Total probability mass of tokens to consider at each step."""


"""Total probability mass of tokens to consider at each step."""
    presence_penalty: float = 0.0
    """Penalizes repeated tokens."""
    frequency_penalty: float = 0.0
    """Penalizes repeated tokens according to frequency."""
    repetition_penalties_include_prompt: Optional[bool] = False
    """Flag deciding whether presence penalty or frequency penalty are
    updated from the prompt."""
    use_multiplicative_presence_penalty: Optional[bool] = False
    """Flag deciding whether presence penalty is applied
    multiplicatively (True) or additively (False)."""
    penalty_bias: Optional[str] = None
    """Penalty bias for the completion."""
    penalty_exceptions: Optional[List[str]] = None
    """List of strings that may be generated without penalty,
    regardless of other penalty settings"""
    penalty_exceptions_include_stop_sequences: Optional[bool] = None
    """Should stop_sequences be included in penalty_exceptions."""
    best_of: Optional[int] = None
    """returns the one with the "best of" results
    (highest log probability per token)
    """
    n: int = 1
    """How many completions to generate for each prompt."""
    logit_bias: Optional[Dict[int, float]] = None
    """The logit bias allows to influence the likelihood of generating tokens."""
    log_probs: Optional[int] = None
    """Number of top log probabilities to be returned for each generated token."""
    tokens: Optional[bool] = False
    """return tokens of completion."""
    disable_optimizations: Optional[bool] = False
    minimum_tokens: Optional[int] = 0
    """Generate at least this number of tokens."""
    echo: bool = False
    """Echo the prompt in the completion."""


echo: bool = False
    """Echo the prompt in the completion."""
    use_multiplicative_frequency_penalty: bool = False
    sequence_penalty: float = 0.0
    sequence_penalty_min_length: int = 2
    use_multiplicative_sequence_penalty: bool = False
    completion_bias_inclusion: Optional[Sequence[str]] = None
    completion_bias_inclusion_first_token_only: bool = False
    completion_bias_exclusion: Optional[Sequence[str]] = None
    completion_bias_exclusion_first_token_only: bool = False
    """Only consider the first token for the completion_bias_exclusion."""
    contextual_control_threshold: Optional[float] = None
    """If set to None, attention control parameters only apply to those tokens that have
    explicitly been set in the request.
    If set to a non-None value, control parameters are also applied to similar tokens.
    """
    control_log_additive: Optional[bool] = True
    """True: apply control by adding the log(control_factor) to attention scores.
    False: (attention_scores - - attention_scores.min(-1)) * control_factor
    """
    repetition_penalties_include_completion: bool = True
    """Flag deciding whether presence penalty or frequency penalty
    are updated from the completion."""
    raw_completion: bool = False
    """Force the raw completion of the model to be returned."""
    stop_sequences: Optional[List[str]] = None
    """Stop sequences to use."""
    # Client params
    aleph_alpha_api_key: Optional[str] = None
    """API key for Aleph Alpha API."""
    host: str = "https://api.aleph-alpha.com"
    """The hostname of the API host.
    The default one is "https://api.aleph-alpha.com")"""
    hosting: Optional[str] = None


hosting: Optional[str] = None
    """Determines in which datacenters the request may be processed.
    You can either set the parameter to "aleph-alpha" or omit it (defaulting to None).
    Not setting this value, or setting it to None, gives us maximal
    flexibility in processing your request in our
    own datacenters and on servers hosted with other providers.
    Choose this option for maximal availability.
    Setting it to "aleph-alpha" allows us to only process the
    request in our own datacenters.
    Choose this option for maximal data privacy."""
    request_timeout_seconds: int = 305
    """Client timeout that will be set for HTTP requests in the
    `requests` library's API calls.
    Server will close all requests after 300 seconds with an internal server error."""
    total_retries: int = 8
    """The number of retries made in case requests fail with certain retryable
    status codes. If the last
    retry fails a corresponding exception is raised. Note, that between retries
    an exponential backoff
    is applied, starting with 0.5 s after the first retry and doubling for
    each retry made. So with the
    default setting of 8 retries a total wait time of 63.5 s is added
    between the retries."""
    nice: bool = False
    """Setting this to True, will signal to the API that you intend to be
    nice to other users
    by de-prioritizing your request below concurrent ones."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""


"""Validate that api key and python package exists in environment."""
        values["aleph_alpha_api_key"] = convert_to_secret_str(
            get_from_dict_or_env(values, "aleph_alpha_api_key", "ALEPH_ALPHA_API_KEY")
        )
        try:
            from aleph_alpha_client import Client
            values["client"] = Client(
                token=values["aleph_alpha_api_key"].get_secret_value(),
                host=values["host"],
                hosting=values["hosting"],
                request_timeout_seconds=values["request_timeout_seconds"],
                total_retries=values["total_retries"],
                nice=values["nice"],
            )
        except ImportError:
            raise ImportError(
                "Could not import aleph_alpha_client python package. "
                "Please install it with `pip install aleph_alpha_client`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling the Aleph Alpha API."""
        return {
            "maximum_tokens": self.maximum_tokens,
            "temperature": self.temperature,
            "top_k": self.top_k,
            "top_p": self.top_p,
            "presence_penalty": self.presence_penalty,
            "frequency_penalty": self.frequency_penalty,
            "n": self.n,
            "repetition_penalties_include_prompt": self.repetition_penalties_include_prompt,  # noqa: E501
            "use_multiplicative_presence_penalty": self.use_multiplicative_presence_penalty,  # noqa: E501
            "penalty_bias": self.penalty_bias,
            "penalty_exceptions": self.penalty_exceptions,
            "penalty_exceptions_include_stop_sequences": self.penalty_exceptions_include_stop_sequences,  # noqa: E501


"best_of": self.best_of,
            "logit_bias": self.logit_bias,
            "log_probs": self.log_probs,
            "tokens": self.tokens,
            "disable_optimizations": self.disable_optimizations,
            "minimum_tokens": self.minimum_tokens,
            "echo": self.echo,
            "use_multiplicative_frequency_penalty": self.use_multiplicative_frequency_penalty,  # noqa: E501
            "sequence_penalty": self.sequence_penalty,
            "sequence_penalty_min_length": self.sequence_penalty_min_length,
            "use_multiplicative_sequence_penalty": self.use_multiplicative_sequence_penalty,  # noqa: E501
            "completion_bias_inclusion": self.completion_bias_inclusion,
            "completion_bias_inclusion_first_token_only": self.completion_bias_inclusion_first_token_only,  # noqa: E501
            "completion_bias_exclusion": self.completion_bias_exclusion,
            "completion_bias_exclusion_first_token_only": self.completion_bias_exclusion_first_token_only,  # noqa: E501
            "contextual_control_threshold": self.contextual_control_threshold,
            "control_log_additive": self.control_log_additive,
            "repetition_penalties_include_completion": self.repetition_penalties_include_completion,  # noqa: E501
            "raw_completion": self.raw_completion,
        }
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model": self.model}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "aleph_alpha"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,


prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Aleph Alpha's completion endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = aleph_alpha("Tell me a joke.")
        """
        from aleph_alpha_client import CompletionRequest, Prompt
        params = self._default_params
        if self.stop_sequences is not None and stop is not None:
            raise ValueError(
                "stop sequences found in both the input and default params."
            )
        elif self.stop_sequences is not None:
            params["stop_sequences"] = self.stop_sequences
        else:
            params["stop_sequences"] = stop
        params = {**params, **kwargs}
        request = CompletionRequest(prompt=Prompt.from_text(prompt), **params)
        response = self.client.complete(model=self.model, request=request)
        text = response.completions[0].completion
        # If stop tokens are provided, Aleph Alpha's endpoint returns them.
        # In order to make this consistent with other endpoints, we strip them.
        if stop is not None or self.stop_sequences is not None:
            text = enforce_stop_tokens(text, params["stop_sequences"])
        return text
if __name__ == "__main__":
    aa = AlephAlpha()
    print(aa("How are you?"))


Source code for langchain.llms.vllm
from typing import Any, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import BaseLLM
from langchain.llms.openai import BaseOpenAI
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema.output import Generation, LLMResult
[docs]class VLLM(BaseLLM):
    """VLLM language model."""
    model: str = ""
    """The name or path of a HuggingFace Transformers model."""
    tensor_parallel_size: Optional[int] = 1
    """The number of GPUs to use for distributed execution with tensor parallelism."""
    trust_remote_code: Optional[bool] = False
    """Trust remote code (e.g., from HuggingFace) when downloading the model
    and tokenizer."""
    n: int = 1
    """Number of output sequences to return for the given prompt."""
    best_of: Optional[int] = None
    """Number of output sequences that are generated from the prompt."""
    presence_penalty: float = 0.0
    """Float that penalizes new tokens based on whether they appear in the
    generated text so far"""
    frequency_penalty: float = 0.0
    """Float that penalizes new tokens based on their frequency in the
    generated text so far"""
    temperature: float = 1.0
    """Float that controls the randomness of the sampling."""
    top_p: float = 1.0
    """Float that controls the cumulative probability of the top tokens to consider."""
    top_k: int = -1
    """Integer that controls the number of top tokens to consider."""
    use_beam_search: bool = False
    """Whether to use beam search instead of sampling."""


"""Whether to use beam search instead of sampling."""
    stop: Optional[List[str]] = None
    """List of strings that stop the generation when they are generated."""
    ignore_eos: bool = False
    """Whether to ignore the EOS token and continue generating tokens after
    the EOS token is generated."""
    max_new_tokens: int = 512
    """Maximum number of tokens to generate per output sequence."""
    logprobs: Optional[int] = None
    """Number of log probabilities to return per output token."""
    dtype: str = "auto"
    """The data type for the model weights and activations."""
    download_dir: Optional[str] = None
    """Directory to download and load the weights. (Default to the default
    cache dir of huggingface)"""
    vllm_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `vllm.LLM` call not explicitly specified."""
    client: Any  #: :meta private:
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that python package exists in environment."""
        try:
            from vllm import LLM as VLLModel
        except ImportError:
            raise ImportError(
                "Could not import vllm python package. "
                "Please install it with `pip install vllm`."
            )
        values["client"] = VLLModel(
            model=values["model"],
            tensor_parallel_size=values["tensor_parallel_size"],
            trust_remote_code=values["trust_remote_code"],
            dtype=values["dtype"],
            download_dir=values["download_dir"],
            **values["vllm_kwargs"],
        )
        return values
    @property


)
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling vllm."""
        return {
            "n": self.n,
            "best_of": self.best_of,
            "max_tokens": self.max_new_tokens,
            "top_k": self.top_k,
            "top_p": self.top_p,
            "temperature": self.temperature,
            "presence_penalty": self.presence_penalty,
            "frequency_penalty": self.frequency_penalty,
            "stop": self.stop,
            "ignore_eos": self.ignore_eos,
            "use_beam_search": self.use_beam_search,
            "logprobs": self.logprobs,
        }
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        """Run the LLM on the given prompt and input."""
        from vllm import SamplingParams
        # build sampling parameters
        params = {**self._default_params, **kwargs, "stop": stop}
        sampling_params = SamplingParams(**params)
        # call the model
        outputs = self.client.generate(prompts, sampling_params)
        generations = []
        for output in outputs:
            text = output.outputs[0].text
            generations.append([Generation(text=text)])
        return LLMResult(generations=generations)
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "vllm"
[docs]class VLLMOpenAI(BaseOpenAI):


[docs]class VLLMOpenAI(BaseOpenAI):
    """vLLM OpenAI-compatible API client"""
    @property
    def _invocation_params(self) -> Dict[str, Any]:
        """Get the parameters used to invoke the model."""
        openai_creds: Dict[str, Any] = {
            "api_key": self.openai_api_key,
            "api_base": self.openai_api_base,
        }
        return {
            "model": self.model_name,
            **openai_creds,
            **self._default_params,
            "logit_bias": None,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "vllm-openai"


Source code for langchain.llms.gooseai
import logging
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Extra, Field, SecretStr, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class GooseAI(LLM):
    """GooseAI large language models.
    To use, you should have the ``openai`` python package installed, and the
    environment variable ``GOOSEAI_API_KEY`` set with your API key.
    Any parameters that are valid to be passed to the openai.create call can be passed
    in, even if not explicitly saved on this class.
    Example:
        .. code-block:: python
            from langchain.llms import GooseAI
            gooseai = GooseAI(model_name="gpt-neo-20b")
    """
    client: Any
    model_name: str = "gpt-neo-20b"
    """Model name to use"""
    temperature: float = 0.7
    """What sampling temperature to use"""
    max_tokens: int = 256
    """The maximum number of tokens to generate in the completion.
    -1 returns as many tokens as possible given the prompt and
    the models maximal context size."""
    top_p: float = 1
    """Total probability mass of tokens to consider at each step."""
    min_tokens: int = 1
    """The minimum number of tokens to generate in the completion."""
    frequency_penalty: float = 0
    """Penalizes repeated tokens according to frequency."""
    presence_penalty: float = 0
    """Penalizes repeated tokens."""


presence_penalty: float = 0
    """Penalizes repeated tokens."""
    n: int = 1
    """How many completions to generate for each prompt."""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not explicitly specified."""
    logit_bias: Optional[Dict[str, float]] = Field(default_factory=dict)
    """Adjust the probability of specific tokens being generated."""
    gooseai_api_key: Optional[SecretStr] = None
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.ignore
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""WARNING! {field_name} is not default parameter.
                    {field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        gooseai_api_key = convert_to_secret_str(
            get_from_dict_or_env(values, "gooseai_api_key", "GOOSEAI_API_KEY")


)
        values["gooseai_api_key"] = gooseai_api_key
        try:
            import openai
            openai.api_key = gooseai_api_key.get_secret_value()
            openai.api_base = "https://api.goose.ai/v1"
            values["client"] = openai.Completion
        except ImportError:
            raise ImportError(
                "Could not import openai python package. "
                "Please install it with `pip install openai`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling GooseAI API."""
        normal_params = {
            "temperature": self.temperature,
            "max_tokens": self.max_tokens,
            "top_p": self.top_p,
            "min_tokens": self.min_tokens,
            "frequency_penalty": self.frequency_penalty,
            "presence_penalty": self.presence_penalty,
            "n": self.n,
            "logit_bias": self.logit_bias,
        }
        return {**normal_params, **self.model_kwargs}
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "gooseai"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the GooseAI API."""
        params = self._default_params


"""Call the GooseAI API."""
        params = self._default_params
        if stop is not None:
            if "stop" in params:
                raise ValueError("`stop` found in both the input and default params.")
            params["stop"] = stop
        params = {**params, **kwargs}
        response = self.client.create(engine=self.model_name, prompt=prompt, **params)
        text = response.choices[0].text
        return text


Source code for langchain.llms.koboldai
import logging
from typing import Any, Dict, List, Optional
import requests
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
logger = logging.getLogger(__name__)
[docs]def clean_url(url: str) -> str:
    """Remove trailing slash and /api from url if present."""
    if url.endswith("/api"):
        return url[:-4]
    elif url.endswith("/"):
        return url[:-1]
    else:
        return url
[docs]class KoboldApiLLM(LLM):
    """Kobold API language model.
    It includes several fields that can be used to control the text generation process.
    To use this class, instantiate it with the required parameters and call it with a
    prompt to generate text. For example:
        kobold = KoboldApiLLM(endpoint="http://localhost:5000")
        result = kobold("Write a story about a dragon.")
    This will send a POST request to the Kobold API with the provided prompt and
    generate text.
    """
    endpoint: str
    """The API endpoint to use for generating text."""
    use_story: Optional[bool] = False
    """ Whether or not to use the story from the KoboldAI GUI when generating text. """
    use_authors_note: Optional[bool] = False
    """Whether to use the author's note from the KoboldAI GUI when generating text.

    This has no effect unless use_story is also enabled.
    """
    use_world_info: Optional[bool] = False
    """Whether to use the world info from the KoboldAI GUI when generating text."""
    use_memory: Optional[bool] = False


use_memory: Optional[bool] = False
    """Whether to use the memory from the KoboldAI GUI when generating text."""
    max_context_length: Optional[int] = 1600
    """Maximum number of tokens to send to the model.

    minimum: 1
    """
    max_length: Optional[int] = 80
    """Number of tokens to generate.

    maximum: 512
    minimum: 1
    """
    rep_pen: Optional[float] = 1.12
    """Base repetition penalty value.

    minimum: 1
    """
    rep_pen_range: Optional[int] = 1024
    """Repetition penalty range.

    minimum: 0
    """
    rep_pen_slope: Optional[float] = 0.9
    """Repetition penalty slope.

    minimum: 0
    """
    temperature: Optional[float] = 0.6
    """Temperature value.

    exclusiveMinimum: 0
    """
    tfs: Optional[float] = 0.9
    """Tail free sampling value.

    maximum: 1
    minimum: 0
    """
    top_a: Optional[float] = 0.9
    """Top-a sampling value.

    minimum: 0
    """
    top_p: Optional[float] = 0.95
    """Top-p sampling value.

    maximum: 1
    minimum: 0
    """
    top_k: Optional[int] = 0
    """Top-k sampling value.

    minimum: 0
    """
    typical: Optional[float] = 0.5
    """Typical sampling value.

    maximum: 1


"""Typical sampling value.

    maximum: 1
    minimum: 0
    """
    @property
    def _llm_type(self) -> str:
        return "koboldai"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the API and return the output.
        Args:
            prompt: The prompt to use for generation.
            stop: A list of strings to stop generation when encountered.
        Returns:
            The generated text.
        Example:
            .. code-block:: python
                from langchain.llms import KoboldApiLLM
                llm = KoboldApiLLM(endpoint="http://localhost:5000")
                llm("Write a story about dragons.")
        """
        data: Dict[str, Any] = {
            "prompt": prompt,
            "use_story": self.use_story,
            "use_authors_note": self.use_authors_note,
            "use_world_info": self.use_world_info,
            "use_memory": self.use_memory,
            "max_context_length": self.max_context_length,
            "max_length": self.max_length,
            "rep_pen": self.rep_pen,
            "rep_pen_range": self.rep_pen_range,
            "rep_pen_slope": self.rep_pen_slope,
            "temperature": self.temperature,
            "tfs": self.tfs,
            "top_a": self.top_a,
            "top_p": self.top_p,
            "top_k": self.top_k,
            "typical": self.typical,
        }


"typical": self.typical,
        }
        if stop is not None:
            data["stop_sequence"] = stop
        response = requests.post(
            f"{clean_url(self.endpoint)}/api/v1/generate", json=data
        )
        response.raise_for_status()
        json_response = response.json()
        if (
            "results" in json_response
            and len(json_response["results"]) > 0
            and "text" in json_response["results"][0]
        ):
            text = json_response["results"][0]["text"].strip()
            if stop is not None:
                for sequence in stop:
                    if text.endswith(sequence):
                        text = text[: -len(sequence)].rstrip()
            return text
        else:
            raise ValueError(
                f"Unexpected response format from Kobold API:  {json_response}"
            )


Source code for langchain.llms.predictionguard
import logging
from typing import Any, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class PredictionGuard(LLM):
    """Prediction Guard large language models.
    To use, you should have the ``predictionguard`` python package installed, and the
    environment variable ``PREDICTIONGUARD_TOKEN`` set with your access token, or pass
    it as a named parameter to the constructor. To use Prediction Guard's API along
    with OpenAI models, set the environment variable ``OPENAI_API_KEY`` with your
    OpenAI API key as well.
    Example:
        .. code-block:: python
            pgllm = PredictionGuard(model="MPT-7B-Instruct",
                                    token="my-access-token",
                                    output={
                                        "type": "boolean"
                                    })
    """
    client: Any  #: :meta private:
    model: Optional[str] = "MPT-7B-Instruct"
    """Model name to use."""
    output: Optional[Dict[str, Any]] = None
    """The output type or structure for controlling the LLM output."""
    max_tokens: int = 256
    """Denotes the number of tokens to predict per generation."""
    temperature: float = 0.75
    """A non-negative float that tunes the degree of randomness in generation."""
    token: Optional[str] = None
    """Your Prediction Guard access token."""
    stop: Optional[List[str]] = None
    class Config:


stop: Optional[List[str]] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that the access token and python package exists in environment."""
        token = get_from_dict_or_env(values, "token", "PREDICTIONGUARD_TOKEN")
        try:
            import predictionguard as pg
            values["client"] = pg.Client(token=token)
        except ImportError:
            raise ImportError(
                "Could not import predictionguard python package. "
                "Please install it with `pip install predictionguard`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling the Prediction Guard API."""
        return {
            "max_tokens": self.max_tokens,
            "temperature": self.temperature,
        }
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model": self.model}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "predictionguard"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Prediction Guard's model API.
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python


The string generated by the model.
        Example:
            .. code-block:: python
                response = pgllm("Tell me a joke.")
        """
        import predictionguard as pg
        params = self._default_params
        if self.stop is not None and stop is not None:
            raise ValueError("`stop` found in both the input and default params.")
        elif self.stop is not None:
            params["stop_sequences"] = self.stop
        else:
            params["stop_sequences"] = stop
        response = pg.Completion.create(
            model=self.model,
            prompt=prompt,
            output=self.output,
            temperature=params["temperature"],
            max_tokens=params["max_tokens"],
            **kwargs,
        )
        text = response["choices"][0]["text"]
        # If stop tokens are provided, Prediction Guard's endpoint returns them.
        # In order to make this consistent with other endpoints, we strip them.
        if stop is not None or self.stop is not None:
            text = enforce_stop_tokens(text, params["stop_sequences"])
        return text


Source code for langchain.llms.ctranslate2
from typing import Any, Dict, List, Optional, Union
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import BaseLLM
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema.output import Generation, LLMResult
[docs]class CTranslate2(BaseLLM):
    """CTranslate2 language model."""
    model_path: str = ""
    """Path to the CTranslate2 model directory."""
    tokenizer_name: str = ""
    """Name of the original Hugging Face model needed to load the proper tokenizer."""
    device: str = "cpu"
    """Device to use (possible values are: cpu, cuda, auto)."""
    device_index: Union[int, List[int]] = 0
    """Device IDs where to place this generator on."""
    compute_type: Union[str, Dict[str, str]] = "default"
    """
    Model computation type or a dictionary mapping a device name to the computation type
    (possible values are: default, auto, int8, int8_float32, int8_float16,
    int8_bfloat16, int16, float16, bfloat16, float32).
    """
    max_length: int = 512
    """Maximum generation length."""
    sampling_topk: int = 1
    """Randomly sample predictions from the top K candidates."""
    sampling_topp: float = 1
    """Keep the most probable tokens whose cumulative probability exceeds this value."""
    sampling_temperature: float = 1
    """Sampling temperature to generate more random samples."""
    client: Any  #: :meta private:
    tokenizer: Any  #: :meta private:


tokenizer: Any  #: :meta private:
    ctranslate2_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """
    Holds any model parameters valid for `ctranslate2.Generator` call not
    explicitly specified.
    """
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that python package exists in environment."""
        try:
            import ctranslate2
        except ImportError:
            raise ImportError(
                "Could not import ctranslate2 python package. "
                "Please install it with `pip install ctranslate2`."
            )
        try:
            import transformers
        except ImportError:
            raise ImportError(
                "Could not import transformers python package. "
                "Please install it with `pip install transformers`."
            )
        values["client"] = ctranslate2.Generator(
            model_path=values["model_path"],
            device=values["device"],
            device_index=values["device_index"],
            compute_type=values["compute_type"],
            **values["ctranslate2_kwargs"],
        )
        values["tokenizer"] = transformers.AutoTokenizer.from_pretrained(
            values["tokenizer_name"]
        )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters."""
        return {
            "max_length": self.max_length,
            "sampling_topk": self.sampling_topk,
            "sampling_topp": self.sampling_topp,
            "sampling_temperature": self.sampling_temperature,
        }
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,


prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        # build sampling parameters
        params = {**self._default_params, **kwargs}
        # call the model
        encoded_prompts = self.tokenizer(prompts)["input_ids"]
        tokenized_prompts = [
            self.tokenizer.convert_ids_to_tokens(encoded_prompt)
            for encoded_prompt in encoded_prompts
        ]
        results = self.client.generate_batch(tokenized_prompts, **params)
        sequences = [result.sequences_ids[0] for result in results]
        decoded_sequences = [self.tokenizer.decode(seq) for seq in sequences]
        generations = []
        for text in decoded_sequences:
            generations.append([Generation(text=text)])
        return LLMResult(generations=generations)
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "ctranslate2"


Source code for langchain.llms.self_hosted_hugging_face
import importlib.util
import logging
from typing import Any, Callable, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.self_hosted import SelfHostedPipeline
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra
DEFAULT_MODEL_ID = "gpt2"
DEFAULT_TASK = "text-generation"
VALID_TASKS = ("text2text-generation", "text-generation", "summarization")
logger = logging.getLogger(__name__)
def _generate_text(
    pipeline: Any,
    prompt: str,
    *args: Any,
    stop: Optional[List[str]] = None,
    **kwargs: Any,
) -> str:
    """Inference function to send to the remote hardware.
    Accepts a Hugging Face pipeline (or more likely,
    a key pointing to such a pipeline on the cluster's object store)
    and returns generated text.
    """
    response = pipeline(prompt, *args, **kwargs)
    if pipeline.task == "text-generation":
        # Text generation return includes the starter text.
        text = response[0]["generated_text"][len(prompt) :]
    elif pipeline.task == "text2text-generation":
        text = response[0]["generated_text"]
    elif pipeline.task == "summarization":
        text = response[0]["summary_text"]
    else:
        raise ValueError(
            f"Got invalid task {pipeline.task}, "
            f"currently only {VALID_TASKS} are supported"
        )
    if stop is not None:
        text = enforce_stop_tokens(text, stop)
    return text
def _load_transformer(


return text
def _load_transformer(
    model_id: str = DEFAULT_MODEL_ID,
    task: str = DEFAULT_TASK,
    device: int = 0,
    model_kwargs: Optional[dict] = None,
) -> Any:
    """Inference function to send to the remote hardware.
    Accepts a huggingface model_id and returns a pipeline for the task.
    """
    from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer
    from transformers import pipeline as hf_pipeline
    _model_kwargs = model_kwargs or {}
    tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
    try:
        if task == "text-generation":
            model = AutoModelForCausalLM.from_pretrained(model_id, **_model_kwargs)
        elif task in ("text2text-generation", "summarization"):
            model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **_model_kwargs)
        else:
            raise ValueError(
                f"Got invalid task {task}, "
                f"currently only {VALID_TASKS} are supported"
            )
    except ImportError as e:
        raise ValueError(
            f"Could not load the {task} model due to missing dependencies."
        ) from e
    if importlib.util.find_spec("torch") is not None:
        import torch
        cuda_device_count = torch.cuda.device_count()
        if device < -1 or (device >= cuda_device_count):
            raise ValueError(
                f"Got device=={device}, "
                f"device is required to be within [-1, {cuda_device_count})"
            )
        if device < 0 and cuda_device_count > 0:
            logger.warning(


if device < 0 and cuda_device_count > 0:
            logger.warning(
                "Device has %d GPUs available. "
                "Provide device={deviceId} to `from_model_id` to use available"
                "GPUs for execution. deviceId is -1 for CPU and "
                "can be a positive integer associated with CUDA device id.",
                cuda_device_count,
            )
    pipeline = hf_pipeline(
        task=task,
        model=model,
        tokenizer=tokenizer,
        device=device,
        model_kwargs=_model_kwargs,
    )
    if pipeline.task not in VALID_TASKS:
        raise ValueError(
            f"Got invalid task {pipeline.task}, "
            f"currently only {VALID_TASKS} are supported"
        )
    return pipeline
[docs]class SelfHostedHuggingFaceLLM(SelfHostedPipeline):
    """HuggingFace Pipeline API to run on self-hosted remote hardware.
    Supported hardware includes auto-launched instances on AWS, GCP, Azure,
    and Lambda, as well as servers specified
    by IP address and SSH credentials (such as on-prem, or another cloud
    like Paperspace, Coreweave, etc.).
    To use, you should have the ``runhouse`` python package installed.
    Only supports `text-generation`, `text2text-generation` and `summarization` for now.
    Example using from_model_id:
        .. code-block:: python
            from langchain.llms import SelfHostedHuggingFaceLLM
            import runhouse as rh
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            hf = SelfHostedHuggingFaceLLM(
                model_id="google/flan-t5-large", task="text2text-generation",


model_id="google/flan-t5-large", task="text2text-generation",
                hardware=gpu
            )
    Example passing fn that generates a pipeline (bc the pipeline is not serializable):
        .. code-block:: python
            from langchain.llms import SelfHostedHuggingFaceLLM
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            import runhouse as rh
            def get_pipeline():
                model_id = "gpt2"
                tokenizer = AutoTokenizer.from_pretrained(model_id)
                model = AutoModelForCausalLM.from_pretrained(model_id)
                pipe = pipeline(
                    "text-generation", model=model, tokenizer=tokenizer
                )
                return pipe
            hf = SelfHostedHuggingFaceLLM(
                model_load_fn=get_pipeline, model_id="gpt2", hardware=gpu)
    """
    model_id: str = DEFAULT_MODEL_ID
    """Hugging Face model_id to load the model."""
    task: str = DEFAULT_TASK
    """Hugging Face task ("text-generation", "text2text-generation" or
    "summarization")."""
    device: int = 0
    """Device to use for inference. -1 for CPU, 0 for GPU, 1 for second GPU, etc."""
    model_kwargs: Optional[dict] = None
    """Keyword arguments to pass to the model."""
    hardware: Any
    """Remote hardware to send the inference function to."""
    model_reqs: List[str] = ["./", "transformers", "torch"]
    """Requirements to install on hardware to inference the model."""
    model_load_fn: Callable = _load_transformer
    """Function to load the model remotely on the server."""
    inference_fn: Callable = _generate_text  #: :meta private:


inference_fn: Callable = _generate_text  #: :meta private:
    """Inference function to send to the remote hardware."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    def __init__(self, **kwargs: Any):
        """Construct the pipeline remotely using an auxiliary function.
        The load function needs to be importable to be imported
        and run on the server, i.e. in a module and not a REPL or closure.
        Then, initialize the remote inference function.
        """
        load_fn_kwargs = {
            "model_id": kwargs.get("model_id", DEFAULT_MODEL_ID),
            "task": kwargs.get("task", DEFAULT_TASK),
            "device": kwargs.get("device", 0),
            "model_kwargs": kwargs.get("model_kwargs", None),
        }
        super().__init__(load_fn_kwargs=load_fn_kwargs, **kwargs)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_id": self.model_id},
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        return "selfhosted_huggingface_pipeline"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        return self.client(
            pipeline=self.pipeline_ref, prompt=prompt, stop=stop, **kwargs
        )


Source code for langchain.llms.replicate
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Extra, Field, root_validator
from langchain.schema.output import GenerationChunk
from langchain.utils import get_from_dict_or_env
if TYPE_CHECKING:
    from replicate.prediction import Prediction
logger = logging.getLogger(__name__)
[docs]class Replicate(LLM):
    """Replicate models.
    To use, you should have the ``replicate`` python package installed,
    and the environment variable ``REPLICATE_API_TOKEN`` set with your API token.
    You can find your token here: https://replicate.com/account
    The model param is required, but any other model parameters can also
    be passed in with the format model_kwargs={model_param: value, ...}
    Example:
        .. code-block:: python
            from langchain.llms import Replicate
            replicate = Replicate(
                model=(
                    "stability-ai/stable-diffusion: "
                    "27b93a2413e7f36cd83da926f3656280b2931564ff050bf9575f1fdf9bcd7478",
                ),
                model_kwargs={"image_dimensions": "512x512"}
            )
    """
    model: str
    model_kwargs: Dict[str, Any] = Field(default_factory=dict, alias="input")
    replicate_api_token: Optional[str] = None
    prompt_key: Optional[str] = None
    version_obj: Any = Field(default=None, exclude=True)
    """Optionally pass in the model version object during initialization to avoid


"""Optionally pass in the model version object during initialization to avoid
        having to make an extra API call to retrieve it during streaming. NOTE: not
        serializable, is excluded from serialization.
    """
    streaming: bool = False
    """Whether to stream the results."""
    stop: List[str] = Field(default_factory=list)
    """Stop sequences to early-terminate generation."""
    class Config:
        """Configuration for this pydantic config."""
        allow_population_by_field_name = True
        extra = Extra.forbid
    @property
    def lc_secrets(self) -> Dict[str, str]:
        return {"replicate_api_token": "REPLICATE_API_TOKEN"}
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return True
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        input = values.pop("input", {})
        if input:
            logger.warning(
                "Init param `input` is deprecated, please use `model_kwargs` instead."
            )
        extra = {**values.pop("model_kwargs", {}), **input}
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""{field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values


values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        replicate_api_token = get_from_dict_or_env(
            values, "replicate_api_token", "REPLICATE_API_TOKEN"
        )
        values["replicate_api_token"] = replicate_api_token
        return values
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {
            "model": self.model,
            "model_kwargs": self.model_kwargs,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of model."""
        return "replicate"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to replicate endpoint."""
        if self.streaming:
            completion: Optional[str] = None
            for chunk in self._stream(
                prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                if completion is None:
                    completion = chunk.text
                else:
                    completion += chunk.text
        else:
            prediction = self._create_prediction(prompt, **kwargs)
            prediction.wait()
            if prediction.status == "failed":
                raise RuntimeError(prediction.error)
            if isinstance(prediction.output, str):
                completion = prediction.output
            else:
                completion = "".join(prediction.output)
        assert completion is not None
        stop_conditions = stop or self.stop
        for s in stop_conditions:


stop_conditions = stop or self.stop
        for s in stop_conditions:
            if s in completion:
                completion = completion[: completion.find(s)]
        return completion
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        prediction = self._create_prediction(prompt, **kwargs)
        stop_conditions = stop or self.stop
        stop_condition_reached = False
        current_completion: str = ""
        for output in prediction.output_iterator():
            current_completion += output
            # test for stop conditions, if specified
            for s in stop_conditions:
                if s in current_completion:
                    prediction.cancel()
                    stop_condition_reached = True
                    # Potentially some tokens that should still be yielded before ending
                    # stream.
                    stop_index = max(output.find(s), 0)
                    output = output[:stop_index]
                    if not output:
                        break
            if output:
                yield GenerationChunk(text=output)
                if run_manager:
                    run_manager.on_llm_new_token(
                        output,
                        verbose=self.verbose,
                    )
            if stop_condition_reached:
                break
    def _create_prediction(self, prompt: str, **kwargs: Any) -> Prediction:
        try:
            import replicate as replicate_python
        except ImportError:
            raise ImportError(
                "Could not import replicate python package. "
                "Please install it with `pip install replicate`."
            )
        # get the model and version
        if self.version_obj is None:
            model_str, version_str = self.model.split(":")


model_str, version_str = self.model.split(":")
            model = replicate_python.models.get(model_str)
            self.version_obj = model.versions.get(version_str)
        if self.prompt_key is None:
            # sort through the openapi schema to get the name of the first input
            input_properties = sorted(
                self.version_obj.openapi_schema["components"]["schemas"]["Input"][
                    "properties"
                ].items(),
                key=lambda item: item[1].get("x-order", 0),
            )
            self.prompt_key = input_properties[0][0]
        input_: Dict = {
            self.prompt_key: prompt,
            **self.model_kwargs,
            **kwargs,
        }
        return replicate_python.predictions.create(
            version=self.version_obj, input=input_
        )


Source code for langchain.llms.octoai_endpoint
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, root_validator
from langchain.utils import get_from_dict_or_env
[docs]class OctoAIEndpoint(LLM):
    """OctoAI LLM Endpoints.
    OctoAIEndpoint is a class to interact with OctoAI
     Compute Service large language model endpoints.
    To use, you should have the ``octoai`` python package installed, and the
    environment variable ``OCTOAI_API_TOKEN`` set with your API token, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms.octoai_endpoint  import OctoAIEndpoint
            OctoAIEndpoint(
                octoai_api_token="octoai-api-key",
                endpoint_url="https://mpt-7b-demo-f1kzsig6xes9.octoai.run/generate",
                model_kwargs={
                    "max_new_tokens": 200,
                    "temperature": 0.75,
                    "top_p": 0.95,
                    "repetition_penalty": 1,
                    "seed": None,
                    "stop": [],
                },
            )
            from langchain.llms.octoai_endpoint  import OctoAIEndpoint
            OctoAIEndpoint(
                octoai_api_token="octoai-api-key",


octoai_api_token="octoai-api-key",
                endpoint_url="https://llama-2-7b-chat-demo-kk0powt97tmb.octoai.run/v1/chat/completions",
                model_kwargs={
                    "model": "llama-2-7b-chat",
                    "messages": [
                        {
                            "role": "system",
                            "content": "Below is an instruction that describes a task.
                                Write a response that completes the request."
                        }
                    ],
                    "stream": False,
                    "max_tokens": 256
                }
            )
    """
    endpoint_url: Optional[str] = None
    """Endpoint URL to use."""
    model_kwargs: Optional[dict] = None
    """Keyword arguments to pass to the model."""
    octoai_api_token: Optional[str] = None
    """OCTOAI API Token"""
    streaming: bool = False
    """Whether to generate a stream of tokens asynchronously"""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator(allow_reuse=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        octoai_api_token = get_from_dict_or_env(
            values, "octoai_api_token", "OCTOAI_API_TOKEN"
        )
        values["endpoint_url"] = get_from_dict_or_env(
            values, "endpoint_url", "ENDPOINT_URL"
        )
        values["octoai_api_token"] = octoai_api_token
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""


"""Get the identifying parameters."""
        _model_kwargs = self.model_kwargs or {}
        return {
            **{"endpoint_url": self.endpoint_url},
            **{"model_kwargs": _model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "octoai_endpoint"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to OctoAI's inference endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        """
        _model_kwargs = self.model_kwargs or {}
        try:
            # Initialize the OctoAI client
            from octoai import client
            octoai_client = client.Client(token=self.octoai_api_token)
            if "model" in _model_kwargs:
                parameter_payload = _model_kwargs
                parameter_payload["messages"].append(
                    {"role": "user", "content": prompt}
                )
                # Send the request using the OctoAI client
                output = octoai_client.infer(self.endpoint_url, parameter_payload)
                text = output.get("choices")[0].get("message").get("content")
            else:
                # Prepare the payload JSON
                parameter_payload = {"inputs": prompt, "parameters": _model_kwargs}
                # Send the request using the OctoAI client
                resp_json = octoai_client.infer(self.endpoint_url, parameter_payload)


resp_json = octoai_client.infer(self.endpoint_url, parameter_payload)
                text = resp_json["generated_text"]
        except Exception as e:
            # Handle any errors raised by the inference endpoint
            raise ValueError(f"Error raised by the inference endpoint: {e}") from e
        if stop is not None:
            # Apply stop tokens when making calls to OctoAI
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.baidu_qianfan_endpoint
from __future__ import annotations
import logging
from typing import (
    Any,
    AsyncIterator,
    Dict,
    Iterator,
    List,
    Optional,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema.output import GenerationChunk
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class QianfanLLMEndpoint(LLM):
    """Baidu Qianfan hosted open source or customized models.
    To use, you should have the ``qianfan`` python package installed, and
    the environment variable ``qianfan_ak`` and ``qianfan_sk`` set with
    your API key and Secret Key.
    ak, sk are required parameters which you could get from
    https://cloud.baidu.com/product/wenxinworkshop
    Example:
        .. code-block:: python
            from langchain.llms import QianfanLLMEndpoint
            qianfan_model = QianfanLLMEndpoint(model="ERNIE-Bot",
                endpoint="your_endpoint", qianfan_ak="your_ak", qianfan_sk="your_sk")
    """
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    client: Any
    qianfan_ak: Optional[str] = None
    qianfan_sk: Optional[str] = None
    streaming: Optional[bool] = False
    """Whether to stream the results or not."""
    model: str = "ERNIE-Bot-turbo"
    """Model name.


"""Model name.
    you could get from https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Nlks5zkzu

    preset models are mapping to an endpoint.
    `model` will be ignored if `endpoint` is set
    """
    endpoint: Optional[str] = None
    """Endpoint of the Qianfan LLM, required if custom model used."""
    request_timeout: Optional[int] = 60
    """request timeout for chat http requests"""
    top_p: Optional[float] = 0.8
    temperature: Optional[float] = 0.95
    penalty_score: Optional[float] = 1
    """Model params, only supported in ERNIE-Bot and ERNIE-Bot-turbo.
    In the case of other model, passing these params will not affect the result.
    """
    @root_validator()
    def validate_enviroment(cls, values: Dict) -> Dict:
        values["qianfan_ak"] = get_from_dict_or_env(
            values,
            "qianfan_ak",
            "QIANFAN_AK",
        )
        values["qianfan_sk"] = get_from_dict_or_env(
            values,
            "qianfan_sk",
            "QIANFAN_SK",
        )
        params = {
            "ak": values["qianfan_ak"],
            "sk": values["qianfan_sk"],
            "model": values["model"],
        }
        if values["endpoint"] is not None and values["endpoint"] != "":
            params["endpoint"] = values["endpoint"]
        try:
            import qianfan
            values["client"] = qianfan.Completion(**params)
        except ImportError:
            raise ImportError(


except ImportError:
            raise ImportError(
                "qianfan package not found, please install it with "
                "`pip install qianfan`"
            )
        return values
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        return {
            **{"endpoint": self.endpoint, "model": self.model},
            **super()._identifying_params,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "baidu-qianfan-endpoint"
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling Qianfan API."""
        normal_params = {
            "model": self.model,
            "endpoint": self.endpoint,
            "stream": self.streaming,
            "request_timeout": self.request_timeout,
            "top_p": self.top_p,
            "temperature": self.temperature,
            "penalty_score": self.penalty_score,
        }
        return {**normal_params, **self.model_kwargs}
    def _convert_prompt_msg_params(
        self,
        prompt: str,
        **kwargs: Any,
    ) -> dict:
        if "streaming" in kwargs:
            kwargs["stream"] = kwargs.pop("streaming")
        return {
            **{"prompt": prompt, "model": self.model},
            **self._default_params,
            **kwargs,
        }
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:


**kwargs: Any,
    ) -> str:
        """Call out to an qianfan models endpoint for each generation with a prompt.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = qianfan_model("Tell me a joke.")
        """
        if self.streaming:
            completion = ""
            for chunk in self._stream(prompt, stop, run_manager, **kwargs):
                completion += chunk.text
            return completion
        params = self._convert_prompt_msg_params(prompt, **kwargs)
        response_payload = self.client.do(**params)
        return response_payload["result"]
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        if self.streaming:
            completion = ""
            async for chunk in self._astream(prompt, stop, run_manager, **kwargs):
                completion += chunk.text
            return completion
        params = self._convert_prompt_msg_params(prompt, **kwargs)
        response_payload = await self.client.ado(**params)
        return response_payload["result"]
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        params = self._convert_prompt_msg_params(prompt, **{**kwargs, "stream": True})


for res in self.client.do(**params):
            if res:
                chunk = GenerationChunk(text=res["result"])
                yield chunk
                if run_manager:
                    run_manager.on_llm_new_token(chunk.text)
    async def _astream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> AsyncIterator[GenerationChunk]:
        params = self._convert_prompt_msg_params(prompt, **{**kwargs, "stream": True})
        async for res in await self.client.ado(**params):
            if res:
                chunk = GenerationChunk(text=res["result"])
                yield chunk
                if run_manager:
                    await run_manager.on_llm_new_token(chunk.text)


Source code for langchain.llms.javelin_ai_gateway
from __future__ import annotations
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import BaseModel, Extra
# Ignoring type because below is valid pydantic code
# Unexpected keyword argument "extra" for "__init_subclass__" of "object"
[docs]class Params(BaseModel, extra=Extra.allow):  # type: ignore[call-arg]
    """Parameters for the Javelin AI Gateway LLM."""
    temperature: float = 0.0
    stop: Optional[List[str]] = None
    max_tokens: Optional[int] = None
[docs]class JavelinAIGateway(LLM):
    """Javelin AI Gateway LLMs.
    To use, you should have the ``javelin_sdk`` python package installed.
    For more information, see https://docs.getjavelin.io
    Example:
        .. code-block:: python
            from langchain.llms import JavelinAIGateway
            completions = JavelinAIGateway(
                gateway_uri="<your-javelin-ai-gateway-uri>",
                route="<your-javelin-ai-gateway-completions-route>",
                params={
                    "temperature": 0.1
                }
            )
    """
    route: str
    """The route to use for the Javelin AI Gateway API."""
    client: Optional[Any] = None
    """The Javelin AI Gateway client."""
    gateway_uri: Optional[str] = None
    """The URI of the Javelin AI Gateway API."""


"""The URI of the Javelin AI Gateway API."""
    params: Optional[Params] = None
    """Parameters for the Javelin AI Gateway API."""
    javelin_api_key: Optional[str] = None
    """The API key for the Javelin AI Gateway API."""
    def __init__(self, **kwargs: Any):
        try:
            from javelin_sdk import (
                JavelinClient,
                UnauthorizedError,
            )
        except ImportError:
            raise ImportError(
                "Could not import javelin_sdk python package. "
                "Please install it with `pip install javelin_sdk`."
            )
        super().__init__(**kwargs)
        if self.gateway_uri:
            try:
                self.client = JavelinClient(
                    base_url=self.gateway_uri, api_key=self.javelin_api_key
                )
            except UnauthorizedError as e:
                raise ValueError("Javelin: Incorrect API Key.") from e
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling Javelin AI Gateway API."""
        params: Dict[str, Any] = {
            "gateway_uri": self.gateway_uri,
            "route": self.route,
            "javelin_api_key": self.javelin_api_key,
            **(self.params.dict() if self.params else {}),
        }
        return params
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return self._default_params
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,


run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the Javelin AI Gateway API."""
        data: Dict[str, Any] = {
            "prompt": prompt,
            **(self.params.dict() if self.params else {}),
        }
        if s := (stop or (self.params.stop if self.params else None)):
            data["stop"] = s
        if self.client is not None:
            resp = self.client.query_route(self.route, query_body=data)
        else:
            raise ValueError("Javelin client is not initialized.")
        resp_dict = resp.dict()
        try:
            return resp_dict["llm_response"]["choices"][0]["text"]
        except KeyError:
            return ""
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call async the Javelin AI Gateway API."""
        data: Dict[str, Any] = {
            "prompt": prompt,
            **(self.params.dict() if self.params else {}),
        }
        if s := (stop or (self.params.stop if self.params else None)):
            data["stop"] = s
        if self.client is not None:
            resp = await self.client.aquery_route(self.route, query_body=data)
        else:
            raise ValueError("Javelin client is not initialized.")
        resp_dict = resp.dict()
        try:
            return resp_dict["llm_response"]["choices"][0]["text"]
        except KeyError:
            return ""
    @property


except KeyError:
            return ""
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "javelin-ai-gateway"


Source code for langchain.llms.self_hosted
import importlib.util
import logging
import pickle
from typing import Any, Callable, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra
logger = logging.getLogger(__name__)
def _generate_text(
    pipeline: Any,
    prompt: str,
    *args: Any,
    stop: Optional[List[str]] = None,
    **kwargs: Any,
) -> str:
    """Inference function to send to the remote hardware.
    Accepts a pipeline callable (or, more likely,
    a key pointing to the model on the cluster's object store)
    and returns text predictions for each document
    in the batch.
    """
    text = pipeline(prompt, *args, **kwargs)
    if stop is not None:
        text = enforce_stop_tokens(text, stop)
    return text
def _send_pipeline_to_device(pipeline: Any, device: int) -> Any:
    """Send a pipeline to a device on the cluster."""
    if isinstance(pipeline, str):
        with open(pipeline, "rb") as f:
            pipeline = pickle.load(f)
    if importlib.util.find_spec("torch") is not None:
        import torch
        cuda_device_count = torch.cuda.device_count()
        if device < -1 or (device >= cuda_device_count):
            raise ValueError(
                f"Got device=={device}, "
                f"device is required to be within [-1, {cuda_device_count})"
            )
        if device < 0 and cuda_device_count > 0:
            logger.warning(