return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_id": self.model_id},
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "deepinfra"
    def _url(self) -> str:
        return f"https://api.deepinfra.com/v1/inference/{self.model_id}"
    def _headers(self) -> Dict:
        return {
            "Authorization": f"bearer {self.deepinfra_api_token}",
            "Content-Type": "application/json",
        }
    def _body(self, prompt: str, kwargs: Any) -> Dict:
        model_kwargs = self.model_kwargs or {}
        model_kwargs = {**model_kwargs, **kwargs}
        return {
            "input": prompt,
            **model_kwargs,
        }
    def _handle_status(self, code: int, text: Any) -> None:
        if code >= 500:
            raise Exception(f"DeepInfra Server: Error {code}")
        elif code >= 400:
            raise ValueError(f"DeepInfra received an invalid payload: {text}")
        elif code != 200:
            raise Exception(
                f"DeepInfra returned an unexpected response with status "
                f"{code}: {text}"
            )
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:


**kwargs: Any,
    ) -> str:
        """Call out to DeepInfra's inference API endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = di("Tell me a joke.")
        """
        request = Requests(headers=self._headers())
        response = request.post(url=self._url(), data=self._body(prompt, kwargs))
        self._handle_status(response.status_code, response.text)
        data = response.json()
        return data["results"][0]["generated_text"]
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        request = Requests(headers=self._headers())
        async with request.apost(
            url=self._url(), data=self._body(prompt, kwargs)
        ) as response:
            self._handle_status(response.status, response.text)
            data = await response.json()
            return data["results"][0]["generated_text"]
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        request = Requests(headers=self._headers())
        response = request.post(
            url=self._url(), data=self._body(prompt, {**kwargs, "stream": True})
        )


)
        self._handle_status(response.status_code, response.text)
        for line in _parse_stream(response.iter_lines()):
            chunk = _handle_sse_line(line)
            if chunk:
                yield chunk
                if run_manager:
                    run_manager.on_llm_new_token(chunk.text)
    async def _astream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> AsyncIterator[GenerationChunk]:
        request = Requests(headers=self._headers())
        async with request.apost(
            url=self._url(), data=self._body(prompt, {**kwargs, "stream": True})
        ) as response:
            self._handle_status(response.status, response.text)
            async for line in _parse_stream_async(response.content):
                chunk = _handle_sse_line(line)
                if chunk:
                    yield chunk
                    if run_manager:
                        await run_manager.on_llm_new_token(chunk.text)
def _parse_stream(rbody: Iterator[bytes]) -> Iterator[str]:
    for line in rbody:
        _line = _parse_stream_helper(line)
        if _line is not None:
            yield _line
async def _parse_stream_async(rbody: aiohttp.StreamReader) -> AsyncIterator[str]:
    async for line in rbody:
        _line = _parse_stream_helper(line)
        if _line is not None:
            yield _line
def _parse_stream_helper(line: bytes) -> Optional[str]:
    if line and line.startswith(b"data:"):
        if line.startswith(b"data: "):
            # SSE event may be valid when it contain whitespace


# SSE event may be valid when it contain whitespace
            line = line[len(b"data: ") :]
        else:
            line = line[len(b"data:") :]
        if line.strip() == b"[DONE]":
            # return here will cause GeneratorExit exception in urllib3
            # and it will close http connection with TCP Reset
            return None
        else:
            return line.decode("utf-8")
    return None
def _handle_sse_line(line: str) -> Optional[GenerationChunk]:
    try:
        obj = json.loads(line)
        return GenerationChunk(
            text=obj.get("token", {}).get("text"),
        )
    except Exception:
        return None


Source code for langchain.llms.titan_takeoff_pro
from typing import Any, Iterator, List, Mapping, Optional
import requests
from requests.exceptions import ConnectionError
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.schema.output import GenerationChunk
[docs]class TitanTakeoffPro(LLM):
    base_url: Optional[str] = "http://localhost:3000"
    """Specifies the baseURL to use for the Titan Takeoff Pro API.
    Default = http://localhost:3000.
    """
    max_new_tokens: Optional[int] = None
    """Maximum tokens generated."""
    min_new_tokens: Optional[int] = None
    """Minimum tokens generated."""
    sampling_topk: Optional[int] = None
    """Sample predictions from the top K most probable candidates."""
    sampling_topp: Optional[float] = None
    """Sample from predictions whose cumulative probability exceeds this value.
    """
    sampling_temperature: Optional[float] = None
    """Sample with randomness. Bigger temperatures are associated with
    more randomness and 'creativity'.
    """
    repetition_penalty: Optional[float] = None
    """Penalise the generation of tokens that have been generated before.
    Set to > 1 to penalize.
    """
    regex_string: Optional[str] = None
    """A regex string for constrained generation."""
    no_repeat_ngram_size: Optional[int] = None
    """Prevent repetitions of ngrams of this size. Default = 0 (turned off)."""
    streaming: bool = False
    """Whether to stream the output. Default = False."""
    @property
    def _default_params(self) -> Mapping[str, Any]:


@property
    def _default_params(self) -> Mapping[str, Any]:
        """Get the default parameters for calling Titan Takeoff Server (Pro)."""
        return {
            **(
                {"regex_string": self.regex_string}
                if self.regex_string is not None
                else {}
            ),
            **(
                {"sampling_temperature": self.sampling_temperature}
                if self.sampling_temperature is not None
                else {}
            ),
            **(
                {"sampling_topp": self.sampling_topp}
                if self.sampling_topp is not None
                else {}
            ),
            **(
                {"repetition_penalty": self.repetition_penalty}
                if self.repetition_penalty is not None
                else {}
            ),
            **(
                {"max_new_tokens": self.max_new_tokens}
                if self.max_new_tokens is not None
                else {}
            ),
            **(
                {"min_new_tokens": self.min_new_tokens}
                if self.min_new_tokens is not None
                else {}
            ),
            **(
                {"sampling_topk": self.sampling_topk}
                if self.sampling_topk is not None
                else {}
            ),
            **(
                {"no_repeat_ngram_size": self.no_repeat_ngram_size}
                if self.no_repeat_ngram_size is not None
                else {}
            ),
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "titan_takeoff_pro"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,


**kwargs: Any,
    ) -> str:
        """Call out to Titan Takeoff (Pro) generate endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                prompt = "What is the capital of the United Kingdom?"
                response = model(prompt)
        """
        try:
            if self.streaming:
                text_output = ""
                for chunk in self._stream(
                    prompt=prompt,
                    stop=stop,
                    run_manager=run_manager,
                ):
                    text_output += chunk.text
                return text_output
            url = f"{self.base_url}/generate"
            params = {"text": prompt, **self._default_params}
            response = requests.post(url, json=params)
            response.raise_for_status()
            response.encoding = "utf-8"
            text = ""
            if "text" in response.json():
                text = response.json()["text"]
                text = text.replace("</s>", "")
            else:
                raise ValueError("Something went wrong.")
            if stop is not None:
                text = enforce_stop_tokens(text, stop)
            return text
        except ConnectionError:
            raise ConnectionError(
                "Could not connect to Titan Takeoff (Pro) server. \
                Please make sure that the server is running."
            )
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:


**kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        """Call out to Titan Takeoff (Pro) stream endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Yields:
            A dictionary like object containing a string token.
        Example:
            .. code-block:: python
                prompt = "What is the capital of the United Kingdom?"
                response = model(prompt)
        """
        url = f"{self.base_url}/generate_stream"
        params = {"text": prompt, **self._default_params}
        response = requests.post(url, json=params, stream=True)
        response.encoding = "utf-8"
        buffer = ""
        for text in response.iter_content(chunk_size=1, decode_unicode=True):
            buffer += text
            if "data:" in buffer:
                # Remove the first instance of "data:" from the buffer.
                if buffer.startswith("data:"):
                    buffer = ""
                if len(buffer.split("data:", 1)) == 2:
                    content, _ = buffer.split("data:", 1)
                    buffer = content.rstrip("\n")
                # Trim the buffer to only have content after the "data:" part.
                if buffer:  # Ensure that there's content to process.
                    chunk = GenerationChunk(text=buffer)
                    buffer = ""  # Reset buffer for the next set of data.
                    yield chunk
                    if run_manager:
                        run_manager.on_llm_new_token(token=chunk.text)
        # Yield any remaining content in the buffer.
        if buffer:
            chunk = GenerationChunk(text=buffer.replace("</s>", ""))
            yield chunk


chunk = GenerationChunk(text=buffer.replace("</s>", ""))
            yield chunk
            if run_manager:
                run_manager.on_llm_new_token(token=chunk.text)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {"base_url": self.base_url, **{}, **self._default_params}


Source code for langchain.llms.yandex
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.load.serializable import Serializable
from langchain.pydantic_v1 import root_validator
from langchain.utils import get_from_dict_or_env
class _BaseYandexGPT(Serializable):
    iam_token: str = ""
    """Yandex Cloud IAM token for service account
    with the `ai.languageModels.user` role"""
    api_key: str = ""
    """Yandex Cloud Api Key for service account
    with the `ai.languageModels.user` role"""
    model_name: str = "general"
    """Model name to use."""
    temperature: float = 0.6
    """What sampling temperature to use.
    Should be a double number between 0 (inclusive) and 1 (inclusive)."""
    max_tokens: int = 7400
    """Sets the maximum limit on the total number of tokens
    used for both the input prompt and the generated response.
    Must be greater than zero and not exceed 7400 tokens."""
    stop: Optional[List[str]] = None
    """Sequences when completion generation will stop."""
    url: str = "llm.api.cloud.yandex.net:443"
    """The url of the API."""
    @property
    def _llm_type(self) -> str:
        return "yandex_gpt"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that iam token exists in environment."""


"""Validate that iam token exists in environment."""
        iam_token = get_from_dict_or_env(values, "iam_token", "YC_IAM_TOKEN", "")
        values["iam_token"] = iam_token
        api_key = get_from_dict_or_env(values, "api_key", "YC_API_KEY", "")
        values["api_key"] = api_key
        if api_key == "" and iam_token == "":
            raise ValueError("Either 'YC_API_KEY' or 'YC_IAM_TOKEN' must be provided.")
        return values
[docs]class YandexGPT(_BaseYandexGPT, LLM):
    """Yandex large language models.
    To use, you should have the ``yandexcloud`` python package installed.
    There are two authentication options for the service account
    with the ``ai.languageModels.user`` role:
        - You can specify the token in a constructor parameter `iam_token`
        or in an environment variable `YC_IAM_TOKEN`.
        - You can specify the key in a constructor parameter `api_key`
        or in an environment variable `YC_API_KEY`.
    Example:
        .. code-block:: python
            from langchain.llms import YandexGPT
            yandex_gpt = YandexGPT(iam_token="t1.9eu...")
    """
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model_name,
            "temperature": self.temperature,
            "max_tokens": self.max_tokens,
            "stop": self.stop,
        }
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,


prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the Yandex GPT model and return the output.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = YandexGPT("Tell me a joke.")
        """
        try:
            import grpc
            from google.protobuf.wrappers_pb2 import DoubleValue, Int64Value
            from yandex.cloud.ai.llm.v1alpha.llm_pb2 import GenerationOptions
            from yandex.cloud.ai.llm.v1alpha.llm_service_pb2 import InstructRequest
            from yandex.cloud.ai.llm.v1alpha.llm_service_pb2_grpc import (
                TextGenerationServiceStub,
            )
        except ImportError as e:
            raise ImportError(
                "Please install YandexCloud SDK" " with `pip install yandexcloud`."
            ) from e
        channel_credentials = grpc.ssl_channel_credentials()
        channel = grpc.secure_channel(self.url, channel_credentials)
        request = InstructRequest(
            model=self.model_name,
            request_text=prompt,
            generation_options=GenerationOptions(
                temperature=DoubleValue(value=self.temperature),
                max_tokens=Int64Value(value=self.max_tokens),
            ),
        )
        stub = TextGenerationServiceStub(channel)
        if self.iam_token:
            metadata = (("authorization", f"Bearer {self.iam_token}"),)
        else:


else:
            metadata = (("authorization", f"Api-Key {self.api_key}"),)
        res = stub.Instruct(request, metadata=metadata)
        text = list(res)[0].alternatives[0].text
        if stop is not None:
            text = enforce_stop_tokens(text, stop)
        return text
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Async call the Yandex GPT model and return the output.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        """
        try:
            import asyncio
            import grpc
            from google.protobuf.wrappers_pb2 import DoubleValue, Int64Value
            from yandex.cloud.ai.llm.v1alpha.llm_pb2 import GenerationOptions
            from yandex.cloud.ai.llm.v1alpha.llm_service_pb2 import (
                InstructRequest,
                InstructResponse,
            )
            from yandex.cloud.ai.llm.v1alpha.llm_service_pb2_grpc import (
                TextGenerationAsyncServiceStub,
            )
            from yandex.cloud.operation.operation_service_pb2 import GetOperationRequest
            from yandex.cloud.operation.operation_service_pb2_grpc import (
                OperationServiceStub,
            )
        except ImportError as e:
            raise ImportError(
                "Please install YandexCloud SDK" " with `pip install yandexcloud`."
            ) from e
        operation_api_url = "operation.api.cloud.yandex.net:443"


operation_api_url = "operation.api.cloud.yandex.net:443"
        channel_credentials = grpc.ssl_channel_credentials()
        async with grpc.aio.secure_channel(self.url, channel_credentials) as channel:
            request = InstructRequest(
                model=self.model_name,
                request_text=prompt,
                generation_options=GenerationOptions(
                    temperature=DoubleValue(value=self.temperature),
                    max_tokens=Int64Value(value=self.max_tokens),
                ),
            )
            stub = TextGenerationAsyncServiceStub(channel)
            if self.iam_token:
                metadata = (("authorization", f"Bearer {self.iam_token}"),)
            else:
                metadata = (("authorization", f"Api-Key {self.api_key}"),)
            operation = await stub.Instruct(request, metadata=metadata)
            async with grpc.aio.secure_channel(
                operation_api_url, channel_credentials
            ) as operation_channel:
                operation_stub = OperationServiceStub(operation_channel)
                while not operation.done:
                    await asyncio.sleep(1)
                    operation_request = GetOperationRequest(operation_id=operation.id)
                    operation = await operation_stub.Get(
                        operation_request, metadata=metadata
                    )
            instruct_response = InstructResponse()
            operation.response.Unpack(instruct_response)
            text = instruct_response.alternatives[0].text
            if stop is not None:
                text = enforce_stop_tokens(text, stop)
            return text


Source code for langchain.llms.deepsparse
# flake8: noqa
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
from langchain.pydantic_v1 import root_validator
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.schema.output import GenerationChunk
[docs]class DeepSparse(LLM):
    """Neural Magic DeepSparse LLM interface.
    To use, you should have the ``deepsparse`` or ``deepsparse-nightly``
    python package installed. See https://github.com/neuralmagic/deepsparse
    This interface let's you deploy optimized LLMs straight from the
    [SparseZoo](https://sparsezoo.neuralmagic.com/?useCase=text_generation)
    Example:
        .. code-block:: python
            from langchain.llms import DeepSparse
            llm = DeepSparse(model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none")
    """  # noqa: E501
    pipeline: Any  #: :meta private:
    model: str
    """The path to a model file or directory or the name of a SparseZoo model stub."""
    model_config: Optional[Dict[str, Any]] = None
    """Keyword arguments passed to the pipeline construction.
    Common parameters are sequence_length, prompt_sequence_length"""
    generation_config: Union[None, str, Dict] = None
    """GenerationConfig dictionary consisting of parameters used to control
    sequences generated for each prompt. Common parameters are:
    max_length, max_new_tokens, num_return_sequences, output_scores,


max_length, max_new_tokens, num_return_sequences, output_scores,
    top_p, top_k, repetition_penalty."""
    streaming: bool = False
    """Whether to stream the results, token by token."""
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {
            "model": self.model,
            "model_config": self.model_config,
            "generation_config": self.generation_config,
            "streaming": self.streaming,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "deepsparse"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that ``deepsparse`` package is installed."""
        try:
            from deepsparse import Pipeline
        except ImportError:
            raise ImportError(
                "Could not import `deepsparse` package. "
                "Please install it with `pip install deepsparse`"
            )
        model_config = values["model_config"] or {}
        values["pipeline"] = Pipeline.create(
            task="text_generation",
            model_path=values["model"],
            **model_config,
        )
        return values
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Generate text from a prompt.
        Args:
            prompt: The prompt to generate text from.
            stop: A list of strings to stop generation when encountered.
        Returns:
            The generated text.
        Example:


Returns:
            The generated text.
        Example:
            .. code-block:: python
                from langchain.llms import DeepSparse
                llm = DeepSparse(model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none")
                llm("Tell me a joke.")
        """
        if self.streaming:
            combined_output = ""
            for chunk in self._stream(
                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                combined_output += chunk.text
            text = combined_output
        else:
            text = (
                self.pipeline(
                    sequences=prompt, generation_config=self.generation_config
                )
                .generations[0]
                .text
            )
        if stop is not None:
            text = enforce_stop_tokens(text, stop)
        return text
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Generate text from a prompt.
        Args:
            prompt: The prompt to generate text from.
            stop: A list of strings to stop generation when encountered.
        Returns:
            The generated text.
        Example:
            .. code-block:: python
                from langchain.llms import DeepSparse
                llm = DeepSparse(model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none")
                llm("Tell me a joke.")
        """
        if self.streaming:
            combined_output = ""


"""
        if self.streaming:
            combined_output = ""
            async for chunk in self._astream(
                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                combined_output += chunk.text
            text = combined_output
        else:
            text = (
                self.pipeline(
                    sequences=prompt, generation_config=self.generation_config
                )
                .generations[0]
                .text
            )
        if stop is not None:
            text = enforce_stop_tokens(text, stop)
        return text
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        """Yields results objects as they are generated in real time.
        It also calls the callback manager's on_llm_new_token event with
        similar parameters to the OpenAI LLM class method of the same name.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            A generator representing the stream of tokens being generated.
        Yields:
            A dictionary like object containing a string token.
        Example:
            .. code-block:: python
                from langchain.llms import DeepSparse
                llm = DeepSparse(
                    model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none",
                    streaming=True
                )
                for chunk in llm.stream("Tell me a joke",
                        stop=["'","\n"]):
                    print(chunk, end='', flush=True)


stop=["'","\n"]):
                    print(chunk, end='', flush=True)
        """
        inference = self.pipeline(
            sequences=prompt, generation_config=self.generation_config, streaming=True
        )
        for token in inference:
            chunk = GenerationChunk(text=token.generations[0].text)
            yield chunk
            if run_manager:
                run_manager.on_llm_new_token(token=chunk.text)
    async def _astream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> AsyncIterator[GenerationChunk]:
        """Yields results objects as they are generated in real time.
        It also calls the callback manager's on_llm_new_token event with
        similar parameters to the OpenAI LLM class method of the same name.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            A generator representing the stream of tokens being generated.
        Yields:
            A dictionary like object containing a string token.
        Example:
            .. code-block:: python
                from langchain.llms import DeepSparse
                llm = DeepSparse(
                    model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none",
                    streaming=True
                )
                for chunk in llm.stream("Tell me a joke",
                        stop=["'","\n"]):
                    print(chunk, end='', flush=True)
        """
        inference = self.pipeline(
            sequences=prompt, generation_config=self.generation_config, streaming=True
        )


sequences=prompt, generation_config=self.generation_config, streaming=True
        )
        for token in inference:
            chunk = GenerationChunk(text=token.generations[0].text)
            yield chunk
            if run_manager:
                await run_manager.on_llm_new_token(token=chunk.text)


Source code for langchain.llms.baseten
import logging
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field
logger = logging.getLogger(__name__)
[docs]class Baseten(LLM):
    """Baseten models.
    To use, you should have the ``baseten`` python package installed,
    and run ``baseten.login()`` with your Baseten API key.
    The required ``model`` param can be either a model id or model
    version id. Using a model version ID will result in
    slightly faster invocation.
    Any other model parameters can also
    be passed in with the format input={model_param: value, ...}
    The Baseten model must accept a dictionary of input with the key
    "prompt" and return a dictionary with a key "data" which maps
    to a list of response strings.
    Example:
        .. code-block:: python
            from langchain.llms import Baseten
            my_model = Baseten(model="MODEL_ID")
            output = my_model("prompt")
    """
    model: str
    input: Dict[str, Any] = Field(default_factory=dict)
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of model."""
        return "baseten"
    def _call(
        self,


return "baseten"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Baseten deployed model endpoint."""
        try:
            import baseten
        except ImportError as exc:
            raise ImportError(
                "Could not import Baseten Python package. "
                "Please install it with `pip install baseten`."
            ) from exc
        # get the model and version
        try:
            model = baseten.deployed_model_version_id(self.model)
            response = model.predict({"prompt": prompt, **kwargs})
        except baseten.common.core.ApiError:
            model = baseten.deployed_model_id(self.model)
            response = model.predict({"prompt": prompt, **kwargs})
        return "".join(response)


Source code for langchain.llms.vertexai
from __future__ import annotations
from concurrent.futures import Executor, ThreadPoolExecutor
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    ClassVar,
    Dict,
    Iterator,
    List,
    Optional,
    Union,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import BaseLLM, create_base_retry_decorator
from langchain.pydantic_v1 import BaseModel, Field, root_validator
from langchain.schema import (
    Generation,
    LLMResult,
)
from langchain.schema.output import GenerationChunk
from langchain.utilities.vertexai import (
    get_client_info,
    init_vertexai,
    raise_vertex_import_error,
)
if TYPE_CHECKING:
    from google.cloud.aiplatform.gapic import (
        PredictionServiceAsyncClient,
        PredictionServiceClient,
    )
    from vertexai.language_models._language_models import (
        TextGenerationResponse,
        _LanguageModel,
    )
def _response_to_generation(
    response: TextGenerationResponse,
) -> GenerationChunk:
    """Convert a stream response to a generation chunk."""
    try:
        generation_info = {
            "is_blocked": response.is_blocked,
            "safety_attributes": response.safety_attributes,
        }
    except Exception:
        generation_info = None
    return GenerationChunk(text=response.text, generation_info=generation_info)
[docs]def is_codey_model(model_name: str) -> bool:
    """Returns True if the model name is a Codey model.
    Args:
        model_name: The model name to check.


Args:
        model_name: The model name to check.
    Returns: True if the model name is a Codey model.
    """
    return "code" in model_name
def _create_retry_decorator(
    llm: VertexAI,
    *,
    run_manager: Optional[
        Union[AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun]
    ] = None,
) -> Callable[[Any], Any]:
    import google.api_core
    errors = [
        google.api_core.exceptions.ResourceExhausted,
        google.api_core.exceptions.ServiceUnavailable,
        google.api_core.exceptions.Aborted,
        google.api_core.exceptions.DeadlineExceeded,
    ]
    decorator = create_base_retry_decorator(
        error_types=errors, max_retries=llm.max_retries, run_manager=run_manager
    )
    return decorator
[docs]def completion_with_retry(
    llm: VertexAI,
    *args: Any,
    run_manager: Optional[CallbackManagerForLLMRun] = None,
    **kwargs: Any,
) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)
    @retry_decorator
    def _completion_with_retry(*args: Any, **kwargs: Any) -> Any:
        return llm.client.predict(*args, **kwargs)
    return _completion_with_retry(*args, **kwargs)
[docs]def stream_completion_with_retry(
    llm: VertexAI,
    *args: Any,
    run_manager: Optional[CallbackManagerForLLMRun] = None,
    **kwargs: Any,
) -> Any:
    """Use tenacity to retry the completion call."""


) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)
    @retry_decorator
    def _completion_with_retry(*args: Any, **kwargs: Any) -> Any:
        return llm.client.predict_streaming(*args, **kwargs)
    return _completion_with_retry(*args, **kwargs)
[docs]async def acompletion_with_retry(
    llm: VertexAI,
    *args: Any,
    run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
    **kwargs: Any,
) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)
    @retry_decorator
    async def _acompletion_with_retry(*args: Any, **kwargs: Any) -> Any:
        return await llm.client.predict_async(*args, **kwargs)
    return await _acompletion_with_retry(*args, **kwargs)
class _VertexAIBase(BaseModel):
    project: Optional[str] = None
    "The default GCP project to use when making Vertex API calls."
    location: str = "us-central1"
    "The default location to use when making API calls."
    request_parallelism: int = 5
    "The amount of parallelism allowed for requests issued to VertexAI models. "
    "Default is 5."
    max_retries: int = 6
    """The maximum number of retries to make when generating."""
    task_executor: ClassVar[Optional[Executor]] = Field(default=None, exclude=True)
    stop: Optional[List[str]] = None
    "Optional list of stop words to use when generating."


"Optional list of stop words to use when generating."
    model_name: Optional[str] = None
    "Underlying model name."
    @classmethod
    def _get_task_executor(cls, request_parallelism: int = 5) -> Executor:
        if cls.task_executor is None:
            cls.task_executor = ThreadPoolExecutor(max_workers=request_parallelism)
        return cls.task_executor
class _VertexAICommon(_VertexAIBase):
    client: "_LanguageModel" = None  #: :meta private:
    model_name: str
    "Underlying model name."
    temperature: float = 0.0
    "Sampling temperature, it controls the degree of randomness in token selection."
    max_output_tokens: int = 128
    "Token limit determines the maximum amount of text output from one prompt."
    top_p: float = 0.95
    "Tokens are selected from most probable to least until the sum of their "
    "probabilities equals the top-p value. Top-p is ignored for Codey models."
    top_k: int = 40
    "How the model selects tokens for output, the next token is selected from "
    "among the top-k most probable tokens. Top-k is ignored for Codey models."
    credentials: Any = Field(default=None, exclude=True)
    "The default custom credentials (google.auth.credentials.Credentials) to use "
    "when making API calls. If not provided, credentials will be ascertained from "
    "the environment."
    n: int = 1
    """How many completions to generate for each prompt."""
    streaming: bool = False
    """Whether to stream the results or not."""
    @property
    def _llm_type(self) -> str:
        return "vertexai"
    @property


return "vertexai"
    @property
    def is_codey_model(self) -> bool:
        return is_codey_model(self.model_name)
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}
    @property
    def _default_params(self) -> Dict[str, Any]:
        if self.is_codey_model:
            return {
                "temperature": self.temperature,
                "max_output_tokens": self.max_output_tokens,
            }
        else:
            return {
                "temperature": self.temperature,
                "max_output_tokens": self.max_output_tokens,
                "top_k": self.top_k,
                "top_p": self.top_p,
                "candidate_count": self.n,
            }
    @classmethod
    def _try_init_vertexai(cls, values: Dict) -> None:
        allowed_params = ["project", "location", "credentials"]
        params = {k: v for k, v in values.items() if k in allowed_params}
        init_vertexai(**params)
        return None
    def _prepare_params(
        self,
        stop: Optional[List[str]] = None,
        stream: bool = False,
        **kwargs: Any,
    ) -> dict:
        stop_sequences = stop or self.stop
        params_mapping = {"n": "candidate_count"}
        params = {params_mapping.get(k, k): v for k, v in kwargs.items()}
        params = {**self._default_params, "stop_sequences": stop_sequences, **params}
        if stream or self.streaming:
            params.pop("candidate_count")
        return params


if stream or self.streaming:
            params.pop("candidate_count")
        return params
[docs]class VertexAI(_VertexAICommon, BaseLLM):
    """Google Vertex AI large language models."""
    model_name: str = "text-bison"
    "The name of the Vertex AI large language model."
    tuned_model_name: Optional[str] = None
    "The name of a tuned model. If provided, model_name is ignored."
[docs]    @classmethod
    def is_lc_serializable(self) -> bool:
        return True
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that the python package exists in environment."""
        cls._try_init_vertexai(values)
        tuned_model_name = values.get("tuned_model_name")
        model_name = values["model_name"]
        try:
            if not is_codey_model(model_name):
                from vertexai.preview.language_models import TextGenerationModel
                if tuned_model_name:
                    values["client"] = TextGenerationModel.get_tuned_model(
                        tuned_model_name
                    )
                else:
                    values["client"] = TextGenerationModel.from_pretrained(model_name)
            else:
                from vertexai.preview.language_models import CodeGenerationModel
                if tuned_model_name:
                    values["client"] = CodeGenerationModel.get_tuned_model(
                        tuned_model_name
                    )
                else:
                    values["client"] = CodeGenerationModel.from_pretrained(model_name)
        except ImportError:
            raise_vertex_import_error()
        if values["streaming"] and values["n"] > 1:
            raise ValueError("Only one candidate can be generated with streaming!")
        return values
[docs]    def get_num_tokens(self, text: str) -> int:


[docs]    def get_num_tokens(self, text: str) -> int:
        """Get the number of tokens present in the text.
        Useful for checking if an input will fit in a model's context window.
        Args:
            text: The string input to tokenize.
        Returns:
            The integer number of tokens in the text.
        """
        try:
            result = self.client.count_tokens(text)
        except AttributeError:
            raise NotImplementedError(
                "Your google-cloud-aiplatform version didn't implement count_tokens."
                "Please, install it with pip install google-cloud-aiplatform>=1.35.0"
            )
        return result.total_tokens
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        stream: Optional[bool] = None,
        **kwargs: Any,
    ) -> LLMResult:
        should_stream = stream if stream is not None else self.streaming
        params = self._prepare_params(stop=stop, stream=should_stream, **kwargs)
        generations = []
        for prompt in prompts:
            if should_stream:
                generation = GenerationChunk(text="")
                for chunk in self._stream(
                    prompt, stop=stop, run_manager=run_manager, **kwargs
                ):
                    generation += chunk
                generations.append([generation])
            else:
                res = completion_with_retry(
                    self, prompt, run_manager=run_manager, **params
                )
                if self.is_codey_model:
                    generations.append([_response_to_generation(res)])
                else:
                    generations.append(
                        [_response_to_generation(r) for r in res.candidates]
                    )


[_response_to_generation(r) for r in res.candidates]
                    )
        return LLMResult(generations=generations)
    async def _agenerate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        params = self._prepare_params(stop=stop, **kwargs)
        generations = []
        for prompt in prompts:
            res = await acompletion_with_retry(
                self, prompt, run_manager=run_manager, **params
            )
            generations.append([_response_to_generation(r) for r in res.candidates])
        return LLMResult(generations=generations)
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        params = self._prepare_params(stop=stop, stream=True, **kwargs)
        for stream_resp in stream_completion_with_retry(
            self, prompt, run_manager=run_manager, **params
        ):
            chunk = _response_to_generation(stream_resp)
            yield chunk
            if run_manager:
                run_manager.on_llm_new_token(
                    chunk.text,
                    chunk=chunk,
                    verbose=self.verbose,
                )
[docs]class VertexAIModelGarden(_VertexAIBase, BaseLLM):
    """Large language models served from Vertex AI Model Garden."""
    client: "PredictionServiceClient" = None  #: :meta private:


client: "PredictionServiceClient" = None  #: :meta private:
    async_client: "PredictionServiceAsyncClient" = None  #: :meta private:
    endpoint_id: str
    "A name of an endpoint where the model has been deployed."
    allowed_model_args: Optional[List[str]] = None
    """Allowed optional args to be passed to the model."""
    prompt_arg: str = "prompt"
    result_arg: str = "generated_text"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that the python package exists in environment."""
        try:
            from google.api_core.client_options import ClientOptions
            from google.cloud.aiplatform.gapic import (
                PredictionServiceAsyncClient,
                PredictionServiceClient,
            )
        except ImportError:
            raise_vertex_import_error()
        if values["project"] is None:
            raise ValueError(
                "A GCP project should be provided to run inference on Model Garden!"
            )
        client_options = ClientOptions(
            api_endpoint=f"{values['location']}-aiplatform.googleapis.com"
        )
        client_info = get_client_info(module="vertex-ai-model-garden")
        values["client"] = PredictionServiceClient(
            client_options=client_options, client_info=client_info
        )
        values["async_client"] = PredictionServiceAsyncClient(
            client_options=client_options, client_info=client_info
        )
        return values
    @property
    def _llm_type(self) -> str:
        return "vertexai_model_garden"
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,


run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        """Run the LLM on the given prompt and input."""
        try:
            from google.protobuf import json_format
            from google.protobuf.struct_pb2 import Value
        except ImportError:
            raise ImportError(
                "protobuf package not found, please install it with"
                " `pip install protobuf`"
            )
        instances = []
        for prompt in prompts:
            if self.allowed_model_args:
                instance = {
                    k: v for k, v in kwargs.items() if k in self.allowed_model_args
                }
            else:
                instance = {}
            instance[self.prompt_arg] = prompt
            instances.append(instance)
        predict_instances = [
            json_format.ParseDict(instance_dict, Value()) for instance_dict in instances
        ]
        endpoint = self.client.endpoint_path(
            project=self.project, location=self.location, endpoint=self.endpoint_id
        )
        response = self.client.predict(endpoint=endpoint, instances=predict_instances)
        generations: List[List[Generation]] = []
        for result in response.predictions:
            generations.append(
                [Generation(text=prediction[self.result_arg]) for prediction in result]
            )
        return LLMResult(generations=generations)
    async def _agenerate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        """Run the LLM on the given prompt and input."""
        try:
            from google.protobuf import json_format
            from google.protobuf.struct_pb2 import Value


from google.protobuf import json_format
            from google.protobuf.struct_pb2 import Value
        except ImportError:
            raise ImportError(
                "protobuf package not found, please install it with"
                " `pip install protobuf`"
            )
        instances = []
        for prompt in prompts:
            if self.allowed_model_args:
                instance = {
                    k: v for k, v in kwargs.items() if k in self.allowed_model_args
                }
            else:
                instance = {}
            instance[self.prompt_arg] = prompt
            instances.append(instance)
        predict_instances = [
            json_format.ParseDict(instance_dict, Value()) for instance_dict in instances
        ]
        endpoint = self.async_client.endpoint_path(
            project=self.project, location=self.location, endpoint=self.endpoint_id
        )
        response = await self.async_client.predict(
            endpoint=endpoint, instances=predict_instances
        )
        generations: List[List[Generation]] = []
        for result in response.predictions:
            generations.append(
                [Generation(text=prediction[self.result_arg]) for prediction in result]
            )
        return LLMResult(generations=generations)


Source code for langchain.llms.google_palm
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms import BaseLLM
from langchain.pydantic_v1 import BaseModel, root_validator
from langchain.schema import Generation, LLMResult
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator() -> Callable[[Any], Any]:
    """Returns a tenacity retry decorator, preconfigured to handle PaLM exceptions"""
    try:
        import google.api_core.exceptions
    except ImportError:
        raise ImportError(
            "Could not import google-api-core python package. "
            "Please install it with `pip install google-api-core`."
        )
    multiplier = 2
    min_seconds = 1
    max_seconds = 60
    max_retries = 10
    return retry(
        reraise=True,
        stop=stop_after_attempt(max_retries),
        wait=wait_exponential(multiplier=multiplier, min=min_seconds, max=max_seconds),
        retry=(
            retry_if_exception_type(google.api_core.exceptions.ResourceExhausted)
            | retry_if_exception_type(google.api_core.exceptions.ServiceUnavailable)
            | retry_if_exception_type(google.api_core.exceptions.GoogleAPIError)
        ),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
[docs]def generate_with_retry(llm: GooglePalm, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""


"""Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    def _generate_with_retry(**kwargs: Any) -> Any:
        return llm.client.generate_text(**kwargs)
    return _generate_with_retry(**kwargs)
def _strip_erroneous_leading_spaces(text: str) -> str:
    """Strip erroneous leading spaces from text.
    The PaLM API will sometimes erroneously return a single leading space in all
    lines > 1. This function strips that space.
    """
    has_leading_space = all(not line or line[0] == " " for line in text.split("\n")[1:])
    if has_leading_space:
        return text.replace("\n ", "\n")
    else:
        return text
[docs]class GooglePalm(BaseLLM, BaseModel):
    """Google PaLM models."""
    client: Any  #: :meta private:
    google_api_key: Optional[str]
    model_name: str = "models/text-bison-001"
    """Model name to use."""
    temperature: float = 0.7
    """Run inference with this temperature. Must by in the closed interval
       [0.0, 1.0]."""
    top_p: Optional[float] = None
    """Decode using nucleus sampling: consider the smallest set of tokens whose
       probability sum is at least top_p. Must be in the closed interval [0.0, 1.0]."""
    top_k: Optional[int] = None
    """Decode using top-k sampling: consider the set of top_k most probable tokens.
       Must be positive."""
    max_output_tokens: Optional[int] = None
    """Maximum number of tokens to include in a candidate. Must be greater than zero.


"""Maximum number of tokens to include in a candidate. Must be greater than zero.
       If unset, will default to 64."""
    n: int = 1
    """Number of chat completions to generate for each prompt. Note that the API may
       not return the full n completions if duplicates are generated."""
    @property
    def lc_secrets(self) -> Dict[str, str]:
        return {"google_api_key": "GOOGLE_API_KEY"}
[docs]    @classmethod
    def is_lc_serializable(self) -> bool:
        return True
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate api key, python package exists."""
        google_api_key = get_from_dict_or_env(
            values, "google_api_key", "GOOGLE_API_KEY"
        )
        try:
            import google.generativeai as genai
            genai.configure(api_key=google_api_key)
        except ImportError:
            raise ImportError(
                "Could not import google-generativeai python package. "
                "Please install it with `pip install google-generativeai`."
            )
        values["client"] = genai
        if values["temperature"] is not None and not 0 <= values["temperature"] <= 1:
            raise ValueError("temperature must be in the range [0.0, 1.0]")
        if values["top_p"] is not None and not 0 <= values["top_p"] <= 1:
            raise ValueError("top_p must be in the range [0.0, 1.0]")
        if values["top_k"] is not None and values["top_k"] <= 0:
            raise ValueError("top_k must be positive")


raise ValueError("top_k must be positive")
        if values["max_output_tokens"] is not None and values["max_output_tokens"] <= 0:
            raise ValueError("max_output_tokens must be greater than zero")
        return values
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        generations = []
        for prompt in prompts:
            completion = generate_with_retry(
                self,
                model=self.model_name,
                prompt=prompt,
                stop_sequences=stop,
                temperature=self.temperature,
                top_p=self.top_p,
                top_k=self.top_k,
                max_output_tokens=self.max_output_tokens,
                candidate_count=self.n,
                **kwargs,
            )
            prompt_generations = []
            for candidate in completion.candidates:
                raw_text = candidate["output"]
                stripped_text = _strip_erroneous_leading_spaces(raw_text)
                prompt_generations.append(Generation(text=stripped_text))
            generations.append(prompt_generations)
        return LLMResult(generations=generations)
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "google_palm"


Source code for langchain.llms.beam
import base64
import json
import logging
import subprocess
import textwrap
import time
from typing import Any, Dict, List, Mapping, Optional
import requests
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Extra, Field, root_validator
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
DEFAULT_NUM_TRIES = 10
DEFAULT_SLEEP_TIME = 4
[docs]class Beam(LLM):
    """Beam API for gpt2 large language model.
    To use, you should have the ``beam-sdk`` python package installed,
    and the environment variable ``BEAM_CLIENT_ID`` set with your client id
    and ``BEAM_CLIENT_SECRET`` set with your client secret. Information on how
    to get this is available here: https://docs.beam.cloud/account/api-keys.
    The wrapper can then be called as follows, where the name, cpu, memory, gpu,
    python version, and python packages can be updated accordingly. Once deployed,
    the instance can be called.
    Example:
        .. code-block:: python
            llm = Beam(model_name="gpt2",
                name="langchain-gpt2",
                cpu=8,
                memory="32Gi",
                gpu="A10G",
                python_version="python3.8",
                python_packages=[
                    "diffusers[torch]>=0.10",
                    "transformers",
                    "torch",
                    "pillow",
                    "accelerate",
                    "safetensors",
                    "xformers",],
                max_length=50)
            llm._deploy()


max_length=50)
            llm._deploy()
            call_result = llm._call(input)
    """
    model_name: str = ""
    name: str = ""
    cpu: str = ""
    memory: str = ""
    gpu: str = ""
    python_version: str = ""
    python_packages: List[str] = []
    max_length: str = ""
    url: str = ""
    """model endpoint to use"""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not
    explicitly specified."""
    beam_client_id: str = ""
    beam_client_secret: str = ""
    app_id: Optional[str] = None
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""{field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:


@root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        beam_client_id = get_from_dict_or_env(
            values, "beam_client_id", "BEAM_CLIENT_ID"
        )
        beam_client_secret = get_from_dict_or_env(
            values, "beam_client_secret", "BEAM_CLIENT_SECRET"
        )
        values["beam_client_id"] = beam_client_id
        values["beam_client_secret"] = beam_client_secret
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model_name,
            "name": self.name,
            "cpu": self.cpu,
            "memory": self.memory,
            "gpu": self.gpu,
            "python_version": self.python_version,
            "python_packages": self.python_packages,
            "max_length": self.max_length,
            "model_kwargs": self.model_kwargs,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "beam"
[docs]    def app_creation(self) -> None:
        """Creates a Python file which will contain your Beam app definition."""
        script = textwrap.dedent(
            """\
        import beam
        # The environment your code will run on
        app = beam.App(
            name="{name}",
            cpu={cpu},
            memory="{memory}",
            gpu="{gpu}",
            python_version="{python_version}",
            python_packages={python_packages},
        )
        app.Trigger.RestAPI(


python_packages={python_packages},
        )
        app.Trigger.RestAPI(
            inputs={{"prompt": beam.Types.String(), "max_length": beam.Types.String()}},
            outputs={{"text": beam.Types.String()}},
            handler="run.py:beam_langchain",
        )
        """
        )
        script_name = "app.py"
        with open(script_name, "w") as file:
            file.write(
                script.format(
                    name=self.name,
                    cpu=self.cpu,
                    memory=self.memory,
                    gpu=self.gpu,
                    python_version=self.python_version,
                    python_packages=self.python_packages,
                )
            )
[docs]    def run_creation(self) -> None:
        """Creates a Python file which will be deployed on beam."""
        script = textwrap.dedent(
            """
        import os
        import transformers
        from transformers import GPT2LMHeadModel, GPT2Tokenizer
        model_name = "{model_name}"
        def beam_langchain(**inputs):
            prompt = inputs["prompt"]
            length = inputs["max_length"]
            tokenizer = GPT2Tokenizer.from_pretrained(model_name)
            model = GPT2LMHeadModel.from_pretrained(model_name)
            encodedPrompt = tokenizer.encode(prompt, return_tensors='pt')
            outputs = model.generate(encodedPrompt, max_length=int(length),
              do_sample=True, pad_token_id=tokenizer.eos_token_id)
            output = tokenizer.decode(outputs[0], skip_special_tokens=True)
            print(output)
            return {{"text": output}}
        """
        )
        script_name = "run.py"
        with open(script_name, "w") as file:
            file.write(script.format(model_name=self.model_name))


file.write(script.format(model_name=self.model_name))
    def _deploy(self) -> str:
        """Call to Beam."""
        try:
            import beam  # type: ignore
            if beam.__path__ == "":
                raise ImportError
        except ImportError:
            raise ImportError(
                "Could not import beam python package. "
                "Please install it with `curl "
                "https://raw.githubusercontent.com/slai-labs"
                "/get-beam/main/get-beam.sh -sSfL | sh`."
            )
        self.app_creation()
        self.run_creation()
        process = subprocess.run(
            "beam deploy app.py", shell=True, capture_output=True, text=True
        )
        if process.returncode == 0:
            output = process.stdout
            logger.info(output)
            lines = output.split("\n")
            for line in lines:
                if line.startswith(" i  Send requests to: https://apps.beam.cloud/"):
                    self.app_id = line.split("/")[-1]
                    self.url = line.split(":")[1].strip()
                    return self.app_id
            raise ValueError(
                f"""Failed to retrieve the appID from the deployment output.
                Deployment output: {output}"""
            )
        else:
            raise ValueError(f"Deployment failed. Error: {process.stderr}")
    @property
    def authorization(self) -> str:
        if self.beam_client_id:
            credential_str = self.beam_client_id + ":" + self.beam_client_secret
        else:
            credential_str = self.beam_client_secret
        return base64.b64encode(credential_str.encode()).decode()
    def _call(
        self,
        prompt: str,
        stop: Optional[list] = None,


self,
        prompt: str,
        stop: Optional[list] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Beam."""
        url = "https://apps.beam.cloud/" + self.app_id if self.app_id else self.url
        payload = {"prompt": prompt, "max_length": self.max_length}
        payload.update(kwargs)
        headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Authorization": "Basic " + self.authorization,
            "Connection": "keep-alive",
            "Content-Type": "application/json",
        }
        for _ in range(DEFAULT_NUM_TRIES):
            request = requests.post(url, headers=headers, data=json.dumps(payload))
            if request.status_code == 200:
                return request.json()["text"]
            time.sleep(DEFAULT_SLEEP_TIME)
        logger.warning("Unable to successfully call model.")
        return ""


Source code for langchain.llms.anthropic
import re
import warnings
from typing import (
    Any,
    AsyncIterator,
    Callable,
    Dict,
    Iterator,
    List,
    Mapping,
    Optional,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field, SecretStr, root_validator
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.output import GenerationChunk
from langchain.schema.prompt import PromptValue
from langchain.utils import (
    check_package_version,
    get_from_dict_or_env,
    get_pydantic_field_names,
)
from langchain.utils.utils import build_extra_kwargs, convert_to_secret_str
class _AnthropicCommon(BaseLanguageModel):
    client: Any = None  #: :meta private:
    async_client: Any = None  #: :meta private:
    model: str = Field(default="claude-2", alias="model_name")
    """Model name to use."""
    max_tokens_to_sample: int = Field(default=256, alias="max_tokens")
    """Denotes the number of tokens to predict per generation."""
    temperature: Optional[float] = None
    """A non-negative float that tunes the degree of randomness in generation."""
    top_k: Optional[int] = None
    """Number of most likely tokens to consider at each step."""
    top_p: Optional[float] = None
    """Total probability mass of tokens to consider at each step."""
    streaming: bool = False
    """Whether to stream the results."""
    default_request_timeout: Optional[float] = None


"""Whether to stream the results."""
    default_request_timeout: Optional[float] = None
    """Timeout for requests to Anthropic Completion API. Default is 600 seconds."""
    anthropic_api_url: Optional[str] = None
    anthropic_api_key: Optional[SecretStr] = None
    HUMAN_PROMPT: Optional[str] = None
    AI_PROMPT: Optional[str] = None
    count_tokens: Optional[Callable[[str], int]] = None
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    @root_validator(pre=True)
    def build_extra(cls, values: Dict) -> Dict:
        extra = values.get("model_kwargs", {})
        all_required_field_names = get_pydantic_field_names(cls)
        values["model_kwargs"] = build_extra_kwargs(
            extra, values, all_required_field_names
        )
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        values["anthropic_api_key"] = convert_to_secret_str(
            get_from_dict_or_env(values, "anthropic_api_key", "ANTHROPIC_API_KEY")
        )
        # Get custom api url from environment.
        values["anthropic_api_url"] = get_from_dict_or_env(
            values,
            "anthropic_api_url",
            "ANTHROPIC_API_URL",
            default="https://api.anthropic.com",
        )
        try:
            import anthropic
            check_package_version("anthropic", gte_version="0.3")
            values["client"] = anthropic.Anthropic(
                base_url=values["anthropic_api_url"],
                api_key=values["anthropic_api_key"].get_secret_value(),


api_key=values["anthropic_api_key"].get_secret_value(),
                timeout=values["default_request_timeout"],
            )
            values["async_client"] = anthropic.AsyncAnthropic(
                base_url=values["anthropic_api_url"],
                api_key=values["anthropic_api_key"].get_secret_value(),
                timeout=values["default_request_timeout"],
            )
            values["HUMAN_PROMPT"] = anthropic.HUMAN_PROMPT
            values["AI_PROMPT"] = anthropic.AI_PROMPT
            values["count_tokens"] = values["client"].count_tokens
        except ImportError:
            raise ImportError(
                "Could not import anthropic python package. "
                "Please it install it with `pip install anthropic`."
            )
        return values
    @property
    def _default_params(self) -> Mapping[str, Any]:
        """Get the default parameters for calling Anthropic API."""
        d = {
            "max_tokens_to_sample": self.max_tokens_to_sample,
            "model": self.model,
        }
        if self.temperature is not None:
            d["temperature"] = self.temperature
        if self.top_k is not None:
            d["top_k"] = self.top_k
        if self.top_p is not None:
            d["top_p"] = self.top_p
        return {**d, **self.model_kwargs}
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{}, **self._default_params}
    def _get_anthropic_stop(self, stop: Optional[List[str]] = None) -> List[str]:
        if not self.HUMAN_PROMPT or not self.AI_PROMPT:
            raise NameError("Please ensure the anthropic package is loaded")


raise NameError("Please ensure the anthropic package is loaded")
        if stop is None:
            stop = []
        # Never want model to invent new turns of Human / Assistant dialog.
        stop.extend([self.HUMAN_PROMPT])
        return stop
[docs]class Anthropic(LLM, _AnthropicCommon):
    """Anthropic large language models.
    To use, you should have the ``anthropic`` python package installed, and the
    environment variable ``ANTHROPIC_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            import anthropic
            from langchain.llms import Anthropic
            model = Anthropic(model="<model_name>", anthropic_api_key="my-api-key")
            # Simplest invocation, automatically wrapped with HUMAN_PROMPT
            # and AI_PROMPT.
            response = model("What are the biggest risks facing humanity?")
            # Or if you want to use the chat mode, build a few-shot-prompt, or
            # put words in the Assistant's mouth, use HUMAN_PROMPT and AI_PROMPT:
            raw_prompt = "What are the biggest risks facing humanity?"
            prompt = f"{anthropic.HUMAN_PROMPT} {prompt}{anthropic.AI_PROMPT}"
            response = model(prompt)
    """
    class Config:
        """Configuration for this pydantic object."""
        allow_population_by_field_name = True
        arbitrary_types_allowed = True
    @root_validator()
    def raise_warning(cls, values: Dict) -> Dict:
        """Raise warning that this class is deprecated."""
        warnings.warn(
            "This Anthropic LLM is deprecated. "


warnings.warn(
            "This Anthropic LLM is deprecated. "
            "Please use `from langchain.chat_models import ChatAnthropic` instead"
        )
        return values
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "anthropic-llm"
    def _wrap_prompt(self, prompt: str) -> str:
        if not self.HUMAN_PROMPT or not self.AI_PROMPT:
            raise NameError("Please ensure the anthropic package is loaded")
        if prompt.startswith(self.HUMAN_PROMPT):
            return prompt  # Already wrapped.
        # Guard against common errors in specifying wrong number of newlines.
        corrected_prompt, n_subs = re.subn(r"^\n*Human:", self.HUMAN_PROMPT, prompt)
        if n_subs == 1:
            return corrected_prompt
        # As a last resort, wrap the prompt ourselves to emulate instruct-style.
        return f"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT} Sure, here you go:\n"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        r"""Call out to Anthropic's completion endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                prompt = "What are the biggest risks facing humanity?"
                prompt = f"\n\nHuman: {prompt}\n\nAssistant:"


prompt = f"\n\nHuman: {prompt}\n\nAssistant:"
                response = model(prompt)
        """
        if self.streaming:
            completion = ""
            for chunk in self._stream(
                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                completion += chunk.text
            return completion
        stop = self._get_anthropic_stop(stop)
        params = {**self._default_params, **kwargs}
        response = self.client.completions.create(
            prompt=self._wrap_prompt(prompt),
            stop_sequences=stop,
            **params,
        )
        return response.completion
[docs]    def convert_prompt(self, prompt: PromptValue) -> str:
        return self._wrap_prompt(prompt.to_string())
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Anthropic's completion endpoint asynchronously."""
        if self.streaming:
            completion = ""
            async for chunk in self._astream(
                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                completion += chunk.text
            return completion
        stop = self._get_anthropic_stop(stop)
        params = {**self._default_params, **kwargs}
        response = await self.async_client.completions.create(
            prompt=self._wrap_prompt(prompt),
            stop_sequences=stop,
            **params,
        )
        return response.completion
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,


prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        r"""Call Anthropic completion_stream and return the resulting generator.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            A generator representing the stream of tokens from Anthropic.
        Example:
            .. code-block:: python
                prompt = "Write a poem about a stream."
                prompt = f"\n\nHuman: {prompt}\n\nAssistant:"
                generator = anthropic.stream(prompt)
                for token in generator:
                    yield token
        """
        stop = self._get_anthropic_stop(stop)
        params = {**self._default_params, **kwargs}
        for token in self.client.completions.create(
            prompt=self._wrap_prompt(prompt), stop_sequences=stop, stream=True, **params
        ):
            chunk = GenerationChunk(text=token.completion)
            yield chunk
            if run_manager:
                run_manager.on_llm_new_token(chunk.text, chunk=chunk)
    async def _astream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> AsyncIterator[GenerationChunk]:
        r"""Call Anthropic completion_stream and return the resulting generator.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            A generator representing the stream of tokens from Anthropic.


Returns:
            A generator representing the stream of tokens from Anthropic.
        Example:
            .. code-block:: python
                prompt = "Write a poem about a stream."
                prompt = f"\n\nHuman: {prompt}\n\nAssistant:"
                generator = anthropic.stream(prompt)
                for token in generator:
                    yield token
        """
        stop = self._get_anthropic_stop(stop)
        params = {**self._default_params, **kwargs}
        async for token in await self.async_client.completions.create(
            prompt=self._wrap_prompt(prompt),
            stop_sequences=stop,
            stream=True,
            **params,
        ):
            chunk = GenerationChunk(text=token.completion)
            yield chunk
            if run_manager:
                await run_manager.on_llm_new_token(chunk.text, chunk=chunk)
[docs]    def get_num_tokens(self, text: str) -> int:
        """Calculate number of tokens."""
        if not self.count_tokens:
            raise NameError("Please ensure the anthropic package is loaded")
        return self.count_tokens(text)


Source code for langchain.llms.ctransformers
from functools import partial
from typing import Any, Dict, List, Optional, Sequence
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import root_validator
[docs]class CTransformers(LLM):
    """C Transformers LLM models.
    To use, you should have the ``ctransformers`` python package installed.
    See https://github.com/marella/ctransformers
    Example:
        .. code-block:: python
            from langchain.llms import CTransformers
            llm = CTransformers(model="/path/to/ggml-gpt-2.bin", model_type="gpt2")
    """
    client: Any  #: :meta private:
    model: str
    """The path to a model file or directory or the name of a Hugging Face Hub
    model repo."""
    model_type: Optional[str] = None
    """The model type."""
    model_file: Optional[str] = None
    """The name of the model file in repo or directory."""
    config: Optional[Dict[str, Any]] = None
    """The config parameters.
    See https://github.com/marella/ctransformers#config"""
    lib: Optional[str] = None
    """The path to a shared library or one of `avx2`, `avx`, `basic`."""
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {
            "model": self.model,
            "model_type": self.model_type,
            "model_file": self.model_file,


"model_type": self.model_type,
            "model_file": self.model_file,
            "config": self.config,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "ctransformers"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that ``ctransformers`` package is installed."""
        try:
            from ctransformers import AutoModelForCausalLM
        except ImportError:
            raise ImportError(
                "Could not import `ctransformers` package. "
                "Please install it with `pip install ctransformers`"
            )
        config = values["config"] or {}
        values["client"] = AutoModelForCausalLM.from_pretrained(
            values["model"],
            model_type=values["model_type"],
            model_file=values["model_file"],
            lib=values["lib"],
            **config,
        )
        return values
    def _call(
        self,
        prompt: str,
        stop: Optional[Sequence[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Generate text from a prompt.
        Args:
            prompt: The prompt to generate text from.
            stop: A list of sequences to stop generation when encountered.
        Returns:
            The generated text.
        Example:
            .. code-block:: python
                response = llm("Tell me a joke.")
        """
        text = []
        _run_manager = run_manager or CallbackManagerForLLMRun.get_noop_manager()


_run_manager = run_manager or CallbackManagerForLLMRun.get_noop_manager()
        for chunk in self.client(prompt, stop=stop, stream=True):
            text.append(chunk)
            _run_manager.on_llm_new_token(chunk, verbose=self.verbose)
        return "".join(text)
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Asynchronous Call out to CTransformers generate method.
        Very helpful when streaming (like with websockets!)
        Args:
            prompt: The prompt to pass into the model.
            stop: A list of strings to stop generation when encountered.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = llm("Once upon a time, ")
        """
        text_callback = None
        if run_manager:
            text_callback = partial(run_manager.on_llm_new_token, verbose=self.verbose)
        text = ""
        for token in self.client(prompt, stop=stop, stream=True):
            if text_callback:
                await text_callback(token)
            text += token
        return text


Source code for langchain.llms.petals
import logging
from typing import Any, Dict, List, Mapping, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.pydantic_v1 import Extra, Field, root_validator
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class Petals(LLM):
    """Petals Bloom models.
    To use, you should have the ``petals`` python package installed, and the
    environment variable ``HUGGINGFACE_API_KEY`` set with your API key.
    Any parameters that are valid to be passed to the call can be passed
    in, even if not explicitly saved on this class.
    Example:
        .. code-block:: python
            from langchain.llms import petals
            petals = Petals()
    """
    client: Any
    """The client to use for the API calls."""
    tokenizer: Any
    """The tokenizer to use for the API calls."""
    model_name: str = "bigscience/bloom-petals"
    """The model to use."""
    temperature: float = 0.7
    """What sampling temperature to use"""
    max_new_tokens: int = 256
    """The maximum number of new tokens to generate in the completion."""
    top_p: float = 0.9
    """The cumulative probability for top-p sampling."""
    top_k: Optional[int] = None
    """The number of highest probability vocabulary tokens
    to keep for top-k-filtering."""
    do_sample: bool = True
    """Whether or not to use sampling; use greedy decoding otherwise."""
    max_length: Optional[int] = None


max_length: Optional[int] = None
    """The maximum length of the sequence to be generated."""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call
    not explicitly specified."""
    huggingface_api_key: Optional[str] = None
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""WARNING! {field_name} is not default parameter.
                    {field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        huggingface_api_key = get_from_dict_or_env(
            values, "huggingface_api_key", "HUGGINGFACE_API_KEY"
        )
        try:
            from petals import AutoDistributedModelForCausalLM
            from transformers import AutoTokenizer
            model_name = values["model_name"]


from transformers import AutoTokenizer
            model_name = values["model_name"]
            values["tokenizer"] = AutoTokenizer.from_pretrained(model_name)
            values["client"] = AutoDistributedModelForCausalLM.from_pretrained(
                model_name
            )
            values["huggingface_api_key"] = huggingface_api_key
        except ImportError:
            raise ImportError(
                "Could not import transformers or petals python package."
                "Please install with `pip install -U transformers petals`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling Petals API."""
        normal_params = {
            "temperature": self.temperature,
            "max_new_tokens": self.max_new_tokens,
            "top_p": self.top_p,
            "top_k": self.top_k,
            "do_sample": self.do_sample,
            "max_length": self.max_length,
        }
        return {**normal_params, **self.model_kwargs}
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "petals"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the Petals API."""
        params = self._default_params
        params = {**params, **kwargs}


params = self._default_params
        params = {**params, **kwargs}
        inputs = self.tokenizer(prompt, return_tensors="pt")["input_ids"]
        outputs = self.client.generate(inputs, **params)
        text = self.tokenizer.decode(outputs[0])
        if stop is not None:
            # I believe this is required since the stop tokens
            # are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.tongyi
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
from requests.exceptions import HTTPError
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema import Generation, LLMResult
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator(llm: Tongyi) -> Callable[[Any], Any]:
    min_seconds = 1
    max_seconds = 4
    # Wait 2^x * 1 second between each retry starting with
    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards
    return retry(
        reraise=True,
        stop=stop_after_attempt(llm.max_retries),
        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),
        retry=(retry_if_exception_type(HTTPError)),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
[docs]def generate_with_retry(llm: Tongyi, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    def _generate_with_retry(**_kwargs: Any) -> Any:
        resp = llm.client.call(**_kwargs)
        if resp.status_code == 200:
            return resp
        elif resp.status_code in [400, 401]:
            raise ValueError(


elif resp.status_code in [400, 401]:
            raise ValueError(
                f"status_code: {resp.status_code} \n "
                f"code: {resp.code} \n message: {resp.message}"
            )
        else:
            raise HTTPError(
                f"HTTP error occurred: status_code: {resp.status_code} \n "
                f"code: {resp.code} \n message: {resp.message}",
                response=resp,
            )
    return _generate_with_retry(**kwargs)
[docs]def stream_generate_with_retry(llm: Tongyi, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    def _stream_generate_with_retry(**_kwargs: Any) -> Any:
        stream_resps = []
        resps = llm.client.call(**_kwargs)
        for resp in resps:
            if resp.status_code == 200:
                stream_resps.append(resp)
            elif resp.status_code in [400, 401]:
                raise ValueError(
                    f"status_code: {resp.status_code} \n "
                    f"code: {resp.code} \n message: {resp.message}"
                )
            else:
                raise HTTPError(
                    f"HTTP error occurred: status_code: {resp.status_code} \n "
                    f"code: {resp.code} \n message: {resp.message}",
                    response=resp,
                )
        return stream_resps
    return _stream_generate_with_retry(**kwargs)
[docs]class Tongyi(LLM):
    """Tongyi Qwen large language models.


"""Tongyi Qwen large language models.
    To use, you should have the ``dashscope`` python package installed, and the
    environment variable ``DASHSCOPE_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms import Tongyi
            Tongyi = tongyi()
    """
    @property
    def lc_secrets(self) -> Dict[str, str]:
        return {"dashscope_api_key": "DASHSCOPE_API_KEY"}
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return True
    client: Any  #: :meta private:
    model_name: str = "qwen-plus-v1"
    """Model name to use."""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    top_p: float = 0.8
    """Total probability mass of tokens to consider at each step."""
    dashscope_api_key: Optional[str] = None
    """Dashscope api key provide by alicloud."""
    n: int = 1
    """How many completions to generate for each prompt."""
    streaming: bool = False
    """Whether to stream the results or not."""
    max_retries: int = 10
    """Maximum number of retries to make when generating."""
    prefix_messages: List = Field(default_factory=list)
    """Series of messages for Chat input."""
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "tongyi"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:


@root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        get_from_dict_or_env(values, "dashscope_api_key", "DASHSCOPE_API_KEY")
        try:
            import dashscope
        except ImportError:
            raise ImportError(
                "Could not import dashscope python package. "
                "Please install it with `pip install dashscope`."
            )
        try:
            values["client"] = dashscope.Generation
        except AttributeError:
            raise ValueError(
                "`dashscope` has no `Generation` attribute, this is likely "
                "due to an old version of the dashscope package. Try upgrading it "
                "with `pip install --upgrade dashscope`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling OpenAI API."""
        normal_params = {
            "top_p": self.top_p,
        }
        return {**normal_params, **self.model_kwargs}
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Tongyi's generate endpoint.
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = tongyi("Tell me a joke.")
        """
        params: Dict[str, Any] = {
            **{"model": self.model_name},
            **self._default_params,


**{"model": self.model_name},
            **self._default_params,
            **kwargs,
        }
        completion = generate_with_retry(
            self,
            prompt=prompt,
            **params,
        )
        return completion["output"]["text"]
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        generations = []
        params: Dict[str, Any] = {
            **{"model": self.model_name},
            **self._default_params,
            **kwargs,
        }
        if self.streaming:
            if len(prompts) > 1:
                raise ValueError("Cannot stream results with multiple prompts.")
            params["stream"] = True
            for stream_resp in stream_generate_with_retry(
                self, prompt=prompts[0], **params
            ):
                generations.append(
                    [
                        Generation(
                            text=stream_resp["output"]["text"],
                            generation_info=dict(
                                finish_reason=stream_resp["output"]["finish_reason"],
                            ),
                        )
                    ]
                )
        else:
            for prompt in prompts:
                completion = generate_with_retry(
                    self,
                    prompt=prompt,
                    **params,
                )
                generations.append(
                    [
                        Generation(
                            text=completion["output"]["text"],
                            generation_info=dict(
                                finish_reason=completion["output"]["finish_reason"],
                            ),
                        )
                    ]
                )
        return LLMResult(generations=generations)


Source code for langchain.llms.fake
import asyncio
import time
from typing import Any, AsyncIterator, Iterator, List, Mapping, Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.schema.language_model import LanguageModelInput
from langchain.schema.runnable import RunnableConfig
[docs]class FakeListLLM(LLM):
    """Fake LLM for testing purposes."""
    responses: List[str]
    sleep: Optional[float] = None
    i: int = 0
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "fake-list"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Return next response"""
        response = self.responses[self.i]
        if self.i < len(self.responses) - 1:
            self.i += 1
        else:
            self.i = 0
        return response
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Return next response"""
        response = self.responses[self.i]
        if self.i < len(self.responses) - 1:
            self.i += 1
        else:
            self.i = 0
        return response
    @property


else:
            self.i = 0
        return response
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        return {"responses": self.responses}
[docs]class FakeStreamingListLLM(FakeListLLM):
    """Fake streaming list LLM for testing purposes."""
[docs]    def stream(
        self,
        input: LanguageModelInput,
        config: Optional[RunnableConfig] = None,
        *,
        stop: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> Iterator[str]:
        result = self.invoke(input, config)
        for c in result:
            if self.sleep is not None:
                time.sleep(self.sleep)
            yield c
[docs]    async def astream(
        self,
        input: LanguageModelInput,
        config: Optional[RunnableConfig] = None,
        *,
        stop: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> AsyncIterator[str]:
        result = await self.ainvoke(input, config)
        for c in result:
            if self.sleep is not None:
                await asyncio.sleep(self.sleep)
            yield c