docs = []
        for thread_id in thread_ids:
            doc = self.process_thread(thread_id, include_images, include_messages)
            if doc is not None:
                docs.append(doc)
        return docs
[docs]    def process_thread(
        self, thread_id: str, include_images: bool, include_messages: bool
    ) -> Optional[Document]:
        thread = self.quip_client.get_thread(thread_id)
        thread_id = thread["thread"]["id"]
        title = thread["thread"]["title"]
        link = thread["thread"]["link"]
        update_ts = thread["thread"]["updated_usec"]
        sanitized_title = QuipLoader._sanitize_title(title)
        logger.info(
            f"processing thread {thread_id} title {sanitized_title} "
            f"link {link} update_ts {update_ts}"
        )
        if "html" in thread:
            # Parse the document
            try:
                tree = self.quip_client.parse_document_html(thread["html"])
            except xml.etree.cElementTree.ParseError as e:
                logger.error(f"Error parsing thread {title} {thread_id}, skipping, {e}")
                return None
            metadata = {
                "title": sanitized_title,
                "update_ts": update_ts,
                "id": thread_id,
                "source": link,
            }
            # Download each image and replace with the new URL
            text = ""
            if include_images:
                text = self.process_thread_images(tree)
            if include_messages:
                text = text + "/n" + self.process_thread_messages(thread_id)
            return Document(
                page_content=thread["html"] + text,
                metadata=metadata,
            )
        return None


metadata=metadata,
            )
        return None
[docs]    def process_thread_images(self, tree: ElementTree) -> str:
        text = ""
        try:
            from PIL import Image
            from pytesseract import pytesseract
        except ImportError:
            raise ImportError(
                "`Pillow or pytesseract` package not found, "
                "please run "
                "`pip install Pillow` or `pip install pytesseract`"
            )
        for img in tree.iter("img"):
            src = img.get("src")
            if not src or not src.startswith("/blob"):
                continue
            _, _, thread_id, blob_id = src.split("/")
            blob_response = self.quip_client.get_blob(thread_id, blob_id)
            try:
                image = Image.open(BytesIO(blob_response.read()))
                text = text + "\n" + pytesseract.image_to_string(image)
            except OSError as e:
                logger.error(f"failed to convert image to text, {e}")
                raise e
        return text
[docs]    def process_thread_messages(self, thread_id: str) -> str:
        max_created_usec = None
        messages = []
        while True:
            chunk = self.quip_client.get_messages(
                thread_id, max_created_usec=max_created_usec, count=100
            )
            messages.extend(chunk)
            if chunk:
                max_created_usec = chunk[-1]["created_usec"] - 1
            else:
                break
        messages.reverse()
        texts = [message["text"] for message in messages]
        return "\n".join(texts)
    @staticmethod
    def _sanitize_title(title: str) -> str:
        sanitized_title = re.sub(r"\s", " ", title)


sanitized_title = re.sub(r"\s", " ", title)
        sanitized_title = re.sub(r"(?u)[^- \w.]", "", sanitized_title)
        if len(sanitized_title) > _MAXIMUM_TITLE_LENGTH:
            sanitized_title = sanitized_title[:_MAXIMUM_TITLE_LENGTH]
        return sanitized_title


Source code for langchain.document_loaders.word_document
"""Loads word documents."""
import os
import tempfile
from abc import ABC
from typing import List
from urllib.parse import urlparse
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class Docx2txtLoader(BaseLoader, ABC):
    """Load `DOCX` file using `docx2txt` and chunks at character level.
    Defaults to check for local file, but if the file is a web path, it will download it
    to a temporary file, and use that, then clean up the temporary file after completion
    """
[docs]    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        if "~" in self.file_path:
            self.file_path = os.path.expanduser(self.file_path)
        # If the file is a web path, download it to a temporary file, and use that
        if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
            r = requests.get(self.file_path)
            if r.status_code != 200:
                raise ValueError(
                    "Check the url of your file; returned status code %s"
                    % r.status_code
                )
            self.web_path = self.file_path
            self.temp_file = tempfile.NamedTemporaryFile()
            self.temp_file.write(r.content)
            self.file_path = self.temp_file.name
        elif not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file or url" % self.file_path)
    def __del__(self) -> None:
        if hasattr(self, "temp_file"):


if hasattr(self, "temp_file"):
            self.temp_file.close()
[docs]    def load(self) -> List[Document]:
        """Load given path as single page."""
        import docx2txt
        return [
            Document(
                page_content=docx2txt.process(self.file_path),
                metadata={"source": self.file_path},
            )
        ]
    @staticmethod
    def _is_valid_url(url: str) -> bool:
        """Check if the url is valid."""
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)
[docs]class UnstructuredWordDocumentLoader(UnstructuredFileLoader):
    """Load `Microsoft Word` file using `Unstructured`.
    Works with both .docx and .doc files.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain.document_loaders import UnstructuredWordDocumentLoader
    loader = UnstructuredWordDocumentLoader(
        "example.docx", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-docx
    """
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.file_utils.filetype import FileType, detect_filetype


from unstructured.file_utils.filetype import FileType, detect_filetype
        unstructured_version = tuple(
            [int(x) for x in __unstructured_version__.split(".")]
        )
        # NOTE(MthwRobinson) - magic will raise an import error if the libmagic
        # system dependency isn't installed. If it's not installed, we'll just
        # check the file extension
        try:
            import magic  # noqa: F401
            is_doc = detect_filetype(self.file_path) == FileType.DOC
        except ImportError:
            _, extension = os.path.splitext(str(self.file_path))
            is_doc = extension == ".doc"
        if is_doc and unstructured_version < (0, 4, 11):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning .doc files is only supported in unstructured>=0.4.11. "
                "Please upgrade the unstructured package and try again."
            )
        if is_doc:
            from unstructured.partition.doc import partition_doc
            return partition_doc(filename=self.file_path, **self.unstructured_kwargs)
        else:
            from unstructured.partition.docx import partition_docx
            return partition_docx(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.s3_file
from __future__ import annotations
import os
import tempfile
from typing import TYPE_CHECKING, List, Optional, Union
from langchain.document_loaders.unstructured import UnstructuredBaseLoader
if TYPE_CHECKING:
    import botocore
[docs]class S3FileLoader(UnstructuredBaseLoader):
    """Load from `Amazon AWS S3` file."""
[docs]    def __init__(
        self,
        bucket: str,
        key: str,
        *,
        region_name: Optional[str] = None,
        api_version: Optional[str] = None,
        use_ssl: Optional[bool] = True,
        verify: Union[str, bool, None] = None,
        endpoint_url: Optional[str] = None,
        aws_access_key_id: Optional[str] = None,
        aws_secret_access_key: Optional[str] = None,
        aws_session_token: Optional[str] = None,
        boto_config: Optional[botocore.client.Config] = None,
    ):
        """Initialize with bucket and key name.
        :param bucket: The name of the S3 bucket.
        :param key: The key of the S3 object.
        :param region_name: The name of the region associated with the client.
            A client is associated with a single region.
        :param api_version: The API version to use.  By default, botocore will
            use the latest API version when creating a client.  You only need
            to specify this parameter if you want to use a previous API version
            of the client.
        :param use_ssl: Whether or not to use SSL.  By default, SSL is used.
            Note that not all services support non-ssl connections.


Note that not all services support non-ssl connections.
        :param verify: Whether or not to verify SSL certificates.
            By default SSL certificates are verified.  You can provide the
            following values:
            * False - do not validate SSL certificates.  SSL will still be
              used (unless use_ssl is False), but SSL certificates
              will not be verified.
            * path/to/cert/bundle.pem - A filename of the CA cert bundle to
              uses.  You can specify this argument if you want to use a
              different CA cert bundle than the one used by botocore.
        :param endpoint_url: The complete URL to use for the constructed
            client.  Normally, botocore will automatically construct the
            appropriate URL to use when communicating with a service.  You can
            specify a complete URL (including the "http/https" scheme) to
            override this behavior.  If this value is provided, then
            ``use_ssl`` is ignored.
        :param aws_access_key_id: The access key to use when creating
            the client.  This is entirely optional, and if not provided,
            the credentials configured for the session will automatically
            be used.  You only need to provide this argument if you want
            to override the credentials used for this specific client.
        :param aws_secret_access_key: The secret key to use when creating
            the client.  Same semantics as aws_access_key_id above.
        :param aws_session_token: The session token to use when creating
            the client.  Same semantics as aws_access_key_id above.
        :type boto_config: botocore.client.Config
        :param boto_config: Advanced boto3 client configuration options. If a value
            is specified in the client config, its value will take precedence


is specified in the client config, its value will take precedence
            over environment variables and configuration values, but not over
            a value passed explicitly to the method. If a default config
            object is set on the session, the config object used when creating
            the client will be the result of calling ``merge()`` on the
            default config with the config provided to this call.
        """
        super().__init__()
        self.bucket = bucket
        self.key = key
        self.region_name = region_name
        self.api_version = api_version
        self.use_ssl = use_ssl
        self.verify = verify
        self.endpoint_url = endpoint_url
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.aws_session_token = aws_session_token
        self.boto_config = boto_config
    def _get_elements(self) -> List:
        """Get elements."""
        from unstructured.partition.auto import partition
        try:
            import boto3
        except ImportError:
            raise ImportError(
                "Could not import `boto3` python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.client(
            "s3",
            region_name=self.region_name,
            api_version=self.api_version,
            use_ssl=self.use_ssl,
            verify=self.verify,
            endpoint_url=self.endpoint_url,
            aws_access_key_id=self.aws_access_key_id,
            aws_secret_access_key=self.aws_secret_access_key,
            aws_session_token=self.aws_session_token,
            config=self.boto_config,
        )
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.key}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)


os.makedirs(os.path.dirname(file_path), exist_ok=True)
            s3.download_file(self.bucket, self.key, file_path)
            return partition(filename=file_path)
    def _get_metadata(self) -> dict:
        return {"source": f"s3://{self.bucket}/{self.key}"}


Source code for langchain.document_loaders.trello
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env
if TYPE_CHECKING:
    from trello import Board, Card, TrelloClient
[docs]class TrelloLoader(BaseLoader):
    """Load cards from a `Trello` board."""
[docs]    def __init__(
        self,
        client: TrelloClient,
        board_name: str,
        *,
        include_card_name: bool = True,
        include_comments: bool = True,
        include_checklist: bool = True,
        card_filter: Literal["closed", "open", "all"] = "all",
        extra_metadata: Tuple[str, ...] = ("due_date", "labels", "list", "closed"),
    ):
        """Initialize Trello loader.
        Args:
            client: Trello API client.
            board_name: The name of the Trello board.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        self.client = client
        self.board_name = board_name
        self.include_card_name = include_card_name
        self.include_comments = include_comments


self.include_card_name = include_card_name
        self.include_comments = include_comments
        self.include_checklist = include_checklist
        self.extra_metadata = extra_metadata
        self.card_filter = card_filter
[docs]    @classmethod
    def from_credentials(
        cls,
        board_name: str,
        *,
        api_key: Optional[str] = None,
        token: Optional[str] = None,
        **kwargs: Any,
    ) -> TrelloLoader:
        """Convenience constructor that builds TrelloClient init param for you.
        Args:
            board_name: The name of the Trello board.
            api_key: Trello API key. Can also be specified as environment variable
                TRELLO_API_KEY.
            token: Trello token. Can also be specified as environment variable
                TRELLO_TOKEN.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        try:
            from trello import TrelloClient  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import trello python package. "
                "Please install it with `pip install py-trello`."
            ) from ex
        api_key = api_key or get_from_env("api_key", "TRELLO_API_KEY")


token = token or get_from_env("token", "TRELLO_TOKEN")
        client = TrelloClient(api_key=api_key, token=token)
        return cls(client, board_name, **kwargs)
[docs]    def load(self) -> List[Document]:
        """Loads all cards from the specified Trello board.
        You can filter the cards, metadata and text included by using the optional
            parameters.
         Returns:
            A list of documents, one for each card in the board.
        """
        try:
            from bs4 import BeautifulSoup  # noqa: F401
        except ImportError as ex:
            raise ImportError(
                "`beautifulsoup4` package not found, please run"
                " `pip install beautifulsoup4`"
            ) from ex
        board = self._get_board()
        # Create a dictionary with the list IDs as keys and the list names as values
        list_dict = {list_item.id: list_item.name for list_item in board.list_lists()}
        # Get Cards on the board
        cards = board.get_cards(card_filter=self.card_filter)
        return [self._card_to_doc(card, list_dict) for card in cards]
    def _get_board(self) -> Board:
        # Find the first board with a matching name
        board = next(
            (b for b in self.client.list_boards() if b.name == self.board_name), None
        )
        if not board:
            raise ValueError(f"Board `{self.board_name}` not found.")
        return board
    def _card_to_doc(self, card: Card, list_dict: dict) -> Document:
        from bs4 import BeautifulSoup  # type: ignore
        text_content = ""
        if self.include_card_name:
            text_content = card.name + "\n"


if self.include_card_name:
            text_content = card.name + "\n"
        if card.description.strip():
            text_content += BeautifulSoup(card.description, "lxml").get_text()
        if self.include_checklist:
            # Get all the checklist items on the card
            for checklist in card.checklists:
                if checklist.items:
                    items = [
                        f"{item['name']}:{item['state']}" for item in checklist.items
                    ]
                    text_content += f"\n{checklist.name}\n" + "\n".join(items)
        if self.include_comments:
            # Get all the comments on the card
            comments = [
                BeautifulSoup(comment["data"]["text"], "lxml").get_text()
                for comment in card.comments
            ]
            text_content += "Comments:" + "\n".join(comments)
        # Default metadata fields
        metadata = {
            "title": card.name,
            "id": card.id,
            "url": card.url,
        }
        # Extra metadata fields. Card object is not subscriptable.
        if "labels" in self.extra_metadata:
            metadata["labels"] = [label.name for label in card.labels]
        if "list" in self.extra_metadata:
            if card.list_id in list_dict:
                metadata["list"] = list_dict[card.list_id]
        if "closed" in self.extra_metadata:
            metadata["closed"] = card.closed
        if "due_date" in self.extra_metadata:
            metadata["due_date"] = card.due_date
        return Document(page_content=text_content, metadata=metadata)


Source code for langchain.document_loaders.url_selenium
"""Loader that uses Selenium to load a page, then uses unstructured to load the html.
"""
import logging
from typing import TYPE_CHECKING, List, Literal, Optional, Union
if TYPE_CHECKING:
    from selenium.webdriver import Chrome, Firefox
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class SeleniumURLLoader(BaseLoader):
    """Load `HTML` pages with `Selenium` and parse with `Unstructured`.
    This is useful for loading pages that require javascript to render.
    Attributes:
        urls (List[str]): List of URLs to load.
        continue_on_failure (bool): If True, continue loading other URLs on failure.
        browser (str): The browser to use, either 'chrome' or 'firefox'.
        binary_location (Optional[str]): The location of the browser binary.
        executable_path (Optional[str]): The path to the browser executable.
        headless (bool): If True, the browser will run in headless mode.
        arguments [List[str]]: List of arguments to pass to the browser.
    """
[docs]    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        browser: Literal["chrome", "firefox"] = "chrome",
        binary_location: Optional[str] = None,
        executable_path: Optional[str] = None,
        headless: bool = True,
        arguments: List[str] = [],
    ):
        """Load a list of URLs using Selenium and unstructured."""
        try:
            import selenium  # noqa:F401
        except ImportError:
            raise ImportError(


import selenium  # noqa:F401
        except ImportError:
            raise ImportError(
                "selenium package not found, please install it with "
                "`pip install selenium`"
            )
        try:
            import unstructured  # noqa:F401
        except ImportError:
            raise ImportError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.browser = browser
        self.binary_location = binary_location
        self.executable_path = executable_path
        self.headless = headless
        self.arguments = arguments
    def _get_driver(self) -> Union["Chrome", "Firefox"]:
        """Create and return a WebDriver instance based on the specified browser.
        Raises:
            ValueError: If an invalid browser is specified.
        Returns:
            Union[Chrome, Firefox]: A WebDriver instance for the specified browser.
        """
        if self.browser.lower() == "chrome":
            from selenium.webdriver import Chrome
            from selenium.webdriver.chrome.options import Options as ChromeOptions
            from selenium.webdriver.chrome.service import Service
            chrome_options = ChromeOptions()
            for arg in self.arguments:
                chrome_options.add_argument(arg)
            if self.headless:
                chrome_options.add_argument("--headless")
                chrome_options.add_argument("--no-sandbox")
            if self.binary_location is not None:
                chrome_options.binary_location = self.binary_location
            if self.executable_path is None:
                return Chrome(options=chrome_options)
            return Chrome(
                options=chrome_options,
                service=Service(executable_path=self.executable_path),
            )
        elif self.browser.lower() == "firefox":
            from selenium.webdriver import Firefox


elif self.browser.lower() == "firefox":
            from selenium.webdriver import Firefox
            from selenium.webdriver.firefox.options import Options as FirefoxOptions
            from selenium.webdriver.firefox.service import Service
            firefox_options = FirefoxOptions()
            for arg in self.arguments:
                firefox_options.add_argument(arg)
            if self.headless:
                firefox_options.add_argument("--headless")
            if self.binary_location is not None:
                firefox_options.binary_location = self.binary_location
            if self.executable_path is None:
                return Firefox(options=firefox_options)
            return Firefox(
                options=firefox_options,
                service=Service(executable_path=self.executable_path),
            )
        else:
            raise ValueError("Invalid browser specified. Use 'chrome' or 'firefox'.")
    def _build_metadata(self, url: str, driver: Union["Chrome", "Firefox"]) -> dict:
        from selenium.common.exceptions import NoSuchElementException
        from selenium.webdriver.common.by import By
        """Build metadata based on the contents of the webpage"""
        metadata = {
            "source": url,
            "title": "No title found.",
            "description": "No description found.",
            "language": "No language found.",
        }
        if title := driver.title:
            metadata["title"] = title
        try:
            if description := driver.find_element(
                By.XPATH, '//meta[@name="description"]'
            ):
                metadata["description"] = (
                    description.get_attribute("content") or "No description found."
                )
        except NoSuchElementException:
            pass
        try:
            if html_tag := driver.find_element(By.TAG_NAME, "html"):
                metadata["language"] = (
                    html_tag.get_attribute("lang") or "No language found."
                )
        except NoSuchElementException:


)
        except NoSuchElementException:
            pass
        return metadata
[docs]    def load(self) -> List[Document]:
        """Load the specified URLs using Selenium and create Document instances.
        Returns:
            List[Document]: A list of Document instances with loaded content.
        """
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        driver = self._get_driver()
        for url in self.urls:
            try:
                driver.get(url)
                page_content = driver.page_source
                elements = partition_html(text=page_content)
                text = "\n\n".join([str(el) for el in elements])
                metadata = self._build_metadata(url, driver)
                docs.append(Document(page_content=text, metadata=metadata))
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                else:
                    raise e
        driver.quit()
        return docs


Source code for langchain.document_loaders.wikipedia
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.wikipedia import WikipediaAPIWrapper
[docs]class WikipediaLoader(BaseLoader):
    """Load from `Wikipedia`.
    The hard limit on the number of downloaded Documents is 300 for now.
    Each wiki page represents one Document.
    """
[docs]    def __init__(
        self,
        query: str,
        lang: str = "en",
        load_max_docs: Optional[int] = 100,
        load_all_available_meta: Optional[bool] = False,
        doc_content_chars_max: Optional[int] = 4000,
    ):
        """
        Initializes a new instance of the WikipediaLoader class.
        Args:
            query (str): The query string to search on Wikipedia.
            lang (str, optional): The language code for the Wikipedia language edition.
                Defaults to "en".
            load_max_docs (int, optional): The maximum number of documents to load.
                Defaults to 100.
            load_all_available_meta (bool, optional): Indicates whether to load all
                available metadata for each document. Defaults to False.
            doc_content_chars_max (int, optional): The maximum number of characters
                for the document content. Defaults to 4000.
        """
        self.query = query
        self.lang = lang
        self.load_max_docs = load_max_docs
        self.load_all_available_meta = load_all_available_meta
        self.doc_content_chars_max = doc_content_chars_max
[docs]    def load(self) -> List[Document]:
        """
        Loads the query result from Wikipedia into a list of Documents.
        Returns:


Loads the query result from Wikipedia into a list of Documents.
        Returns:
            List[Document]: A list of Document objects representing the loaded
                Wikipedia pages.
        """
        client = WikipediaAPIWrapper(
            lang=self.lang,
            top_k_results=self.load_max_docs,
            load_all_available_meta=self.load_all_available_meta,
            doc_content_chars_max=self.doc_content_chars_max,
        )
        docs = client.load(self.query)
        return docs


Source code for langchain.document_loaders.tensorflow_datasets
from typing import Callable, Dict, Iterator, List, Optional
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
from langchain.utilities.tensorflow_datasets import TensorflowDatasets
[docs]class TensorflowDatasetLoader(BaseLoader):
    """Load from `TensorFlow Dataset`.
    Attributes:
        dataset_name: the name of the dataset to load
        split_name: the name of the split to load.
        load_max_docs: a limit to the number of loaded documents. Defaults to 100.
        sample_to_document_function: a function that converts a dataset sample
          into a Document
    Example:
        .. code-block:: python
            from langchain.document_loaders import TensorflowDatasetLoader
            def mlqaen_example_to_document(example: dict) -> Document:
                return Document(
                    page_content=decode_to_str(example["context"]),
                    metadata={
                        "id": decode_to_str(example["id"]),
                        "title": decode_to_str(example["title"]),
                        "question": decode_to_str(example["question"]),
                        "answer": decode_to_str(example["answers"]["text"][0]),
                    },
                )
            tsds_client = TensorflowDatasetLoader(
                    dataset_name="mlqa/en",
                    split_name="test",
                    load_max_docs=100,
                    sample_to_document_function=mlqaen_example_to_document,
                )
    """
[docs]    def __init__(
        self,
        dataset_name: str,
        split_name: str,
        load_max_docs: Optional[int] = 100,
        sample_to_document_function: Optional[Callable[[Dict], Document]] = None,
    ):
        """Initialize the TensorflowDatasetLoader.
        Args:


):
        """Initialize the TensorflowDatasetLoader.
        Args:
            dataset_name: the name of the dataset to load
            split_name: the name of the split to load.
            load_max_docs: a limit to the number of loaded documents. Defaults to 100.
            sample_to_document_function: a function that converts a dataset sample
                into a Document.
        """
        self.dataset_name: str = dataset_name
        self.split_name: str = split_name
        self.load_max_docs = load_max_docs
        """The maximum number of documents to load."""
        self.sample_to_document_function: Optional[
            Callable[[Dict], Document]
        ] = sample_to_document_function
        """Custom function that transform a dataset sample into a Document."""
        self._tfds_client = TensorflowDatasets(
            dataset_name=self.dataset_name,
            split_name=self.split_name,
            load_max_docs=self.load_max_docs,
            sample_to_document_function=self.sample_to_document_function,
        )
[docs]    def lazy_load(self) -> Iterator[Document]:
        yield from self._tfds_client.lazy_load()
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())


Source code for langchain.document_loaders.tsv
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredTSVLoader(UnstructuredFileLoader):
    """Load `TSV` files using `Unstructured`.
    Like other
    Unstructured loaders, UnstructuredTSVLoader can be used in both
    "single" and "elements" mode. If you use the loader in "elements"
    mode, the TSV file will be a single Unstructured Table element.
    If you use the loader in "elements" mode, an HTML representation
    of the table will be available in the "text_as_html" key in the
    document metadata.
    Examples
    --------
    from langchain.document_loaders.tsv import UnstructuredTSVLoader
    loader = UnstructuredTSVLoader("stanley-cups.tsv", mode="elements")
    docs = loader.load()
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.7.6")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.tsv import partition_tsv
        return partition_tsv(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.brave_search
from typing import Iterator, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.brave_search import BraveSearchWrapper
[docs]class BraveSearchLoader(BaseLoader):
    """Load with `Brave Search` engine."""
[docs]    def __init__(self, query: str, api_key: str, search_kwargs: Optional[dict] = None):
        """Initializes the BraveLoader.
        Args:
            query: The query to search for.
            api_key: The API key to use.
            search_kwargs: The search kwargs to use.
        """
        self.query = query
        self.api_key = api_key
        self.search_kwargs = search_kwargs or {}
[docs]    def load(self) -> List[Document]:
        brave_client = BraveSearchWrapper(
            api_key=self.api_key,
            search_kwargs=self.search_kwargs,
        )
        return brave_client.download_documents(self.query)
[docs]    def lazy_load(self) -> Iterator[Document]:
        for doc in self.load():
            yield doc


Source code for langchain.document_loaders.rst
"""Loads RST files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredRSTLoader(UnstructuredFileLoader):
    """Load `RST` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain.document_loaders import UnstructuredRSTLoader
    loader = UnstructuredRSTLoader(
        "example.rst", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-rst
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Initialize with a file path.
        Args:
            file_path: The path to the file to load.
            mode: The mode to use for partitioning. See unstructured for details.
                Defaults to "single".
            **unstructured_kwargs: Additional keyword arguments to pass
                to unstructured.
        """
        validate_unstructured_version(min_unstructured_version="0.7.5")


"""
        validate_unstructured_version(min_unstructured_version="0.7.5")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.rst import partition_rst
        return partition_rst(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.rss
import logging
from typing import Any, Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.news import NewsURLLoader
logger = logging.getLogger(__name__)
[docs]class RSSFeedLoader(BaseLoader):
    """Load news articles from `RSS` feeds using `Unstructured`.
    Args:
        urls: URLs for RSS feeds to load. Each articles in the feed is loaded into its own document.
        opml: OPML file to load feed urls from. Only one of urls or opml should be provided.  The value
        can be a URL string, or OPML markup contents as byte or string.
        continue_on_failure: If True, continue loading documents even if
            loading fails for a particular URL.
        show_progress_bar: If True, use tqdm to show a loading progress bar. Requires
            tqdm to be installed, ``pip install tqdm``.
        **newsloader_kwargs: Any additional named arguments to pass to
            NewsURLLoader.
    Example:
        .. code-block:: python
            from langchain.document_loaders import RSSFeedLoader
            loader = RSSFeedLoader(
                urls=["<url-1>", "<url-2>"],
            )
            docs = loader.load()
    The loader uses feedparser to parse RSS feeds.  The feedparser library is not installed by default so you should
    install it if using this loader:
    https://pythonhosted.org/feedparser/
    If you use OPML, you should also install listparser:
    https://pythonhosted.org/listparser/
    Finally, newspaper is used to process each article:
    https://newspaper.readthedocs.io/en/latest/


https://newspaper.readthedocs.io/en/latest/
    """  # noqa: E501
[docs]    def __init__(
        self,
        urls: Optional[Sequence[str]] = None,
        opml: Optional[str] = None,
        continue_on_failure: bool = True,
        show_progress_bar: bool = False,
        **newsloader_kwargs: Any,
    ) -> None:
        """Initialize with urls or OPML."""
        if (urls is None) == (
            opml is None
        ):  # This is True if both are None or neither is None
            raise ValueError(
                "Provide either the urls or the opml argument, but not both."
            )
        self.urls = urls
        self.opml = opml
        self.continue_on_failure = continue_on_failure
        self.show_progress_bar = show_progress_bar
        self.newsloader_kwargs = newsloader_kwargs
[docs]    def load(self) -> List[Document]:
        iter = self.lazy_load()
        if self.show_progress_bar:
            try:
                from tqdm import tqdm
            except ImportError as e:
                raise ImportError(
                    "Package tqdm must be installed if show_progress_bar=True. "
                    "Please install with 'pip install tqdm' or set "
                    "show_progress_bar=False."
                ) from e
            iter = tqdm(iter)
        return list(iter)
    @property
    def _get_urls(self) -> Sequence[str]:
        if self.urls:
            return self.urls
        try:
            import listparser
        except ImportError as e:
            raise ImportError(
                "Package listparser must be installed if the opml arg is used. "
                "Please install with 'pip install listparser' or use the "


"Please install with 'pip install listparser' or use the "
                "urls arg instead."
            ) from e
        rss = listparser.parse(self.opml)
        return [feed.url for feed in rss.feeds]
[docs]    def lazy_load(self) -> Iterator[Document]:
        try:
            import feedparser  # noqa:F401
        except ImportError:
            raise ImportError(
                "feedparser package not found, please install it with "
                "`pip install feedparser`"
            )
        for url in self._get_urls:
            try:
                feed = feedparser.parse(url)
                if getattr(feed, "bozo", False):
                    raise ValueError(
                        f"Error fetching {url}, exception: {feed.bozo_exception}"
                    )
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching {url}, exception: {e}")
                    continue
                else:
                    raise e
            try:
                for entry in feed.entries:
                    loader = NewsURLLoader(
                        urls=[entry.link],
                        **self.newsloader_kwargs,
                    )
                    article = loader.load()[0]
                    article.metadata["feed"] = url
                    yield article
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error processing entry {entry.link}, exception: {e}")
                    continue
                else:
                    raise e


Source code for langchain.document_loaders.epub
from typing import List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEPubLoader(UnstructuredFileLoader):
    """Load `EPub` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain.document_loaders import UnstructuredEPubLoader
    loader = UnstructuredEPubLoader(
        "example.epub", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-epub
    """
    def _get_elements(self) -> List:
        min_unstructured_version = "0.5.4"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning epub files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        from unstructured.partition.epub import partition_epub
        return partition_epub(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.mastodon
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import mastodon
def _dependable_mastodon_import() -> mastodon:
    try:
        import mastodon
    except ImportError:
        raise ImportError(
            "Mastodon.py package not found, "
            "please install it with `pip install Mastodon.py`"
        )
    return mastodon
[docs]class MastodonTootsLoader(BaseLoader):
    """Load the `Mastodon` 'toots'."""
[docs]    def __init__(
        self,
        mastodon_accounts: Sequence[str],
        number_toots: Optional[int] = 100,
        exclude_replies: bool = False,
        access_token: Optional[str] = None,
        api_base_url: str = "https://mastodon.social",
    ):
        """Instantiate Mastodon toots loader.
        Args:
            mastodon_accounts: The list of Mastodon accounts to query.
            number_toots: How many toots to pull for each account. Defaults to 100.
            exclude_replies: Whether to exclude reply toots from the load.
                Defaults to False.
            access_token: An access token if toots are loaded as a Mastodon app. Can
                also be specified via the environment variables "MASTODON_ACCESS_TOKEN".
            api_base_url: A Mastodon API base URL to talk to, if not using the default.
                Defaults to "https://mastodon.social".
        """


Defaults to "https://mastodon.social".
        """
        mastodon = _dependable_mastodon_import()
        access_token = access_token or os.environ.get("MASTODON_ACCESS_TOKEN")
        self.api = mastodon.Mastodon(
            access_token=access_token, api_base_url=api_base_url
        )
        self.mastodon_accounts = mastodon_accounts
        self.number_toots = number_toots
        self.exclude_replies = exclude_replies
[docs]    def load(self) -> List[Document]:
        """Load toots into documents."""
        results: List[Document] = []
        for account in self.mastodon_accounts:
            user = self.api.account_lookup(account)
            toots = self.api.account_statuses(
                user.id,
                only_media=False,
                pinned=False,
                exclude_replies=self.exclude_replies,
                exclude_reblogs=True,
                limit=self.number_toots,
            )
            docs = self._format_toots(toots, user)
            results.extend(docs)
        return results
    def _format_toots(
        self, toots: List[Dict[str, Any]], user_info: dict
    ) -> Iterable[Document]:
        """Format toots into documents.
        Adding user info, and selected toot fields into the metadata.
        """
        for toot in toots:
            metadata = {
                "created_at": toot["created_at"],
                "user_info": user_info,
                "is_reply": toot["in_reply_to_id"] is not None,
            }
            yield Document(
                page_content=toot["content"],
                metadata=metadata,
            )


Source code for langchain.document_loaders.gcs_directory
from typing import Callable, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.gcs_file import GCSFileLoader
from langchain.utilities.vertexai import get_client_info
[docs]class GCSDirectoryLoader(BaseLoader):
    """Load from GCS directory."""
[docs]    def __init__(
        self,
        project_name: str,
        bucket: str,
        prefix: str = "",
        loader_func: Optional[Callable[[str], BaseLoader]] = None,
    ):
        """Initialize with bucket and key name.
        Args:
            project_name: The ID of the project for the GCS bucket.
            bucket: The name of the GCS bucket.
            prefix: The prefix of the GCS bucket.
            loader_func: A loader function that instantiates a loader based on a
                file_path argument. If nothing is provided, the  GCSFileLoader
                would use its default loader.
        """
        self.project_name = project_name
        self.bucket = bucket
        self.prefix = prefix
        self._loader_func = loader_func
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ImportError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        client = storage.Client(
            project=self.project_name,
            client_info=get_client_info(module="google-cloud-storage"),
        )
        docs = []
        for blob in client.list_blobs(self.bucket, prefix=self.prefix):


for blob in client.list_blobs(self.bucket, prefix=self.prefix):
            # we shall just skip directories since GCSFileLoader creates
            # intermediate directories on the fly
            if blob.name.endswith("/"):
                continue
            loader = GCSFileLoader(
                self.project_name, self.bucket, blob.name, loader_func=self._loader_func
            )
            docs.extend(loader.load())
        return docs


Source code for langchain.document_loaders.web_base
"""Web base loader class."""
import asyncio
import logging
import warnings
from typing import Any, Dict, Iterator, List, Optional, Sequence, Union
import aiohttp
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
default_header_template = {
    "User-Agent": "",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
    ";q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://www.google.com/",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
}
def _build_metadata(soup: Any, url: str) -> dict:
    """Build metadata from BeautifulSoup output."""
    metadata = {"source": url}
    if title := soup.find("title"):
        metadata["title"] = title.get_text()
    if description := soup.find("meta", attrs={"name": "description"}):
        metadata["description"] = description.get("content", "No description found.")
    if html := soup.find("html"):
        metadata["language"] = html.get("lang", "No language found.")
    return metadata
[docs]class WebBaseLoader(BaseLoader):
    """Load HTML pages using `urllib` and parse them with `BeautifulSoup'."""
[docs]    def __init__(
        self,
        web_path: Union[str, Sequence[str]] = "",
        header_template: Optional[dict] = None,
        verify_ssl: bool = True,
        proxies: Optional[dict] = None,


verify_ssl: bool = True,
        proxies: Optional[dict] = None,
        continue_on_failure: bool = False,
        autoset_encoding: bool = True,
        encoding: Optional[str] = None,
        web_paths: Sequence[str] = (),
        requests_per_second: int = 2,
        default_parser: str = "html.parser",
        requests_kwargs: Optional[Dict[str, Any]] = None,
        raise_for_status: bool = False,
        bs_get_text_kwargs: Optional[Dict[str, Any]] = None,
        bs_kwargs: Optional[Dict[str, Any]] = None,
        session: Any = None,
    ) -> None:
        """Initialize loader.
        Args:
            web_paths: Web paths to load from.
            requests_per_second: Max number of concurrent requests to make.
            default_parser: Default parser to use for BeautifulSoup.
            requests_kwargs: kwargs for requests
            raise_for_status: Raise an exception if http status code denotes an error.
            bs_get_text_kwargs: kwargs for beatifulsoup4 get_text
            bs_kwargs: kwargs for beatifulsoup4 web page parsing
        """
        # web_path kept for backwards-compatibility.
        if web_path and web_paths:
            raise ValueError(
                "Received web_path and web_paths. Only one can be specified. "
                "web_path is deprecated, web_paths should be used."
            )
        if web_paths:
            self.web_paths = list(web_paths)
        elif isinstance(web_path, str):
            self.web_paths = [web_path]
        elif isinstance(web_path, Sequence):
            self.web_paths = list(web_path)
        else:
            raise TypeError(
                f"web_path must be str or Sequence[str] got ({type(web_path)}) or"


f" web_paths must be Sequence[str] got ({type(web_paths)})"
            )
        self.requests_per_second = requests_per_second
        self.default_parser = default_parser
        self.requests_kwargs = requests_kwargs or {}
        self.raise_for_status = raise_for_status
        self.bs_get_text_kwargs = bs_get_text_kwargs or {}
        self.bs_kwargs = bs_kwargs or {}
        if session:
            self.session = session
        else:
            session = requests.Session()
            header_template = header_template or default_header_template.copy()
            if not header_template.get("User-Agent"):
                try:
                    from fake_useragent import UserAgent
                    header_template["User-Agent"] = UserAgent().random
                except ImportError:
                    logger.info(
                        "fake_useragent not found, using default user agent."
                        "To get a realistic header for requests, "
                        "`pip install fake_useragent`."
                    )
            session.headers = dict(header_template)
            session.verify = verify_ssl
            if proxies:
                session.proxies.update(proxies)
            self.session = session
        self.continue_on_failure = continue_on_failure
        self.autoset_encoding = autoset_encoding
        self.encoding = encoding
    @property
    def web_path(self) -> str:
        if len(self.web_paths) > 1:
            raise ValueError("Multiple webpaths found.")
        return self.web_paths[0]
    async def _fetch(
        self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5
    ) -> str:
        async with aiohttp.ClientSession() as session:
            for i in range(retries):
                try:
                    async with session.get(
                        url,
                        headers=self.session.headers,


async with session.get(
                        url,
                        headers=self.session.headers,
                        ssl=None if self.session.verify else False,
                    ) as response:
                        return await response.text()
                except aiohttp.ClientConnectionError as e:
                    if i == retries - 1:
                        raise
                    else:
                        logger.warning(
                            f"Error fetching {url} with attempt "
                            f"{i + 1}/{retries}: {e}. Retrying..."
                        )
                        await asyncio.sleep(cooldown * backoff**i)
        raise ValueError("retry count exceeded")
    async def _fetch_with_rate_limit(
        self, url: str, semaphore: asyncio.Semaphore
    ) -> str:
        async with semaphore:
            try:
                return await self._fetch(url)
            except Exception as e:
                if self.continue_on_failure:
                    logger.warning(
                        f"Error fetching {url}, skipping due to"
                        f" continue_on_failure=True"
                    )
                    return ""
                logger.exception(
                    f"Error fetching {url} and aborting, use continue_on_failure=True "
                    "to continue loading urls after encountering an error."
                )
                raise e
[docs]    async def fetch_all(self, urls: List[str]) -> Any:
        """Fetch all urls concurrently with rate limiting."""
        semaphore = asyncio.Semaphore(self.requests_per_second)
        tasks = []
        for url in urls:
            task = asyncio.ensure_future(self._fetch_with_rate_limit(url, semaphore))
            tasks.append(task)
        try:
            from tqdm.asyncio import tqdm_asyncio
            return await tqdm_asyncio.gather(
                *tasks, desc="Fetching pages", ascii=True, mininterval=1
            )
        except ImportError:


)
        except ImportError:
            warnings.warn("For better logging of progress, `pip install tqdm`")
            return await asyncio.gather(*tasks)
    @staticmethod
    def _check_parser(parser: str) -> None:
        """Check that parser is valid for bs4."""
        valid_parsers = ["html.parser", "lxml", "xml", "lxml-xml", "html5lib"]
        if parser not in valid_parsers:
            raise ValueError(
                "`parser` must be one of " + ", ".join(valid_parsers) + "."
            )
[docs]    def scrape_all(self, urls: List[str], parser: Union[str, None] = None) -> List[Any]:
        """Fetch all urls, then return soups for all results."""
        from bs4 import BeautifulSoup
        results = asyncio.run(self.fetch_all(urls))
        final_results = []
        for i, result in enumerate(results):
            url = urls[i]
            if parser is None:
                if url.endswith(".xml"):
                    parser = "xml"
                else:
                    parser = self.default_parser
                self._check_parser(parser)
            final_results.append(BeautifulSoup(result, parser, **self.bs_kwargs))
        return final_results
    def _scrape(
        self,
        url: str,
        parser: Union[str, None] = None,
        bs_kwargs: Optional[dict] = None,
    ) -> Any:
        from bs4 import BeautifulSoup
        if parser is None:
            if url.endswith(".xml"):
                parser = "xml"
            else:
                parser = self.default_parser
        self._check_parser(parser)
        html_doc = self.session.get(url, **self.requests_kwargs)
        if self.raise_for_status:


if self.raise_for_status:
            html_doc.raise_for_status()
        if self.encoding is not None:
            html_doc.encoding = self.encoding
        elif self.autoset_encoding:
            html_doc.encoding = html_doc.apparent_encoding
        return BeautifulSoup(html_doc.text, parser, **(bs_kwargs or {}))
[docs]    def scrape(self, parser: Union[str, None] = None) -> Any:
        """Scrape data from webpage and return it in BeautifulSoup format."""
        if parser is None:
            parser = self.default_parser
        return self._scrape(self.web_path, parser=parser, bs_kwargs=self.bs_kwargs)
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load text from the url(s) in web_path."""
        for path in self.web_paths:
            soup = self._scrape(path, bs_kwargs=self.bs_kwargs)
            text = soup.get_text(**self.bs_get_text_kwargs)
            metadata = _build_metadata(soup, path)
            yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load text from the url(s) in web_path."""
        return list(self.lazy_load())
[docs]    def aload(self) -> List[Document]:
        """Load text from the urls in web_path async into Documents."""
        results = self.scrape_all(self.web_paths)
        docs = []
        for path, soup in zip(self.web_paths, results):
            text = soup.get_text(**self.bs_get_text_kwargs)
            metadata = _build_metadata(soup, path)
            docs.append(Document(page_content=text, metadata=metadata))
        return docs


Source code for langchain.document_loaders.pubmed
from typing import Iterator, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.pubmed import PubMedAPIWrapper
[docs]class PubMedLoader(BaseLoader):
    """Load from the `PubMed` biomedical library.
    Attributes:
        query: The query to be passed to the PubMed API.
        load_max_docs: The maximum number of documents to load.
    """
[docs]    def __init__(
        self,
        query: str,
        load_max_docs: Optional[int] = 3,
    ):
        """Initialize the PubMedLoader.
        Args:
            query: The query to be passed to the PubMed API.
            load_max_docs: The maximum number of documents to load.
              Defaults to 3.
        """
        self.query = query
        self.load_max_docs = load_max_docs
        self._client = PubMedAPIWrapper(
            top_k_results=load_max_docs,
        )
[docs]    def load(self) -> List[Document]:
        return list(self._client.lazy_load_docs(self.query))
[docs]    def lazy_load(self) -> Iterator[Document]:
        for doc in self._client.lazy_load_docs(self.query):
            yield doc


Source code for langchain.document_loaders.slack_directory
import json
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class SlackDirectoryLoader(BaseLoader):
    """Load from a `Slack` directory dump."""
[docs]    def __init__(self, zip_path: str, workspace_url: Optional[str] = None):
        """Initialize the SlackDirectoryLoader.
        Args:
            zip_path (str): The path to the Slack directory dump zip file.
            workspace_url (Optional[str]): The Slack workspace URL.
              Including the URL will turn
              sources into links. Defaults to None.
        """
        self.zip_path = Path(zip_path)
        self.workspace_url = workspace_url
        self.channel_id_map = self._get_channel_id_map(self.zip_path)
    @staticmethod
    def _get_channel_id_map(zip_path: Path) -> Dict[str, str]:
        """Get a dictionary mapping channel names to their respective IDs."""
        with zipfile.ZipFile(zip_path, "r") as zip_file:
            try:
                with zip_file.open("channels.json", "r") as f:
                    channels = json.load(f)
                return {channel["name"]: channel["id"] for channel in channels}
            except KeyError:
                return {}
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the Slack directory dump."""
        docs = []
        with zipfile.ZipFile(self.zip_path, "r") as zip_file:
            for channel_path in zip_file.namelist():
                channel_name = Path(channel_path).parent.name
                if not channel_name:
                    continue
                if channel_path.endswith(".json"):


if not channel_name:
                    continue
                if channel_path.endswith(".json"):
                    messages = self._read_json(zip_file, channel_path)
                    for message in messages:
                        document = self._convert_message_to_document(
                            message, channel_name
                        )
                        docs.append(document)
        return docs
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        return data
    def _convert_message_to_document(
        self, message: dict, channel_name: str
    ) -> Document:
        """
        Convert a message to a Document object.
        Args:
            message (dict): A message in the form of a dictionary.
            channel_name (str): The name of the channel the message belongs to.
        Returns:
            Document: A Document object representing the message.
        """
        text = message.get("text", "")
        metadata = self._get_message_metadata(message, channel_name)
        return Document(
            page_content=text,
            metadata=metadata,
        )
    def _get_message_metadata(self, message: dict, channel_name: str) -> dict:
        """Create and return metadata for a given message and channel."""
        timestamp = message.get("ts", "")
        user = message.get("user", "")
        source = self._get_message_source(channel_name, user, timestamp)
        return {
            "source": source,
            "channel": channel_name,
            "timestamp": timestamp,
            "user": user,
        }


"timestamp": timestamp,
            "user": user,
        }
    def _get_message_source(self, channel_name: str, user: str, timestamp: str) -> str:
        """
        Get the message source as a string.
        Args:
            channel_name (str): The name of the channel the message belongs to.
            user (str): The user ID who sent the message.
            timestamp (str): The timestamp of the message.
        Returns:
            str: The message source.
        """
        if self.workspace_url:
            channel_id = self.channel_id_map.get(channel_name, "")
            return (
                f"{self.workspace_url}/archives/{channel_id}"
                + f"/p{timestamp.replace('.', '')}"
            )
        else:
            return f"{channel_name} - {user} - {timestamp}"


Source code for langchain.document_loaders.arcgis_loader
"""Document Loader for ArcGIS FeatureLayers."""
from __future__ import annotations
import json
import re
import warnings
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import arcgis
_NOT_PROVIDED = "(Not Provided)"
[docs]class ArcGISLoader(BaseLoader):
    """Load records from an ArcGIS FeatureLayer."""
[docs]    def __init__(
        self,
        layer: Union[str, arcgis.features.FeatureLayer],
        gis: Optional[arcgis.gis.GIS] = None,
        where: str = "1=1",
        out_fields: Optional[Union[List[str], str]] = None,
        return_geometry: bool = False,
        result_record_count: Optional[int] = None,
        lyr_desc: Optional[str] = None,
        **kwargs: Any,
    ):
        try:
            import arcgis
        except ImportError as e:
            raise ImportError(
                "arcgis is required to use the ArcGIS Loader. "
                "Install it with pip or conda."
            ) from e
        try:
            from bs4 import BeautifulSoup  # type: ignore
            self.BEAUTIFULSOUP = BeautifulSoup
        except ImportError:
            warnings.warn("BeautifulSoup not found. HTML will not be parsed.")
            self.BEAUTIFULSOUP = None
        self.gis = gis or arcgis.gis.GIS()
        if isinstance(layer, str):
            self.url = layer
            self.layer = arcgis.features.FeatureLayer(layer, gis=gis)
        else:


else:
            self.url = layer.url
            self.layer = layer
        self.layer_properties = self._get_layer_properties(lyr_desc)
        self.where = where
        if isinstance(out_fields, str):
            self.out_fields = out_fields
        elif out_fields is None:
            self.out_fields = "*"
        else:
            self.out_fields = ",".join(out_fields)
        self.return_geometry = return_geometry
        self.result_record_count = result_record_count
        self.return_all_records = not isinstance(result_record_count, int)
        query_params = dict(
            where=self.where,
            out_fields=self.out_fields,
            return_geometry=self.return_geometry,
            return_all_records=self.return_all_records,
            result_record_count=self.result_record_count,
        )
        query_params.update(kwargs)
        self.query_params = query_params
    def _get_layer_properties(self, lyr_desc: Optional[str] = None) -> dict:
        """Get the layer properties from the FeatureLayer."""
        import arcgis
        layer_number_pattern = re.compile(r"/\d+$")
        props = self.layer.properties
        if lyr_desc is None:
            # retrieve description from the FeatureLayer if not provided
            try:
                if self.BEAUTIFULSOUP:
                    lyr_desc = self.BEAUTIFULSOUP(props["description"]).text
                else:
                    lyr_desc = props["description"]
                lyr_desc = lyr_desc or _NOT_PROVIDED
            except KeyError:
                lyr_desc = _NOT_PROVIDED
        try:
            item_id = props["serviceItemId"]
            item = self.gis.content.get(item_id) or arcgis.features.FeatureLayer(
                re.sub(layer_number_pattern, "", self.url),
            )
            try:
                raw_desc = item.description


)
            try:
                raw_desc = item.description
            except AttributeError:
                raw_desc = item.properties.description
            if self.BEAUTIFULSOUP:
                item_desc = self.BEAUTIFULSOUP(raw_desc).text
            else:
                item_desc = raw_desc
            item_desc = item_desc or _NOT_PROVIDED
        except KeyError:
            item_desc = _NOT_PROVIDED
        return {
            "layer_description": lyr_desc,
            "item_description": item_desc,
            "layer_properties": props,
        }
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load records from FeatureLayer."""
        query_response = self.layer.query(**self.query_params)
        features = (feature.as_dict for feature in query_response)
        for feature in features:
            attributes = feature["attributes"]
            page_content = json.dumps(attributes)
            metadata = {
                "accessed": f"{datetime.now(timezone.utc).isoformat()}Z",
                "name": self.layer_properties["layer_properties"]["name"],
                "url": self.url,
                "layer_description": self.layer_properties["layer_description"],
                "item_description": self.layer_properties["item_description"],
                "layer_properties": self.layer_properties["layer_properties"],
            }
            if self.return_geometry:
                try:
                    metadata["geometry"] = feature["geometry"]
                except KeyError:
                    warnings.warn(
                        "Geometry could not be retrieved from the feature layer."
                    )
            yield Document(page_content=page_content, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load all records from FeatureLayer."""
        return list(self.lazy_load())


Source code for langchain.document_loaders.chatgpt
import datetime
import json
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]def concatenate_rows(message: dict, title: str) -> str:
    """
    Combine message information in a readable format ready to be used.
    Args:
        message: Message to be concatenated
        title: Title of the conversation
    Returns:
        Concatenated message
    """
    if not message:
        return ""
    sender = message["author"]["role"] if message["author"] else "unknown"
    text = message["content"]["parts"][0]
    date = datetime.datetime.fromtimestamp(message["create_time"]).strftime(
        "%Y-%m-%d %H:%M:%S"
    )
    return f"{title} - {sender} on {date}: {text}\n\n"
[docs]class ChatGPTLoader(BaseLoader):
    """Load conversations from exported `ChatGPT` data."""
[docs]    def __init__(self, log_file: str, num_logs: int = -1):
        """Initialize a class object.
        Args:
            log_file: Path to the log file
            num_logs: Number of logs to load. If 0, load all logs.
        """
        self.log_file = log_file
        self.num_logs = num_logs
[docs]    def load(self) -> List[Document]:
        with open(self.log_file, encoding="utf8") as f:
            data = json.load(f)[: self.num_logs] if self.num_logs else json.load(f)
        documents = []
        for d in data:
            title = d["title"]
            messages = d["mapping"]


title = d["title"]
            messages = d["mapping"]
            text = "".join(
                [
                    concatenate_rows(messages[key]["message"], title)
                    for idx, key in enumerate(messages)
                    if not (
                        idx == 0
                        and messages[key]["message"]["author"]["role"] == "system"
                    )
                ]
            )
            metadata = {"source": str(self.log_file)}
            documents.append(Document(page_content=text, metadata=metadata))
        return documents


Source code for langchain.document_loaders.concurrent
from __future__ import annotations
import concurrent.futures
from pathlib import Path
from typing import Iterator, Literal, Optional, Sequence, Union
from langchain.document_loaders.base import BaseBlobParser
from langchain.document_loaders.blob_loaders import BlobLoader, FileSystemBlobLoader
from langchain.document_loaders.generic import GenericLoader
from langchain.document_loaders.parsers.registry import get_parser
from langchain.schema import Document
_PathLike = Union[str, Path]
DEFAULT = Literal["default"]
[docs]class ConcurrentLoader(GenericLoader):
    """Load and pars Documents concurrently."""
[docs]    def __init__(
        self, blob_loader: BlobLoader, blob_parser: BaseBlobParser, num_workers: int = 4
    ) -> None:
        super().__init__(blob_loader, blob_parser)
        self.num_workers = num_workers
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Load documents lazily with concurrent parsing."""
        with concurrent.futures.ThreadPoolExecutor(
            max_workers=self.num_workers
        ) as executor:
            futures = {
                executor.submit(self.blob_parser.lazy_parse, blob)
                for blob in self.blob_loader.yield_blobs()
            }
            for future in concurrent.futures.as_completed(futures):
                yield from future.result()
[docs]    @classmethod
    def from_filesystem(
        cls,
        path: _PathLike,
        *,
        glob: str = "**/[!.]*",
        exclude: Sequence[str] = (),
        suffixes: Optional[Sequence[str]] = None,
        show_progress: bool = False,
        parser: Union[DEFAULT, BaseBlobParser] = "default",
        num_workers: int = 4,


num_workers: int = 4,
    ) -> ConcurrentLoader:
        """
        Create a concurrent generic document loader using a
        filesystem blob loader.
        Args:
            path: The path to the directory to load documents from.
            glob: The glob pattern to use to find documents.
            suffixes: The suffixes to use to filter documents. If None, all files
                      matching the glob will be loaded.
            exclude: A list of patterns to exclude from the loader.
            show_progress: Whether to show a progress bar or not (requires tqdm).
                           Proxies to the file system loader.
            parser: A blob parser which knows how to parse blobs into documents
            num_workers: Max number of concurrent workers to use.
        """
        blob_loader = FileSystemBlobLoader(
            path,
            glob=glob,
            exclude=exclude,
            suffixes=suffixes,
            show_progress=show_progress,
        )
        if isinstance(parser, str):
            blob_parser = get_parser(parser)
        else:
            blob_parser = parser
        return cls(blob_loader, blob_parser, num_workers=num_workers)


Source code for langchain.document_loaders.markdown
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredMarkdownLoader(UnstructuredFileLoader):
    """Load `Markdown` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain.document_loaders import UnstructuredMarkdownLoader
    loader = UnstructuredMarkdownLoader(
        "example.md", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-md
    """
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.partition.md import partition_md
        # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
        # versions of unstructured like 0.4.17-dev1
        _unstructured_version = __unstructured_version__.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        if unstructured_version < (0, 4, 16):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "


f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning markdown files is only supported in unstructured>=0.4.16."
            )
        return partition_md(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.blackboard
import contextlib
import re
from pathlib import Path
from typing import Any, List, Optional, Tuple
from urllib.parse import unquote
from langchain.docstore.document import Document
from langchain.document_loaders.directory import DirectoryLoader
from langchain.document_loaders.pdf import PyPDFLoader
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class BlackboardLoader(WebBaseLoader):
    """Load a `Blackboard` course.
    This loader is not compatible with all Blackboard courses. It is only
    compatible with courses that use the new Blackboard interface.
    To use this loader, you must have the BbRouter cookie. You can get this
    cookie by logging into the course and then copying the value of the
    BbRouter cookie from the browser's developer tools.
    Example:
        .. code-block:: python
            from langchain.document_loaders import BlackboardLoader
            loader = BlackboardLoader(
                blackboard_course_url="https://blackboard.example.com/webapps/blackboard/execute/announcement?method=search&context=course_entry&course_id=_123456_1",
                bbrouter="expires:12345...",
            )
            documents = loader.load()
    """  # noqa: E501
[docs]    def __init__(
        self,
        blackboard_course_url: str,
        bbrouter: str,
        load_all_recursively: bool = True,
        basic_auth: Optional[Tuple[str, str]] = None,
        cookies: Optional[dict] = None,
        continue_on_failure: bool = False,
    ):
        """Initialize with blackboard course url.
        The BbRouter cookie is required for most blackboard courses.
        Args:


The BbRouter cookie is required for most blackboard courses.
        Args:
            blackboard_course_url: Blackboard course url.
            bbrouter: BbRouter cookie.
            load_all_recursively: If True, load all documents recursively.
            basic_auth: Basic auth credentials.
            cookies: Cookies.
            continue_on_failure: whether to continue loading the sitemap if an error
                occurs loading a url, emitting a warning instead of raising an
                exception. Setting this to True makes the loader more robust, but also
                may result in missing data. Default: False
        Raises:
            ValueError: If blackboard course url is invalid.
        """
        super().__init__(
            web_paths=(blackboard_course_url), continue_on_failure=continue_on_failure
        )
        # Get base url
        try:
            self.base_url = blackboard_course_url.split("/webapps/blackboard")[0]
        except IndexError:
            raise IndexError(
                "Invalid blackboard course url. "
                "Please provide a url that starts with "
                "https://<blackboard_url>/webapps/blackboard"
            )
        if basic_auth is not None:
            self.session.auth = basic_auth
        # Combine cookies
        if cookies is None:
            cookies = {}
        cookies.update({"BbRouter": bbrouter})
        self.session.cookies.update(cookies)
        self.load_all_recursively = load_all_recursively
        self.check_bs4()
[docs]    def check_bs4(self) -> None:
        """Check if BeautifulSoup4 is installed.
        Raises:
            ImportError: If BeautifulSoup4 is not installed.
        """
        try:
            import bs4  # noqa: F401
        except ImportError:
            raise ImportError(


except ImportError:
            raise ImportError(
                "BeautifulSoup4 is required for BlackboardLoader. "
                "Please install it with `pip install beautifulsoup4`."
            )
[docs]    def load(self) -> List[Document]:
        """Load data into Document objects.
        Returns:
            List of Documents.
        """
        if self.load_all_recursively:
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = self.base_url + path
                print(f"Fetching documents from {url}")
                soup_info = self._scrape(url)
                with contextlib.suppress(ValueError):
                    documents.extend(self._get_documents(soup_info))
            return documents
        else:
            print(f"Fetching documents from {self.web_path}")
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            return self._get_documents(soup_info)
    def _get_folder_path(self, soup: Any) -> str:
        """Get the folder path to save the Documents in.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            Folder path.
        """
        # Get the course name
        course_name = soup.find("span", {"id": "crumb_1"})
        if course_name is None:
            raise ValueError("No course name found.")
        course_name = course_name.text.strip()
        # Prepare the folder path
        course_name_clean = (
            unquote(course_name)
            .replace(" ", "_")
            .replace("/", "_")
            .replace(":", "_")


.replace("/", "_")
            .replace(":", "_")
            .replace(",", "_")
            .replace("?", "_")
            .replace("'", "_")
            .replace("!", "_")
            .replace('"', "_")
        )
        # Get the folder path
        folder_path = Path(".") / course_name_clean
        return str(folder_path)
    def _get_documents(self, soup: Any) -> List[Document]:
        """Fetch content from page and return Documents.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of documents.
        """
        attachments = self._get_attachments(soup)
        self._download_attachments(attachments)
        documents = self._load_documents()
        return documents
    def _get_attachments(self, soup: Any) -> List[str]:
        """Get all attachments from a page.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of attachments.
        """
        from bs4 import BeautifulSoup, Tag
        # Get content list
        content_list = soup.find("ul", {"class": "contentList"})
        if content_list is None:
            raise ValueError("No content list found.")
        content_list: BeautifulSoup  # type: ignore
        # Get all attachments
        attachments = []
        for attachment in content_list.find_all("ul", {"class": "attachments"}):
            attachment: Tag  # type: ignore
            for link in attachment.find_all("a"):
                link: Tag  # type: ignore
                href = link.get("href")
                # Only add if href is not None and does not start with #
                if href is not None and not href.startswith("#"):
                    attachments.append(href)
        return attachments


attachments.append(href)
        return attachments
    def _download_attachments(self, attachments: List[str]) -> None:
        """Download all attachments.
        Args:
            attachments: List of attachments.
        """
        # Make sure the folder exists
        Path(self.folder_path).mkdir(parents=True, exist_ok=True)
        # Download all attachments
        for attachment in attachments:
            self.download(attachment)
    def _load_documents(self) -> List[Document]:
        """Load all documents in the folder.
        Returns:
            List of documents.
        """
        # Create the document loader
        loader = DirectoryLoader(
            path=self.folder_path,
            glob="*.pdf",
            loader_cls=PyPDFLoader,  # type: ignore
        )
        # Load the documents
        documents = loader.load()
        # Return all documents
        return documents
    def _get_paths(self, soup: Any) -> List[str]:
        """Get all relative paths in the navbar."""
        relative_paths = []
        course_menu = soup.find("ul", {"class": "courseMenu"})
        if course_menu is None:
            raise ValueError("No course menu found.")
        for link in course_menu.find_all("a"):
            href = link.get("href")
            if href is not None and href.startswith("/"):
                relative_paths.append(href)
        return relative_paths
[docs]    def download(self, path: str) -> None:
        """Download a file from an url.
        Args:
            path: Path to the file.
        """
        # Get the file content
        response = self.session.get(self.base_url + path, allow_redirects=True)
        # Get the filename
        filename = self.parse_filename(response.url)
        # Write the file to disk


filename = self.parse_filename(response.url)
        # Write the file to disk
        with open(Path(self.folder_path) / filename, "wb") as f:
            f.write(response.content)
[docs]    def parse_filename(self, url: str) -> str:
        """Parse the filename from an url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        """
        if (url_path := Path(url)) and url_path.suffix == ".pdf":
            return url_path.name
        else:
            return self._parse_filename_from_url(url)
    def _parse_filename_from_url(self, url: str) -> str:
        """Parse the filename from an url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        Raises:
            ValueError: If the filename could not be parsed.
        """
        filename_matches = re.search(r"filename%2A%3DUTF-8%27%27(.+)", url)
        if filename_matches:
            filename = filename_matches.group(1)
        else:
            raise ValueError(f"Could not parse filename from {url}")
        if ".pdf" not in filename:
            raise ValueError(f"Incorrect file type: {filename}")
        filename = filename.split(".pdf")[0] + ".pdf"
        filename = unquote(filename)
        filename = filename.replace("%20", " ")
        return filename
if __name__ == "__main__":
    loader = BlackboardLoader(
        "https://<YOUR BLACKBOARD URL"
        " HERE>/webapps/blackboard/content/listContent.jsp?course_id=_<YOUR COURSE ID"


" HERE>_1&content_id=_<YOUR CONTENT ID HERE>_1&mode=reset",
        "<YOUR BBROUTER COOKIE HERE>",
        load_all_recursively=True,
    )
    documents = loader.load()
    print(f"Loaded {len(documents)} pages of PDFs from {loader.web_path}")


Source code for langchain.document_loaders.news
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, Iterator, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class NewsURLLoader(BaseLoader):
    """Load news articles from URLs using `Unstructured`.
    Args:
        urls: URLs to load. Each is loaded into its own document.
        text_mode: If True, extract text from URL and use that for page content.
            Otherwise, extract raw HTML.
        nlp: If True, perform NLP on the extracted contents, like providing a summary
            and extracting keywords.
        continue_on_failure: If True, continue loading documents even if
            loading fails for a particular URL.
        show_progress_bar: If True, use tqdm to show a loading progress bar. Requires
            tqdm to be installed, ``pip install tqdm``.
        **newspaper_kwargs: Any additional named arguments to pass to
            newspaper.Article().
    Example:
        .. code-block:: python
            from langchain.document_loaders import NewsURLLoader
            loader = NewsURLLoader(
                urls=["<url-1>", "<url-2>"],
            )
            docs = loader.load()
    Newspaper reference:
        https://newspaper.readthedocs.io/en/latest/
    """
[docs]    def __init__(
        self,
        urls: List[str],
        text_mode: bool = True,
        nlp: bool = False,
        continue_on_failure: bool = True,
        show_progress_bar: bool = False,
        **newspaper_kwargs: Any,
    ) -> None:
        """Initialize with file path."""
        try:


) -> None:
        """Initialize with file path."""
        try:
            import newspaper  # noqa:F401
            self.__version = newspaper.__version__
        except ImportError:
            raise ImportError(
                "newspaper package not found, please install it with "
                "`pip install newspaper3k`"
            )
        self.urls = urls
        self.text_mode = text_mode
        self.nlp = nlp
        self.continue_on_failure = continue_on_failure
        self.newspaper_kwargs = newspaper_kwargs
        self.show_progress_bar = show_progress_bar
[docs]    def load(self) -> List[Document]:
        iter = self.lazy_load()
        if self.show_progress_bar:
            try:
                from tqdm import tqdm
            except ImportError as e:
                raise ImportError(
                    "Package tqdm must be installed if show_progress_bar=True. "
                    "Please install with 'pip install tqdm' or set "
                    "show_progress_bar=False."
                ) from e
            iter = tqdm(iter)
        return list(iter)
[docs]    def lazy_load(self) -> Iterator[Document]:
        try:
            from newspaper import Article
        except ImportError as e:
            raise ImportError(
                "Cannot import newspaper, please install with `pip install newspaper3k`"
            ) from e
        for url in self.urls:
            try:
                article = Article(url, **self.newspaper_kwargs)
                article.download()
                article.parse()
                if self.nlp:
                    article.nlp()
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                    continue
                else:
                    raise e
            metadata = {


continue
                else:
                    raise e
            metadata = {
                "title": getattr(article, "title", ""),
                "link": getattr(article, "url", getattr(article, "canonical_link", "")),
                "authors": getattr(article, "authors", []),
                "language": getattr(article, "meta_lang", ""),
                "description": getattr(article, "meta_description", ""),
                "publish_date": getattr(article, "publish_date", ""),
            }
            if self.text_mode:
                content = article.text
            else:
                content = article.html
            if self.nlp:
                metadata["keywords"] = getattr(article, "keywords", [])
                metadata["summary"] = getattr(article, "summary", "")
            yield Document(page_content=content, metadata=metadata)


Source code for langchain.document_loaders.gcs_file
import os
import tempfile
from typing import Callable, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
from langchain.utilities.vertexai import get_client_info
[docs]class GCSFileLoader(BaseLoader):
    """Load from GCS file."""
[docs]    def __init__(
        self,
        project_name: str,
        bucket: str,
        blob: str,
        loader_func: Optional[Callable[[str], BaseLoader]] = None,
    ):
        """Initialize with bucket and key name.
        Args:
            project_name: The name of the project to load
            bucket: The name of the GCS bucket.
            blob: The name of the GCS blob to load.
            loader_func: A loader function that instantiates a loader based on a
                file_path argument. If nothing is provided, the
                UnstructuredFileLoader is used.
        Examples:
            To use an alternative PDF loader:
            >> from from langchain.document_loaders import PyPDFLoader
            >> loader = GCSFileLoader(..., loader_func=PyPDFLoader)
            To use UnstructuredFileLoader with additional arguments:
            >> loader = GCSFileLoader(...,
            >>      loader_func=lambda x: UnstructuredFileLoader(x, mode="elements"))
        """
        self.bucket = bucket
        self.blob = blob
        self.project_name = project_name
        def default_loader_func(file_path: str) -> BaseLoader:
            return UnstructuredFileLoader(file_path)
        self._loader_func = loader_func if loader_func else default_loader_func
[docs]    def load(self) -> List[Document]:


[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ImportError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        # Initialise a client
        storage_client = storage.Client(
            self.project_name, client_info=get_client_info("google-cloud-storage")
        )
        # Create a bucket object for our bucket
        bucket = storage_client.get_bucket(self.bucket)
        # Create a blob object from the filepath
        blob = bucket.blob(self.blob)
        # retrieve custom metadata associated with the blob
        metadata = bucket.get_blob(self.blob).metadata
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.blob}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            blob.download_to_filename(file_path)
            loader = self._loader_func(file_path)
            docs = loader.load()
            for doc in docs:
                if "source" in doc.metadata:
                    doc.metadata["source"] = f"gs://{self.bucket}/{self.blob}"
                if metadata:
                    doc.metadata.update(metadata)
            return docs


Source code for langchain.document_loaders.imsdb
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class IMSDbLoader(WebBaseLoader):
    """Load `IMSDb` webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        text = soup.select_one("td[class='scrtext']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.telegram
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
if TYPE_CHECKING:
    import pandas as pd
    from telethon.hints import EntityLike
[docs]def concatenate_rows(row: dict) -> str:
    """Combine message information in a readable format ready to be used."""
    date = row["date"]
    sender = row["from"]
    text = row["text"]
    return f"{sender} on {date}: {text}\n\n"
[docs]class TelegramChatFileLoader(BaseLoader):
    """Load from `Telegram chat` dump."""
[docs]    def __init__(self, path: str):
        """Initialize with a path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        text = "".join(
            concatenate_rows(message)
            for message in d["messages"]
            if message["type"] == "message" and isinstance(message["text"], str)
        )
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]
[docs]def text_to_docs(text: Union[str, List[str]]) -> List[Document]:
    """Convert a string or list of strings to a list of Documents with metadata."""
    if isinstance(text, str):
        # Take a single string as one page


if isinstance(text, str):
        # Take a single string as one page
        text = [text]
    page_docs = [Document(page_content=page) for page in text]
    # Add page numbers as metadata
    for i, doc in enumerate(page_docs):
        doc.metadata["page"] = i + 1
    # Split pages into chunks
    doc_chunks = []
    for doc in page_docs:
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=800,
            separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
            chunk_overlap=20,
        )
        chunks = text_splitter.split_text(doc.page_content)
        for i, chunk in enumerate(chunks):
            doc = Document(
                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
            )
            # Add sources a metadata
            doc.metadata["source"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"
            doc_chunks.append(doc)
    return doc_chunks
[docs]class TelegramChatApiLoader(BaseLoader):
    """Load `Telegram` chat json directory dump."""
[docs]    def __init__(
        self,
        chat_entity: Optional[EntityLike] = None,
        api_id: Optional[int] = None,
        api_hash: Optional[str] = None,
        username: Optional[str] = None,
        file_path: str = "telegram_data.json",
    ):
        """Initialize with API parameters.
        Args:
            chat_entity: The chat entity to fetch data from.
            api_id: The API ID.
            api_hash: The API hash.
            username: The username.