continue
                texts.append(text)
            except requests.HTTPError as e:
                if e.response.status_code == 404:
                    print(f"Attachment not found at {absolute_url}")
                    continue
                else:
                    raise
        return texts
[docs]    def process_pdf(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from pdf2image import convert_from_bytes  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract` or `pdf2image` package not found, "
                "please run `pip install pytesseract pdf2image`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            images = convert_from_bytes(response.content)
        except ValueError:
            return text
        for i, image in enumerate(images):
            image_text = pytesseract.image_to_string(image, lang=ocr_languages)
            text += f"Page {i + 1}:\n{image_text}\n\n"
        return text
[docs]    def process_image(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract` or `Pillow` package not found, "


"`pytesseract` or `Pillow` package not found, "
                "please run `pip install pytesseract Pillow`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            image = Image.open(BytesIO(response.content))
        except OSError:
            return text
        return pytesseract.image_to_string(image, lang=ocr_languages)
[docs]    def process_doc(self, link: str) -> str:
        try:
            import docx2txt  # noqa: F401
        except ImportError:
            raise ImportError(
                "`docx2txt` package not found, please run `pip install docx2txt`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        file_data = BytesIO(response.content)
        return docx2txt.process(file_data)
[docs]    def process_xls(self, link: str) -> str:
        import io
        import os
        try:
            import xlrd  # noqa: F401
        except ImportError:
            raise ImportError("`xlrd` package not found, please run `pip install xlrd`")
        try:
            import pandas as pd
        except ImportError:
            raise ImportError(
                "`pandas` package not found, please run `pip install pandas`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""


response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        filename = os.path.basename(link)
        # Getting the whole content of the url after filename,
        # Example: ".csv?version=2&modificationDate=1631800010678&cacheVersion=1&api=v2"
        file_extension = os.path.splitext(filename)[1]
        if file_extension.startswith(
            ".csv"
        ):  # if the extension found in the url is ".csv"
            content_string = response.content.decode("utf-8")
            df = pd.read_csv(io.StringIO(content_string))
            text += df.to_string(index=False, header=False) + "\n\n"
        else:
            workbook = xlrd.open_workbook(file_contents=response.content)
            for sheet in workbook.sheets():
                text += f"{sheet.name}:\n"
                for row in range(sheet.nrows):
                    for col in range(sheet.ncols):
                        text += f"{sheet.cell_value(row, col)}\t"
                    text += "\n"
                text += "\n"
        return text
[docs]    def process_svg(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
            from reportlab.graphics import renderPM  # noqa: F401
            from svglib.svglib import svg2rlg  # noqa: F401
        except ImportError:
            raise ImportError(


except ImportError:
            raise ImportError(
                "`pytesseract`, `Pillow`, `reportlab` or `svglib` package not found, "
                "please run `pip install pytesseract Pillow reportlab svglib`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        drawing = svg2rlg(BytesIO(response.content))
        img_data = BytesIO()
        renderPM.drawToFile(drawing, img_data, fmt="PNG")
        img_data.seek(0)
        image = Image.open(img_data)
        return pytesseract.image_to_string(image, lang=ocr_languages)


Source code for langchain.document_loaders.async_html
import asyncio
import logging
import warnings
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, Iterator, List, Optional, Union, cast
import aiohttp
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
default_header_template = {
    "User-Agent": "",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
    ";q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://www.google.com/",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
}
def _build_metadata(soup: Any, url: str) -> dict:
    """Build metadata from BeautifulSoup output."""
    metadata = {"source": url}
    if title := soup.find("title"):
        metadata["title"] = title.get_text()
    if description := soup.find("meta", attrs={"name": "description"}):
        metadata["description"] = description.get("content", "No description found.")
    if html := soup.find("html"):
        metadata["language"] = html.get("lang", "No language found.")
    return metadata
[docs]class AsyncHtmlLoader(BaseLoader):
    """Load `HTML` asynchronously."""
[docs]    def __init__(
        self,
        web_path: Union[str, List[str]],
        header_template: Optional[dict] = None,
        verify_ssl: Optional[bool] = True,
        proxies: Optional[dict] = None,
        autoset_encoding: bool = True,


autoset_encoding: bool = True,
        encoding: Optional[str] = None,
        default_parser: str = "html.parser",
        requests_per_second: int = 2,
        requests_kwargs: Optional[Dict[str, Any]] = None,
        raise_for_status: bool = False,
        ignore_load_errors: bool = False,
    ):
        """Initialize with a webpage path."""
        # TODO: Deprecate web_path in favor of web_paths, and remove this
        # left like this because there are a number of loaders that expect single
        # urls
        if isinstance(web_path, str):
            self.web_paths = [web_path]
        elif isinstance(web_path, List):
            self.web_paths = web_path
        headers = header_template or default_header_template
        if not headers.get("User-Agent"):
            try:
                from fake_useragent import UserAgent
                headers["User-Agent"] = UserAgent().random
            except ImportError:
                logger.info(
                    "fake_useragent not found, using default user agent."
                    "To get a realistic header for requests, "
                    "`pip install fake_useragent`."
                )
        self.session = requests.Session()
        self.session.headers = dict(headers)
        self.session.verify = verify_ssl
        if proxies:
            self.session.proxies.update(proxies)
        self.requests_per_second = requests_per_second
        self.default_parser = default_parser
        self.requests_kwargs = requests_kwargs or {}
        self.raise_for_status = raise_for_status
        self.autoset_encoding = autoset_encoding
        self.encoding = encoding
        self.ignore_load_errors = ignore_load_errors
    def _fetch_valid_connection_docs(self, url: str) -> Any:
        if self.ignore_load_errors:
            try:


if self.ignore_load_errors:
            try:
                return self.session.get(url, **self.requests_kwargs)
            except Exception as e:
                warnings.warn(str(e))
                return None
        return self.session.get(url, **self.requests_kwargs)
    @staticmethod
    def _check_parser(parser: str) -> None:
        """Check that parser is valid for bs4."""
        valid_parsers = ["html.parser", "lxml", "xml", "lxml-xml", "html5lib"]
        if parser not in valid_parsers:
            raise ValueError(
                "`parser` must be one of " + ", ".join(valid_parsers) + "."
            )
    def _scrape(
        self,
        url: str,
        parser: Union[str, None] = None,
        bs_kwargs: Optional[dict] = None,
    ) -> Any:
        from bs4 import BeautifulSoup
        if parser is None:
            if url.endswith(".xml"):
                parser = "xml"
            else:
                parser = self.default_parser
        self._check_parser(parser)
        html_doc = self._fetch_valid_connection_docs(url)
        if not getattr(html_doc, "ok", False):
            return None
        if self.raise_for_status:
            html_doc.raise_for_status()
        if self.encoding is not None:
            html_doc.encoding = self.encoding
        elif self.autoset_encoding:
            html_doc.encoding = html_doc.apparent_encoding
        return BeautifulSoup(html_doc.text, parser, **(bs_kwargs or {}))
    async def _fetch(
        self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5
    ) -> str:


) -> str:
        async with aiohttp.ClientSession() as session:
            for i in range(retries):
                try:
                    async with session.get(
                        url,
                        headers=self.session.headers,
                        ssl=None if self.session.verify else False,
                    ) as response:
                        try:
                            text = await response.text()
                        except UnicodeDecodeError:
                            logger.error(f"Failed to decode content from {url}")
                            text = ""
                        return text
                except aiohttp.ClientConnectionError as e:
                    if i == retries - 1 and self.ignore_load_errors:
                        logger.warning(f"Error fetching {url} after {retries} retries.")
                        return ""
                    elif i == retries - 1:
                        raise
                    else:
                        logger.warning(
                            f"Error fetching {url} with attempt "
                            f"{i + 1}/{retries}: {e}. Retrying..."
                        )
                        await asyncio.sleep(cooldown * backoff**i)
        raise ValueError("retry count exceeded")
    async def _fetch_with_rate_limit(
        self, url: str, semaphore: asyncio.Semaphore
    ) -> str:
        async with semaphore:
            return await self._fetch(url)
[docs]    async def fetch_all(self, urls: List[str]) -> Any:
        """Fetch all urls concurrently with rate limiting."""
        semaphore = asyncio.Semaphore(self.requests_per_second)
        tasks = []
        for url in urls:
            task = asyncio.ensure_future(self._fetch_with_rate_limit(url, semaphore))
            tasks.append(task)
        try:
            from tqdm.asyncio import tqdm_asyncio
            return await tqdm_asyncio.gather(
                *tasks, desc="Fetching pages", ascii=True, mininterval=1
            )


*tasks, desc="Fetching pages", ascii=True, mininterval=1
            )
        except ImportError:
            warnings.warn("For better logging of progress, `pip install tqdm`")
            return await asyncio.gather(*tasks)
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load text from the url(s) in web_path."""
        for doc in self.load():
            yield doc
[docs]    def load(self) -> List[Document]:
        """Load text from the url(s) in web_path."""
        try:
            # Raises RuntimeError if there is no current event loop.
            asyncio.get_running_loop()
            # If there is a current event loop, we need to run the async code
            # in a separate loop, in a separate thread.
            with ThreadPoolExecutor(max_workers=1) as executor:
                future = executor.submit(asyncio.run, self.fetch_all(self.web_paths))
                results = future.result()
        except RuntimeError:
            results = asyncio.run(self.fetch_all(self.web_paths))
        docs = []
        for i, text in enumerate(cast(List[str], results)):
            soup = self._scrape(self.web_paths[i])
            if not soup:
                continue
            metadata = _build_metadata(soup, self.web_paths[i])
            docs.append(Document(page_content=text, metadata=metadata))
        return docs


Source code for langchain.document_loaders.etherscan
import os
import re
from typing import Iterator, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class EtherscanLoader(BaseLoader):
    """Load transactions from `Ethereum` mainnet.
    The Loader use Etherscan API to interact with Ethereum mainnet.
    ETHERSCAN_API_KEY environment variable must be set use this loader.
    """
[docs]    def __init__(
        self,
        account_address: str,
        api_key: str = "docs-demo",
        filter: str = "normal_transaction",
        page: int = 1,
        offset: int = 10,
        start_block: int = 0,
        end_block: int = 99999999,
        sort: str = "desc",
    ):
        self.account_address = account_address
        self.api_key = os.environ.get("ETHERSCAN_API_KEY") or api_key
        self.filter = filter
        self.page = page
        self.offset = offset
        self.start_block = start_block
        self.end_block = end_block
        self.sort = sort
        if not self.api_key:
            raise ValueError("Etherscan API key not provided")
        if not re.match(r"^0x[a-fA-F0-9]{40}$", self.account_address):
            raise ValueError(f"Invalid contract address {self.account_address}")
        if filter not in [
            "normal_transaction",
            "internal_transaction",
            "erc20_transaction",
            "eth_balance",
            "erc721_transaction",
            "erc1155_transaction",
        ]:
            raise ValueError(f"Invalid filter {filter}")


]:
            raise ValueError(f"Invalid filter {filter}")
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load Documents from table."""
        result = []
        if self.filter == "normal_transaction":
            result = self.getNormTx()
        elif self.filter == "internal_transaction":
            result = self.getInternalTx()
        elif self.filter == "erc20_transaction":
            result = self.getERC20Tx()
        elif self.filter == "eth_balance":
            result = self.getEthBalance()
        elif self.filter == "erc721_transaction":
            result = self.getERC721Tx()
        elif self.filter == "erc1155_transaction":
            result = self.getERC1155Tx()
        else:
            raise ValueError(f"Invalid filter {filter}")
        for doc in result:
            yield doc
[docs]    def load(self) -> List[Document]:
        """Load transactions from spcifc account by Etherscan."""
        return list(self.lazy_load())
[docs]    def getNormTx(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=txlist&address={self.account_address}"
            f"&startblock={self.start_block}&endblock={self.end_block}&page={self.page}"
            f"&offset={self.offset}&sort={self.sort}&apikey={self.api_key}"
        )
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        items = response.json()["result"]
        result = []
        if len(items) == 0:
            return [Document(page_content="")]


if len(items) == 0:
            return [Document(page_content="")]
        for item in items:
            content = str(item)
            metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
            result.append(Document(page_content=content, metadata=metadata))
        print(len(result))
        return result
[docs]    def getEthBalance(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=balance"
            f"&address={self.account_address}&tag=latest&apikey={self.api_key}"
        )
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        return [Document(page_content=response.json()["result"])]
[docs]    def getInternalTx(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=txlistinternal"
            f"&address={self.account_address}&startblock={self.start_block}"
            f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
            f"&sort={self.sort}&apikey={self.api_key}"
        )
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        items = response.json()["result"]
        result = []
        if len(items) == 0:
            return [Document(page_content="")]
        for item in items:
            content = str(item)


for item in items:
            content = str(item)
            metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
            result.append(Document(page_content=content, metadata=metadata))
        return result
[docs]    def getERC20Tx(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=tokentx"
            f"&address={self.account_address}&startblock={self.start_block}"
            f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
            f"&sort={self.sort}&apikey={self.api_key}"
        )
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        items = response.json()["result"]
        result = []
        if len(items) == 0:
            return [Document(page_content="")]
        for item in items:
            content = str(item)
            metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
            result.append(Document(page_content=content, metadata=metadata))
        return result
[docs]    def getERC721Tx(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=tokennfttx"
            f"&address={self.account_address}&startblock={self.start_block}"
            f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
            f"&sort={self.sort}&apikey={self.api_key}"
        )
        try:


)
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        items = response.json()["result"]
        result = []
        if len(items) == 0:
            return [Document(page_content="")]
        for item in items:
            content = str(item)
            metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
            result.append(Document(page_content=content, metadata=metadata))
        return result
[docs]    def getERC1155Tx(self) -> List[Document]:
        url = (
            f"https://api.etherscan.io/api?module=account&action=token1155tx"
            f"&address={self.account_address}&startblock={self.start_block}"
            f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
            f"&sort={self.sort}&apikey={self.api_key}"
        )
        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            print("Error occurred while making the request:", e)
        items = response.json()["result"]
        result = []
        if len(items) == 0:
            return [Document(page_content="")]
        for item in items:
            content = str(item)
            metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
            result.append(Document(page_content=content, metadata=metadata))
        return result


Source code for langchain.document_loaders.bilibili
import json
import re
import warnings
from typing import List, Tuple
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BiliBiliLoader(BaseLoader):
    """Load `BiliBili` video transcripts."""
[docs]    def __init__(self, video_urls: List[str]):
        """Initialize with bilibili url.
        Args:
            video_urls: List of bilibili urls.
        """
        self.video_urls = video_urls
[docs]    def load(self) -> List[Document]:
        """Load Documents from bilibili url."""
        results = []
        for url in self.video_urls:
            transcript, video_info = self._get_bilibili_subs_and_info(url)
            doc = Document(page_content=transcript, metadata=video_info)
            results.append(doc)
        return results
    def _get_bilibili_subs_and_info(self, url: str) -> Tuple[str, dict]:
        try:
            from bilibili_api import sync, video
        except ImportError:
            raise ImportError(
                "requests package not found, please install it with "
                "`pip install bilibili-api-python`"
            )
        bvid = re.search(r"BV\w+", url)
        if bvid is not None:
            v = video.Video(bvid=bvid.group())
        else:
            aid = re.search(r"av[0-9]+", url)
            if aid is not None:
                try:
                    v = video.Video(aid=int(aid.group()[2:]))
                except AttributeError:
                    raise ValueError(f"{url} is not bilibili url.")
            else:


raise ValueError(f"{url} is not bilibili url.")
            else:
                raise ValueError(f"{url} is not bilibili url.")
        video_info = sync(v.get_info())
        video_info.update({"url": url})
        sub = sync(v.get_subtitle(video_info["cid"]))
        # Get subtitle url
        sub_list = sub["subtitles"]
        if sub_list:
            sub_url = sub_list[0]["subtitle_url"]
            if not sub_url.startswith("http"):
                sub_url = "https:" + sub_url
            result = requests.get(sub_url)
            raw_sub_titles = json.loads(result.content)["body"]
            raw_transcript = " ".join([c["content"] for c in raw_sub_titles])
            raw_transcript_with_meta_info = (
                f"Video Title: {video_info['title']},"
                f"description: {video_info['desc']}\n\n"
                f"Transcript: {raw_transcript}"
            )
            return raw_transcript_with_meta_info, video_info
        else:
            raw_transcript = ""
            warnings.warn(
                f"""
                No subtitles found for video: {url}.
                Return Empty transcript.
                """
            )
            return raw_transcript, video_info


Source code for langchain.document_loaders.docusaurus
"""Load Documents from Docusarus Documentation"""
from typing import Any, List, Optional
from langchain.document_loaders.sitemap import SitemapLoader
[docs]class DocusaurusLoader(SitemapLoader):
    """
    Loader that leverages the SitemapLoader to loop through the generated pages of a
    Docusaurus Documentation website and extracts the content by looking for specific
    HTML tags. By default, the parser searches for the main content of the Docusaurus
    page, which is normally the <article>. You also have the option to define your own
    custom HTML tags by providing them as a list, for example: ["div", ".main", "a"].
    """
[docs]    def __init__(
        self,
        url: str,
        custom_html_tags: Optional[List[str]] = None,
        **kwargs: Any,
    ):
        """
        Initialize DocusaurusLoader
        Args:
            url: The base URL of the Docusaurus website.
            custom_html_tags: Optional custom html tags to extract content from pages.
            kwargs: Additional args to extend the underlying SitemapLoader, for example:
                filter_urls, blocksize, meta_function, is_local, continue_on_failure
        """
        if not kwargs.get("is_local"):
            url = f"{url}/sitemap.xml"
        self.custom_html_tags = custom_html_tags or ["main article"]
        super().__init__(
            url,
            parsing_function=kwargs.get("parsing_function") or self._parsing_function,
            **kwargs,
        )
    def _parsing_function(self, content: Any) -> str:
        """Parses specific elements from a Docusarus page."""


"""Parses specific elements from a Docusarus page."""
        relevant_elements = content.select(",".join(self.custom_html_tags))
        for element in relevant_elements:
            if element not in relevant_elements:
                element.decompose()
        return str(content.get_text())


Source code for langchain.document_loaders.json_loader
import json
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class JSONLoader(BaseLoader):
    """Load a `JSON` file using a `jq` schema.
    Example:
        [{"text": ...}, {"text": ...}, {"text": ...}] -> schema = .[].text
        {"key": [{"text": ...}, {"text": ...}, {"text": ...}]} -> schema = .key[].text
        ["", "", ""] -> schema = .[]
    """
[docs]    def __init__(
        self,
        file_path: Union[str, Path],
        jq_schema: str,
        content_key: Optional[str] = None,
        metadata_func: Optional[Callable[[Dict, Dict], Dict]] = None,
        text_content: bool = True,
        json_lines: bool = False,
    ):
        """Initialize the JSONLoader.
        Args:
            file_path (Union[str, Path]): The path to the JSON or JSON Lines file.
            jq_schema (str): The jq schema to use to extract the data or text from
                the JSON.
            content_key (str): The key to use to extract the content from the JSON if
                the jq_schema results to a list of objects (dict).
            metadata_func (Callable[Dict, Dict]): A function that takes in the JSON
                object extracted by the jq_schema and the default metadata and returns
                a dict of the updated metadata.
            text_content (bool): Boolean flag to indicate whether the content is in
                string format, default to True.
            json_lines (bool): Boolean flag to indicate whether the input is in


json_lines (bool): Boolean flag to indicate whether the input is in
                JSON Lines format.
        """
        try:
            import jq  # noqa:F401
        except ImportError:
            raise ImportError(
                "jq package not found, please install it with `pip install jq`"
            )
        self.file_path = Path(file_path).resolve()
        self._jq_schema = jq.compile(jq_schema)
        self._content_key = content_key
        self._metadata_func = metadata_func
        self._text_content = text_content
        self._json_lines = json_lines
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the JSON file."""
        docs: List[Document] = []
        if self._json_lines:
            with self.file_path.open(encoding="utf-8") as f:
                for line in f:
                    line = line.strip()
                    if line:
                        self._parse(line, docs)
        else:
            self._parse(self.file_path.read_text(encoding="utf-8"), docs)
        return docs
    def _parse(self, content: str, docs: List[Document]) -> None:
        """Convert given content to documents."""
        data = self._jq_schema.input(json.loads(content))
        # Perform some validation
        # This is not a perfect validation, but it should catch most cases
        # and prevent the user from getting a cryptic error later on.
        if self._content_key is not None:
            self._validate_content_key(data)
        if self._metadata_func is not None:
            self._validate_metadata_func(data)
        for i, sample in enumerate(data, len(docs) + 1):
            text = self._get_text(sample=sample)


text = self._get_text(sample=sample)
            metadata = self._get_metadata(
                sample=sample, source=str(self.file_path), seq_num=i
            )
            docs.append(Document(page_content=text, metadata=metadata))
    def _get_text(self, sample: Any) -> str:
        """Convert sample to string format"""
        if self._content_key is not None:
            content = sample.get(self._content_key)
        else:
            content = sample
        if self._text_content and not isinstance(content, str):
            raise ValueError(
                f"Expected page_content is string, got {type(content)} instead. \
                    Set `text_content=False` if the desired input for \
                    `page_content` is not a string"
            )
        # In case the text is None, set it to an empty string
        elif isinstance(content, str):
            return content
        elif isinstance(content, dict):
            return json.dumps(content) if content else ""
        else:
            return str(content) if content is not None else ""
    def _get_metadata(
        self, sample: Dict[str, Any], **additional_fields: Any
    ) -> Dict[str, Any]:
        """
        Return a metadata dictionary base on the existence of metadata_func
        :param sample: single data payload
        :param additional_fields: key-word arguments to be added as metadata values
        :return:
        """
        if self._metadata_func is not None:
            return self._metadata_func(sample, additional_fields)
        else:
            return additional_fields
    def _validate_content_key(self, data: Any) -> None:
        """Check if a content key is valid"""
        sample = data.first()
        if not isinstance(sample, dict):
            raise ValueError(


if not isinstance(sample, dict):
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict), \
                    so sample must be a dict but got `{type(sample)}`"
            )
        if sample.get(self._content_key) is None:
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict) \
                    with the key `{self._content_key}`"
            )
    def _validate_metadata_func(self, data: Any) -> None:
        """Check if the metadata_func output is valid"""
        sample = data.first()
        if self._metadata_func is not None:
            sample_metadata = self._metadata_func(sample, {})
            if not isinstance(sample_metadata, dict):
                raise ValueError(
                    f"Expected the metadata_func to return a dict but got \
                        `{type(sample_metadata)}`"
                )


Source code for langchain.document_loaders.readthedocs
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Tuple, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    from bs4 import NavigableString
    from bs4.element import Comment, Tag
[docs]class ReadTheDocsLoader(BaseLoader):
    """Load `ReadTheDocs` documentation directory."""
[docs]    def __init__(
        self,
        path: Union[str, Path],
        encoding: Optional[str] = None,
        errors: Optional[str] = None,
        custom_html_tag: Optional[Tuple[str, dict]] = None,
        patterns: Sequence[str] = ("*.htm", "*.html"),
        exclude_links_ratio: float = 1.0,
        **kwargs: Optional[Any],
    ):
        """
        Initialize ReadTheDocsLoader
        The loader loops over all files under `path` and extracts the actual content of
        the files by retrieving main html tags. Default main html tags include
        `<main id="main-content>`, <`div role="main>`, and `<article role="main">`. You
        can also define your own html tags by passing custom_html_tag, e.g.
        `("div", "class=main")`. The loader iterates html tags with the order of
        custom html tags (if exists) and default html tags. If any of the tags is not
        empty, the loop will break and retrieve the content out of that tag.
        Args:
            path: The location of pulled readthedocs folder.
            encoding: The encoding with which to open the documents.


encoding: The encoding with which to open the documents.
            errors: Specify how encoding and decoding errors are to be handled—this
                cannot be used in binary mode.
            custom_html_tag: Optional custom html tag to retrieve the content from
                files.
            patterns: The file patterns to load, passed to `glob.rglob`.
            exclude_links_ratio: The ratio of links:content to exclude pages from.
                This is to reduce the frequency at which index pages make their
                way into retrieved results. Recommended: 0.5
            kwargs: named arguments passed to `bs4.BeautifulSoup`.
        """
        try:
            from bs4 import BeautifulSoup
        except ImportError:
            raise ImportError(
                "Could not import python packages. "
                "Please install it with `pip install beautifulsoup4`. "
            )
        try:
            _ = BeautifulSoup(
                "Parser builder library test.",
                "html.parser",
                **kwargs,
            )
        except Exception as e:
            raise ValueError("Parsing kwargs do not appear valid") from e
        self.file_path = Path(path)
        self.encoding = encoding
        self.errors = errors
        self.custom_html_tag = custom_html_tag
        self.patterns = patterns
        self.bs_kwargs = kwargs
        self.exclude_links_ratio = exclude_links_ratio
[docs]    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader for Documents."""
        for file_pattern in self.patterns:
            for p in self.file_path.rglob(file_pattern):
                if p.is_dir():
                    continue
                with open(p, encoding=self.encoding, errors=self.errors) as f:
                    text = self._clean_data(f.read())


text = self._clean_data(f.read())
                yield Document(page_content=text, metadata={"source": str(p)})
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())
    def _clean_data(self, data: str) -> str:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(data, "html.parser", **self.bs_kwargs)
        # default tags
        html_tags = [
            ("div", {"role": "main"}),
            ("main", {"id": "main-content"}),
        ]
        if self.custom_html_tag is not None:
            html_tags.append(self.custom_html_tag)
        element = None
        # reversed order. check the custom one first
        for tag, attrs in html_tags[::-1]:
            element = soup.find(tag, attrs)
            # if found, break
            if element is not None:
                break
        if element is not None and _get_link_ratio(element) <= self.exclude_links_ratio:
            text = _get_clean_text(element)
        else:
            text = ""
        # trim empty lines
        return "\n".join([t for t in text.split("\n") if t])
def _get_clean_text(element: Tag) -> str:
    """Returns cleaned text with newlines preserved and irrelevant elements removed."""
    elements_to_skip = [
        "script",
        "noscript",
        "canvas",
        "meta",
        "svg",
        "map",
        "area",
        "audio",
        "source",
        "track",
        "video",
        "embed",
        "object",
        "param",
        "picture",
        "iframe",
        "frame",
        "frameset",


"iframe",
        "frame",
        "frameset",
        "noframes",
        "applet",
        "form",
        "button",
        "select",
        "base",
        "style",
        "img",
    ]
    newline_elements = [
        "p",
        "div",
        "ul",
        "ol",
        "li",
        "h1",
        "h2",
        "h3",
        "h4",
        "h5",
        "h6",
        "pre",
        "table",
        "tr",
    ]
    text = _process_element(element, elements_to_skip, newline_elements)
    return text.strip()
def _get_link_ratio(section: Tag) -> float:
    links = section.find_all("a")
    total_text = "".join(str(s) for s in section.stripped_strings)
    if len(total_text) == 0:
        return 0
    link_text = "".join(
        str(string.string.strip())
        for link in links
        for string in link.strings
        if string
    )
    return len(link_text) / len(total_text)
def _process_element(
    element: Union[Tag, NavigableString, Comment],
    elements_to_skip: List[str],
    newline_elements: List[str],
) -> str:
    """
    Traverse through HTML tree recursively to preserve newline and skip
    unwanted (code/binary) elements
    """
    from bs4 import NavigableString
    from bs4.element import Comment, Tag
    tag_name = getattr(element, "name", None)
    if isinstance(element, Comment) or tag_name in elements_to_skip:
        return ""


if isinstance(element, Comment) or tag_name in elements_to_skip:
        return ""
    elif isinstance(element, NavigableString):
        return element
    elif tag_name == "br":
        return "\n"
    elif tag_name in newline_elements:
        return (
            "".join(
                _process_element(child, elements_to_skip, newline_elements)
                for child in element.children
                if isinstance(child, (Tag, NavigableString, Comment))
            )
            + "\n"
        )
    else:
        return "".join(
            _process_element(child, elements_to_skip, newline_elements)
            for child in element.children
            if isinstance(child, (Tag, NavigableString, Comment))
        )


Source code for langchain.document_loaders.chromium
import asyncio
import logging
from typing import Iterator, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class AsyncChromiumLoader(BaseLoader):
    """Scrape HTML pages from URLs using a
    headless instance of the Chromium."""
[docs]    def __init__(
        self,
        urls: List[str],
    ):
        """
        Initialize the loader with a list of URL paths.
        Args:
            urls (List[str]): A list of URLs to scrape content from.
        Raises:
            ImportError: If the required 'playwright' package is not installed.
        """
        self.urls = urls
        try:
            import playwright  # noqa: F401
        except ImportError:
            raise ImportError(
                "playwright is required for AsyncChromiumLoader. "
                "Please install it with `pip install playwright`."
            )
[docs]    async def ascrape_playwright(self, url: str) -> str:
        """
        Asynchronously scrape the content of a given URL using Playwright's async API.
        Args:
            url (str): The URL to scrape.
        Returns:
            str: The scraped HTML content or an error message if an exception occurs.
        """
        from playwright.async_api import async_playwright
        logger.info("Starting scraping...")
        results = ""
        async with async_playwright() as p:
            browser = await p.chromium.launch(headless=True)
            try:
                page = await browser.new_page()
                await page.goto(url)
                results = await page.content()  # Simply get the HTML content
                logger.info("Content scraped")


logger.info("Content scraped")
            except Exception as e:
                results = f"Error: {e}"
            await browser.close()
        return results
[docs]    def lazy_load(self) -> Iterator[Document]:
        """
        Lazily load text content from the provided URLs.
        This method yields Documents one at a time as they're scraped,
        instead of waiting to scrape all URLs before returning.
        Yields:
            Document: The scraped content encapsulated within a Document object.
        """
        for url in self.urls:
            html_content = asyncio.run(self.ascrape_playwright(url))
            metadata = {"source": url}
            yield Document(page_content=html_content, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """
        Load and return all Documents from the provided URLs.
        Returns:
            List[Document]: A list of Document objects
            containing the scraped content from each URL.
        """
        return list(self.lazy_load())


Source code for langchain.document_loaders.notebook
"""Loads .ipynb notebook files."""
import json
from pathlib import Path
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]def concatenate_cells(
    cell: dict, include_outputs: bool, max_output_length: int, traceback: bool
) -> str:
    """Combine cells information in a readable format ready to be used.
    Args:
        cell: A dictionary
        include_outputs: Whether to include the outputs of the cell.
        max_output_length: Maximum length of the output to be displayed.
        traceback: Whether to return a traceback of the error.
    Returns:
        A string with the cell information.
    """
    cell_type = cell["cell_type"]
    source = cell["source"]
    output = cell["outputs"]
    if include_outputs and cell_type == "code" and output:
        if "ename" in output[0].keys():
            error_name = output[0]["ename"]
            error_value = output[0]["evalue"]
            if traceback:
                traceback = output[0]["traceback"]
                return (
                    f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
                    f" with description '{error_value}'\n"
                    f"and traceback '{traceback}'\n\n"
                )
            else:
                return (
                    f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
                    f"with description '{error_value}'\n\n"
                )
        elif output[0]["output_type"] == "stream":
            output = output[0]["text"]


output = output[0]["text"]
            min_output = min(max_output_length, len(output))
            return (
                f"'{cell_type}' cell: '{source}'\n with "
                f"output: '{output[:min_output]}'\n\n"
            )
    else:
        return f"'{cell_type}' cell: '{source}'\n\n"
    return ""
[docs]def remove_newlines(x: Any) -> Any:
    """Recursively remove newlines, no matter the data structure they are stored in."""
    import pandas as pd
    if isinstance(x, str):
        return x.replace("\n", "")
    elif isinstance(x, list):
        return [remove_newlines(elem) for elem in x]
    elif isinstance(x, pd.DataFrame):
        return x.applymap(remove_newlines)
    else:
        return x
[docs]class NotebookLoader(BaseLoader):
    """Load `Jupyter notebook` (.ipynb) files."""
[docs]    def __init__(
        self,
        path: str,
        include_outputs: bool = False,
        max_output_length: int = 10,
        remove_newline: bool = False,
        traceback: bool = False,
    ):
        """Initialize with a path.
        Args:
            path: The path to load the notebook from.
            include_outputs: Whether to include the outputs of the cell.
                Defaults to False.
            max_output_length: Maximum length of the output to be displayed.
                Defaults to 10.
            remove_newline: Whether to remove newlines from the notebook.
                Defaults to False.
            traceback: Whether to return a traceback of the error.
                Defaults to False.
        """
        self.file_path = path


Defaults to False.
        """
        self.file_path = path
        self.include_outputs = include_outputs
        self.max_output_length = max_output_length
        self.remove_newline = remove_newline
        self.traceback = traceback
[docs]    def load(
        self,
    ) -> List[Document]:
        """Load documents."""
        try:
            import pandas as pd
        except ImportError:
            raise ImportError(
                "pandas is needed for Notebook Loader, "
                "please install with `pip install pandas`"
            )
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        data = pd.json_normalize(d["cells"])
        filtered_data = data[["cell_type", "source", "outputs"]]
        if self.remove_newline:
            filtered_data = filtered_data.applymap(remove_newlines)
        text = filtered_data.apply(
            lambda x: concatenate_cells(
                x, self.include_outputs, self.max_output_length, self.traceback
            ),
            axis=1,
        ).str.cat(sep=" ")
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.acreom
import re
from pathlib import Path
from typing import Iterator, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class AcreomLoader(BaseLoader):
    """Load `acreom` vault from a directory."""
    FRONT_MATTER_REGEX = re.compile(r"^---\n(.*?)\n---\n", re.MULTILINE | re.DOTALL)
    """Regex to match front matter metadata in markdown files."""
[docs]    def __init__(
        self, path: str, encoding: str = "UTF-8", collect_metadata: bool = True
    ):
        """Initialize the loader."""
        self.file_path = path
        """Path to the directory containing the markdown files."""
        self.encoding = encoding
        """Encoding to use when reading the files."""
        self.collect_metadata = collect_metadata
        """Whether to collect metadata from the front matter."""
    def _parse_front_matter(self, content: str) -> dict:
        """Parse front matter metadata from the content and return it as a dict."""
        if not self.collect_metadata:
            return {}
        match = self.FRONT_MATTER_REGEX.search(content)
        front_matter = {}
        if match:
            lines = match.group(1).split("\n")
            for line in lines:
                if ":" in line:
                    key, value = line.split(":", 1)
                    front_matter[key.strip()] = value.strip()
                else:
                    # Skip lines without a colon
                    continue
        return front_matter
    def _remove_front_matter(self, content: str) -> str:
        """Remove front matter metadata from the given content."""
        if not self.collect_metadata:
            return content


if not self.collect_metadata:
            return content
        return self.FRONT_MATTER_REGEX.sub("", content)
    def _process_acreom_content(self, content: str) -> str:
        # remove acreom specific elements from content that
        # do not contribute to the context of current document
        content = re.sub(r"\s*-\s\[\s\]\s.*|\s*\[\s\]\s.*", "", content)  # rm tasks
        content = re.sub(r"#", "", content)  # rm hashtags
        content = re.sub(r"\[\[.*?\]\]", "", content)  # rm doclinks
        return content
[docs]    def lazy_load(self) -> Iterator[Document]:
        ps = list(Path(self.file_path).glob("**/*.md"))
        for p in ps:
            with open(p, encoding=self.encoding) as f:
                text = f.read()
            front_matter = self._parse_front_matter(text)
            text = self._remove_front_matter(text)
            text = self._process_acreom_content(text)
            metadata = {
                "source": str(p.name),
                "path": str(p),
                **front_matter,
            }
            yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())


Source code for langchain.document_loaders.git
import os
from typing import Callable, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GitLoader(BaseLoader):
    """Load `Git` repository files.
    The Repository can be local on disk available at `repo_path`,
    or remote at `clone_url` that will be cloned to `repo_path`.
    Currently, supports only text files.
    Each document represents one file in the repository. The `path` points to
    the local Git repository, and the `branch` specifies the branch to load
    files from. By default, it loads from the `main` branch.
    """
[docs]    def __init__(
        self,
        repo_path: str,
        clone_url: Optional[str] = None,
        branch: Optional[str] = "main",
        file_filter: Optional[Callable[[str], bool]] = None,
    ):
        """
        Args:
            repo_path: The path to the Git repository.
            clone_url: Optional. The URL to clone the repository from.
            branch: Optional. The branch to load files from. Defaults to `main`.
            file_filter: Optional. A function that takes a file path and returns
              a boolean indicating whether to load the file. Defaults to None.
        """
        self.repo_path = repo_path
        self.clone_url = clone_url
        self.branch = branch
        self.file_filter = file_filter
[docs]    def load(self) -> List[Document]:
        try:
            from git import Blob, Repo  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import git python package. "


raise ImportError(
                "Could not import git python package. "
                "Please install it with `pip install GitPython`."
            ) from ex
        if not os.path.exists(self.repo_path) and self.clone_url is None:
            raise ValueError(f"Path {self.repo_path} does not exist")
        elif self.clone_url:
            # If the repo_path already contains a git repository, verify that it's the
            # same repository as the one we're trying to clone.
            if os.path.isdir(os.path.join(self.repo_path, ".git")):
                repo = Repo(self.repo_path)
                # If the existing repository is not the same as the one we're trying to
                # clone, raise an error.
                if repo.remotes.origin.url != self.clone_url:
                    raise ValueError(
                        "A different repository is already cloned at this path."
                    )
            else:
                repo = Repo.clone_from(self.clone_url, self.repo_path)
            repo.git.checkout(self.branch)
        else:
            repo = Repo(self.repo_path)
            repo.git.checkout(self.branch)
        docs: List[Document] = []
        for item in repo.tree().traverse():
            if not isinstance(item, Blob):
                continue
            file_path = os.path.join(self.repo_path, item.path)
            ignored_files = repo.ignored([file_path])  # type: ignore
            if len(ignored_files):
                continue
            # uses filter to skip files
            if self.file_filter and not self.file_filter(file_path):
                continue
            rel_file_path = os.path.relpath(file_path, self.repo_path)
            try:
                with open(file_path, "rb") as f:
                    content = f.read()
                    file_type = os.path.splitext(item.name)[1]


content = f.read()
                    file_type = os.path.splitext(item.name)[1]
                    # loads only text files
                    try:
                        text_content = content.decode("utf-8")
                    except UnicodeDecodeError:
                        continue
                    metadata = {
                        "source": rel_file_path,
                        "file_path": rel_file_path,
                        "file_name": item.name,
                        "file_type": file_type,
                    }
                    doc = Document(page_content=text_content, metadata=metadata)
                    docs.append(doc)
            except Exception as e:
                print(f"Error reading file {file_path}: {e}")
        return docs


Source code for langchain.document_loaders.arxiv
from typing import Any, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.arxiv import ArxivAPIWrapper
[docs]class ArxivLoader(BaseLoader):
    """Load a query result from `Arxiv`.
    The loader converts the original PDF format into the text.
    Args:
        Supports all arguments of `ArxivAPIWrapper`.
    """
[docs]    def __init__(
        self, query: str, doc_content_chars_max: Optional[int] = None, **kwargs: Any
    ):
        self.query = query
        self.client = ArxivAPIWrapper(
            doc_content_chars_max=doc_content_chars_max, **kwargs
        )
[docs]    def load(self) -> List[Document]:
        return self.client.load(self.query)


Source code for langchain.document_loaders.email
import os
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEmailLoader(UnstructuredFileLoader):
    """Load email files using `Unstructured`.
    Works with both
    .eml and .msg files. You can process attachments in addition to the
    e-mail message itself by passing process_attachments=True into the
    constructor for the loader. By default, attachments will be processed
    with the unstructured partition function. If you already know the document
    types of the attachments, you can specify another partitioning function
    with the attachment partitioner kwarg.
    Example
    -------
    from langchain.document_loaders import UnstructuredEmailLoader
    loader = UnstructuredEmailLoader("example_data/fake-email.eml", mode="elements")
    loader.load()
    Example
    -------
    from langchain.document_loaders import UnstructuredEmailLoader
    loader = UnstructuredEmailLoader(
        "example_data/fake-email-attachment.eml",
        mode="elements",
        process_attachments=True,
    )
    loader.load()
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        process_attachments = unstructured_kwargs.get("process_attachments")
        attachment_partitioner = unstructured_kwargs.get("attachment_partitioner")
        if process_attachments and attachment_partitioner is None:
            from unstructured.partition.auto import partition
            unstructured_kwargs["attachment_partitioner"] = partition


unstructured_kwargs["attachment_partitioner"] = partition
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.file_utils.filetype import FileType, detect_filetype
        filetype = detect_filetype(self.file_path)
        if filetype == FileType.EML:
            from unstructured.partition.email import partition_email
            return partition_email(filename=self.file_path, **self.unstructured_kwargs)
        elif satisfies_min_unstructured_version("0.5.8") and filetype == FileType.MSG:
            from unstructured.partition.msg import partition_msg
            return partition_msg(filename=self.file_path, **self.unstructured_kwargs)
        else:
            raise ValueError(
                f"Filetype {filetype} is not supported in UnstructuredEmailLoader."
            )
[docs]class OutlookMessageLoader(BaseLoader):
    """
    Loads Outlook Message files using extract_msg.
    https://github.com/TeamMsgExtractor/msg-extractor
    """
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path.
        Args:
            file_path: The path to the Outlook Message file.
        """
        self.file_path = file_path
        if not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file" % self.file_path)
        try:
            import extract_msg  # noqa:F401
        except ImportError:
            raise ImportError(
                "extract_msg is not installed. Please install it with "
                "`pip install extract_msg`"
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        import extract_msg
        msg = extract_msg.Message(self.file_path)
        return [


import extract_msg
        msg = extract_msg.Message(self.file_path)
        return [
            Document(
                page_content=msg.body,
                metadata={
                    "source": self.file_path,
                    "subject": msg.subject,
                    "sender": msg.sender,
                    "date": msg.date,
                },
            )
        ]


Source code for langchain.document_loaders.srt
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class SRTLoader(BaseLoader):
    """Load `.srt` (subtitle) files."""
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path."""
        try:
            import pysrt  # noqa:F401
        except ImportError:
            raise ImportError(
                "package `pysrt` not found, please install it with `pip install pysrt`"
            )
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load using pysrt file."""
        import pysrt
        parsed_info = pysrt.open(self.file_path)
        text = " ".join([t.text for t in parsed_info])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.text
import logging
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.helpers import detect_file_encodings
logger = logging.getLogger(__name__)
[docs]class TextLoader(BaseLoader):
    """Load text file.
    Args:
        file_path: Path to the file to load.
        encoding: File encoding to use. If `None`, the file will be loaded
        with the default system encoding.
        autodetect_encoding: Whether to try to autodetect the file encoding
            if the specified encoding fails.
    """
[docs]    def __init__(
        self,
        file_path: str,
        encoding: Optional[str] = None,
        autodetect_encoding: bool = False,
    ):
        """Initialize with file path."""
        self.file_path = file_path
        self.encoding = encoding
        self.autodetect_encoding = autodetect_encoding
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        text = ""
        try:
            with open(self.file_path, encoding=self.encoding) as f:
                text = f.read()
        except UnicodeDecodeError as e:
            if self.autodetect_encoding:
                detected_encodings = detect_file_encodings(self.file_path)
                for encoding in detected_encodings:
                    logger.debug(f"Trying encoding: {encoding.encoding}")
                    try:
                        with open(self.file_path, encoding=encoding.encoding) as f:
                            text = f.read()
                        break
                    except UnicodeDecodeError:
                        continue
            else:
                raise RuntimeError(f"Error loading {self.file_path}") from e
        except Exception as e:


except Exception as e:
            raise RuntimeError(f"Error loading {self.file_path}") from e
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.notiondb
from typing import Any, Dict, List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
NOTION_BASE_URL = "https://api.notion.com/v1"
DATABASE_URL = NOTION_BASE_URL + "/databases/{database_id}/query"
PAGE_URL = NOTION_BASE_URL + "/pages/{page_id}"
BLOCK_URL = NOTION_BASE_URL + "/blocks/{block_id}/children"
[docs]class NotionDBLoader(BaseLoader):
    """Load from `Notion DB`.
    Reads content from pages within a Notion Database.
    Args:
        integration_token (str): Notion integration token.
        database_id (str): Notion database id.
        request_timeout_sec (int): Timeout for Notion requests in seconds.
            Defaults to 10.
    """
[docs]    def __init__(
        self,
        integration_token: str,
        database_id: str,
        request_timeout_sec: Optional[int] = 10,
    ) -> None:
        """Initialize with parameters."""
        if not integration_token:
            raise ValueError("integration_token must be provided")
        if not database_id:
            raise ValueError("database_id must be provided")
        self.token = integration_token
        self.database_id = database_id
        self.headers = {
            "Authorization": "Bearer " + self.token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }
        self.request_timeout_sec = request_timeout_sec
[docs]    def load(self) -> List[Document]:
        """Load documents from the Notion database.
        Returns:
            List[Document]: List of documents.


Returns:
            List[Document]: List of documents.
        """
        page_summaries = self._retrieve_page_summaries()
        return list(self.load_page(page_summary) for page_summary in page_summaries)
    def _retrieve_page_summaries(
        self, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[Dict[str, Any]]:
        """Get all the pages from a Notion database."""
        pages: List[Dict[str, Any]] = []
        while True:
            data = self._request(
                DATABASE_URL.format(database_id=self.database_id),
                method="POST",
                query_dict=query_dict,
            )
            pages.extend(data.get("results"))
            if not data.get("has_more"):
                break
            query_dict["start_cursor"] = data.get("next_cursor")
        return pages
[docs]    def load_page(self, page_summary: Dict[str, Any]) -> Document:
        """Read a page.
        Args:
            page_summary: Page summary from Notion API.
        """
        page_id = page_summary["id"]
        # load properties as metadata
        metadata: Dict[str, Any] = {}
        for prop_name, prop_data in page_summary["properties"].items():
            prop_type = prop_data["type"]
            if prop_type == "rich_text":
                value = (
                    prop_data["rich_text"][0]["plain_text"]
                    if prop_data["rich_text"]
                    else None
                )
            elif prop_type == "title":
                value = (
                    prop_data["title"][0]["plain_text"] if prop_data["title"] else None
                )
            elif prop_type == "multi_select":
                value = (


)
            elif prop_type == "multi_select":
                value = (
                    [item["name"] for item in prop_data["multi_select"]]
                    if prop_data["multi_select"]
                    else []
                )
            elif prop_type == "url":
                value = prop_data["url"]
            elif prop_type == "unique_id":
                value = (
                    f'{prop_data["unique_id"]["prefix"]}-{prop_data["unique_id"]["number"]}'
                    if prop_data["unique_id"]
                    else None
                )
            elif prop_type == "status":
                value = prop_data["status"]["name"] if prop_data["status"] else None
            elif prop_type == "people":
                value = (
                    [item["name"] for item in prop_data["people"]]
                    if prop_data["people"]
                    else []
                )
            elif prop_type == "date":
                value = prop_data["date"] if prop_data["date"] else None
            elif prop_type == "last_edited_time":
                value = (
                    prop_data["last_edited_time"]
                    if prop_data["last_edited_time"]
                    else None
                )
            elif prop_type == "created_time":
                value = prop_data["created_time"] if prop_data["created_time"] else None
            elif prop_type == "checkbox":
                value = prop_data["checkbox"]
            elif prop_type == "email":
                value = prop_data["email"]
            elif prop_type == "number":
                value = prop_data["number"]
            elif prop_type == "select":
                value = prop_data["select"]["name"] if prop_data["select"] else None
            else:
                value = None
            metadata[prop_name.lower()] = value


else:
                value = None
            metadata[prop_name.lower()] = value
        metadata["id"] = page_id
        return Document(page_content=self._load_blocks(page_id), metadata=metadata)
    def _load_blocks(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block and its children."""
        result_lines_arr: List[str] = []
        cur_block_id: str = block_id
        while cur_block_id:
            data = self._request(BLOCK_URL.format(block_id=cur_block_id))
            for result in data["results"]:
                result_obj = result[result["type"]]
                if "rich_text" not in result_obj:
                    continue
                cur_result_text_arr: List[str] = []
                for rich_text in result_obj["rich_text"]:
                    if "text" in rich_text:
                        cur_result_text_arr.append(
                            "\t" * num_tabs + rich_text["text"]["content"]
                        )
                if result["has_children"]:
                    children_text = self._load_blocks(
                        result["id"], num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)
                result_lines_arr.append("\n".join(cur_result_text_arr))
            cur_block_id = data.get("next_cursor")
        return "\n".join(result_lines_arr)
    def _request(
        self, url: str, method: str = "GET", query_dict: Dict[str, Any] = {}
    ) -> Any:
        res = requests.request(
            method,
            url,
            headers=self.headers,
            json=query_dict,
            timeout=self.request_timeout_sec,
        )
        res.raise_for_status()
        return res.json()


Source code for langchain.document_loaders.rocksetdb
from typing import Any, Callable, Iterator, List, Optional, Tuple
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
[docs]def default_joiner(docs: List[Tuple[str, Any]]) -> str:
    """Default joiner for content columns."""
    return "\n".join([doc[1] for doc in docs])
[docs]class ColumnNotFoundError(Exception):
    """Column not found error."""
    def __init__(self, missing_key: str, query: str):
        super().__init__(f'Column "{missing_key}" not selected in query:\n{query}')
[docs]class RocksetLoader(BaseLoader):
    """Load from a `Rockset` database.
    To use, you should have the `rockset` python package installed.
    Example:
        .. code-block:: python
            # This code will load 3 records from the "langchain_demo"
            # collection as Documents, with the `text` column used as
            # the content
            from langchain.document_loaders import RocksetLoader
            from rockset import RocksetClient, Regions, models
            loader = RocksetLoader(
                RocksetClient(Regions.usw2a1, "<api key>"),
                models.QueryRequestSql(
                    query="select * from langchain_demo limit 3"
                ),
                ["text"]
            )
        )
    """
[docs]    def __init__(
        self,
        client: Any,
        query: Any,
        content_keys: List[str],
        metadata_keys: Optional[List[str]] = None,
        content_columns_joiner: Callable[[List[Tuple[str, Any]]], str] = default_joiner,
    ):


):
        """Initialize with Rockset client.
        Args:
            client: Rockset client object.
            query: Rockset query object.
            content_keys: The collection columns to be written into the `page_content`
                of the Documents.
            metadata_keys: The collection columns to be written into the `metadata` of
                the Documents. By default, this is all the keys in the document.
            content_columns_joiner: Method that joins content_keys and its values into a
                string. It's method that takes in a List[Tuple[str, Any]]],
                representing a list of tuples of (column name, column value).
                By default, this is a method that joins each column value with a new
                line. This method is only relevant if there are multiple content_keys.
        """
        try:
            from rockset import QueryPaginator, RocksetClient
            from rockset.models import QueryRequestSql
        except ImportError:
            raise ImportError(
                "Could not import rockset client python package. "
                "Please install it with `pip install rockset`."
            )
        if not isinstance(client, RocksetClient):
            raise ValueError(
                f"client should be an instance of rockset.RocksetClient, "
                f"got {type(client)}"
            )
        if not isinstance(query, QueryRequestSql):
            raise ValueError(
                f"query should be an instance of rockset.model.QueryRequestSql, "
                f"got {type(query)}"
            )
        self.client = client
        self.query = query
        self.content_keys = content_keys
        self.content_columns_joiner = content_columns_joiner
        self.metadata_keys = metadata_keys
        self.paginator = QueryPaginator
        self.request_model = QueryRequestSql
        try:


self.paginator = QueryPaginator
        self.request_model = QueryRequestSql
        try:
            self.client.set_application("langchain")
        except AttributeError:
            # ignore
            pass
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        query_results = self.client.Queries.query(
            sql=self.query
        ).results  # execute the SQL query
        for doc in query_results:  # for each doc in the response
            try:
                yield Document(
                    page_content=self.content_columns_joiner(
                        [(col, doc[col]) for col in self.content_keys]
                    ),
                    metadata={col: doc[col] for col in self.metadata_keys}
                    if self.metadata_keys is not None
                    else doc,
                )  # try to yield the Document
            except (
                KeyError
            ) as e:  # either content_columns or metadata_columns is invalid
                raise ColumnNotFoundError(
                    e.args[0], self.query
                )  # raise that the column isn't in the db schema


Source code for langchain.document_loaders.college_confidential
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class CollegeConfidentialLoader(WebBaseLoader):
    """Load `College Confidential` webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpages as Documents."""
        soup = self.scrape()
        text = soup.select_one("main[class='skin-handler']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.xml
"""Loads Microsoft Excel files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredXMLLoader(UnstructuredFileLoader):
    """Load `XML` file using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain.document_loaders import UnstructuredXMLLoader
    loader = UnstructuredXMLLoader(
        "example.xml", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-xml
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.6.7")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.xml import partition_xml
        return partition_xml(filename=self.file_path, **self.unstructured_kwargs)


Source code for langchain.document_loaders.spreedly
import json
import urllib.request
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
SPREEDLY_ENDPOINTS = {
    "gateways_options": "https://core.spreedly.com/v1/gateways_options.json",
    "gateways": "https://core.spreedly.com/v1/gateways.json",
    "receivers_options": "https://core.spreedly.com/v1/receivers_options.json",
    "receivers": "https://core.spreedly.com/v1/receivers.json",
    "payment_methods": "https://core.spreedly.com/v1/payment_methods.json",
    "certificates": "https://core.spreedly.com/v1/certificates.json",
    "transactions": "https://core.spreedly.com/v1/transactions.json",
    "environments": "https://core.spreedly.com/v1/environments.json",
}
[docs]class SpreedlyLoader(BaseLoader):
    """Load from `Spreedly` API."""
[docs]    def __init__(self, access_token: str, resource: str) -> None:
        """Initialize with an access token and a resource.
        Args:
            access_token: The access token.
            resource: The resource.
        """
        self.access_token = access_token
        self.resource = resource
        self.headers = {
            "Authorization": f"Bearer {self.access_token}",
            "Accept": "application/json",
        }
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:


with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = SPREEDLY_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()


Source code for langchain.document_loaders.gutenberg
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GutenbergLoader(BaseLoader):
    """Load from `Gutenberg.org`."""
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path."""
        if not file_path.startswith("https://www.gutenberg.org"):
            raise ValueError("file path must start with 'https://www.gutenberg.org'")
        if not file_path.endswith(".txt"):
            raise ValueError("file path must end with '.txt'")
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from urllib.request import urlopen
        elements = urlopen(self.file_path)
        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.base_o365
"""Base class for all loaders that uses O365 Package"""
from __future__ import annotations
import logging
import os
import tempfile
from abc import abstractmethod
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Dict, Iterable, List, Sequence, Union
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.blob_loaders.file_system import FileSystemBlobLoader
from langchain.document_loaders.blob_loaders.schema import Blob
from langchain.pydantic_v1 import BaseModel, BaseSettings, Field, FilePath, SecretStr
if TYPE_CHECKING:
    from O365 import Account
    from O365.drive import Drive, Folder
logger = logging.getLogger(__name__)
CHUNK_SIZE = 1024 * 1024 * 5
class _O365Settings(BaseSettings):
    client_id: str = Field(..., env="O365_CLIENT_ID")
    client_secret: SecretStr = Field(..., env="O365_CLIENT_SECRET")
    class Config:
        env_prefix = ""
        case_sentive = False
        env_file = ".env"
class _O365TokenStorage(BaseSettings):
    token_path: FilePath = Path.home() / ".credentials" / "o365_token.txt"
class _FileType(str, Enum):
    DOC = "doc"
    DOCX = "docx"
    PDF = "pdf"
[docs]def fetch_mime_types(file_types: Sequence[_FileType]) -> Dict[str, str]:
    """Fetch the mime types for the specified file types."""
    mime_types_mapping = {}
    for file_type in file_types:
        if file_type.value == "doc":
            mime_types_mapping[file_type.value] = "application/msword"
        elif file_type.value == "docx":


elif file_type.value == "docx":
            mime_types_mapping[
                file_type.value
            ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"  # noqa: E501
        elif file_type.value == "pdf":
            mime_types_mapping[file_type.value] = "application/pdf"
    return mime_types_mapping
[docs]class O365BaseLoader(BaseLoader, BaseModel):
    """Base class for all loaders that uses O365 Package"""
    settings: _O365Settings = Field(default_factory=_O365Settings)
    """Settings for the Office365 API client."""
    auth_with_token: bool = False
    """Whether to authenticate with a token or not. Defaults to False."""
    chunk_size: Union[int, str] = CHUNK_SIZE
    """Number of bytes to retrieve from each api call to the server. int or 'auto'."""
    @property
    @abstractmethod
    def _file_types(self) -> Sequence[_FileType]:
        """Return supported file types."""
    @property
    def _fetch_mime_types(self) -> Dict[str, str]:
        """Return a dict of supported file types to corresponding mime types."""
        return fetch_mime_types(self._file_types)
    @property
    @abstractmethod
    def _scopes(self) -> List[str]:
        """Return required scopes."""
    def _load_from_folder(self, folder: Folder) -> Iterable[Blob]:
        """Lazily load all files from a specified folder of the configured MIME type.
        Args:
            folder: The Folder instance from which the files are to be loaded. This
                Folder instance should represent a directory in a file system where the
                files are stored.
        Yields:
            An iterator that yields Blob instances, which are binary representations of
                the files loaded from the folder.


the files loaded from the folder.
        """
        file_mime_types = self._fetch_mime_types
        items = folder.get_items()
        with tempfile.TemporaryDirectory() as temp_dir:
            os.makedirs(os.path.dirname(temp_dir), exist_ok=True)
            for file in items:
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        file.download(to_path=temp_dir, chunk_size=self.chunk_size)
            loader = FileSystemBlobLoader(path=temp_dir)
            yield from loader.yield_blobs()
    def _load_from_object_ids(
        self, drive: Drive, object_ids: List[str]
    ) -> Iterable[Blob]:
        """Lazily load files specified by their object_ids from a drive.
        Load files into the system as binary large objects (Blobs) and return Iterable.
        Args:
            drive: The Drive instance from which the files are to be loaded. This Drive
                instance should represent a cloud storage service or similar storage
                system where the files are stored.
            object_ids: A list of object_id strings. Each object_id represents a unique
                identifier for a file in the drive.
        Yields:
            An iterator that yields Blob instances, which are binary representations of
            the files loaded from the drive using the specified object_ids.
        """
        file_mime_types = self._fetch_mime_types
        with tempfile.TemporaryDirectory() as temp_dir:
            for object_id in object_ids:
                file = drive.get_item(object_id)
                if not file:
                    logging.warning(
                        "There isn't a file with"
                        f"object_id {object_id} in drive {drive}."
                    )
                    continue
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):


if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        file.download(to_path=temp_dir, chunk_size=self.chunk_size)
            loader = FileSystemBlobLoader(path=temp_dir)
            yield from loader.yield_blobs()
    def _auth(self) -> Account:
        """Authenticates the OneDrive API client
        Returns:
            The authenticated Account object.
        """
        try:
            from O365 import Account, FileSystemTokenBackend
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        if self.auth_with_token:
            token_storage = _O365TokenStorage()
            token_path = token_storage.token_path
            token_backend = FileSystemTokenBackend(
                token_path=token_path.parent, token_filename=token_path.name
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=self._scopes,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
        else:
            token_backend = FileSystemTokenBackend(
                token_path=Path.home() / ".credentials"
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=self._scopes,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
            # make the auth
            account.authenticate()
        return account


Source code for langchain.document_loaders.lakefs
import os
import tempfile
import urllib.parse
from typing import Any, List, Optional
from urllib.parse import urljoin
import requests
from requests.auth import HTTPBasicAuth
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredBaseLoader
from langchain.schema import Document
[docs]class LakeFSClient:
[docs]    def __init__(
        self,
        lakefs_access_key: str,
        lakefs_secret_key: str,
        lakefs_endpoint: str,
    ):
        self.__endpoint = "/".join([lakefs_endpoint, "api", "v1/"])
        self.__auth = HTTPBasicAuth(lakefs_access_key, lakefs_secret_key)
        try:
            health_check = requests.get(
                urljoin(self.__endpoint, "healthcheck"), auth=self.__auth
            )
            health_check.raise_for_status()
        except Exception:
            raise ValueError(
                "lakeFS server isn't accessible. Make sure lakeFS is running."
            )
[docs]    def ls_objects(
        self, repo: str, ref: str, path: str, presign: Optional[bool]
    ) -> List:
        qp = {"prefix": path, "presign": presign}
        eqp = urllib.parse.urlencode(qp)
        objects_ls_endpoint = urljoin(
            self.__endpoint, f"repositories/{repo}/refs/{ref}/objects/ls?{eqp}"
        )
        olsr = requests.get(objects_ls_endpoint, auth=self.__auth)
        olsr.raise_for_status()
        olsr_json = olsr.json()
        return list(
            map(


return list(
            map(
                lambda res: (res["path"], res["physical_address"]), olsr_json["results"]
            )
        )
[docs]    def is_presign_supported(self) -> bool:
        config_endpoint = self.__endpoint + "config"
        response = requests.get(config_endpoint, auth=self.__auth)
        response.raise_for_status()
        config = response.json()
        return config["storage_config"]["pre_sign_support"]
[docs]class LakeFSLoader(BaseLoader):
    """Load from `lakeFS`."""
    repo: str
    ref: str
    path: str
[docs]    def __init__(
        self,
        lakefs_access_key: str,
        lakefs_secret_key: str,
        lakefs_endpoint: str,
        repo: Optional[str] = None,
        ref: Optional[str] = "main",
        path: Optional[str] = "",
    ):
        """
        :param lakefs_access_key: [required] lakeFS server's access key
        :param lakefs_secret_key: [required] lakeFS server's secret key
        :param lakefs_endpoint: [required] lakeFS server's endpoint address,
               ex: https://example.my-lakefs.com
        :param repo: [optional, default = ''] target repository
        :param ref: [optional, default = 'main'] target ref (branch name,
               tag, or commit ID)
        :param path: [optional, default = ''] target path
        """
        self.__lakefs_client = LakeFSClient(
            lakefs_access_key, lakefs_secret_key, lakefs_endpoint
        )
        self.repo = "" if repo is None or repo == "" else str(repo)


self.repo = "" if repo is None or repo == "" else str(repo)
        self.ref = "main" if ref is None or ref == "" else str(ref)
        self.path = "" if path is None else str(path)
[docs]    def set_path(self, path: str) -> None:
        self.path = path
[docs]    def set_ref(self, ref: str) -> None:
        self.ref = ref
[docs]    def set_repo(self, repo: str) -> None:
        self.repo = repo
[docs]    def load(self) -> List[Document]:
        self.__validate_instance()
        presigned = self.__lakefs_client.is_presign_supported()
        docs: List[Document] = []
        objs = self.__lakefs_client.ls_objects(
            repo=self.repo, ref=self.ref, path=self.path, presign=presigned
        )
        for obj in objs:
            lakefs_unstructured_loader = UnstructuredLakeFSLoader(
                obj[1], self.repo, self.ref, obj[0], presigned
            )
            docs.extend(lakefs_unstructured_loader.load())
        return docs
    def __validate_instance(self) -> None:
        if self.repo is None or self.repo == "":
            raise ValueError(
                "no repository was provided. use `set_repo` to specify a repository"
            )
        if self.ref is None or self.ref == "":
            raise ValueError("no ref was provided. use `set_ref` to specify a ref")
        if self.path is None:
            raise ValueError("no path was provided. use `set_path` to specify a path")
[docs]class UnstructuredLakeFSLoader(UnstructuredBaseLoader):
[docs]    def __init__(
        self,


[docs]    def __init__(
        self,
        url: str,
        repo: str,
        ref: str = "main",
        path: str = "",
        presign: bool = True,
        **unstructured_kwargs: Any,
    ):
        """
        Args:
        :param lakefs_access_key:
        :param lakefs_secret_key:
        :param lakefs_endpoint:
        :param repo:
        :param ref:
        """
        super().__init__(**unstructured_kwargs)
        self.url = url
        self.repo = repo
        self.ref = ref
        self.path = path
        self.presign = presign
    def _get_metadata(self) -> dict:
        return {"repo": self.repo, "ref": self.ref, "path": self.path}
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        local_prefix = "local://"
        if self.presign:
            with tempfile.TemporaryDirectory() as temp_dir:
                file_path = f"{temp_dir}/{self.path.split('/')[-1]}"
                os.makedirs(os.path.dirname(file_path), exist_ok=True)
                response = requests.get(self.url)
                response.raise_for_status()
                with open(file_path, mode="wb") as file:
                    file.write(response.content)
                return partition(filename=file_path)
        elif not self.url.startswith(local_prefix):
            raise ValueError(
                "Non pre-signed URLs are supported only with 'local' blockstore"
            )
        else:
            local_path = self.url[len(local_prefix) :]
            return partition(filename=local_path)


Source code for langchain.document_loaders.azure_blob_storage_container
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.azure_blob_storage_file import (
    AzureBlobStorageFileLoader,
)
from langchain.document_loaders.base import BaseLoader
[docs]class AzureBlobStorageContainerLoader(BaseLoader):
    """Load from `Azure Blob Storage` container."""
[docs]    def __init__(self, conn_str: str, container: str, prefix: str = ""):
        """Initialize with connection string, container and blob prefix."""
        self.conn_str = conn_str
        """Connection string for Azure Blob Storage."""
        self.container = container
        """Container name."""
        self.prefix = prefix
        """Prefix for blob names."""
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from azure.storage.blob import ContainerClient
        except ImportError as exc:
            raise ImportError(
                "Could not import azure storage blob python package. "
                "Please install it with `pip install azure-storage-blob`."
            ) from exc
        container = ContainerClient.from_connection_string(
            conn_str=self.conn_str, container_name=self.container
        )
        docs = []
        blob_list = container.list_blobs(name_starts_with=self.prefix)
        for blob in blob_list:
            loader = AzureBlobStorageFileLoader(
                self.conn_str,
                self.container,
                blob.name,  # type: ignore
            )
            docs.extend(loader.load())
        return docs


Source code for langchain.document_loaders.quip
import logging
import re
import xml.etree.cElementTree
import xml.sax.saxutils
from io import BytesIO
from typing import List, Optional, Sequence
from xml.etree.ElementTree import ElementTree
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
_MAXIMUM_TITLE_LENGTH = 64
[docs]class QuipLoader(BaseLoader):
    """Load `Quip` pages.
    Port of https://github.com/quip/quip-api/tree/master/samples/baqup
    """
[docs]    def __init__(
        self, api_url: str, access_token: str, request_timeout: Optional[int] = 60
    ):
        """
        Args:
            api_url: https://platform.quip.com
            access_token: token of access quip API. Please refer:
            https://quip.com/dev/automation/documentation/current#section/Authentication/Get-Access-to-Quip's-APIs
            request_timeout: timeout of request, default 60s.
        """
        try:
            from quip_api.quip import QuipClient
        except ImportError:
            raise ImportError(
                "`quip_api` package not found, please run " "`pip install quip_api`"
            )
        self.quip_client = QuipClient(
            access_token=access_token, base_url=api_url, request_timeout=request_timeout
        )
[docs]    def load(
        self,
        folder_ids: Optional[List[str]] = None,
        thread_ids: Optional[List[str]] = None,
        max_docs: Optional[int] = 1000,
        include_all_folders: bool = False,
        include_comments: bool = False,


include_all_folders: bool = False,
        include_comments: bool = False,
        include_images: bool = False,
    ) -> List[Document]:
        """
        Args:
            :param folder_ids: List of specific folder IDs to load, defaults to None
            :param thread_ids: List of specific thread IDs to load, defaults to None
            :param max_docs: Maximum number of docs to retrieve in total, defaults 1000
            :param include_all_folders: Include all folders that your access_token
                   can access, but doesn't include your private folder
            :param include_comments: Include comments, defaults to False
            :param include_images: Include images, defaults to False
        """
        if not folder_ids and not thread_ids and not include_all_folders:
            raise ValueError(
                "Must specify at least one among `folder_ids`, `thread_ids` "
                "or set `include_all`_folders as True"
            )
        thread_ids = thread_ids or []
        if folder_ids:
            for folder_id in folder_ids:
                self.get_thread_ids_by_folder_id(folder_id, 0, thread_ids)
        if include_all_folders:
            user = self.quip_client.get_authenticated_user()
            if "group_folder_ids" in user:
                self.get_thread_ids_by_folder_id(
                    user["group_folder_ids"], 0, thread_ids
                )
            if "shared_folder_ids" in user:
                self.get_thread_ids_by_folder_id(
                    user["shared_folder_ids"], 0, thread_ids
                )
        thread_ids = list(set(thread_ids[:max_docs]))
        return self.process_threads(thread_ids, include_images, include_comments)
[docs]    def get_thread_ids_by_folder_id(


[docs]    def get_thread_ids_by_folder_id(
        self, folder_id: str, depth: int, thread_ids: List[str]
    ) -> None:
        """Get thread ids by folder id and update in thread_ids"""
        from quip_api.quip import HTTPError, QuipError
        try:
            folder = self.quip_client.get_folder(folder_id)
        except QuipError as e:
            if e.code == 403:
                logging.warning(
                    f"depth {depth}, Skipped over restricted folder {folder_id}, {e}"
                )
            else:
                logging.warning(
                    f"depth {depth}, Skipped over folder {folder_id} "
                    f"due to unknown error {e.code}"
                )
            return
        except HTTPError as e:
            logging.warning(
                f"depth {depth}, Skipped over folder {folder_id} "
                f"due to HTTP error {e.code}"
            )
            return
        title = folder["folder"].get("title", "Folder %s" % folder_id)
        logging.info(f"depth {depth}, Processing folder {title}")
        for child in folder["children"]:
            if "folder_id" in child:
                self.get_thread_ids_by_folder_id(
                    child["folder_id"], depth + 1, thread_ids
                )
            elif "thread_id" in child:
                thread_ids.append(child["thread_id"])
[docs]    def process_threads(
        self, thread_ids: Sequence[str], include_images: bool, include_messages: bool
    ) -> List[Document]:
        """Process a list of thread into a list of documents."""
        docs = []
        for thread_id in thread_ids: