"""Module for downloading and analyzing the shooting victims database."""

import gzip
import tempfile
from dataclasses import dataclass
from typing import Literal, Optional

import boto3
import carto2gpd
import geopandas as gpd
import numpy as np
import pandas as pd
import requests
import simplejson as json
from dotenv import find_dotenv, load_dotenv
from loguru import logger
from pydantic import BaseModel, Field, validator
from shapely.geometry import Point

from . import DATA_DIR, EPSG
from .courts import CourtInfoByIncident
from .geo import *
from .streets import StreetHotSpots
from .utils import validate_data_schema

class Geometry(Point):
    """
    Shapely point geometry.
    """

    @classmethod
    def __get_validators__(cls):
        yield cls.validate

    @classmethod
    def validate(cls, v):
        if not isinstance(v, Point):
            raise TypeError("shapely point object required")
        return v

    @classmethod
    def __modify_schema__(cls, field_schema, field):
        pass

def upload_to_s3(data, filename):
    """Upload data to a public AWS s3 bucket."""

    # Load the credentials
    load_dotenv(find_dotenv())

    # Initialize the s3 resource
    s3 = boto3.client("s3")

    # Compress JSON
    json_str = data.to_json() + "\n"
    json_bytes = json_str.encode("utf-8")

    with tempfile.TemporaryDirectory() as tmpdir:
        tmpfile = f"{tmpdir}/{filename}"
        with gzip.open(tmpfile, "w") as fout:
            fout.write(json_bytes)

        # Upload to s3
        BUCKET = "gun-violence-dashboard"
        s3.upload_file(
            tmpfile,
            BUCKET,
            filename,
            ExtraArgs={
                "ContentType": "application/json",
                "ContentEncoding": "gzip",
                "ACL": "public-read",
            },
        )

def carto2gpd_post(url, table_name, where=None, fields=None):
    """Query carto API with a post call"""

    # Get the fields
    if fields is None:
        fields = "*"
    else:
        if "the_geom" not in fields:
            fields.append("the_geom")
        fields = ",".join(fields)

    # Build the query
    query = f"SELECT {fields} FROM {table_name}"
    if where:
        query += f" WHERE {where}"

    # Make the request
    params = dict(q=query, format="geojson", skipfields=["cartodb_id"])
    r = requests.post(url, data=params)

    if r.status_code == 200:
        return gpd.GeoDataFrame.from_features(r.json(), crs="EPSG:4326")
    else:
        raise ValueError("Error querying carto API")

def add_geographic_info(df):
    """Add geographic info."""

    # Get a fresh copy
    df = df.copy().reset_index(drop=True)

    # The original length
    original_length = len(df)

    # Check city limits
    city_limits = get_city_limits().to_crs(df.crs)
    outside_limits = ~df.geometry.within(city_limits.squeeze().geometry)
    missing = outside_limits.sum()

    # Set missing geometry to null
    logger.info(f"{missing} shootings outside city limits")
    if missing > 0:
        df.loc[outside_limits, "geometry"] = np.nan

    # Try to replace any missing geometries from criminal incidents
    dc_key_list = ", ".join(
        df.loc[df.geometry.isnull(), "dc_key"].apply(lambda x: f"'{x}'")
    )

    # Query with a post request
    url = "https://phl.carto.com/api/v2/sql"
    table_name = "incidents_part1_part2"
    where = f"dc_key IN ( {dc_key_list} )"
    incidents = carto2gpd_post(url, table_name, where=where, fields=["dc_key"]).to_crs(
        df.crs
    )
    incidents["dc_key"] = incidents["dc_key"].astype(str)

    # Did we get any matches
    matches = len(incidents)
    logger.info(f"Found {matches} matches for {missing} missing geometries")

    # Merge
    if matches > 0:
        missing_sel = df.geometry.isnull()
        missing = df.loc[missing_sel]
        df2 = missing.drop(columns=["geometry"]).merge(
            incidents[["dc_key", "geometry"]].drop_duplicates(subset=["dc_key"]),
            on="dc_key",
            how="left",
        )
        df = pd.concat([df.loc[~missing_sel], df2]).reset_index(drop=True)

    def _add_geo_info(data, geo):
        out = gpd.sjoin(data, geo, how="left", predicate="within")

        # NOTE: sometimes this will match multiple geo boundaries
        # REMOVE THEM
        duplicated = out.index.duplicated()
        if duplicated.sum():
            out = out.loc[~duplicated]

        return out.drop(labels=["index_right"], axis=1)

    # Add geographic columns
    geo_funcs = [
        get_zip_codes,
        get_police_districts,
        get_council_districts,
        get_neighborhoods,
        get_school_catchments,
        get_pa_house_districts,
        get_pa_senate_districts,
    ]
    for geo_func in geo_funcs:
        df = df.pipe(_add_geo_info, geo_func().to_crs(df.crs))

    # if geo columns are missing, geometry should be empty point
    df.loc[df["neighborhood"].isnull(), "geometry"] = np.nan

    # Check the length
    if len(df) != original_length:
        raise ValueError("Length of data has changed; this shouldn't happen!")

    return df

def load_existing_shootings_data():
    """Load existing shootings data."""
    files = sorted((DATA_DIR / "processed").glob("shootings_20*.json"))
    return pd.concat([gpd.read_file(f) for f in files])

class ShootingVictimsSchema(BaseModel):
    """Schema for the shooting victims dataset."""

    dc_key: str = Field(
        title="Incident number",
        description="The unique incident number assigned by the Police Department.",
    )
    race: Literal["B", "H", "W", "A", "Other/Unknown"] = Field(
        title="Race/Ethnicity",
        description=(
            "The race/ethnicity of the shooting victim. "
            "Allowed values include: 'B' = Black, Non-Hispanic, 'H' = Hispanic, "
            "'W' = White, Non-Hispanic, 'A' = Asian, and 'Other/Unknown'"
        ),
    )
    sex: Literal["M", "F"] = Field(
        title="Sex", description="The sex of the shooting victim."
    )
    fatal: Literal[True, False] = Field(
        title="Fatal?", description="Whether the incident was fatal."
    )
    date: str = Field(
        title="Date",
        description="The datetime of the incident in the format 'Y/m/d H:M:S'",
    )
    age_group: Literal[
        "Younger than 18", "18 to 30", "31 to 45", "Older than 45", "Unknown"
    ] = Field(tile="Age group", description="The victim's age group (or unknown).")

    has_court_case: Literal[True, False] = Field(
        title="Associated Court Case?",
        description="Does the incident number have an associated court case?",
    )

    # Not all ages are known
    age: Optional[float] = Field(
        default=None,
        title="Age",
        description="The victim's age; missing in some cases.",
    )

    # Optional geographic add-ons
    geometry: Optional[Geometry] = Field(
        default=None,
        description="The lat/lng point location of the shooting incident; missing in some cases.",
    )
    street_name: Optional[str] = Field(
        default=None,
        title="Street name",
        description="The name of the street the incident occurred on, if available.",
    )
    block_number: Optional[float] = Field(
        default=None,
        title="Block number",
        description="The block number where the incident occurred, if available.",
    )
    zip_code: Optional[str] = Field(
        default=None,
        title="ZIP Code",
        description="The ZIP code where the incident occurred, if available.",
    )
    council_district: Optional[str] = Field(
        default=None,
        title="Council district",
        description="The council district where the incident occurred, if available.",
    )
    police_district: Optional[str] = Field(
        default=None,
        title="Police district",
        description="The police district where the incident occurred, if available.",
    )
    neighborhood: Optional[str] = Field(
        default=None,
        title="Neighborhood name",
        description="The name of the neighborhood where the incident occurred, if available.",
    )
    school_name: Optional[str] = Field(
        default=None,
        title="School catchment",
        description="The elementary school catchment where the incident occurred, if available.",
    )
    house_district: Optional[str] = Field(
        default=None,
        title="PA House district",
        description="The PA House district where the incident occurred, if available.",
    )
    senate_district: Optional[str] = Field(
        default=None,
        title="Block number",
        description="The PA Senate district where the incident occurred, if available.",
    )
    segment_id: Optional[str] = Field(
        default=None,
        title="Block number",
        description="The ID of the street segment where the incident occurred, if available.",
    )

    @validator("dc_key")
    def verify_dc_key(cls, v):
        if not isinstance(v, str):
            assert not np.isnan(v), "cannot be NaN"
        else:
            assert not v.endswith(".0"), "bad string formatting"
        return v

@dataclass
class ShootingVictimsData:
    """Class for downloading and analyzing the shooting victims
    database from Open Data Philly."""

    debug: bool = False
    ignore_checks: bool = False

    ENDPOINT: str = "https://phl.carto.com/api/v2/sql"
    TABLE_NAME: str = "shootings"

    @validate_data_schema(ShootingVictimsSchema)
    def get(self) -> gpd.GeoDataFrame:
        """Download and return the formatted data."""

        if self.debug:
            logger.debug("Downloading shooting victims database")

        # Raw data from carto
        df = carto2gpd.get(self.ENDPOINT, self.TABLE_NAME)

        # Verify DC key first
        missing_dc_keys = df["dc_key"].isnull()
        if missing_dc_keys.sum() and not self.ignore_checks:
            n = missing_dc_keys.sum()
            raise ValueError(f"Found {n} rows with missing DC keys")

        # Format
        df = (
            df.assign(
                time=lambda df: df.time.replace("<Null>", np.nan).fillna("00:00:00"),
                date=lambda df: pd.to_datetime(
                    df.date_.str.slice(0, 10).str.cat(df.time, sep=" ")
                ),
                dc_key=lambda df: df.dc_key.astype(float).astype(int).astype(str),
                year=lambda df: df.date.dt.year,
                race=lambda df: df.race.fillna("Other/Unknown"),
                age=lambda df: df.age.astype(float),
                age_group=lambda df: np.select(
                    [
                        df.age <= 17,
                        (df.age > 17) & (df.age <= 30),
                        (df.age > 30) & (df.age <= 45),
                        (df.age > 45),
                    ],
                    ["Younger than 18", "18 to 30", "31 to 45", "Older than 45"],
                    default="Unknown",
                ),
                fatal=lambda df: df.fatal.apply(lambda x: True if x == 1 else False),
            )
            .assign(
                race=lambda df: df.race.where(df.latino != 1, other="H"),
            )
            .drop(labels=["point_x", "point_y", "date_", "time", "objectid"], axis=1)
            .sort_values("date", ascending=False)
            .reset_index(drop=True)
            .assign(
                date=lambda df: df.date.dt.strftime("%Y/%m/%d %H:%M:%S")
            )  # Convert date back to string
            .to_crs(epsg=EPSG)
        )

        # Add the other category for race/ethnicity
        main_race_categories = ["H", "W", "B", "A"]
        sel = df.race.isin(main_race_categories)
        df.loc[~sel, "race"] = "Other/Unknown"

        # CHECKS
        if not self.ignore_checks:
            old_df = load_existing_shootings_data()
            TOLERANCE = 100

            # Check for too many rows
            if len(df) - len(old_df) > TOLERANCE:
                logger.info(f"Length of new data: {len(df)}")
                logger.info(f"Length of old data: {len(old_df)}")
                raise ValueError(
                    "New data seems to have too many rows...please manually confirm new data is correct."
                )

            # Check for too few rows
            TOLERANCE = 10
            if len(old_df) - len(df) > TOLERANCE:
                logger.info(f"Length of new data: {len(df)}")
                logger.info(f"Length of old data: {len(old_df)}")
                raise ValueError(
                    "New data seems to have too few rows...please manually confirm new data is correct."
                )

        # Add geographic info
        df = add_geographic_info(df)

        # Handle NaN/None
        df = df.assign(
            geometry=lambda df: df.geometry.fillna(Point()),
        )

        # Value-added info for hot spots and court info
        hotspots = StreetHotSpots(debug=self.debug)
        courts = CourtInfoByIncident(debug=self.debug)
        df = (
            df.pipe(hotspots.merge)
            .pipe(courts.merge)
            .assign(segment_id=lambda df: df.segment_id.replace("", np.nan))
        )

        # Trim to the schema fields
        fields = ShootingVictimsSchema.__fields__.keys()
        df = df[fields]

        return df

    def save(self, data):
        """Save annual, processed data files."""

        # Get the years from the date
        years = pd.to_datetime(data["date"]).dt.year

        # Get unique years
        # IMPORTANT: this must be int so it is JSON serializable
        unique_years = [int(year) for year in sorted(np.unique(years), reverse=True)]
        json.dump(unique_years, (DATA_DIR / "processed" / "data_years.json").open("w"))

        # Save each year's data to separate file
        for year in unique_years:
            if self.debug:
                logger.debug(f"Saving {year} shootings as a GeoJSON file")

            # Get data for this year
            # Save in EPSG = 4326
            data_yr = data.loc[years == year].to_crs(epsg=4326)

            data_yr.to_file(
                DATA_DIR / "processed" / f"shootings_{year}.json",
                driver="GeoJSON",
                index=False,
            )

            # Save to s3
            upload_to_s3(data_yr, f"shootings_{year}.json")


import random

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from common.ld_utils import unsort as unsort_list

# pytorch implementation for np.nanmean
# https://github.com/pytorch/pytorch/issues/21987#issuecomment-539402619
def nanmean(v, *args, inplace=False, **kwargs):
    if not inplace:
        v = v.clone()
    is_nan = torch.isnan(v)
    v[is_nan] = 0
    return v.sum(*args, **kwargs) / (~is_nan).float().sum(*args, **kwargs)

def grad_norm(model):
    # compute norm of gradient for a model
    total_norm = None
    for p in model.parameters():
        if p.grad is not None:
            if total_norm is None:
                total_norm = 0
            param_norm = p.grad.detach().data.norm(2)
            total_norm += param_norm.item() ** 2

    if total_norm is not None:
        total_norm = total_norm ** (1.0 / 2)
    else:
        total_norm = 0.0
    return total_norm

def pad_tensor_list(v_list: list):
    dev = v_list[0].device
    num_meshes = len(v_list)
    num_dim = 1 if len(v_list[0].shape) == 1 else v_list[0].shape[1]
    v_len_list = []
    for verts in v_list:
        v_len_list.append(verts.shape[0])

    pad_len = max(v_len_list)
    dtype = v_list[0].dtype
    if num_dim == 1:
        padded_tensor = torch.zeros(num_meshes, pad_len, dtype=dtype)
    else:
        padded_tensor = torch.zeros(num_meshes, pad_len, num_dim, dtype=dtype)
    for idx, (verts, v_len) in enumerate(zip(v_list, v_len_list)):
        padded_tensor[idx, :v_len] = verts
    padded_tensor = padded_tensor.to(dev)
    v_len_list = torch.LongTensor(v_len_list).to(dev)
    return padded_tensor, v_len_list

def unpad_vtensor(
    vtensor: (torch.Tensor), lens: (torch.LongTensor, torch.cuda.LongTensor)
):
    tensors_list = []
    for verts, vlen in zip(vtensor, lens):
        tensors_list.append(verts[:vlen])
    return tensors_list

def one_hot_embedding(labels, num_classes):
    """Embedding labels to one-hot form.
    Args:
      labels: (LongTensor) class labels, sized [N, D1, D2, ..].
      num_classes: (int) number of classes.
    Returns:
      (tensor) encoded labels, sized [N, D1, D2, .., Dk, #classes].
    """
    y = torch.eye(num_classes).float()
    return y[labels]

def unsort(ten, sort_idx):
    """
    Unsort a tensor of shape (N, *) using the sort_idx list(N).
    Return a tensor of the pre-sorting order in shape (N, *)
    """
    assert isinstance(ten, torch.Tensor)
    assert isinstance(sort_idx, list)
    assert ten.shape[0] == len(sort_idx)

    out_list = list(torch.chunk(ten, ten.size(0), dim=0))
    out_list = unsort_list(out_list, sort_idx)
    out_list = torch.cat(out_list, dim=0)
    return out_list

def softargmax_kd(inputs_hm, temperature):
    """
    Perform softargmax on a k-dimensional heatmap
    Inputs: (N, D1, D2, .., Dk)
    Outputs: (N, K)
    """
    assert isinstance(inputs_hm, torch.Tensor)
    assert isinstance(temperature, float)
    assert len(inputs_hm.shape) >= 2
    dev = inputs_hm.device
    batch_size = inputs_hm.shape[0]
    num_dim = len(inputs_hm.shape) - 1
    dims = inputs_hm.shape[1:]
    lins = [torch.arange(dim).view(-1, 1).to(dev) + 0.5 for dim in dims]
    points_kd = lins[0]
    for lin in lins[1:]:
        points_kd = all_comb(points_kd, lin)

    points_kd = points_kd.unsqueeze(0)
    hms = inputs_hm.view(batch_size, -1, 1)

    norm_hms = F.softmax(hms * temperature, dim=1)
    weighted_points = (points_kd * norm_hms).sum(dim=1)
    assert weighted_points.shape[0] == batch_size
    assert weighted_points.shape[1] == num_dim
    return weighted_points

def gumbel_sample_kd(hm, num_samples, tau, logit_scale, hard):
    """
    hm: (N, D1, D2, D3, .., Dk), e.g., (N, x, y, z)
    num_samples: int, num samples to draw from each heatmap (N*samples) samples at the end
    tau: float, tau term for gumbel
    logit_scale: scale back logits before log(logits)
    hard: gives one hot softmax prob or not in gumbel.

    Outputs:
        samples: (N, samples, k)
    """
    batch_size = hm.shape[0]
    hm_size = hm.shape[1]
    assert np.all(np.array(hm.shape[1:]) == hm_size)
    num_dim = len(hm.shape[1:])
    dev = hm.device

    assert (
        num_dim == 2 or num_dim == 3
    ), "This function only supports 2d/3d heatmaps; verify for higher dim"
    hm = hm.reshape(batch_size, 1, -1)
    prob_dim = hm.shape[2]
    mesh_grids = torch.meshgrid([torch.arange(hm_size) for _ in range(num_dim)])
    mesh_grids = torch.stack(mesh_grids, dim=-1).to(dev).float()
    mesh_grids = mesh_grids.view(1, 1, prob_dim, num_dim)

    hm = hm.expand(batch_size, num_samples, prob_dim).reshape(
        batch_size * num_samples, prob_dim
    )
    samples = F.gumbel_softmax(logit_scale * hm, tau=tau, hard=hard, dim=1)
    samples = samples.view(batch_size, num_samples, prob_dim, 1)
    samples = samples * mesh_grids
    samples = samples.sum(dim=2)
    return samples

def gumbel_sample_kd_iter(hm, num_samples, tau, logit_scale, hard, chunk_size):
    """
    Helper for gumbel_sample_kd to avoid memory requirements by splitting the batch into chunks
    hm: (N, D1, D2, D3, .., Dk), e.g., (N, x, y, z)
    num_samples: int, num samples to draw from each heatmap (N*samples) samples at the end
    tau: float, tau term for gumbel
    logit_scale: scale back logits before log(logits)
    hard: gives one hot softmax prob or not in gumbel.

    Outputs:
        samples: (N, samples, k)
    """
    hm_splits = list(hm.split(chunk_size))
    sample_list = []
    for curr_hm in hm_splits:
        sample_list.append(
            gumbel_sample_kd(curr_hm, num_samples, tau, logit_scale, hard)
        )
    samples = torch.cat(sample_list, dim=0)
    return samples

def fetch_comb_index(num, dev, comb_type):
    if comb_type == "lower":
        comb_idx = torch.LongTensor(
            [(i, j) for i in range(num) for j in range(num) if i < j]
        ).to(dev)
    elif comb_type == "lower_diag":
        comb_idx = torch.LongTensor(
            [(i, j) for i in range(num) for j in range(num) if i <= j]
        ).to(dev)
    elif comb_type == "diag":
        comb_idx = torch.LongTensor(
            [(i, j) for i in range(num) for j in range(num) if i == j]
        ).to(dev)
    elif comb_type == "off_diag":
        comb_idx = torch.LongTensor(
            [(i, j) for i in range(num) for j in range(num) if i != j]
        ).to(dev)
    elif comb_type == "full":
        comb_idx = torch.LongTensor(
            [(i, j) for i in range(num) for j in range(num)]
        ).to(dev)
    else:
        assert False
    return comb_idx

def all_comb(X, Y):
    """
    Returns all possible combinations of elements in X and Y.
    X: (n_x, d_x)
    Y: (n_y, d_y)
    Output: Z: (n_x*x_y, d_x+d_y)
    Example:
    X = tensor([[8, 8, 8],
                [7, 5, 9]])
    Y = tensor([[3, 8, 7, 7],
                [3, 7, 9, 9],
                [6, 4, 3, 7]])
    Z = tensor([[8, 8, 8, 3, 8, 7, 7],
                [8, 8, 8, 3, 7, 9, 9],
                [8, 8, 8, 6, 4, 3, 7],
                [7, 5, 9, 3, 8, 7, 7],
                [7, 5, 9, 3, 7, 9, 9],
                [7, 5, 9, 6, 4, 3, 7]])
    """
    assert len(X.size()) == 2
    assert len(Y.size()) == 2
    X1 = X.unsqueeze(1)
    Y1 = Y.unsqueeze(0)
    X2 = X1.repeat(1, Y.shape[0], 1)
    Y2 = Y1.repeat(X.shape[0], 1, 1)
    Z = torch.cat([X2, Y2], -1)
    Z = Z.view(-1, Z.shape[-1])
    return Z

def toggle_parameters(model, requires_grad):
    """
    Set all weights to requires_grad or not.
    """
    for param in model.parameters():
        param.requires_grad = requires_grad

def detach_tensor(ten):
    """This function move tensor to cpu and convert to numpy"""
    if isinstance(ten, torch.Tensor):
        return ten.cpu().detach().numpy()
    return ten

def count_model_parameters(model):
    """
    Return the amount of parameters that requries gradients.
    """
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

def reset_all_seeds(seed):
    """Reset all seeds for reproduciability."""
    random.seed(seed)
    torch.manual_seed(seed)
    np.random.seed(seed)

def get_activation(name):
    """This function return an activation constructor by name."""
    if name == "tanh":
        return nn.Tanh()
    elif name == "sigmoid":
        return nn.Sigmoid()
    elif name == "relu":
        return nn.ReLU()
    elif name == "selu":
        return nn.SELU()
    elif name == "relu6":
        return nn.ReLU6()
    elif name == "softplus":
        return nn.Softplus()
    elif name == "softshrink":
        return nn.Softshrink()
    else:
        print("Undefined activation: %s" % (name))
        assert False

def stack_ll_tensors(tensor_list_list):
    """
    Recursively stack a list of lists of lists .. whose elements are tensors with the same shape
    """
    if isinstance(tensor_list_list, torch.Tensor):
        return tensor_list_list
    assert isinstance(tensor_list_list, list)
    if isinstance(tensor_list_list[0], torch.Tensor):
        return torch.stack(tensor_list_list)

    stacked_tensor = []
    for tensor_list in tensor_list_list:
        stacked_tensor.append(stack_ll_tensors(tensor_list))
    stacked_tensor = torch.stack(stacked_tensor)
    return stacked_tensor

def get_optim(name):
    """This function return an optimizer constructor by name."""
    if name == "adam":
        return optim.Adam
    elif name == "rmsprop":
        return optim.RMSprop
    elif name == "sgd":
        return optim.SGD
    else:
        print("Undefined optim: %s" % (name))
        assert False

def decay_lr(optimizer, gamma):
    """
    Decay the learning rate by gamma
    """
    assert isinstance(gamma, float)
    assert 0 <= gamma and gamma <= 1.0
    for param_group in optimizer.param_groups:
        param_group["lr"] *= gamma


from __future__ import annotations

import json
import time
import uuid
import email
import inspect
import logging
import platform
import warnings
import email.utils
from types import TracebackType
from random import random
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Type,
    Union,
    Generic,
    Mapping,
    TypeVar,
    Iterable,
    Iterator,
    Optional,
    Generator,
    AsyncIterator,
    cast,
    overload,
)
from functools import lru_cache
from typing_extensions import Literal, get_args, get_origin

import anyio
import httpx
import distro
import pydantic
from httpx import URL, Limits
from pydantic import PrivateAttr

from . import _exceptions
from ._qs import Querystring
from ._types import (
    NOT_GIVEN,
    Body,
    Omit,
    Query,
    ModelT,
    Headers,
    Timeout,
    NoneType,
    NotGiven,
    ResponseT,
    Transport,
    AnyMapping,
    ProxiesTypes,
    RequestFiles,
    AsyncTransport,
    RequestOptions,
    UnknownResponse,
    ModelBuilderProtocol,
)
from ._utils import is_dict, is_given, is_mapping
from ._compat import model_copy, model_dump
from ._models import (
    BaseModel,
    GenericModel,
    FinalRequestOptions,
    validate_type,
    construct_type,
)
from ._streaming import Stream, AsyncStream
from ._exceptions import (
    APIStatusError,
    APITimeoutError,
    APIConnectionError,
    APIResponseValidationError,
)

log: logging.Logger = logging.getLogger(__name__)

# TODO: make base page type vars covariant
SyncPageT = TypeVar("SyncPageT", bound="BaseSyncPage[Any]")
AsyncPageT = TypeVar("AsyncPageT", bound="BaseAsyncPage[Any]")

_T = TypeVar("_T")
_T_co = TypeVar("_T_co", covariant=True)

_StreamT = TypeVar("_StreamT", bound=Stream[Any])
_AsyncStreamT = TypeVar("_AsyncStreamT", bound=AsyncStream[Any])

if TYPE_CHECKING:
    from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT
else:
    try:
        from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT
    except ImportError:
        # taken from https://github.com/encode/httpx/blob/3ba5fe0d7ac70222590e759c31442b1cab263791/httpx/_config.py#L366
        HTTPX_DEFAULT_TIMEOUT = Timeout(5.0)

# default timeout is 10 minutes
DEFAULT_TIMEOUT = Timeout(timeout=600.0, connect=5.0)
DEFAULT_MAX_RETRIES = 2
DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20)

class MissingStreamClassError(TypeError):
    def __init__(self) -> None:
        super().__init__(
            "The `stream` argument was set to `True` but the `stream_cls` argument was not given. See `anthropic._streaming` for reference",
        )

class PageInfo:
    """Stores the necesary information to build the request to retrieve the next page.

    Either `url` or `params` must be set.
    """

    url: URL | NotGiven
    params: Query | NotGiven

    @overload
    def __init__(
        self,
        *,
        url: URL,
    ) -> None:
        ...

    @overload
    def __init__(
        self,
        *,
        params: Query,
    ) -> None:
        ...

    def __init__(
        self,
        *,
        url: URL | NotGiven = NOT_GIVEN,
        params: Query | NotGiven = NOT_GIVEN,
    ) -> None:
        self.url = url
        self.params = params

class BasePage(GenericModel, Generic[ModelT]):
    """
    Defines the core interface for pagination.

    Type Args:
        ModelT: The pydantic model that represents an item in the response.

    Methods:
        has_next_page(): Check if there is another page available
        next_page_info(): Get the necesary information to make a request for the next page
    """

    _options: FinalRequestOptions = PrivateAttr()
    _model: Type[ModelT] = PrivateAttr()

    def has_next_page(self) -> bool:
        items = self._get_page_items()
        if not items:
            return False
        return self.next_page_info() is not None

    def next_page_info(self) -> Optional[PageInfo]:
        ...

    def _get_page_items(self) -> Iterable[ModelT]:  # type: ignore[empty-body]
        ...

    def _params_from_url(self, url: URL) -> httpx.QueryParams:
        # TODO: do we have to preprocess params here?
        return httpx.QueryParams(cast(Any, self._options.params)).merge(url.params)

    def _info_to_options(self, info: PageInfo) -> FinalRequestOptions:
        options = model_copy(self._options)

        if not isinstance(info.params, NotGiven):
            options.params = {**options.params, **info.params}
            return options

        if not isinstance(info.url, NotGiven):
            params = self._params_from_url(info.url)
            url = info.url.copy_with(params=params)
            options.params = dict(url.params)
            options.url = str(url)
            return options

        raise ValueError("Unexpected PageInfo state")

class BaseSyncPage(BasePage[ModelT], Generic[ModelT]):
    _client: SyncAPIClient = pydantic.PrivateAttr()

    def _set_private_attributes(
        self,
        client: SyncAPIClient,
        model: Type[ModelT],
        options: FinalRequestOptions,
    ) -> None:
        self._model = model
        self._client = client
        self._options = options

    # Pydantic uses a custom `__iter__` method to support casting BaseModels
    # to dictionaries. e.g. dict(model).
    # As we want to support `for item in page`, this is inherently incompatible
    # with the default pydantic behaviour. It is not possible to support both
    # use cases at once. Fortunately, this is not a big deal as all other pydantic
    # methods should continue to work as expected as there is an alternative method
    # to cast a model to a dictionary, model.dict(), which is used internally
    # by pydantic.
    def __iter__(self) -> Iterator[ModelT]:  # type: ignore
        for page in self.iter_pages():
            for item in page._get_page_items():
                yield item

    def iter_pages(self: SyncPageT) -> Iterator[SyncPageT]:
        page = self
        while True:
            yield page
            if page.has_next_page():
                page = page.get_next_page()
            else:
                return

    def get_next_page(self: SyncPageT) -> SyncPageT:
        info = self.next_page_info()
        if not info:
            raise RuntimeError(
                "No next page expected; please check `.has_next_page()` before calling `.get_next_page()`."
            )

        options = self._info_to_options(info)
        return self._client._request_api_list(self._model, page=self.__class__, options=options)

class AsyncPaginator(Generic[ModelT, AsyncPageT]):
    def __init__(
        self,
        client: AsyncAPIClient,
        options: FinalRequestOptions,
        page_cls: Type[AsyncPageT],
        model: Type[ModelT],
    ) -> None:
        self._model = model
        self._client = client
        self._options = options
        self._page_cls = page_cls

    def __await__(self) -> Generator[Any, None, AsyncPageT]:
        return self._get_page().__await__()

    async def _get_page(self) -> AsyncPageT:
        page = await self._client.request(self._page_cls, self._options)
        page._set_private_attributes(  # pyright: ignore[reportPrivateUsage]
            model=self._model,
            options=self._options,
            client=self._client,
        )
        return page

    async def __aiter__(self) -> AsyncIterator[ModelT]:
        # https://github.com/microsoft/pyright/issues/3464
        page = cast(
            AsyncPageT,
            await self,  # type: ignore
        )
        async for item in page:
            yield item

class BaseAsyncPage(BasePage[ModelT], Generic[ModelT]):
    _client: AsyncAPIClient = pydantic.PrivateAttr()

    def _set_private_attributes(
        self,
        model: Type[ModelT],
        client: AsyncAPIClient,
        options: FinalRequestOptions,
    ) -> None:
        self._model = model
        self._client = client
        self._options = options

    async def __aiter__(self) -> AsyncIterator[ModelT]:
        async for page in self.iter_pages():
            for item in page._get_page_items():
                yield item

    async def iter_pages(self: AsyncPageT) -> AsyncIterator[AsyncPageT]:
        page = self
        while True:
            yield page
            if page.has_next_page():
                page = await page.get_next_page()
            else:
                return

    async def get_next_page(self: AsyncPageT) -> AsyncPageT:
        info = self.next_page_info()
        if not info:
            raise RuntimeError(
                "No next page expected; please check `.has_next_page()` before calling `.get_next_page()`."
            )

        options = self._info_to_options(info)
        return await self._client._request_api_list(self._model, page=self.__class__, options=options)

_HttpxClientT = TypeVar("_HttpxClientT", bound=Union[httpx.Client, httpx.AsyncClient])

class BaseClient(Generic[_HttpxClientT]):
    _client: _HttpxClientT
    _version: str
    _base_url: URL
    max_retries: int
    timeout: Union[float, Timeout, None]
    _limits: httpx.Limits
    _proxies: ProxiesTypes | None
    _transport: Transport | AsyncTransport | None
    _strict_response_validation: bool
    _idempotency_header: str | None

    def __init__(
        self,
        *,
        version: str,
        base_url: str | URL,
        _strict_response_validation: bool,
        max_retries: int = DEFAULT_MAX_RETRIES,
        timeout: float | Timeout | None = DEFAULT_TIMEOUT,
        limits: httpx.Limits,
        transport: Transport | AsyncTransport | None,
        proxies: ProxiesTypes | None,
        custom_headers: Mapping[str, str] | None = None,
        custom_query: Mapping[str, object] | None = None,
    ) -> None:
        self._version = version
        self._base_url = self._enforce_trailing_slash(URL(base_url))
        self.max_retries = max_retries
        self.timeout = timeout
        self._limits = limits
        self._proxies = proxies
        self._transport = transport
        self._custom_headers = custom_headers or {}
        self._custom_query = custom_query or {}
        self._strict_response_validation = _strict_response_validation
        self._idempotency_header = None

    def _enforce_trailing_slash(self, url: URL) -> URL:
        if url.raw_path.endswith(b"/"):
            return url
        return url.copy_with(raw_path=url.raw_path + b"/")

    def _make_status_error_from_response(
        self,
        response: httpx.Response,
    ) -> APIStatusError:
        err_text = response.text.strip()
        body = err_text

        try:
            body = json.loads(err_text)
            err_msg = f"Error code: {response.status_code} - {body}"
        except Exception:
            err_msg = err_text or f"Error code: {response.status_code}"

        return self._make_status_error(err_msg, body=body, response=response)

    def _make_status_error(
        self,
        err_msg: str,
        *,
        body: object,
        response: httpx.Response,
    ) -> _exceptions.APIStatusError:
        raise NotImplementedError()

    def _remaining_retries(
        self,
        remaining_retries: Optional[int],
        options: FinalRequestOptions,
    ) -> int:
        return remaining_retries if remaining_retries is not None else options.get_max_retries(self.max_retries)

    def _build_headers(self, options: FinalRequestOptions) -> httpx.Headers:
        custom_headers = options.headers or {}
        headers_dict = _merge_mappings(self.default_headers, custom_headers)
        self._validate_headers(headers_dict, custom_headers)

        headers = httpx.Headers(headers_dict)

        idempotency_header = self._idempotency_header
        if idempotency_header and options.method.lower() != "get" and idempotency_header not in headers:
            if not options.idempotency_key:
                options.idempotency_key = self._idempotency_key()

            headers[idempotency_header] = options.idempotency_key

        return headers

    def _prepare_request(
        self,
        request: httpx.Request,  # noqa: ARG002
    ) -> None:
        """This method is used as a callback for mutating the `Request` object
        after it has been constructed.

        This is useful for cases where you want to add certain headers based off of
        the request properties, e.g. `url`, `method` etc.
        """
        return None

    def _prepare_url(self, url: str) -> URL:
        """
        Merge a URL argument together with any 'base_url' on the client,
        to create the URL used for the outgoing request.
        """
        # Copied from httpx's `_merge_url` method.
        merge_url = URL(url)
        if merge_url.is_relative_url:
            merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b"/")
            return self.base_url.copy_with(raw_path=merge_raw_path)

        return merge_url

    def _build_request(
        self,
        options: FinalRequestOptions,
    ) -> httpx.Request:
        if log.isEnabledFor(logging.DEBUG):
            log.debug("Request options: %s", model_dump(options, exclude_unset=True))

        kwargs: dict[str, Any] = {}

        json_data = options.json_data
        if options.extra_json is not None:
            if json_data is None:
                json_data = cast(Body, options.extra_json)
            elif is_mapping(json_data):
                json_data = _merge_mappings(json_data, options.extra_json)
            else:
                raise RuntimeError(f"Unexpected JSON data type, {type(json_data)}, cannot merge with `extra_body`")

        headers = self._build_headers(options)
        params = _merge_mappings(self._custom_query, options.params)

        # If the given Content-Type header is multipart/form-data then it
        # has to be removed so that httpx can generate the header with
        # additional information for us as it has to be in this form
        # for the server to be able to correctly parse the request:
        # multipart/form-data; boundary=---abc--
        if headers.get("Content-Type") == "multipart/form-data":
            headers.pop("Content-Type")

            # As we are now sending multipart/form-data instead of application/json
            # we need to tell httpx to use it, https://www.python-httpx.org/advanced/#multipart-file-encoding
            if json_data:
                if not is_dict(json_data):
                    raise TypeError(
                        f"Expected query input to be a dictionary for multipart requests but got {type(json_data)} instead."
                    )
                kwargs["data"] = self._serialize_multipartform(json_data)

        # TODO: report this error to httpx
        request = self._client.build_request(  # pyright: ignore[reportUnknownMemberType]
            headers=headers,
            timeout=self.timeout if isinstance(options.timeout, NotGiven) else options.timeout,
            method=options.method,
            url=self._prepare_url(options.url),
            # the `Query` type that we use is incompatible with qs'
            # `Params` type as it needs to be typed as `Mapping[str, object]`
            # so that passing a `TypedDict` doesn't cause an error.
            # https://github.com/microsoft/pyright/issues/3526#event-6715453066
            params=self.qs.stringify(cast(Mapping[str, Any], params)) if params else None,
            json=json_data,
            files=options.files,
            **kwargs,
        )
        self._prepare_request(request)
        return request

    def _serialize_multipartform(self, data: Mapping[object, object]) -> dict[str, object]:
        items = self.qs.stringify_items(
            # TODO: type ignore is required as stringify_items is well typed but we can't be
            # well typed without heavy validation.
            data,  # type: ignore
            array_format="brackets",
        )
        serialized: dict[str, object] = {}
        for key, value in items:
            if key in serialized:
                raise ValueError(f"Duplicate key encountered: {key}; This behaviour is not supported")
            serialized[key] = value
        return serialized

    def _extract_stream_chunk_type(self, stream_cls: type) -> type:
        args = get_args(stream_cls)
        if not args:
            raise TypeError(
                f"Expected stream_cls to have been given a generic type argument, e.g. Stream[Foo] but received {stream_cls}",
            )
        return cast(type, args[0])

    def _process_response(
        self,
        *,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,  # noqa: ARG002
        response: httpx.Response,
    ) -> ResponseT:
        if cast_to is NoneType:
            return cast(ResponseT, None)

        if cast_to == str:
            return cast(ResponseT, response.text)

        origin = get_origin(cast_to) or cast_to

        if inspect.isclass(origin) and issubclass(origin, httpx.Response):
            # Because of the invariance of our ResponseT TypeVar, users can subclass httpx.Response
            # and pass that class to our request functions. We cannot change the variance to be either
            # covariant or contravariant as that makes our usage of ResponseT illegal. We could construct
            # the response class ourselves but that is something that should be supported directly in httpx
            # as it would be easy to incorrectly construct the Response object due to the multitude of arguments.
            if cast_to != httpx.Response:
                raise ValueError(f"Subclasses of httpx.Response cannot be passed to `cast_to`")
            return cast(ResponseT, response)

        # The check here is necessary as we are subverting the the type system
        # with casts as the relationship between TypeVars and Types are very strict
        # which means we must return *exactly* what was input or transform it in a
        # way that retains the TypeVar state. As we cannot do that in this function
        # then we have to resort to using `cast`. At the time of writing, we know this
        # to be safe as we have handled all the types that could be bound to the
        # `ResponseT` TypeVar, however if that TypeVar is ever updated in the future, then
        # this function would become unsafe but a type checker would not report an error.
        if (
            cast_to is not UnknownResponse
            and not origin is list
            and not origin is dict
            and not origin is Union
            and not issubclass(origin, BaseModel)
        ):
            raise RuntimeError(
                f"Invalid state, expected {cast_to} to be a subclass type of {BaseModel}, {dict}, {list} or {Union}."
            )

        # split is required to handle cases where additional information is included
        # in the response, e.g. application/json; charset=utf-8
        content_type, *_ = response.headers.get("content-type").split(";")
        if content_type != "application/json":
            if self._strict_response_validation:
                raise APIResponseValidationError(
                    response=response,
                    message=f"Expected Content-Type response header to be `application/json` but received `{content_type}` instead.",
                    body=response.text,
                )

            # If the API responds with content that isn't JSON then we just return
            # the (decoded) text without performing any parsing so that you can still
            # handle the response however you need to.
            return response.text  # type: ignore

        data = response.json()

        try:
            return self._process_response_data(data=data, cast_to=cast_to, response=response)
        except pydantic.ValidationError as err:
            raise APIResponseValidationError(response=response, body=data) from err

    def _process_response_data(
        self,
        *,
        data: object,
        cast_to: type[ResponseT],
        response: httpx.Response,
    ) -> ResponseT:
        if data is None:
            return cast(ResponseT, None)

        if cast_to is UnknownResponse:
            return cast(ResponseT, data)

        if inspect.isclass(cast_to) and issubclass(cast_to, ModelBuilderProtocol):
            return cast(ResponseT, cast_to.build(response=response, data=data))

        if self._strict_response_validation:
            return cast(ResponseT, validate_type(type_=cast_to, value=data))

        return cast(ResponseT, construct_type(type_=cast_to, value=data))

    @property
    def qs(self) -> Querystring:
        return Querystring()

    @property
    def custom_auth(self) -> httpx.Auth | None:
        return None

    @property
    def auth_headers(self) -> dict[str, str]:
        return {}

    @property
    def default_headers(self) -> dict[str, str | Omit]:
        return {
            "Accept": "application/json",
            "Content-Type": "application/json",
            "User-Agent": self.user_agent,
            **self.platform_headers(),
            **self.auth_headers,
            **self._custom_headers,
        }

    def _validate_headers(
        self,
        headers: Headers,  # noqa: ARG002
        custom_headers: Headers,  # noqa: ARG002
    ) -> None:
        """Validate the given default headers and custom headers.

        Does nothing by default.
        """
        return

    @property
    def user_agent(self) -> str:
        return f"{self.__class__.__name__}/Python {self._version}"

    @property
    def base_url(self) -> URL:
        return self._base_url

    @base_url.setter
    def base_url(self, url: URL | str) -> None:
        self._client.base_url = url if isinstance(url, URL) else URL(url)

    @lru_cache(maxsize=None)
    def platform_headers(self) -> Dict[str, str]:
        return {
            "X-Stainless-Lang": "python",
            "X-Stainless-Package-Version": self._version,
            "X-Stainless-OS": str(get_platform()),
            "X-Stainless-Arch": str(get_architecture()),
            "X-Stainless-Runtime": platform.python_implementation(),
            "X-Stainless-Runtime-Version": platform.python_version(),
        }

    def _calculate_retry_timeout(
        self,
        remaining_retries: int,
        options: FinalRequestOptions,
        response_headers: Optional[httpx.Headers] = None,
    ) -> float:
        max_retries = options.get_max_retries(self.max_retries)
        try:
            # About the Retry-After header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
            #
            # <http-date>". See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After#syntax for
            # details.
            if response_headers is not None:
                retry_header = response_headers.get("retry-after")
                try:
                    retry_after = int(retry_header)
                except Exception:
                    retry_date_tuple = email.utils.parsedate_tz(retry_header)
                    if retry_date_tuple is None:
                        retry_after = -1
                    else:
                        retry_date = email.utils.mktime_tz(retry_date_tuple)
                        retry_after = int(retry_date - time.time())
            else:
                retry_after = -1

        except Exception:
            retry_after = -1

        # If the API asks us to wait a certain amount of time (and it's a reasonable amount), just do what it says.
        if 0 < retry_after <= 60:
            return retry_after

        initial_retry_delay = 0.5
        max_retry_delay = 2.0
        nb_retries = max_retries - remaining_retries

        # Apply exponential backoff, but not more than the max.
        sleep_seconds = min(initial_retry_delay * pow(nb_retries - 1, 2), max_retry_delay)

        # Apply some jitter, plus-or-minus half a second.
        jitter = random() - 0.5
        timeout = sleep_seconds + jitter
        return timeout if timeout >= 0 else 0

    def _should_retry(self, response: httpx.Response) -> bool:
        # Note: this is not a standard header
        should_retry_header = response.headers.get("x-should-retry")

        # If the server explicitly says whether or not to retry, obey.
        if should_retry_header == "true":
            return True
        if should_retry_header == "false":
            return False

        # Retry on request timeouts.
        if response.status_code == 408:
            return True

        # Retry on lock timeouts.
        if response.status_code == 409:
            return True

        # Retry on rate limits.
        if response.status_code == 429:
            return True

        # Retry internal errors.
        if response.status_code >= 500:
            return True

        return False

    def _idempotency_key(self) -> str:
        return f"stainless-python-retry-{uuid.uuid4()}"

class SyncAPIClient(BaseClient[httpx.Client]):
    _client: httpx.Client
    _has_custom_http_client: bool
    _default_stream_cls: type[Stream[Any]] | None = None

    def __init__(
        self,
        *,
        version: str,
        base_url: str | URL,
        max_retries: int = DEFAULT_MAX_RETRIES,
        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,
        transport: Transport | None = None,
        proxies: ProxiesTypes | None = None,
        limits: Limits | None = None,
        http_client: httpx.Client | None = None,
        custom_headers: Mapping[str, str] | None = None,
        custom_query: Mapping[str, object] | None = None,
        _strict_response_validation: bool,
    ) -> None:
        if limits is not None:
            warnings.warn(
                "The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `connection_pool_limits`")
        else:
            limits = DEFAULT_LIMITS

        if transport is not None:
            warnings.warn(
                "The `transport` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `transport`")

        if proxies is not None:
            warnings.warn(
                "The `proxies` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `proxies`")

        if not is_given(timeout):
            # if the user passed in a custom http client with a non-default
            # timeout set then we use that timeout.
            #
            # note: there is an edge case here where the user passes in a client
            # where they've explicitly set the timeout to match the default timeout
            # as this check is structural, meaning that we'll think they didn't
            # pass in a timeout and will ignore it
            if http_client and http_client.timeout != HTTPX_DEFAULT_TIMEOUT:
                timeout = http_client.timeout
            else:
                timeout = DEFAULT_TIMEOUT

        super().__init__(
            version=version,
            limits=limits,
            # cast to a valid type because mypy doesn't understand our type narrowing
            timeout=cast(Timeout, timeout),
            proxies=proxies,
            base_url=base_url,
            transport=transport,
            max_retries=max_retries,
            custom_query=custom_query,
            custom_headers=custom_headers,
            _strict_response_validation=_strict_response_validation,
        )
        self._client = http_client or httpx.Client(
            base_url=base_url,
            # cast to a valid type because mypy doesn't understand our type narrowing
            timeout=cast(Timeout, timeout),
            proxies=proxies,
            transport=transport,
            limits=limits,
        )
        self._has_custom_http_client = bool(http_client)

    def is_closed(self) -> bool:
        return self._client.is_closed

    def close(self) -> None:
        """Close the underlying HTTPX client.

        The client will *not* be usable after this.
        """
        # If an error is thrown while constructing a client, self._client
        # may not be present
        if hasattr(self, "_client"):
            self._client.close()

    def __enter__(self: _T) -> _T:
        return self

    def __exit__(
        self,
        exc_type: type[BaseException] | None,
        exc: BaseException | None,
        exc_tb: TracebackType | None,
    ) -> None:
        self.close()

    @overload
    def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        remaining_retries: Optional[int] = None,
        *,
        stream: Literal[True],
        stream_cls: Type[_StreamT],
    ) -> _StreamT:
        ...

    @overload
    def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        remaining_retries: Optional[int] = None,
        *,
        stream: Literal[False] = False,
    ) -> ResponseT:
        ...

    @overload
    def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        remaining_retries: Optional[int] = None,
        *,
        stream: bool = False,
        stream_cls: Type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        ...

    def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        remaining_retries: Optional[int] = None,
        *,
        stream: bool = False,
        stream_cls: type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        return self._request(
            cast_to=cast_to,
            options=options,
            stream=stream,
            stream_cls=stream_cls,
            remaining_retries=remaining_retries,
        )

    def _request(
        self,
        *,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        remaining_retries: int | None,
        stream: bool,
        stream_cls: type[_StreamT] | None,
    ) -> ResponseT | _StreamT:
        retries = self._remaining_retries(remaining_retries, options)
        request = self._build_request(options)

        try:
            response = self._client.send(request, auth=self.custom_auth, stream=stream)
            log.debug(
                'HTTP Request: %s %s "%i %s"', request.method, request.url, response.status_code, response.reason_phrase
            )
            response.raise_for_status()
        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code
            if retries > 0 and self._should_retry(err.response):
                return self._retry_request(
                    options,
                    cast_to,
                    retries,
                    err.response.headers,
                    stream=stream,
                    stream_cls=stream_cls,
                )

            # If the response is streamed then we need to explicitly read the response
            # to completion before attempting to access the response text.
            err.response.read()
            raise self._make_status_error_from_response(err.response) from None
        except httpx.TimeoutException as err:
            if retries > 0:
                return self._retry_request(options, cast_to, retries, stream=stream, stream_cls=stream_cls)
            raise APITimeoutError(request=request) from err
        except Exception as err:
            if retries > 0:
                return self._retry_request(options, cast_to, retries, stream=stream, stream_cls=stream_cls)
            raise APIConnectionError(request=request) from err

        if stream:
            if stream_cls:
                return stream_cls(cast_to=self._extract_stream_chunk_type(stream_cls), response=response, client=self)

            stream_cls = cast("type[_StreamT] | None", self._default_stream_cls)
            if stream_cls is None:
                raise MissingStreamClassError()
            return stream_cls(cast_to=cast_to, response=response, client=self)

        return self._process_response(cast_to=cast_to, options=options, response=response)

    def _retry_request(
        self,
        options: FinalRequestOptions,
        cast_to: Type[ResponseT],
        remaining_retries: int,
        response_headers: Optional[httpx.Headers] = None,
        *,
        stream: bool,
        stream_cls: type[_StreamT] | None,
    ) -> ResponseT | _StreamT:
        remaining = remaining_retries - 1
        timeout = self._calculate_retry_timeout(remaining, options, response_headers)
        log.info("Retrying request to %s in %f seconds", options.url, timeout)

        # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a
        # different thread if necessary.
        time.sleep(timeout)

        return self._request(
            options=options,
            cast_to=cast_to,
            remaining_retries=remaining,
            stream=stream,
            stream_cls=stream_cls,
        )

    def _request_api_list(
        self,
        model: Type[ModelT],
        page: Type[SyncPageT],
        options: FinalRequestOptions,
    ) -> SyncPageT:
        resp = self.request(page, options, stream=False)
        resp._set_private_attributes(  # pyright: ignore[reportPrivateUsage]
            client=self,
            model=model,
            options=options,
        )
        return resp

    @overload
    def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: Literal[False] = False,
    ) -> ResponseT:
        ...

    @overload
    def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: Literal[True],
        stream_cls: type[_StreamT],
    ) -> _StreamT:
        ...

    @overload
    def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: bool,
        stream_cls: type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        ...

    def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: bool = False,
        stream_cls: type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        opts = FinalRequestOptions.construct(method="get", url=path, **options)
        # cast is required because mypy complains about returning Any even though
        # it understands the type variables
        return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

    @overload
    def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
        files: RequestFiles | None = None,
        stream: Literal[False] = False,
    ) -> ResponseT:
        ...

    @overload
    def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
        files: RequestFiles | None = None,
        stream: Literal[True],
        stream_cls: type[_StreamT],
    ) -> _StreamT:
        ...

    @overload
    def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
        files: RequestFiles | None = None,
        stream: bool,
        stream_cls: type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        ...

    def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
        files: RequestFiles | None = None,
        stream: bool = False,
        stream_cls: type[_StreamT] | None = None,
    ) -> ResponseT | _StreamT:
        opts = FinalRequestOptions.construct(method="post", url=path, json_data=body, files=files, **options)
        return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

    def patch(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="patch", url=path, json_data=body, **options)
        return self.request(cast_to, opts)

    def put(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="put", url=path, json_data=body, files=files, **options)
        return self.request(cast_to, opts)

    def delete(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="delete", url=path, json_data=body, **options)
        return self.request(cast_to, opts)

    def get_api_list(
        self,
        path: str,
        *,
        model: Type[ModelT],
        page: Type[SyncPageT],
        body: Body | None = None,
        options: RequestOptions = {},
        method: str = "get",
    ) -> SyncPageT:
        opts = FinalRequestOptions.construct(method=method, url=path, json_data=body, **options)
        return self._request_api_list(model, page, opts)

class AsyncAPIClient(BaseClient[httpx.AsyncClient]):
    _client: httpx.AsyncClient
    _has_custom_http_client: bool
    _default_stream_cls: type[AsyncStream[Any]] | None = None

    def __init__(
        self,
        *,
        version: str,
        base_url: str | URL,
        _strict_response_validation: bool,
        max_retries: int = DEFAULT_MAX_RETRIES,
        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,
        transport: AsyncTransport | None = None,
        proxies: ProxiesTypes | None = None,
        limits: Limits | None = None,
        http_client: httpx.AsyncClient | None = None,
        custom_headers: Mapping[str, str] | None = None,
        custom_query: Mapping[str, object] | None = None,
    ) -> None:
        if limits is not None:
            warnings.warn(
                "The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `connection_pool_limits`")
        else:
            limits = DEFAULT_LIMITS

        if transport is not None:
            warnings.warn(
                "The `transport` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `transport`")

        if proxies is not None:
            warnings.warn(
                "The `proxies` argument is deprecated. The `http_client` argument should be passed instead",
                category=DeprecationWarning,
                stacklevel=3,
            )
            if http_client is not None:
                raise ValueError("The `http_client` argument is mutually exclusive with `proxies`")

        if not is_given(timeout):
            # if the user passed in a custom http client with a non-default
            # timeout set then we use that timeout.
            #
            # note: there is an edge case here where the user passes in a client
            # where they've explicitly set the timeout to match the default timeout
            # as this check is structural, meaning that we'll think they didn't
            # pass in a timeout and will ignore it
            if http_client and http_client.timeout != HTTPX_DEFAULT_TIMEOUT:
                timeout = http_client.timeout
            else:
                timeout = DEFAULT_TIMEOUT

        super().__init__(
            version=version,
            base_url=base_url,
            limits=limits,
            # cast to a valid type because mypy doesn't understand our type narrowing
            timeout=cast(Timeout, timeout),
            proxies=proxies,
            transport=transport,
            max_retries=max_retries,
            custom_query=custom_query,
            custom_headers=custom_headers,
            _strict_response_validation=_strict_response_validation,
        )
        self._client = http_client or httpx.AsyncClient(
            base_url=base_url,
            # cast to a valid type because mypy doesn't understand our type narrowing
            timeout=cast(Timeout, timeout),
            proxies=proxies,
            transport=transport,
            limits=limits,
        )
        self._has_custom_http_client = bool(http_client)

    def is_closed(self) -> bool:
        return self._client.is_closed

    async def close(self) -> None:
        """Close the underlying HTTPX client.

        The client will *not* be usable after this.
        """
        await self._client.aclose()

    async def __aenter__(self: _T) -> _T:
        return self

    async def __aexit__(
        self,
        exc_type: type[BaseException] | None,
        exc: BaseException | None,
        exc_tb: TracebackType | None,
    ) -> None:
        await self.close()

    @overload
    async def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        *,
        stream: Literal[False] = False,
        remaining_retries: Optional[int] = None,
    ) -> ResponseT:
        ...

    @overload
    async def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        *,
        stream: Literal[True],
        stream_cls: type[_AsyncStreamT],
        remaining_retries: Optional[int] = None,
    ) -> _AsyncStreamT:
        ...

    @overload
    async def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        *,
        stream: bool,
        stream_cls: type[_AsyncStreamT] | None = None,
        remaining_retries: Optional[int] = None,
    ) -> ResponseT | _AsyncStreamT:
        ...

    async def request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        *,
        stream: bool = False,
        stream_cls: type[_AsyncStreamT] | None = None,
        remaining_retries: Optional[int] = None,
    ) -> ResponseT | _AsyncStreamT:
        return await self._request(
            cast_to=cast_to,
            options=options,
            stream=stream,
            stream_cls=stream_cls,
            remaining_retries=remaining_retries,
        )

    async def _request(
        self,
        cast_to: Type[ResponseT],
        options: FinalRequestOptions,
        *,
        stream: bool,
        stream_cls: type[_AsyncStreamT] | None,
        remaining_retries: int | None,
    ) -> ResponseT | _AsyncStreamT:
        retries = self._remaining_retries(remaining_retries, options)
        request = self._build_request(options)

        try:
            response = await self._client.send(request, auth=self.custom_auth, stream=stream)
            log.debug(
                'HTTP Request: %s %s "%i %s"', request.method, request.url, response.status_code, response.reason_phrase
            )
            response.raise_for_status()
        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code
            if retries > 0 and self._should_retry(err.response):
                return await self._retry_request(
                    options,
                    cast_to,
                    retries,
                    err.response.headers,
                    stream=stream,
                    stream_cls=stream_cls,
                )

            # If the response is streamed then we need to explicitly read the response
            # to completion before attempting to access the response text.
            await err.response.aread()
            raise self._make_status_error_from_response(err.response) from None
        except httpx.ConnectTimeout as err:
            if retries > 0:
                return await self._retry_request(options, cast_to, retries, stream=stream, stream_cls=stream_cls)
            raise APITimeoutError(request=request) from err
        except httpx.ReadTimeout as err:
            # We explicitly do not retry on ReadTimeout errors as this means
            # that the server processing the request has taken 60 seconds
            # (our default timeout). This likely indicates that something
            # is not working as expected on the server side.
            raise
        except httpx.TimeoutException as err:
            if retries > 0:
                return await self._retry_request(options, cast_to, retries, stream=stream, stream_cls=stream_cls)
            raise APITimeoutError(request=request) from err
        except Exception as err:
            if retries > 0:
                return await self._retry_request(options, cast_to, retries, stream=stream, stream_cls=stream_cls)
            raise APIConnectionError(request=request) from err

        if stream:
            if stream_cls:
                return stream_cls(cast_to=self._extract_stream_chunk_type(stream_cls), response=response, client=self)

            stream_cls = cast("type[_AsyncStreamT] | None", self._default_stream_cls)
            if stream_cls is None:
                raise MissingStreamClassError()
            return stream_cls(cast_to=cast_to, response=response, client=self)

        return self._process_response(cast_to=cast_to, options=options, response=response)

    async def _retry_request(
        self,
        options: FinalRequestOptions,
        cast_to: Type[ResponseT],
        remaining_retries: int,
        response_headers: Optional[httpx.Headers] = None,
        *,
        stream: bool,
        stream_cls: type[_AsyncStreamT] | None,
    ) -> ResponseT | _AsyncStreamT:
        remaining = remaining_retries - 1
        timeout = self._calculate_retry_timeout(remaining, options, response_headers)
        log.info("Retrying request to %s in %f seconds", options.url, timeout)

        await anyio.sleep(timeout)

        return await self._request(
            options=options,
            cast_to=cast_to,
            remaining_retries=remaining,
            stream=stream,
            stream_cls=stream_cls,
        )

    def _request_api_list(
        self,
        model: Type[ModelT],
        page: Type[AsyncPageT],
        options: FinalRequestOptions,
    ) -> AsyncPaginator[ModelT, AsyncPageT]:
        return AsyncPaginator(client=self, options=options, page_cls=page, model=model)

    @overload
    async def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: Literal[False] = False,
    ) -> ResponseT:
        ...

    @overload
    async def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: Literal[True],
        stream_cls: type[_AsyncStreamT],
    ) -> _AsyncStreamT:
        ...

    @overload
    async def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: bool,
        stream_cls: type[_AsyncStreamT] | None = None,
    ) -> ResponseT | _AsyncStreamT:
        ...

    async def get(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        options: RequestOptions = {},
        stream: bool = False,
        stream_cls: type[_AsyncStreamT] | None = None,
    ) -> ResponseT | _AsyncStreamT:
        opts = FinalRequestOptions.construct(method="get", url=path, **options)
        return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)

    @overload
    async def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
        stream: Literal[False] = False,
    ) -> ResponseT:
        ...

    @overload
    async def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
        stream: Literal[True],
        stream_cls: type[_AsyncStreamT],
    ) -> _AsyncStreamT:
        ...

    @overload
    async def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
        stream: bool,
        stream_cls: type[_AsyncStreamT] | None = None,
    ) -> ResponseT | _AsyncStreamT:
        ...

    async def post(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
        stream: bool = False,
        stream_cls: type[_AsyncStreamT] | None = None,
    ) -> ResponseT | _AsyncStreamT:
        opts = FinalRequestOptions.construct(method="post", url=path, json_data=body, files=files, **options)
        return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)

    async def patch(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="patch", url=path, json_data=body, **options)
        return await self.request(cast_to, opts)

    async def put(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        files: RequestFiles | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="put", url=path, json_data=body, files=files, **options)
        return await self.request(cast_to, opts)

    async def delete(
        self,
        path: str,
        *,
        cast_to: Type[ResponseT],
        body: Body | None = None,
        options: RequestOptions = {},
    ) -> ResponseT:
        opts = FinalRequestOptions.construct(method="delete", url=path, json_data=body, **options)
        return await self.request(cast_to, opts)

    def get_api_list(
        self,
        path: str,
        *,
        # TODO: support paginating `str`
        model: Type[ModelT],
        page: Type[AsyncPageT],
        body: Body | None = None,
        options: RequestOptions = {},
        method: str = "get",
    ) -> AsyncPaginator[ModelT, AsyncPageT]:
        opts = FinalRequestOptions.construct(method=method, url=path, json_data=body, **options)
        return self._request_api_list(model, page, opts)

def make_request_options(
    *,
    query: Query | None = None,
    extra_headers: Headers | None = None,
    extra_query: Query | None = None,
    extra_body: Body | None = None,
    idempotency_key: str | None = None,
    timeout: float | None | NotGiven = NOT_GIVEN,
) -> RequestOptions:
    """Create a dict of type RequestOptions without keys of NotGiven values."""
    options: RequestOptions = {}
    if extra_headers is not None:
        options["headers"] = extra_headers

    if extra_body is not None:
        options["extra_json"] = cast(AnyMapping, extra_body)

    if query is not None:
        options["params"] = query

    if extra_query is not None:
        options["params"] = {**options.get("params", {}), **extra_query}

    if not isinstance(timeout, NotGiven):
        options["timeout"] = timeout

    if idempotency_key is not None:
        options["idempotency_key"] = idempotency_key

    return options

class OtherPlatform:
    def __init__(self, name: str) -> None:
        self.name = name

    def __str__(self) -> str:
        return f"Other:{self.name}"

Platform = Union[
    OtherPlatform,
    Literal[
        "MacOS",
        "Linux",
        "Windows",
        "FreeBSD",
        "OpenBSD",
        "iOS",
        "Android",
        "Unknown",
    ],
]

def get_platform() -> Platform:
    system = platform.system().lower()
    platform_name = platform.platform().lower()
    if "iphone" in platform_name or "ipad" in platform_name:
        # Tested using Python3IDE on an iPhone 11 and Pythonista on an iPad 7
        # system is Darwin and platform_name is a string like:
        # - Darwin-21.6.0-iPhone12,1-64bit
        # - Darwin-21.6.0-iPad7,11-64bit
        return "iOS"

    if system == "darwin":
        return "MacOS"

    if system == "windows":
        return "Windows"

    if "android" in platform_name:
        # Tested using Pydroid 3
        # system is Linux and platform_name is a string like 'Linux-5.10.81-android12-9-00001-geba40aecb3b7-ab8534902-aarch64-with-libc'
        return "Android"

    if system == "linux":
        # https://distro.readthedocs.io/en/latest/#distro.id
        distro_id = distro.id()
        if distro_id == "freebsd":
            return "FreeBSD"

        if distro_id == "openbsd":
            return "OpenBSD"

        return "Linux"

    if platform_name:
        return OtherPlatform(platform_name)

    return "Unknown"

class OtherArch:
    def __init__(self, name: str) -> None:
        self.name = name

    def __str__(self) -> str:
        return f"other:{self.name}"

Arch = Union[OtherArch, Literal["x32", "x64", "arm", "arm64", "unknown"]]

def get_architecture() -> Arch:
    python_bitness, _ = platform.architecture()
    machine = platform.machine().lower()
    if machine in ("arm64", "aarch64"):
        return "arm64"

    # TODO: untested
    if machine == "arm":
        return "arm"

    if machine == "x86_64":
        return "x64"

    # TODO: untested
    if python_bitness == "32bit":
        return "x32"

    if machine:
        return OtherArch(machine)

    return "unknown"

def _merge_mappings(
    obj1: Mapping[_T_co, Union[_T, Omit]],
    obj2: Mapping[_T_co, Union[_T, Omit]],
) -> Dict[_T_co, _T]:
    """Merge two mappings of the same type, removing any values that are instances of `Omit`.

    In cases with duplicate keys the second mapping takes precedence.
    """
    merged = {**obj1, **obj2}
    return {key: value for key, value in merged.items() if not isinstance(value, Omit)}