import os
import sys

from jinja2 import Template, Environment, FileSystemLoader
import yaml

DOCS = {
    'zigbee': (28, 'Spec.', 'http://www.zigbee.org/wp-content/uploads/2014/11/docs-05-3474-20-0csg-zigbee-specification.pdf'),
    'zcl': (0, 'ZCL', 'http://www.zigbee.org/~zigbeeor/wp-content/uploads/2014/10/07-5123-06-zigbee-cluster-library-specification.pdf'),
}

def f_href(value):
    return value.replace(' ', '_')

def f_spec(page, doc='zigbee', label=None, title=None):
    offset, title_, url = DOCS[doc]
    label = label or page

    if title is None:
        title = title_

    if not f_spec._template:
        t = '''{{ title }}{% if page %}, p.{% endif %}
               {{ label }}'''
        f_spec._template = env.from_string(t)

    page = page + offset if page else None
    return f_spec._template.render(title=title, url=url, page=page, label=label)

f_spec._template = None

def f_column(item, column):
    if 'template' in column:
        t = column['template']
    else:
        t = '{{ %s }}' % column['column']

    if t in f_column._cache:
        template = f_column._cache[t]
    else:
        template = env.from_string(t)
        f_column._cache[t] = template

    return template.render(**item)

f_column._cache = {}

def f_render(fmt, **context):
    if fmt not in f_render._cache:
        f_render._cache[fmt] = env.from_string(fmt)
    return f_render._cache[fmt].render(**context)

f_render._cache = {}

packets = {}

for fn in os.listdir('packets'):
    if fn.endswith('.yml'):
        with open('packets/%s' % fn) as fp:
            packet = yaml.load(fp)
        packets.setdefault(packet['category'], []).append(packet)

env = Environment(loader=FileSystemLoader('.'))
env.filters['href'] = f_href
env.filters['spec'] = f_spec
env.filters['column'] = f_column
env.filters['render'] = f_render

categories = []
for category, items in packets.items():
    items = sorted(items, key=lambda i: i['type'])
    categories.append({
        'category': category,
        'items': items,
        'order': items[0]['type']
    })

categories = [(c['category'], c['items']) for c in sorted(categories, key=lambda c: c['order'])]

context = {'categories': categories, 'tables': []}

for fn in sorted(os.listdir('tables')):
    if fn.endswith('.yml'):
        with open('tables/%s' % fn) as fp:
            table = yaml.load(fp)

        if 'description' in table:
            template = env.from_string(table['description'])
            table['description'] = template.render(**table)
        context['tables'].append(table)

template = env.get_template('template.html')

with open('index.html', 'w') as fp:
    fp.write(template.render(**context))


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 23 13:57:33 2021

@author: Linda7lll
"""

# %%

# .ini

import configparser
from PIL import Image
import glob
import os.path

# %%
config = configparser.ConfigParser()

# %%

# save images
def save(category,num,datasets):

     if(datasets == 'Comparison Detector'):

        if ((category == "NIL") or (category == "ACTINOMYCES") or (category == "HERPS") or (category == "FLORA") or (category == "CANDIDA") or (category == "TRICHOMONAS")or (category == "Metaplastic") or (category == "Parabasal") or (category == "Superficial-Intermediate")):
         crop.save("./Train/0/"+datasets+"_N_"+str(num)+"_"+category+".bmp")

        else :
         crop.save("./Train/1/"+datasets+"_P_"+str(num)+"_"+category+".bmp")

#%%

z_file = 0
total_file =0
total_cell = 0  # total cell
num = 0  # index cell

#%%

for filename in glob.glob('./train-bmp/*.bmp'):

    im = Image.open(filename)
    inifile_name = os.path.splitext(filename)[0] + '.ini'
    total_file +=1
    #print(inifile_name)

    #read ini file
    read_s = config.read(inifile_name)

    # category, cell_num
    datasets = config.get("main", "dataset")
    cell_number = config.get("main", "cell_number")
    cell_num = int(cell_number)
    print("\nfile_num: {}, Cell_num: {}, file_dataset: {}".format(total_file,cell_num,datasets))

    total_cell += + cell_num

    if cell_num == 0:
       category1 = config.get("main", "category")
       z_file += 1

   #cell_number, coordinate
    for a in range(1, cell_num+1):

        print("\nCell_"+str(a)+" Coordinate: ")
        left = config.get('cell_'+str(a), 'left')
        l = float(left)

        top = config.get('cell_'+str(a), 'top')
        t = float(top)

        right = config.get('cell_'+str(a), 'right')
        r = float(right)

        bottom = config.get('cell_'+str(a), 'bottom')
        b = float(bottom)

        print("left:", l, ", top:", t, ", right:", r, ", bottom:", b)

        # print cell_category, cell_num
        num = num+1
        print("Cell_Last_num:", num)

        category = config.get('cell_'+str(a), 'category')
        print("Cell_Category: ", category)
        print("dataset: ",datasets)

        # crop and save
        area = (l, t, r, b)
        crop = im.crop(area)
        save(category, num, datasets)

print("\n\nnumber of file: ", total_file)  # toplam dosya sayısı
print("Total cells: ", total_cell)  # toplam hücre sayısı
print("number of zero_file:", z_file)  # içinde hücre olmayan dosya sayısı

# %%


"""Pydantic modules for OGC TileMatrixSets (https://www.ogc.org/standards/tms)"""

import math
import warnings
from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union

from pydantic import AnyHttpUrl, BaseModel, Field, PrivateAttr, validator
from pyproj import CRS, Transformer
from pyproj.enums import WktVersion
from pyproj.exceptions import ProjError

from .commons import BoundingBox, Coords, Tile
from .errors import (
    InvalidZoomError,
    NoQuadkeySupport,
    PointOutsideTMSBounds,
    QuadKeyError,
)
from .utils import (
    _parse_tile_arg,
    bbox_to_feature,
    check_quadkey_support,
    meters_per_unit,
    point_in_bbox,
)

NumType = Union[float, int]
BoundsType = Tuple[NumType, NumType]
LL_EPSILON = 1e-11
WGS84_CRS = CRS.from_epsg(4326)

class CRSType(CRS, str):
    """
    A geographic or projected coordinate reference system.
    """

    @classmethod
    def __get_validators__(cls):
        """validator for the type."""
        yield cls.validate

    @classmethod
    def validate(cls, value: Union[CRS, str]) -> CRS:
        """Validate CRS."""
        # If input is a string we tranlate it to CRS
        if not isinstance(value, CRS):
            return CRS.from_user_input(value)

        return value

    @classmethod
    def __modify_schema__(cls, field_schema):
        """Update default schema."""
        field_schema.update(
            anyOf=[
                {"type": "pyproj.CRS"},
                {"type": "string", "minLength": 1, "maxLength": 65536},
            ],
            examples=[
                "CRS.from_epsg(4326)",
                "http://www.opengis.net/def/crs/EPSG/0/3978",
                "urn:ogc:def:crs:EPSG::2193",
            ],
        )

    def __repr__(self):
        """Type representation."""
        return f"CRS({super().__repr__()})"

def CRS_to_uri(crs: CRS) -> str:
    """Convert CRS to URI."""
    epsg_code = crs.to_epsg()
    return f"http://www.opengis.net/def/crs/EPSG/0/{epsg_code}"

def crs_axis_inverted(crs: CRS) -> bool:
    """Check if CRS has inverted AXIS (lat,lon) instead of (lon,lat)."""
    return crs.axis_info[0].abbrev.upper() in ["Y", "LAT", "N"]

class TMSBoundingBox(BaseModel):
    """Bounding box"""

    type: str = Field("BoundingBoxType", const=True)
    crs: CRSType
    lowerCorner: BoundsType
    upperCorner: BoundsType

    class Config:
        """Configure TMSBoundingBox."""

        arbitrary_types_allowed = True
        json_encoders = {CRS: lambda v: CRS_to_uri(v)}

class TileMatrix(BaseModel):
    """Tile matrix"""

    type: str = Field("TileMatrixType", const=True)
    title: Optional[str]
    abstract: Optional[str]
    keywords: Optional[List[str]]
    identifier: str = Field(..., regex=r"^[0-9]+$")
    scaleDenominator: float
    topLeftCorner: BoundsType
    tileWidth: int
    tileHeight: int
    matrixWidth: int
    matrixHeight: int

    class Config:
        """Forbid additional items like variableMatrixWidth."""

        extra = "forbid"

class TileMatrixSet(BaseModel):
    """Tile matrix set"""

    type: str = Field("TileMatrixSetType", const=True)
    title: str
    abstract: Optional[str]
    keywords: Optional[List[str]]
    identifier: str = Field(..., regex=r"^[\w\d_\-]+$")
    supportedCRS: CRSType
    wellKnownScaleSet: Optional[AnyHttpUrl] = None
    boundingBox: Optional[TMSBoundingBox]
    tileMatrix: List[TileMatrix]

    # Private attributes
    _is_quadtree: bool = PrivateAttr()

    # CRS transformation attributes
    _geographic_crs: CRSType = PrivateAttr(default=WGS84_CRS)
    _to_geographic: Transformer = PrivateAttr()
    _from_geographic: Transformer = PrivateAttr()

    class Config:
        """Configure TileMatrixSet."""

        arbitrary_types_allowed = True
        json_encoders = {CRS: lambda v: CRS_to_uri(v)}

    def __init__(self, **data):
        """Create PyProj transforms and check if TileMatrixSet supports quadkeys."""
        super().__init__(**data)

        self._is_quadtree = check_quadkey_support(self.tileMatrix)

        self._geographic_crs = data.get("_geographic_crs", WGS84_CRS)

        try:
            self._to_geographic = Transformer.from_crs(
                self.supportedCRS, self._geographic_crs, always_xy=True
            )
            self._from_geographic = Transformer.from_crs(
                self._geographic_crs, self.supportedCRS, always_xy=True
            )
        except ProjError:
            warnings.warn(
                "Could not create coordinate Transformer from input CRS to the given geographic CRS"
                "some methods might not be available.",
                UserWarning,
            )
            self._to_geographic = None
            self._from_geographic = None

    @validator("tileMatrix")
    def sort_tile_matrices(cls, v):
        """Sort matrices by identifier"""
        return sorted(v, key=lambda m: int(m.identifier))

    def __iter__(self):
        """Iterate over matrices"""
        for matrix in self.tileMatrix:
            yield matrix

    def __repr__(self):
        """Simplify default pydantic model repr."""
        return f"<TileMatrixSet title='{self.title}' identifier='{self.identifier}'>"

    @property
    def crs(self) -> CRS:
        """Fetch CRS from epsg"""
        return self.supportedCRS

    @property
    def rasterio_crs(self):
        """Return rasterio CRS."""

        import rasterio
        from rasterio.env import GDALVersion

        if GDALVersion.runtime().major < 3:
            return rasterio.crs.CRS.from_wkt(self.crs.to_wkt(WktVersion.WKT1_GDAL))
        else:
            return rasterio.crs.CRS.from_wkt(self.crs.to_wkt())

    @property
    def minzoom(self) -> int:
        """TileMatrixSet minimum TileMatrix identifier"""
        return int(self.tileMatrix[0].identifier)

    @property
    def maxzoom(self) -> int:
        """TileMatrixSet maximum TileMatrix identifier"""
        return int(self.tileMatrix[-1].identifier)

    @property
    def _invert_axis(self) -> bool:
        """Check if CRS has inverted AXIS (lat,lon) instead of (lon,lat)."""
        return crs_axis_inverted(self.crs)

    @classmethod
    def custom(
        cls,
        extent: List[float],
        crs: CRS,
        tile_width: int = 256,
        tile_height: int = 256,
        matrix_scale: List = [1, 1],
        extent_crs: Optional[CRS] = None,
        minzoom: int = 0,
        maxzoom: int = 24,
        title: str = "Custom TileMatrixSet",
        identifier: str = "Custom",
        geographic_crs: CRS = WGS84_CRS,
    ):
        """
        Construct a custom TileMatrixSet.

        Attributes
        ----------
        crs: pyproj.CRS
            Tile Matrix Set coordinate reference system
        extent: list
            Bounding box of the Tile Matrix Set, (left, bottom, right, top).
        tile_width: int
            Width of each tile of this tile matrix in pixels (default is 256).
        tile_height: int
            Height of each tile of this tile matrix in pixels (default is 256).
        matrix_scale: list
            Tiling schema coalescence coefficient (default: [1, 1] for EPSG:3857).
            Should be set to [2, 1] for EPSG:4326.
            see: http://docs.opengeospatial.org/is/17-083r2/17-083r2.html#14
        extent_crs: pyproj.CRS
            Extent's coordinate reference system, as a pyproj CRS object.
            (default: same as input crs)
        minzoom: int
            Tile Matrix Set minimum zoom level (default is 0).
        maxzoom: int
            Tile Matrix Set maximum zoom level (default is 24).
        title: str
            Tile Matrix Set title (default is 'Custom TileMatrixSet')
        identifier: str
            Tile Matrix Set identifier (default is 'Custom')
        geographic_crs: pyproj.CRS
            Geographic (lat,lon) coordinate reference system (default is EPSG:4326)

        Returns:
        --------
        TileMatrixSet

        """
        tms: Dict[str, Any] = {
            "title": title,
            "identifier": identifier,
            "supportedCRS": crs,
            "tileMatrix": [],
            "_geographic_crs": geographic_crs,
        }

        is_inverted = crs_axis_inverted(crs)

        if is_inverted:
            tms["boundingBox"] = TMSBoundingBox(
                crs=extent_crs or crs,
                lowerCorner=[extent[1], extent[0]],
                upperCorner=[extent[3], extent[2]],
            )
        else:
            tms["boundingBox"] = TMSBoundingBox(
                crs=extent_crs or crs,
                lowerCorner=[extent[0], extent[1]],
                upperCorner=[extent[2], extent[3]],
            )

        if extent_crs:
            transform = Transformer.from_crs(extent_crs, crs, always_xy=True)
            left, bottom, right, top = extent
            bbox = BoundingBox(
                *transform.transform_bounds(left, bottom, right, top, densify_pts=21)
            )
        else:
            bbox = BoundingBox(*extent)

        x_origin = bbox.left if not is_inverted else bbox.top
        y_origin = bbox.top if not is_inverted else bbox.left

        width = abs(bbox.right - bbox.left)
        height = abs(bbox.top - bbox.bottom)
        mpu = meters_per_unit(crs)
        for zoom in range(minzoom, maxzoom + 1):
            res = max(
                width / (tile_width * matrix_scale[0]) / 2.0 ** zoom,
                height / (tile_height * matrix_scale[1]) / 2.0 ** zoom,
            )
            tms["tileMatrix"].append(
                TileMatrix(
                    **dict(
                        identifier=str(zoom),
                        scaleDenominator=res * mpu / 0.00028,
                        topLeftCorner=[x_origin, y_origin],
                        tileWidth=tile_width,
                        tileHeight=tile_height,
                        matrixWidth=matrix_scale[0] * 2 ** zoom,
                        matrixHeight=matrix_scale[1] * 2 ** zoom,
                    )
                )
            )

        return cls(**tms)

    def matrix(self, zoom: int) -> TileMatrix:
        """Return the TileMatrix for a specific zoom."""
        try:
            tile_matrix = list(
                filter(lambda m: m.identifier == str(zoom), self.tileMatrix)
            )[0]
        except IndexError:
            matrix_scale = list(
                {
                    round(
                        self.tileMatrix[idx].scaleDenominator
                        / self.tileMatrix[idx - 1].scaleDenominator,
                        2,
                    )
                    for idx in range(1, len(self.tileMatrix))
                }
            )
            if len(matrix_scale) > 1:
                raise Exception(
                    f"TileMatrix not found for level: {zoom} - Unable to construct tileMatrix for TMS with variable scale"
                )

            warnings.warn(
                f"TileMatrix not found for level: {zoom} - Creating values from TMS Scale.",
                UserWarning,
            )

            tile_matrix = self.tileMatrix[-1]
            factor = 1 / matrix_scale[0]
            while not str(zoom) == tile_matrix.identifier:
                tile_matrix = TileMatrix(
                    **dict(
                        identifier=str(int(tile_matrix.identifier) + 1),
                        scaleDenominator=tile_matrix.scaleDenominator / factor,
                        topLeftCorner=tile_matrix.topLeftCorner,
                        tileWidth=tile_matrix.tileWidth,
                        tileHeight=tile_matrix.tileHeight,
                        matrixWidth=int(tile_matrix.matrixWidth * factor),
                        matrixHeight=int(tile_matrix.matrixHeight * factor),
                    )
                )

        return tile_matrix

    def _resolution(self, matrix: TileMatrix) -> float:
        """
        Tile resolution for a TileMatrix.

        From note g in http://docs.opengeospatial.org/is/17-083r2/17-083r2.html#table_2:
          The pixel size of the tile can be obtained from the scaleDenominator
          by multiplying the later by 0.28 10-3 / metersPerUnit.

        """
        return matrix.scaleDenominator * 0.28e-3 / meters_per_unit(self.crs)

    def zoom_for_res(
        self,
        res: float,
        max_z: Optional[int] = None,
        zoom_level_strategy: str = "auto",
        min_z: Optional[int] = None,
    ) -> int:
        """Get TMS zoom level corresponding to a specific resolution.

        Args:
            res (float): Resolution in TMS unit.
            max_z (int): Maximum zoom level (default is tms maxzoom).
            zoom_level_strategy (str): Strategy to determine zoom level (same as in GDAL 3.2).
                LOWER will select the zoom level immediately below the theoretical computed non-integral zoom level.
                On the contrary, UPPER will select the immediately above zoom level.
                Defaults to AUTO which selects the closest zoom level.
                ref: https://gdal.org/drivers/raster/cog.html#raster-cog
            min_z (int): Minimum zoom level (default is tms minzoom).

        Returns:
            int: TMS zoom for a given resolution.

        Examples:
            >>> zoom_for_res(430.021)

        """
        if max_z is None:
            max_z = self.maxzoom

        if min_z is None:
            min_z = self.minzoom

        # Freely adapted from https://github.com/OSGeo/gdal/blob/dc38aa64d779ecc45e3cd15b1817b83216cf96b8/gdal/frmts/gtiff/cogdriver.cpp#L272-L305
        for zoom_level in range(min_z, max_z + 1):
            matrix_res = self._resolution(self.matrix(zoom_level))
            if res > matrix_res or abs(res - matrix_res) / matrix_res <= 1e-8:
                break

        if zoom_level > 0 and abs(res - matrix_res) / matrix_res > 1e-8:
            if zoom_level_strategy.lower() == "lower":
                zoom_level = max(zoom_level - 1, min_z)
            elif zoom_level_strategy.lower() == "upper":
                zoom_level = min(zoom_level, max_z)
            elif zoom_level_strategy.lower() == "auto":
                if (self._resolution(self.matrix(max(zoom_level - 1, min_z))) / res) < (
                    res / matrix_res
                ):
                    zoom_level = max(zoom_level - 1, min_z)
            else:
                raise ValueError(
                    f"Invalid strategy: {zoom_level_strategy}. Should be one of lower|upper|auto"
                )

        return zoom_level

    def lnglat(self, x: float, y: float, truncate=False) -> Coords:
        """Transform point(x,y) to geographic longitude and latitude."""
        inside = point_in_bbox(Coords(x, y), self.xy_bbox)
        if not inside:
            warnings.warn(
                f"Point ({x}, {y}) is outside TMS bounds {list(self.xy_bbox)}.",
                PointOutsideTMSBounds,
            )

        lng, lat = self._to_geographic.transform(x, y)

        if truncate:
            lng, lat = self.truncate_lnglat(lng, lat)

        return Coords(lng, lat)

    def xy(self, lng: float, lat: float, truncate=False) -> Coords:
        """Transform geographic longitude and latitude coordinates to TMS CRS."""
        if truncate:
            lng, lat = self.truncate_lnglat(lng, lat)

        inside = point_in_bbox(Coords(lng, lat), self.bbox)
        if not inside:
            warnings.warn(
                f"Point ({lng}, {lat}) is outside TMS bounds {list(self.bbox)}.",
                PointOutsideTMSBounds,
            )

        x, y = self._from_geographic.transform(lng, lat)

        return Coords(x, y)

    def truncate_lnglat(self, lng: float, lat: float) -> Tuple[float, float]:
        """
        Truncate geographic coordinates to TMS geographic bbox.

        Adapted from https://github.com/mapbox/mercantile/blob/master/mercantile/__init__.py

        """
        if lng > self.bbox.right:
            lng = self.bbox.right
        elif lng < self.bbox.left:
            lng = self.bbox.left

        if lat > self.bbox.top:
            lat = self.bbox.top
        elif lat < self.bbox.bottom:
            lat = self.bbox.bottom

        return lng, lat

    def _tile(self, xcoord: float, ycoord: float, zoom: int) -> Tile:
        """
        Get the tile containing a Point (in TMS CRS).

        Parameters
        ----------
        xcoord, ycoord : float
            A `X` and `Y` pair in TMS coordinate reference system.
        zoom : int
            The zoom level.

        Returns
        -------
        Tile

        """
        matrix = self.matrix(zoom)
        res = self._resolution(matrix)

        origin_x = (
            matrix.topLeftCorner[1] if self._invert_axis else matrix.topLeftCorner[0]
        )
        origin_y = (
            matrix.topLeftCorner[0] if self._invert_axis else matrix.topLeftCorner[1]
        )

        xtile = (
            math.floor((xcoord - origin_x) / float(res * matrix.tileWidth))
            if not math.isinf(xcoord)
            else 0
        )
        ytile = (
            math.floor((origin_y - ycoord) / float(res * matrix.tileHeight))
            if not math.isinf(ycoord)
            else 0
        )

        # # avoid out-of-range tiles
        if xtile < 0:
            xtile = 0

        if ytile < 0:
            ytile = 0

        if xtile > matrix.matrixWidth:
            xtile = matrix.matrixWidth

        if ytile > matrix.matrixHeight:
            ytile = matrix.matrixHeight

        return Tile(x=xtile, y=ytile, z=zoom)

    def tile(self, lng: float, lat: float, zoom: int, truncate=False) -> Tile:
        """
        Get the tile for a given geographic longitude and latitude pair.

        Parameters
        ----------
        lng, lat : float
            A longitude and latitude pair in geographic coordinate reference system.
        zoom : int
            The zoom level.
        truncate : bool
            Whether or not to truncate inputs to limits of TMS geographic bounds.

        Returns
        -------
        Tile

        """
        x, y = self.xy(lng, lat, truncate=truncate)
        return self._tile(x, y, zoom)

    def _ul(self, *tile: Tile) -> Coords:
        """
        Return the upper left coordinate of the tile in TMS coordinate reference system.

        Attributes
        ----------
        tile: (x, y, z) tile coordinates or a Tile object we want the upper left coordinates of.

        Returns
        -------
        Coords: The upper left coordinates of the input tile.

        """
        t = _parse_tile_arg(*tile)

        matrix = self.matrix(t.z)
        res = self._resolution(matrix)

        origin_x = (
            matrix.topLeftCorner[1] if self._invert_axis else matrix.topLeftCorner[0]
        )
        origin_y = (
            matrix.topLeftCorner[0] if self._invert_axis else matrix.topLeftCorner[1]
        )

        xcoord = origin_x + t.x * res * matrix.tileWidth
        ycoord = origin_y - t.y * res * matrix.tileHeight
        return Coords(xcoord, ycoord)

    def xy_bounds(self, *tile: Tile) -> BoundingBox:
        """
        Return the bounding box of the tile in TMS coordinate reference system.

        Attributes
        ----------
        tile: A tuple of (x, y, z) tile coordinates or a Tile object we want the bounding box of.

        Returns
        -------
        BoundingBox: The bounding box of the input tile.

        """
        t = _parse_tile_arg(*tile)

        left, top = self._ul(t)
        right, bottom = self._ul(Tile(t.x + 1, t.y + 1, t.z))
        return BoundingBox(left, bottom, right, top)

    def ul(self, *tile: Tile) -> Coords:
        """
        Return the upper left coordinates of the tile in geographic coordinate reference system.

        Attributes
        ----------
        tile (tuple or Tile): (x, y, z) tile coordinates or a Tile object we want the upper left geographic coordinates of.

        Returns
        -------
        Coords: The upper left geographic coordinates of the input tile.

        """
        t = _parse_tile_arg(*tile)

        x, y = self._ul(t)
        return Coords(*self.lnglat(x, y))

    def bounds(self, *tile: Tile) -> BoundingBox:
        """
        Return the bounding box of the tile in geographic coordinate reference system.

        Attributes
        ----------
        tile (tuple or Tile): A tuple of (x, y, z) tile coordinates or a Tile object we want the bounding box of.

        Returns
        -------
        BoundingBox: The bounding box of the input tile.

        """
        t = _parse_tile_arg(*tile)

        left, top = self.ul(t)
        right, bottom = self.ul(Tile(t.x + 1, t.y + 1, t.z))
        return BoundingBox(left, bottom, right, top)

    @property
    def xy_bbox(self):
        """Return TMS bounding box in TileMatrixSet's CRS."""
        if self.boundingBox:
            left = (
                self.boundingBox.lowerCorner[1]
                if self._invert_axis
                else self.boundingBox.lowerCorner[0]
            )
            bottom = (
                self.boundingBox.lowerCorner[0]
                if self._invert_axis
                else self.boundingBox.lowerCorner[1]
            )
            right = (
                self.boundingBox.upperCorner[1]
                if self._invert_axis
                else self.boundingBox.upperCorner[0]
            )
            top = (
                self.boundingBox.upperCorner[0]
                if self._invert_axis
                else self.boundingBox.upperCorner[1]
            )
            if self.boundingBox.crs != self.crs:
                transform = Transformer.from_crs(
                    self.boundingBox.crs, self.crs, always_xy=True
                )
                left, bottom, right, top = transform.transform_bounds(
                    left,
                    bottom,
                    right,
                    top,
                    densify_pts=21,
                )

        else:
            zoom = self.minzoom
            matrix = self.matrix(zoom)
            left, top = self._ul(Tile(0, 0, zoom))
            right, bottom = self._ul(
                Tile(matrix.matrixWidth, matrix.matrixHeight, zoom)
            )

        return BoundingBox(left, bottom, right, top)

    @property
    def bbox(self):
        """Return TMS bounding box in geographic coordinate reference system."""
        left, bottom, right, top = self.xy_bbox
        return BoundingBox(
            *self._to_geographic.transform_bounds(
                left,
                bottom,
                right,
                top,
                densify_pts=21,
            )
        )

    def intersect_tms(self, bbox: BoundingBox) -> bool:
        """Check if a bounds intersects with the TMS bounds."""
        tms_bounds = self.xy_bbox
        return (
            (bbox[0] < tms_bounds[2])
            and (bbox[2] > tms_bounds[0])
            and (bbox[3] > tms_bounds[1])
            and (bbox[1] < tms_bounds[3])
        )

    def tiles(
        self,
        west: float,
        south: float,
        east: float,
        north: float,
        zooms: Sequence[int],
        truncate: bool = False,
    ) -> Iterator[Tile]:
        """
        Get the tiles overlapped by a geographic bounding box

        Original code from https://github.com/mapbox/mercantile/blob/master/mercantile/__init__.py#L424

        Parameters
        ----------
        west, south, east, north : sequence of float
            Bounding values in decimal degrees (geographic CRS).
        zooms : int or sequence of int
            One or more zoom levels.
        truncate : bool, optional
            Whether or not to truncate inputs to web mercator limits.

        Yields
        ------
        Tile

        Notes
        -----
        A small epsilon is used on the south and east parameters so that this
        function yields exactly one tile when given the bounds of that same tile.

        """
        if isinstance(zooms, int):
            zooms = (zooms,)

        if truncate:
            west, south = self.truncate_lnglat(west, south)
            east, north = self.truncate_lnglat(east, north)

        if west > east:
            bbox_west = (self.bbox.left, south, east, north)
            bbox_east = (west, south, self.bbox.right, north)
            bboxes = [bbox_west, bbox_east]
        else:
            bboxes = [(west, south, east, north)]

        for w, s, e, n in bboxes:
            # Clamp bounding values.
            w = max(self.bbox.left, w)
            s = max(self.bbox.bottom, s)
            e = min(self.bbox.right, e)
            n = min(self.bbox.top, n)

            for z in zooms:
                ul_tile = self.tile(
                    w + LL_EPSILON, n - LL_EPSILON, z
                )  # Not in mercantile
                lr_tile = self.tile(e - LL_EPSILON, s + LL_EPSILON, z)

                for i in range(ul_tile.x, lr_tile.x + 1):
                    for j in range(ul_tile.y, lr_tile.y + 1):
                        yield Tile(i, j, z)

    def feature(
        self,
        tile: Tile,
        fid: Optional[str] = None,
        props: Dict = {},
        buffer: Optional[NumType] = None,
        precision: Optional[int] = None,
        projected: bool = False,
    ) -> Dict:
        """
        Get the GeoJSON feature corresponding to a tile.

        Originally from https://github.com/mapbox/mercantile/blob/master/mercantile/__init__.py

        Parameters
        ----------
        tile : Tile or sequence of int
            May be be either an instance of Tile or 3 ints, X, Y, Z.
        fid : str, optional
            A feature id.
        props : dict, optional
            Optional extra feature properties.
        buffer : float, optional
            Optional buffer distance for the GeoJSON polygon.
        precision: float
            If >= 0, geometry coordinates will be rounded to this number of decimal,
            otherwise original coordinate values will be preserved (default).
        projected : bool, optional
            Return coordinates in TMS projection. Default is false.

        Returns
        -------
        dict

        """
        west, south, east, north = self.xy_bounds(tile)

        if not projected:
            west, south, east, north = self._to_geographic.transform_bounds(
                west, south, east, north, densify_pts=21
            )

        if buffer:
            west -= buffer
            south -= buffer
            east += buffer
            north += buffer

        if precision and precision >= 0:
            west, south, east, north = (
                round(v, precision) for v in (west, south, east, north)
            )

        bbox = [min(west, east), min(south, north), max(west, east), max(south, north)]
        geom = bbox_to_feature(west, south, east, north)

        xyz = str(tile)
        feat: Dict[str, Any] = {
            "type": "Feature",
            "bbox": bbox,
            "id": xyz,
            "geometry": geom,
            "properties": {
                "title": f"XYZ tile {xyz}",
                "grid_name": self.identifier,
                "grid_crs": self.crs.to_string(),
            },
        }

        if projected:
            warnings.warn(
                "CRS is no longer part of the GeoJSON specification."
                "Other projection than EPSG:4326 might not be supported.",
                UserWarning,
            )
            feat.update(
                {"crs": {"type": "EPSG", "properties": {"code": self.crs.to_epsg()}}}
            )

        if props:
            feat["properties"].update(props)

        if fid is not None:
            feat["id"] = fid

        return feat

    def quadkey(self, *tile: Tile) -> str:
        """Get the quadkey of a tile

        Parameters
        ----------
        tile : Tile or sequence of int
            May be be either an instance of Tile or 3 ints, X, Y, Z.

        Returns
        -------
        str

        """
        if not self._is_quadtree:
            raise NoQuadkeySupport(
                "This Tile Matrix Set doesn't support 2 x 2 quadkeys."
            )

        t = _parse_tile_arg(*tile)

        qk = []
        for z in range(t.z, self.minzoom, -1):
            digit = 0
            mask = 1 << (z - 1)
            if t.x & mask:
                digit += 1
            if t.y & mask:
                digit += 2
            qk.append(str(digit))

        return "".join(qk)

    def quadkey_to_tile(self, qk: str) -> Tile:
        """Get the tile corresponding to a quadkey

        Parameters
        ----------
        qk : str
            A quadkey string.

        Returns
        -------
        Tile

        """
        if not self._is_quadtree:
            raise NoQuadkeySupport(
                "This Tile Matrix Set doesn't support 2 x 2 quadkeys."
            )

        if len(qk) == 0:
            return Tile(0, 0, 0)

        xtile, ytile = 0, 0
        for i, digit in enumerate(reversed(qk)):
            mask = 1 << i
            if digit == "1":
                xtile = xtile | mask
            elif digit == "2":
                ytile = ytile | mask
            elif digit == "3":
                xtile = xtile | mask
                ytile = ytile | mask
            elif digit != "0":
                raise QuadKeyError("Unexpected quadkey digit: %r", digit)

        return Tile(xtile, ytile, i + 1)

    def minmax(self, zoom: int) -> Dict:
        """Return TileMatrix Extrema.

        Parameters
        ----------
        zoom : int
            The zoom level.

        Returns
        -------
        Dict

        """
        m = self.matrix(zoom)
        return {
            "x": {"min": 0, "max": m.matrixWidth - 1},
            "y": {"min": 0, "max": m.matrixHeight - 1},
        }

    def is_valid(self, *tile: Tile) -> bool:
        """Check if a tile is valid."""
        t = _parse_tile_arg(*tile)

        if t.z < self.minzoom:
            return False

        extrema = self.minmax(t.z)
        validx = extrema["x"]["min"] <= t.x <= extrema["x"]["max"]
        validy = extrema["y"]["min"] <= t.y <= extrema["y"]["max"]

        return validx and validy

    def neighbors(self, *tile: Tile) -> List[Tile]:
        """The neighbors of a tile

        The neighbors function makes no guarantees regarding neighbor tile
        ordering.

        The neighbors function returns up to eight neighboring tiles, where
        tiles will be omitted when they are not valid.

        Parameters
        ----------
        tile : Tile or sequence of int
            May be be either an instance of Tile or 3 ints, X, Y, Z.

        Returns
        -------
        list

        """
        t = _parse_tile_arg(*tile)
        extrema = self.minmax(t.z)

        tiles = []
        for i in [-1, 0, 1]:
            for j in [-1, 0, 1]:
                if i == 0 and j == 0:
                    continue
                elif t.x + i < extrema["x"]["min"] or t.y + j < extrema["y"]["min"]:
                    continue

                elif t.x + i > extrema["x"]["max"] or t.y + j > extrema["y"]["max"]:
                    continue

                tiles.append(Tile(x=t.x + i, y=t.y + j, z=t.z))

        return tiles

    def parent(self, *tile: Tile, zoom: int = None):
        """Get the parent of a tile

        The parent is the tile of one zoom level lower that contains the
        given "child" tile.

        Parameters
        ----------
        tile : Tile or sequence of int
            May be be either an instance of Tile or 3 ints, X, Y, Z.
        zoom : int, optional
            Determines the *zoom* level of the returned parent tile.
            This defaults to one lower than the tile (the immediate parent).

        Returns
        -------
        list: list of Tile

        """
        t = _parse_tile_arg(*tile)

        if t.z == self.minzoom:
            return []

        if zoom is not None and t.z <= zoom:
            raise InvalidZoomError("zoom must be less than that of the input tile")

        target_zoom = t.z - 1 if zoom is None else zoom

        res = self._resolution(self.matrix(t.z)) / 10.0

        bbox = self.xy_bounds(t)
        ul_tile = self._tile(bbox.left + res, bbox.top - res, target_zoom)
        lr_tile = self._tile(bbox.right - res, bbox.bottom + res, target_zoom)

        tiles = []
        for i in range(ul_tile.x, lr_tile.x + 1):
            for j in range(ul_tile.y, lr_tile.y + 1):
                tiles.append(Tile(i, j, target_zoom))

        return tiles

    def children(self, *tile: Tile, zoom: int = None):
        """Get the children of a tile

        The children are ordered: top-left, top-right, bottom-right, bottom-left.

        Parameters
        ----------
        tile : Tile or sequence of int
            May be be either an instance of Tile or 3 ints, X, Y, Z.
        zoom : int, optional
            Determines the *zoom* level of the returned parent tile.
            This defaults to one lower than the tile (the immediate parent).

        Returns
        -------
        list: list of Tile

        """
        t = _parse_tile_arg(*tile)

        if zoom is not None and t.z > zoom:
            raise InvalidZoomError("zoom must be greater than that of the input tile")

        target_zoom = t.z + 1 if zoom is None else zoom

        bbox = self.xy_bounds(t)
        res = self._resolution(self.matrix(t.z)) / 10.0

        ul_tile = self._tile(bbox.left + res, bbox.top - res, target_zoom)
        lr_tile = self._tile(bbox.right - res, bbox.bottom + res, target_zoom)

        tiles = []
        for i in range(ul_tile.x, lr_tile.x + 1):
            for j in range(ul_tile.y, lr_tile.y + 1):
                tiles.append(Tile(i, j, target_zoom))

        return tiles


#! /usr/lib/python3
#coding=utf-8

import requests

'''
爬取笔趣小说网的网页
'''

url = "http://www.biqukan.com/"
url1 = "https://www.biqukan.com/1_1094/5403177.html"

# req = requests.get(url1)
# print(req.text)

'''
爬取高清图片
'''

url3 = "https://unsplash.com/"
req = requests.get(url=url3)
print(req.text)


from PyQt5.QtCore import QObject, pyqtSlot, pyqtSignal, QVariant
from PyQt5.QtQml import QQmlApplicationEngine, QQmlEngine
from requests.exceptions import ConnectionError, ReadTimeout
from PyQt5.QtGui import QGuiApplication, QIcon
from threading import Thread
from pathlib import Path
from tools import tools
from time import sleep
from scrapers import *
import scrapers
import json
import sys
import os
import re

# Manga searcher.
class Searcher(QObject):
    search_manga_data = pyqtSignal(list, str, arguments=["dataSearch", "error"])
    search_manga_controls = pyqtSignal(str, arguments=["jsonControls"])

    def __init__(self):
        QObject.__init__(self)

    # Allows change source.
    @pyqtSlot(str, str)
    def change_manga_source(self, manga_source_name, alias):
        global manga_source
        manga_source = eval(manga_source_name)  # New manga source.

        # Write the config file.
        tools.config_writer("scrapers", "current", value=manga_source_name)
        tools.config_writer("scrapers", "current_alias", value=alias)

        # Reads and sends to QML.
        config_file_read = tools.config_file()
        context.setContextProperty("configFile", config_file_read)

        self.emit_controls()

    # Search manga thread.
    @pyqtSlot(str, int)
    def search_manga(self, parameter_list, page):
        data = Thread(target=self.search_manga_thread, args=[parameter_list, page])
        data.start()

    # Download thumbnail thread.
    @pyqtSlot(str, str)
    def download_thumbnail(self, link, image_name):
        data = Thread(target=tools.download_image, args=[link, thumbnail_dir, image_name])
        data.start()

    # Allows search.
    def search_manga_thread(self, parameter_list, page):
        data = manga_source.search_manga(**json.loads(parameter_list), page=page)

        if type(data) == ConnectionError:
            self.search_manga_data.emit([], "ConnectionError")
        elif type(data) == ReadTimeout:
            self.search_manga_data.emit([], "ReadTimeout")
        elif type(data) == str:
            self.search_manga_data.emit([], data)
        else:
            data = self.check_thumbnail(data)
            self.search_manga_data.emit(data, "")

    # Emit advanced search controls.
    @pyqtSlot()
    def emit_controls(self):
        controls = manga_source.get_search_controls()
        self.search_manga_controls.emit(controls)

    # Check if the thumbnail already exists.
    def check_thumbnail(self, data):
        for thumbnail in data:
            name = re.search(r"[^/]\w+\..{3,4}$", thumbnail["thumbnail"]).group()
            if Path(thumbnail_dir + name).exists():
                thumbnail["thumbnail"] = Path(thumbnail_dir + name).as_uri()

        return data

# TODO: Parse data through Python
# Manga viewer (data)
class Viewer(QObject):
    manga_data = pyqtSignal("QVariant", "QVariant", str, str, bool, arguments=["mangaData", "source", "error", "saved"])

    def __init__(self):
        QObject.__init__(self)

    # Set manga data thread
    @pyqtSlot(str, str, str, bool)
    def set_manga_data(self, url, source, name, forced):
        data = Thread(target=self.set_manga_data_thread, args=[url, source, name, forced])
        data.start()

    @pyqtSlot(str, str, str)
    def save_manga(self, data, source, name):
        name = re.sub(windows_expr, "", name)  # Windows folders support.
        source_config_dir = Path(manga_config_dir, source, name)

        # If the manga config does not exists
        if not Path(source_config_dir).exists():
            os.makedirs(Path(source_config_dir).absolute())

        with open(str(Path(source_config_dir, name)) + ".json", "w") as config:
            json.dump(json.loads(data), config, indent=4, ensure_ascii=False)

    # Set manga data
    def set_manga_data_thread(self, url, source, name, forced):
        name = re.sub(windows_expr, "", name)

        config_manga_file = str(Path(manga_config_dir, source, name, name)) + ".json"
        data_saved = {}

        if Path(config_manga_file).exists():
            with open(config_manga_file) as f:
                data = json.load(f)
                saved = True

            if forced:
                data_saved = data
                data = manga_source.get_manga_data(url)
                saved = False
        else:
            data = manga_source.get_manga_data(url)
            saved = False

        if type(data) == ConnectionError:
            self.manga_data.emit({}, {}, "", "ConnectionError", False)
        elif type(data) == ReadTimeout:
            self.manga_data.emit({}, {}, "", "ReadTimeout", False)
        elif type(data) == str:
            self.manga_data.emit({}, {}, "", data, False)
        else:
            self.manga_data.emit(data, data_saved, manga_source.name, "", saved)

    @pyqtSlot(str, str, str)
    def delete_manga(self, source, name, chapter):
        # Windows folders support.
        chapter = re.sub(windows_expr, "", chapter)
        name = re.sub(windows_expr, "", name)

        chapter_dir = Path(downloads_dir, source, name, chapter)
        chapter_config_dir = Path(config_dir, "manga", source, name, "chapters", chapter)
        chapter_config = str(chapter_config_dir) + ".json"

        for image in os.walk(chapter_dir.absolute()):
            images_name = image[2]

            for image_name in images_name:
                image_path = Path(image[0], image_name)
                os.remove(image_path)

        if os.path.exists(chapter_config):
            os.remove(chapter_config)

# TODO: Advanced Manga reader
class Reader(QObject):
    def __init__(self):
        QObject.__init__(self)

class Library(QObject):
    set_save_manga = pyqtSignal(str, str, str, arguments=["thumbnail", "title", "link"])

    def __init__(self):
        QObject.__init__(self)

    @pyqtSlot()
    def get_saved_mangas(self):
        depth = 2

        for files in tools.walklevel(str(Path(config_dir, "manga")), depth):
            manga_config_path = Path(files[0], "".join(files[2]))

            if manga_config_path.is_file():
                with open(manga_config_path) as f:
                    file = json.load(f)
                if file["bookmarked"]:
                    self.set_save_manga.emit(file["thumbnail"], file["title"], file["link"])

class Downloader(QObject):
    set_images = pyqtSignal(str, int, int, str, int, arguments=[
        "image", "imgWidth", "imgHeight", "url", "_totalPages"])
    downloaded = pyqtSignal(str, arguments="buttonLink")
    download_progress = pyqtSignal(int, int, str, arguments=["nImages", "nDownloads", "mangaID"])

    def __init__(self):
        QObject.__init__(self)
        self.cancel_download = False
        self.pause = False

    # Set images thread
    @pyqtSlot(str, str, str, str)
    def download_manga(self, url, source, name, chapter):
        data = Thread(target=self.download_manga_thread, args=[url, source, name, chapter])
        data.start()

    # Read manga thread
    @pyqtSlot(str, str, str, str, bool)
    def read_manga(self, url, source, name, chapter, downloaded):
        data = Thread(target=self.read_manga_thread, args=[url, source, name, chapter, downloaded])
        data.start()

    @pyqtSlot()
    def cancel_manga(self):
        self.cancel_download = True

    @pyqtSlot()
    def toggle_pause(self):
        self.pause = not self.pause

    @pyqtSlot(str, str, int)
    def set_downloaded(self, name, source, n_chapter):
        name = re.sub(windows_expr, "", name)

        config_manga_file = Path(manga_config_dir, source, name, name + ".json")

        with open(config_manga_file) as config:
            read_config = json.load(config)
            read_config["chapters"]["chapter_" + str(n_chapter)]["downloaded"] = True
            read_config["chapters"]["chapter_" + str(n_chapter)]["downloading"] = False

        with open(config_manga_file, "w") as config:
            json.dump(read_config, config, indent=4, ensure_ascii=False)

    @pyqtSlot(str, str, int, bool)
    def set_queued(self, name, source, n_chapter, queued):
        name = re.sub(windows_expr, "", name)

        config_manga_file = Path(manga_config_dir, source, name, name + ".json")

        with open(config_manga_file) as config:
            read_config = json.load(config)
            read_config["chapters"]["chapter_" + str(n_chapter)]["queued"] = queued

        with open(config_manga_file, "w") as config:
            json.dump(read_config, config, indent=4, ensure_ascii=False)

    @pyqtSlot(str, str, int, bool)
    def set_downloading(self, name, source, n_chapter, downloading):
        name = re.sub(windows_expr, "", name)

        config_manga_file = Path(manga_config_dir, source, name, name + ".json")

        with open(config_manga_file) as config:
            read_config = json.load(config)
            read_config["chapters"]["chapter_" + str(n_chapter)]["downloading"] = downloading

        with open(config_manga_file, "w") as config:
            json.dump(read_config, config, indent=4, ensure_ascii=False)

    @pyqtSlot(str)
    def download_manga_thread(self, url, source, name, chapter):
        mangaID = url + source + name + chapter  # MangaID

        # Windows folders support.
        name = re.sub(windows_expr, "", name)
        chapter = re.sub(windows_expr, "", chapter)

        # Setting the image size.
        images_size = {}
        images_config_path = Path(manga_config_dir, source, name, "chapters")
        if not images_config_path.exists():
            os.makedirs(images_config_path)

        images = manga_source.get_chapters_images(url)
        chapter_dir = Path(downloads_dir, source, name, chapter)

        n_images = len(images)

        # If the chapter directory does not exists.
        if not chapter_dir.exists():
            os.makedirs(chapter_dir)

        # Iterate through images.
        for i in range(n_images):
            while self.pause:  # Sleep for 1 second while self.pause is True
                sleep(1)

            image_name = str(i) + re.search(r"\..{3,4}$", images[i]).group()
            image_config = str(Path(images_config_path, chapter)) + ".json"

            image_path = Path(chapter_dir, image_name)

            image = tools.download_image(images[i], chapter_dir, image_name)
            while not image:
                image = tools.download_image(images[i], chapter_dir, image_name)

            if self.cancel_download:
                self.cancel_download = False
                return None

            if os.path.exists(image_config):
                with open(image_config) as f:
                    config_length = len(json.load(f))
            else:
                config_length = 0

            # If the configuration image exists and it's complete.
            if os.path.exists(image_config) and images_size == {} and config_length == n_images:
                pass
            # Else, create it.
            else:
                width, height = tools.get_image_size(image_path)
                images_size[image_name] = [width, height]
                with open(image_config, "w") as f:
                    json.dump(images_size, f, indent=4)

            self.download_progress.emit(n_images, i + 1, mangaID)  # MangaID

    # Read manga
    def read_manga_thread(self, url, source, name, chapter, downloaded):
        # Windows folders support.
        name = re.sub(windows_expr, "", name)
        chapter = re.sub(windows_expr, "", chapter)

        # Setting the image size.
        images_size = {}
        images_config_path = Path(manga_config_dir, source, name, "chapters")
        image_config = str(Path(images_config_path, chapter)) + ".json"
        if not images_config_path.exists():
            os.makedirs(images_config_path)

        chapter_dir = Path(downloads_dir, source, name, chapter)

        # If the chapter is downloaded.
        if downloaded:
            images = [str(Path(chapter_dir, image)) for image in os.listdir(chapter_dir)]
            images.sort(key=lambda x: int(re.search(r"(\d+)\..{3,4}$", x).groups()[0]))

            for image in images:
                image_name = re.search(r"\w+\..{3,4}$", image).group()

                with open(image_config) as f:
                    sizes = json.load(f)
                    width, height = sizes[image_name]
                self.set_images.emit(Path(image).as_uri(), width, height, url, len(images))
        else:
            pass

config_file = tools.config_file()
scale_factor = config_file["system"]["scale_factor"]
current_scraper = config_file["scrapers"]["current"]

thumbnail_dir, cache_save_dir, downloads_dir, config_dir, manga_config_dir = tools.get_dirs()

scraper_data = {}
manga_source = getattr(__import__("scrapers"), current_scraper)

for scraper in scrapers.__all__:
    scraper_data[getattr(__import__("scrapers"), scraper).name] = scraper

windows_expr = re.compile(r"[\\/:*?\"<>|]")

os.environ["QT_QUICK_CONTROLS_STYLE"] = "Material"
os.environ["QT_QUICK_CONTROLS_MATERIAL_VARIANT"] = "Dense"
os.environ["QT_SCALE_FACTOR"] = str(scale_factor)

application = QGuiApplication(sys.argv)
application.setWindowIcon(QIcon("resources/icon.png"))

manga_searcher = Searcher()
manga_viewer = Viewer()
manga_reader = Reader()
manga_library = Library()
manga_downloader = Downloader()
engine = QQmlApplicationEngine()

# Pass variables to QML
context = engine.rootContext()
context.setContextProperty("MangaSearcher", manga_searcher)
context.setContextProperty("MangaViewer", manga_viewer)
context.setContextProperty("MangaReader", manga_reader)
context.setContextProperty("MangaLibrary", manga_library)
context.setContextProperty("MangaDownloader", manga_downloader)
context.setContextProperty("configFile", config_file)
context.setContextProperty("scraperData", scraper_data)
context.setContextProperty("thumbnailDir", thumbnail_dir)
context.setContextProperty("os", sys.platform)

engine.load("gui/nunnix_manga.qml")
sys.exit(application.exec_())


from collections import deque
def BFS(s):
    D = [0] * (V + 1)
    visit = [False] * (V + 1)
    Q = deque()
    Q.append(s); D[s] = 0

    while Q:
        u = Q.popleft()
        for v in G[u]:
            if D[v] > D[u] + w:
                D[v] += D[u] + w
                Q.append(v)

python/알고리즘 응용/그래프/최단경로.py


from pathlib import Path

import pandas as pd
import prefect
from prefect import Flow, Parameter, case, task
from prefect.executors import LocalDaskExecutor

from config import (
    CONTROLS_STATISTICS_CSV_RESOURCE_ID,
    CONTROLS_STATISTICS_CSV_RESOURCE_TITLE,
    CONTROLS_STATISTICS_DATASET_ID,
    FLEET_SEGMENTS_CSV_RESOURCE_ID,
    FLEET_SEGMENTS_CSV_RESOURCE_TITLE,
    IS_INTEGRATION,
)
from src.pipeline.generic_tasks import extract
from src.pipeline.processing import prepare_df_for_loading
from src.pipeline.shared_tasks.control_flow import check_flow_not_running
from src.pipeline.shared_tasks.datagouv import get_csv_file_object, update_resource

@task(checkpoint=False)
def extract_controls_open_data() -> pd.DataFrame:
    return extract(
        "monitorfish_remote",
        "monitorfish/controls_open_data.sql",
        dtypes={
            "control_year": int,
            "control_month": int,
            "number_controls": int,
            "number_controls_ytd": int,
        },
    )

@task(checkpoint=False)
def extract_fleet_segments_open_data() -> pd.DataFrame:
    return extract(
        "monitorfish_remote",
        "monitorfish/fleet_segments_open_data.sql",
    )

@task(checkpoint=False)
def transform_controls_open_data(controls) -> pd.DataFrame:
    logger = prefect.context.get("logger")
    controls = prepare_df_for_loading(
        controls,
        logger=logger,
        nullable_integer_columns=["target_number_of_controls_year"],
    )
    return controls

@task(checkpoint=False)
def transform_fleet_segments_open_data(fleet_segments) -> pd.DataFrame:
    logger = prefect.context.get("logger")
    fleet_segments = prepare_df_for_loading(
        fleet_segments,
        logger=logger,
        nullable_integer_columns=["year"],
        pg_array_columns=["gears", "fao_areas", "species"],
    )
    return fleet_segments

with Flow("Controls open data", executor=LocalDaskExecutor()) as flow:

    flow_not_running = check_flow_not_running()
    with case(flow_not_running, True):

        dataset_id = Parameter("dataset_id", default=CONTROLS_STATISTICS_DATASET_ID)
        controls_resource_id = Parameter(
            "controls_resource_id", default=CONTROLS_STATISTICS_CSV_RESOURCE_ID
        )
        controls_resource_title = Parameter(
            "controls_resource_title", default=CONTROLS_STATISTICS_CSV_RESOURCE_TITLE
        )

        fleet_segments_resource_id = Parameter(
            "fleet_segments_resource_id", default=FLEET_SEGMENTS_CSV_RESOURCE_ID
        )
        fleet_segments_resource_title = Parameter(
            "fleet_segments_resource_title", default=FLEET_SEGMENTS_CSV_RESOURCE_TITLE
        )

        is_integration = Parameter("is_integration", default=IS_INTEGRATION)

        controls = extract_controls_open_data()
        controls = transform_controls_open_data(controls)
        controls_csv_file = get_csv_file_object(controls)

        fleet_segments = extract_fleet_segments_open_data()
        fleet_segments = transform_fleet_segments_open_data(fleet_segments)
        fleet_segments_csv_file = get_csv_file_object(fleet_segments)

        update_resource(
            dataset_id=dataset_id,
            resource_id=controls_resource_id,
            resource_title=controls_resource_title,
            resource=controls_csv_file,
            mock_update=is_integration,
        )

        update_resource(
            dataset_id=dataset_id,
            resource_id=fleet_segments_resource_id,
            resource_title=fleet_segments_resource_title,
            resource=fleet_segments_csv_file,
            mock_update=is_integration,
        )

flow.file_name = Path(__file__).name


import re
from datetime import datetime, timezone, timedelta

def to_timestamp(dt_str, tz_str):
    retest = re.split('[C:]',tz_str)
    tz_utc = timezone(timedelta(hours=int(retest[1])))
    dt = list(map(int,re.split('[- :]',dt_str)))

    res = datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5], tzinfo=tz_utc)
    re_tz = datetime.timestamp(res)
    return re_tz

# 测试:
t1 = to_timestamp('2015-6-1 08:10:30', 'UTC+7:00')
assert t1 == 1433121030.0, t1

t2 = to_timestamp('2015-5-31 16:10:30', 'UTC-09:00')
assert t2 == 1433121030.0, t2

print('ok')


# imports
import os
from rich import print
import time
from random import seed
from random import randint

# color class

# seed random number generator
seed(1000)

# main
loop = True

while loop == True:
    ask = int(input("Enter first number to generate (type 'q' to quit): "))
    ask2 = int(input('Enter second number to generate: '))
    value = randint(ask, ask2)
    print(value)
    time.sleep(1)
    if ask == 'q':
        loop == False
    else:
        loop == True


from app.models.db import db, Group, UserGroup, Expense, Transaction, Comment
from faker import Faker

faker = Faker()

def seed_expenses():

    expenses1 = Expense(description="Electric", amount=240, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Electric bill for January", group_id=1, user_id=2)
    expenses2 = Expense(description="Flights", amount=400, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Flights to Arubaaaaa", group_id=2, user_id=1)
    expenses3 = Expense(description="Internet", amount=120, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Comcast", group_id=3, user_id=5)
    expenses4 = Expense(description="Groceries", amount=200, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Trip to Trader Joes", group_id=4, user_id=1)
    expenses5 = Expense(description="Uber", amount=30, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Uber to bar", group_id=1, user_id=3)

    db.session.add(expenses1)
    db.session.add(expenses2)
    db.session.add(expenses3)
    db.session.add(expenses4)
    db.session.add(expenses5)

    db.session.commit()

def undo_expenses():
    db.session.execute('TRUNCATE expenses;')
    db.session.commit()


from django.shortcuts import render
from django.core.mail import send_mail
from django.conf import settings
from django.contrib import messages

# Create your views here.
def contact(request):
    if request.method == "POST":

        message_name = request.POST.get('name')
        message_subject = request.POST.get('subject')
        message_email = request.POST.get('email')
        message_message = request.POST.get('message')

        data = {
            'name': message_name,
            'subject': message_subject,
            'email': message_email,
            'message': message_message
        }

        message = """
        From: {}
        New message: {}
        Email: {}
        """.format(data['name'], data['message'], data['email'])
        # send an email
        send_mail(data['subject'], message, '', ['mpt.craig@gmail.com'])
        messages.success(request, f'Your email was send Successfully!')

        return render(request, 'contact/contact.html', {})
    else:
        return render(request, 'contact/contact.html', {})


import json
import urllib
from textwrap import dedent

import netlib.http
from netlib.utils import parse_content_type

def curl_command(flow):
    data = "curl "

    for k, v in flow.request.headers.fields:
        data += "-H '%s:%s' " % (k, v)

    if flow.request.method != "GET":
        data += "-X %s " % flow.request.method

    full_url = flow.request.scheme + "://" + flow.request.host + flow.request.path
    data += "'%s'" % full_url

    if flow.request.content:
        data += " --data-binary '%s'" % flow.request.content

    return data

def python_code(flow):
    code = dedent("""
        import requests

        url = '{url}'
        {headers}{params}{data}
        response = requests.request(
            method='{method}',
            url=url,{args}
        )

        print(response.text)
    """).strip()

    components = map(lambda x: urllib.quote(x, safe=""), flow.request.path_components)
    url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components)

    args = ""
    headers = ""
    if flow.request.headers:
        lines = ["    '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields]
        headers += "\nheaders = {\n%s}\n" % "".join(lines)
        args += "\n    headers=headers,"

    params = ""
    if flow.request.query:
        lines = ["    '%s': '%s',\n" % (k, v) for k, v in flow.request.query]
        params = "\nparams = {\n%s}\n" % "".join(lines)
        args += "\n    params=params,"

    data = ""
    if flow.request.body:
        json_obj = is_json(flow.request.headers, flow.request.body)
        if json_obj:
            # Without the separators field json.dumps() produces
            # trailing white spaces: https://bugs.python.org/issue16333
            data = json.dumps(json_obj, indent=4, separators=(',', ': '))
            data = "\njson = %s\n" % data
            args += "\n    json=json,"
        else:
            data = "\ndata = '''%s'''\n" % flow.request.body
            args += "\n    data=data,"

    code = code.format(
        url=url,
        headers=headers,
        params=params,
        data=data,
        method=flow.request.method,
        args=args,
    )

    return code

def raw_request(flow):
    data = netlib.http.http1.assemble_request(flow.request)
    return data

def is_json(headers, content):
    if headers:
        ct = parse_content_type(headers.get("content-type", ""))
        if ct and "%s/%s" % (ct[0], ct[1]) == "application/json":
            try:
                return json.loads(content)
            except ValueError:
                return False
    return False


#!/usr/bin/env python

import rospy
import numpy as np
import argparse

from UserInputMapper import UserInputMapper
from RobotState import RobotState
import input_handlers
from input_handlers import KinovaJoystickListener, HydraListener, MouseJoystickListener

from AdaTeleopHandler import *

VIEWER_DEFAULT = 'InteractiveMarker'
SIMULATE_DEFAULT = False

def Initialize_Adapy(args):
    """ Initializes robot and environment through adapy, using the specified environment path

    @param env_path path to OpenRAVE environment
    @return environment, robot
    """

    #env_path = '/environments/tablewithobjects_assisttest.env.xml'
    adapy_args = {'sim':args.sim,
                  'attach_viewer':args.viewer,
                  #'env_path':env_path
                  }
    openravepy.RaveInitialize(True, level=openravepy.DebugLevel.Debug)
    #openravepy.misc.InitOpenRAVELogging();
    env, robot = adapy.initialize(**adapy_args)
    Init_Robot(robot)
    return env,robot

def Init_Robot(robot):
    robot.SetActiveDOFs(range(6))
    #self.robot.arm.hand.OpenHand()
    #if (self.sim):
    robot_pose = np.array([[1, 0, 0, 0.409],[0, 1, 0, 0.338],[0, 0, 1, 0.795],[0, 0, 0, 1]])
    robot.SetTransform(robot_pose)
    if (robot.simulated):
      #servo simulator params
      robot.arm.servo_simulator.period=1./200.

def Reset_Robot(robot):
  if robot.simulated:
    num_hand_dofs = len(robot.arm.hand.GetDOFValues())
    inds, pos = robot.configurations.get_configuration('home')
    with robot.GetEnv():
      robot.SetDOFValues(pos, inds)
      robot.arm.hand.SetDOFValues(np.ones(num_hand_dofs)*0.1)
  #else:
    #robot.arm.hand.OpenHand()
  #  robot.arm.PlanToNamedConfiguration('home', execute=True)

if __name__ == "__main__":
    #parser.add_argument('-num', '--mouse-num', help='mouse number given by X in /dev/input/mouseX', type=str)
    parser = argparse.ArgumentParser(description="Direct Teleoperation for Ada")

    parser.add_argument('-s', '--sim', action='store_true', default=SIMULATE_DEFAULT,
                        help='simulation mode')
    parser.add_argument('-v', '--viewer', nargs='?', const=True, default=VIEWER_DEFAULT,
                        help='attach a viewer of the specified type')
    #parser.add_argument('--env-xml', type=str,
                        #help='environment XML file; defaults to an empty environment')
    parser.add_argument('--debug', action='store_true',
                        help='enable debug logging')
    parser.add_argument('-input', '--input-interface-name', help='name of the input interface. Possible choices: ' + str(possible_teleop_interface_names), type=str)
    parser.add_argument('-joy_dofs', '--num-input-dofs', help='number of dofs of input, either 2 or 3', type=int, default=2)
    args = parser.parse_args()

    rospy.init_node('ada_teleoperation', anonymous = True)

    env,robot = Initialize_Adapy(args)
    Reset_Robot(robot)
    ada_teleop = AdaTeleopHandler(env, robot, args.input_interface_name, args.num_input_dofs)
    ada_teleop.ExecuteDirectTeleop()


import sys
from utils.print_color import print_color

from character import Character
def starting_location():
    pc = Character()
    print_color('\nWelcome to the land of mystery what is your name: ', 'green')
    name = raw_input()
    pc.start_pc(name)
    print_color('\nWelcome ' + pc.name, 'green')
    command = raw_input('\nWhat do you want to do? ')
    while 'quit' not in command:
        pc.process_command(command)
        print_color('\n' + pc.name, 'cyan')
        print_color('=: ', 'green')
        command = raw_input()
    raw_input('Thanks for playing, press enter to exit.')

starting_location()


import tkinter
from PIL import ImageTk, Image

#define window
root = tkinter.Tk()
root.title('Image basics')
root.geometry('700x700')

# PNG image
my_img = tkinter.PhotoImage(file='a.png')

my_label = tkinter.Label(root, image=my_img)
my_label.pack()

my_btn = tkinter.Button(root, image=my_img)
my_btn.pack()

# JPEG image
img = ImageTk.PhotoImage(Image.open('b.jpg'))
label = tkinter.Label(root, image = img)
label.pack()

#run root in main looop
root.mainloop()


import pathlib

import nbformat
import pytest
from nbconvert.preprocessors import CellExecutionError
from nbconvert.preprocessors import ExecutePreprocessor

root = pathlib.Path(__file__).parent / "../examples"
fpaths = sorted(root.glob("*.ipynb"))

def idfn(val):
    if isinstance(val, pathlib.Path):
        return val.name

@pytest.mark.parametrize("fpath", fpaths, ids=idfn)
def test_notebook(fpath):
    """Execute each notebook."""
    # Information on running notebooks is found here
    # https://nbconvert.readthedocs.io/en/latest/execute_api.html
    with fpath.open() as fp:
        nb = nbformat.read(fp, as_version=4)

    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

    try:
        ep.preprocess(nb, {"metadata": {"path": fpath.parent}})
    except CellExecutionError:
        msg = 'Error executing the notebook "%s".\n\n' % fpath
        msg += 'See notebook "%s" for the traceback.' % fpath
        print(msg)
        raise
    finally:
        with fpath.open(mode="w", encoding="utf-8") as fp:
            nbformat.write(nb, fp)


from itertools import chain
from django import forms
from django.conf import settings
from django.utils.encoding import smart_unicode, force_unicode
from django.utils.html import escape, conditional_escape
from django.contrib.admin import widgets
from import_export import widgets as import_export_widgets
import os
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from filer.models.imagemodels import Image
from django.db import transaction
from oscar.core.loading import get_model
from import_export import resources, fields
from django.db.models.fields.related import ForeignKey
import functools
import json
from filer import settings as filer_settings
try:
    from django.utils.encoding import force_text
except ImportError:
    from django.utils.encoding import force_unicode as force_text
from django.db.transaction import atomic, savepoint, savepoint_rollback, savepoint_commit  # noqa

ProductImage = get_model('catalogue', 'ProductImage')

def search_file(image_name, folder):
    """ Given a search path, find file with requested name """
    file = None

    for root, directories, file_names in os.walk(folder):
        for file_name in file_names:
            if file_name == image_name:
                if file is None:
                    file = os.path.join(root, file_name)
                else:
                    raise ValueError('The desired image {} is not unique. Duplicate - {}'.
                                     format(file, os.path.join(root, file_name)))
    return file

class MPTTModelChoiceIterator(forms.models.ModelChoiceIterator):
    def choice(self, obj):
        tree_id = getattr(obj, getattr(self.queryset.model._meta, 'tree_id_atrr', 'tree_id'), 0)
        left = getattr(obj, getattr(self.queryset.model._meta, 'left_atrr', 'lft'), 0)
        return super(MPTTModelChoiceIterator, self).choice(obj) + ((tree_id, left),)

class MPTTModelMultipleChoiceField(forms.ModelMultipleChoiceField):
    def label_from_instance(self, obj):
        level = getattr(obj, getattr(self.queryset.model._meta, 'level_attr', 'level'), 0)
        return u'%s %s' % ('-'*level, smart_unicode(obj))

    def _get_choices(self):
        if hasattr(self, '_choices'):
            return self._choices
        return MPTTModelChoiceIterator(self)

    choices = property(_get_choices, forms.ChoiceField._set_choices)

class MPTTFilteredSelectMultiple(widgets.FilteredSelectMultiple):
    def __init__(self, verbose_name, is_stacked, attrs=None, choices=()):
        super(MPTTFilteredSelectMultiple, self).__init__(verbose_name, is_stacked, attrs, choices)

    def render_options(self, choices, selected_choices):
        """
        this is copy'n'pasted from django.forms.widgets Select(Widget)
        change to the for loop and render_option so they will unpack and use our extra tuple of mptt sort fields
        (if you pass in some default choices for this field, make sure they have the extra tuple too!)
        """
        def render_option(option_value, option_label, sort_fields):
            option_value = force_unicode(option_value)
            selected_html = (option_value in selected_choices) and u' selected="selected"' or ''
            return u'<option value="%s" data-tree-id="%s" data-left-value="%s"%s>%s</option>' % (
                escape(option_value),
                sort_fields[0],
                sort_fields[1],
                selected_html,
                conditional_escape(force_unicode(option_label)),
            )
        # Normalize to strings.
        selected_choices = set([force_unicode(v) for v in selected_choices])
        output = []
        for option_value, option_label, sort_fields in chain(self.choices, choices):
            if isinstance(option_label, (list, tuple)):
                output.append(u'<optgroup label="%s">' % escape(force_unicode(option_value)))
                for option in option_label:
                    output.append(render_option(*option))
                output.append(u'</optgroup>')
            else:
                output.append(render_option(option_value, option_label, sort_fields))
        return u'\n'.join(output)

    class Media:
        extend = False
        js = (settings.STATIC_ROOT + "js/mptt_m2m_selectbox.js",
              )

    class Media:
        extend = False
        js = (
            settings.STATIC_URL + "admin/js/core.js",
            settings.STATIC_URL + "admin/js/mptt_m2m_selectbox.js",
            settings.STATIC_URL + "admin/js/SelectFilter2.js",
        )

class ImageForeignKeyWidget(import_export_widgets.ForeignKeyWidget):
    def clean(self, value):
        if value:
            if not os.path.dirname(value):
                folder = os.path.join(settings.MEDIA_ROOT, filer_settings.DEFAULT_FILER_STORAGES['public']['main']['UPLOAD_TO_PREFIX'])
                image = search_file(value, folder)
                value = '/'.join(os.path.relpath(image).split('/')[1:])

            image = self.model.objects.filter(file=value).first()

            if image is None:
                image = Image.objects.create(**{'file': value, self.field: value})
            return image

class ImageManyToManyWidget(import_export_widgets.ManyToManyWidget):
    def __init__(self, model, separator=None, field=None, *args, **kwargs):
        super(ImageManyToManyWidget, self).__init__(model, separator=separator, field=field, *args, **kwargs)
        self.obj = None

    def clean(self, value):
        product_images = []

        if value:
            images = filter(None, value.split(self.separator))
            upload_to = filer_settings.DEFAULT_FILER_STORAGES['public']['main']['UPLOAD_TO_PREFIX']

            for display_order, val in enumerate(images):
                product_image = ProductImage.objects.filter(
                    product=self.obj, display_order=display_order
                ).first()

                current_path = os.getcwd()
                os.chdir(settings.MEDIA_ROOT)

                if not os.path.dirname(val):
                    folder = os.path.join(settings.MEDIA_ROOT, upload_to)
                    image = search_file(val, folder)

                    if image is not None:
                        val = os.path.relpath(image, start=settings.MEDIA_ROOT)
                else:
                    if not os.path.exists(os.path.abspath(val)):
                        raise ValueError('File "{}" does not exist.'.format(val))

                    if not os.path.isfile(os.path.abspath(val)):
                        raise ValueError('Is not file - "{}" '.format(val))

                os.chdir(current_path)

                image = Image.objects.filter(file=val).first()

                if image is None:
                    image = Image.objects.create(file=val, original_filename=val)

                if product_image is None:
                    product_image = ProductImage.objects.filter(product=self.obj, original__file=val).first()

                    if product_image is None:
                        product_image = ProductImage.objects.create(product=self.obj, original=image, display_order=display_order)
                    else:
                        product_image.display_order = display_order
                        product_image.save()
                    product_images.append(product_image)
                else:
                    product_image.original = image
                    product_image.save()
                    product_images.append(product_image)

            ProductImage.objects.filter(product=self.obj).exclude(pk__in=[obj.pk for obj in product_images]).delete()
        else:
            ProductImage.objects.filter(product=self.obj).delete()
        return product_images

class IntermediateModelManyToManyWidget(import_export_widgets.ManyToManyWidget):
    def __init__(self, *args, **kwargs):
        super(IntermediateModelManyToManyWidget, self).__init__(*args, **kwargs)
        self.rel_field = None
        self.intermediate_own_fields = []

    def clean(self, value):
        items = json.loads(value)
        for key, item in enumerate(items[:]):
            items[key][self.field] = self.model.objects.get(**{self.field: item[self.field]})
        return items

    def render(self, value, obj):
        return json.dumps([self.related_object_representation(obj, related_obj) for related_obj in value.all()])

    def related_object_representation(self, obj, related_obj):
        result = {self.field: getattr(related_obj, self.field, None)}

        if self.rel_field.rel.through._meta.auto_created:
            return result

        intermediate_obj = self.rel_field.rel.through.objects.filter(**{
            self.rel_field.m2m_reverse_field_name(): related_obj,
            self.rel_field.m2m_field_name(): obj
        }).first()

        if intermediate_obj is not None:
            for field in self.intermediate_own_fields:
                result[field.name] = force_text(getattr(intermediate_obj, field.name))
        else:
            for field in self.intermediate_own_fields:
                result[field.name] = field.default if field.default else None

        return result

class CharWidget(import_export_widgets.Widget):
    """
    Widget for converting text fields.
    """

    def render(self, value):
        try:
            featured_value = force_text(int(float(value)))
        except ValueError:
            featured_value = force_text(value)
        return featured_value

class ForeignKeyWidget(import_export_widgets.ForeignKeyWidget):
    def clean(self, value):
        return super(ForeignKeyWidget, self).clean(value.strip())

class ManyToManyWidget(import_export_widgets.ManyToManyWidget):
    def clean(self, value, row=None, *args, **kwargs):
        if not value:
            return self.model.objects.none()

        ids = filter(None, value.split(self.separator))
        ids = map(lambda slug: slug.strip(), ids)
        return self.model.objects.filter(**{
            '%s__in' % self.field: ids
        })


import time
import collections
import settings
import re
import codecs

def is_english(s):
    try:
        s.encode('ascii')
        return True
    except UnicodeEncodeError:
        return False

def trim_str(s, stopwords):
    s = s.replace('\n', ' ').replace('\r', ' ')
    s = re.sub(r"http\S+", "", s)
    s = re.sub(r"\d+", "", s)
    s = re.sub(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', ' ', s)
    wordlist = [w for w in s.lower().split() if
                (w not in stopwords) & (len(w) > 1) & (len(w) < 16)]
    return wordlist

def fetch_and_save_feature(desc_file, feature_file, connection, extends_dict, selector):
    """
    从数据库features_t中读取特征和文本并形成文件
    :param desc_file:
    :param feature_file:
    :param connection:
    :param extends_dict:
    :param selector:
    :return:
    """
    type_dict = {}
    next_type = 0
    cursor = connection.cursor()
    cursor.execute("select feedid, streamid, doc, location_type, created from features_t", connection)

    with codecs.open(settings.DEFINITIONS_ROOT + '/resource/stopwords.txt', 'r') as f:
        stoplist = set(f.read().split())

    epoch = int(time.mktime(time.strptime('2008.01.01', '%Y.%m.%d')))
    result_dict = {}
    regular_word = collections.defaultdict(int)
    for feed_id, streamid, doc, l_type, create_time in cursor.fetchall():

        # 生成文当前对doc进行补全
        if extends_dict is not None and (feed_id, streamid) in extends_dict:
            doc = doc + '. ' + extends_dict[(feed_id, streamid)]

        # 时间
        time_from = time.mktime(create_time.timetuple())
        lb_time = str(int((time_from - epoch) / (3600 * 24 * 30 * 6)))

        # 地点
        if l_type not in type_dict:
            type_dict[l_type] = next_type
            print('l_type=' + l_type + '\ttype_id=' + str(next_type))
            next_type += 1

        # 词
        wordlist = trim_str(doc, stoplist) if is_english(doc) else []
        for w in wordlist:
            regular_word[w] += 1

        result_dict[feed_id] = (lb_time, type_dict[l_type], wordlist)

    # 检查词频是否满足要求，分别存放到text.in和features.in中
    feed_id_list = []
    with codecs.open(desc_file, 'w') as ft:
        with codecs.open(feature_file, 'w') as ff:
            for feed_id, (lb_time, l_type, wordlist) in result_dict.items():
                final_words = " ".join([w for w in wordlist if regular_word[w] > 2])
                if len(final_words) > 1:
                    feed_id_list.append(feed_id)
                    ft.write(final_words + "\n")
                    if selector is not None:
                        if selector == 0:
                            ff.write("f=0\n")  # 默认feature
                        elif selector == 1:
                            ff.write("f_time=" + lb_time + "\n")
                        elif selector == 2:
                            ff.write("f_loc_" + str(l_type) + "\n")
                        elif selector == 3:
                            ff.write("f_loc_" + str(l_type) + " f_time=" + lb_time + "\n")
                    else:
                        ff.write("f_loc_" + str(l_type) + " f_time=" + lb_time + "\n")

    regular_set = set([w for w, c in regular_word.items() if c > 2])  # word set
    return feed_id_list, regular_set


'''
Description: Product Generator for the FFW and FFS products.

@author Tracy.L.Hansen@noaa.gov
@version 1.0
'''
import collections
import HydroGenerator

class Product(HydroGenerator.Product):

    def __init__(self) :
        ''' Hazard Types covered
             ('FF.W.Convective',     'Flood'),
             ('FF.W.NonConvective',  'Flood'),
             ('FF.W.BurnScar',       'Flood'),
        '''
        super(Product, self).__init__()

        # Used by the VTECEngineWrapper to access the productGeneratorTable
        self._productGeneratorName = 'FFW_FFS_ProductGenerator'

    def _initialize(self) :
        super(Product, self)._initialize()
        self._productID = 'FFW'
        self._productCategory = 'FFW_FFS'
        self._productName = 'Flash Flood Warning'
        self._purgeHours = -1.0
        self._FFW_ProductName = 'Flash Flood Warning'
        self._FFS_ProductName = 'Flash Flood Statement'
        # Polygon-based, so locations listed will be limited to within the polygon rather than county area
        self._polygonBased = True

    def defineScriptMetadata(self):
        metadata = collections.OrderedDict()
        metadata['author'] = 'Raytheon'
        metadata['description'] = 'Content generator for flash flood warning.'
        metadata['version'] = '1.0'
        return metadata

    def defineDialog(self, eventSet):
        return {}

    def executeFrom(self, dataList, eventSet, productParts=None):
        if isinstance(productParts, list) and len(productParts) > 0:
            dataList = self.correctProduct(dataList, eventSet, productParts, True)
        else:
            self.updateExpireTimes(dataList)
        return dataList

    def execute(self, eventSet, dialogInputMap):
        self._initialize()

        self.logger.info('Start ProductGeneratorTemplate:execute FFW_FFS')

        # Extract information for execution
        self._getVariables(eventSet, dialogInputMap)
        eventSetAttributes = eventSet.getAttributes()

        if not self._inputHazardEvents:
            return []

        productDicts, hazardEvents = self._makeProducts_FromHazardEvents(self._inputHazardEvents, eventSetAttributes)

        return productDicts, hazardEvents

    def _preProcessHazardEvents(self, hazardEvents):
        '''
        Add expirationTime to each event and set
        immediate cause for FF.W.NonConvective prior to VTEC processing
        '''
        for hazardEvent in hazardEvents:
            expirationTime = self._tpc.round(hazardEvent.getEndTime())
            hazardEvent.setExpirationTime(expirationTime)
            if hazardEvent.getHazardType() == 'FF.W.NonConvective':
                immediateCause = self.hydrologicCauseMapping(hazardEvent.get('hydrologicCause'))
                hazardEvent.set('immediateCause', immediateCause)

    def _groupSegments(self, segments):
        '''
         Group the segments into the products
            return a list of productSegmentGroup dictionaries

         Check the pil
          IF FFW -- make a new FFW -- there can only be one segment per FFW
          Group the segments into FFS products with same ETN
        '''
        # For short fused areal products,
        #   we can safely make the assumption of only one hazard/action per segment.
        productSegmentGroups = []
        for segment in segments:
            vtecRecords = self.getVtecRecords(segment)
            for vtecRecord in vtecRecords:  # NOTE there is only one vtecRecord / hazard to process
                pil = vtecRecord.get('pil')
                etn = vtecRecord.get('etn')
                if pil == 'FFW':
                    # Create new FFW
                    productSegmentGroup = self.createProductSegmentGroup(pil, self._FFW_ProductName, 'area', self._vtecEngine, 'counties', False,
                                            [self.createProductSegment(segment, vtecRecords)], etn=etn, formatPolygon=True)
                    productSegmentGroups.append(productSegmentGroup)
                else:  # FFS
                    # See if this record matches the ETN of an existing FFS
                    found = False
                    for productSegmentGroup in productSegmentGroups:
                        if productSegmentGroup.productID == 'FFS' and productSegmentGroup.etn == etn:
                            productSegmentGroup.addProductSegment(self.createProductSegment(segment, vtecRecords))
                            found = True
                    if not found:
                        # Make a new FFS productSegmentGroup
                       productSegmentGroup = self.createProductSegmentGroup(pil, self._FFS_ProductName, 'area', self._vtecEngine, 'counties', True,
                                                [self.createProductSegment(segment, vtecRecords)], etn=etn, formatPolygon=True)
                       productSegmentGroups.append(productSegmentGroup)
        for productSegmentGroup in productSegmentGroups:
            self._addProductParts(productSegmentGroup)
        return productSegmentGroups

    def _addProductParts(self, productSegmentGroup):
        productID = productSegmentGroup.productID
        productSegments = productSegmentGroup.productSegments
        if productID == 'FFW':
            productPartsDict = self._hydroProductParts.productParts_FFW(productSegments)
        elif productID == 'FFS':
            productPartsDict = self._hydroProductParts.productParts_FFS(productSegments)
        productParts = self.createProductParts(productPartsDict)
        productSegmentGroup.setProductParts(productParts)


#-*-encoding=utf-8-*-
from lxml import etree
import async_http
from spider import log_with_time
from spider import jsonp_json
from spider import format_price
import pdb
import re
import json
import time

burl = "http://www.zm7.cn/"
def cats_parser(url, res, rule):
    content = res['text']
    t = etree.HTML(content)
    ret = [burl + i for i in t.xpath(rule)]

    return ret

def pager(task, rule):
    pad = lambda n: "min0-max0-attr0-%d-sort_order-ASC.html" % n
    burl = task['url'][:-5] + '-'
    t = etree.HTML(task['text'])
    pagetxt = t.xpath(rule)

    ret = []

    if not pagetxt:
        log_with_time("bad response: %r" % task['url'])
        return ret

    pagetxt = pagetxt[0].text
    pagenum = re.findall("\d+", pagetxt)
    pagenum = int(pagenum[-1])
    for i in range(1, pagenum+1):
        ret.append(burl+pad(i))
    return ret

re_price = re.compile("\d+\.\d+")
def list_parser(task, rule):
    t = etree.HTML(task['text'])
    nodes = t.xpath(rule['nodes'])
    ret = []
    for node in nodes:
        gid = node.xpath(rule['gid'])
        price = node.xpath(rule['price'])
        if not gid or not price:
            log_with_time("bad response: %r" % task['url'])
            continue
        gid = burl + gid[0]
        price = price[0].text
        price = re_price.search(price).group()
        ret.append((gid, price))
    return ret

def stock_parser(task, rule):
    ret = []
    ostock = re.search("暂时无货", task['text'])

    gid = task["url"]
    price = task["price"]

    if ostock:
        stock = 0
    else:
        stock = 1

    ret = [(gid, price, stock)]
    fret = format_price(ret)
    dps = {}
    for i in fret:
        dps[i[1]] = int(time.time())
    return {"result":fret, "dps":dps}


from Tracker import Tracker
import torch
import torch.nn as nn
import torch.optim as optim
from Tracker import Tracker
from flask import Flask, render_template
from flask import jsonify
import sys
app = Flask(__name__, static_url_path='/static')
class incidenceGuesser(nn.Module):

    def __init__(self):
        super(incidenceGuesser, self).__init__()
        self.rnn = nn.LSTM(60, 7, 1)
        self.ReLu = nn.ReLU()
        self.hn = torch.rand(1,1,7)
        self.cn = torch.rand(1,1,7)

    def forward(self, x):
        x = x.view(-1,1,60)
        output, (hn, cn) = self.rnn(x, (self.hn, self.cn))
        hn = hn.detach()
        cn = cn.detach()
        self.hn = hn
        self.cn = cn
        return self.ReLu(output.view(-1,7))

PATH = './model/model-safed/mk2/cnn.pth'

model = incidenceGuesser()
model.load_state_dict(torch.load(PATH))
model.eval()

def estimate(id, latest = True):
    tracker = Tracker()
    ags = id.split("-")[0]
    tracker.track(id = ags)
    f = open("MinMax.txt", "r")
    MinMax = f.read().split(",")
    f = open(f"data/{id}.csv", "r")
    csv = f.read().split("\n")
    dates = []
    values = []
    MinMax[0] = float(MinMax[0])
    MinMax[1] = float(MinMax[1])
    for x, i in enumerate(csv):
        if i != '':
            dates.append(i.split(",")[1])
            values.append(i.split(",")[0])
    for x, i in enumerate(values):
        if i == '':
            values.pop(x)
        else:
            values[x] = (float(i) - MinMax[0]) / (MinMax[1] - MinMax[0])
    values.reverse()
    latest = values[0:60]
    dates.reverse()
    for x, i in enumerate(dates):
        dates[x] = i.split("T")[0]
    dates = dates[0:60]
    latest.reverse()
    dates.reverse()
    est = model(torch.tensor(latest))
    est = est.tolist()
    print(est, file=sys.stderr)
    latest.extend(est[0]) #remove [0] for non LSTM models readd after deletion
    for x, i in enumerate(latest):
        latest[x] = i * (MinMax[1] - MinMax[0]) + MinMax[0]
    return latest, dates

@app.route('/estimate/latest/<id>', methods = ['GET'])
def EstimateLatest(id = None):
    res = estimate(id)
    return jsonify(res)

@app.route('/track/all', methods = ['POST'])
def trackAlll():
    tracker = Tracker()
    tracker.track()
    return

@app.route("/view/main", methods = ['GET'])
def ViewMain():
    return render_template('main.html')

@app.route('/static/<path:path>')
def send_js(path):
    return app.send_static_file(path)


import requests
import pandas as pd
import multiprocessing as mp
import settings

iex_p_api_token = settings.get_iex_p_api_token()
iex_s_api_token = settings.get_iex_s_api_token()

base_url = 'https://sandbox.iexapis.com/stable/'

quote_keep_keys = ('open',
                   'high',
                   'low',
                   'close',
                   'previousClose',
                   'volume',
                   'previousVolume',
                   'avgTotalVolume',
                   'changePercent')

def filter_for_keys(d, keep_keys):
    """filters out unnecessary keys from dictionary

    Args:
        d (dictionary): original dictionary
        keep_keys (iterable): any iterable that contains the keys we want to keep

    Returns:
        dictionary: contains only needed keeps
    """
    return {key: d[key] for key in keep_keys}

def get_eod_equities_data(ticker):
    """get quote for specific ticker

    Args:
        ticker (string): stock ticker

    Returns:
        dictionary that contains necessary data
    """
    url = base_url + f'stock/{ticker}/quote?token={iex_p_api_token}'
    json_response = requests.get(url).json()

    return filter_for_keys(json_response, quote_keep_keys)

def get_length_list(l, length):
    """ separates list l into lists of length

    Args:
        l (list):
        length (int):

    Yields:
        list: of length
    """
    for i in range(0, len(l), length):
        yield l[i:i + length]

def get_eod_equities_data(tickers):

    d = {}
    for l_tickers in get_length_list(tickers, 100):
        s_symbols = ','.join(l_tickers)

        url = base_url + \
            f'stock/market/batch?symbols={s_symbols}&types=quote&token={iex_p_api_token}'

        json_response = requests.get(url).json()

        d_partial = {key: filter_for_keys(
            value['quote'], quote_keep_keys) for key, value in json_response.items()}

        d.update(d_partial)
    return d


from django.urls import path, include
from mainapp import views
from django.conf import settings
from django.conf.urls.static import static
from django.contrib.auth import views as auth_views

app_name = "mainapp"
urlpatterns = [
	path('', views.index, name='index'),
	path('login/', views.login, name='login'),
	path('signup/', views.signup, name='signup'),
	path('log_out/', views.log_out, name='log_out'),
	path('new_item/', views.new_item, name='new_item'),
	path('closedauction/', views.closedauction, name='closedauction'),
	path('update_profile/', views.update_profile, name='update_profile'),
	path('itempage.html/', views.itempage, name='itempage'),
	path('user_biddings/', views.user_biddings, name='user_biddings'),
]

#Should not have during production.
if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


#!/usr/bin/python3
"""
Defines BaseModel class.
"""
from datetime import datetime
from uuid import uuid4
import models

class BaseModel():
    """
    Base class for all models.
    Attributes:
        id (str): Unique identifier.
        created_at (datetime): Creation timestamp.
        updated_at (datetime): Last update timestamp.
    """

    def __init__(self, *args, **kwargs):
        """
        Initializes a new instance of BaseModel.
        Args:
            *args: Non-keyword arguments (unused).
            **kwargs: Keyword arguments to initialize the instance.
        """
        date = '%Y-%m-%dT%H:%M:%S.%f'
        if kwargs:
            for key, value in kwargs.items():
                if "created_at" == key:
                    self.created_at = datetime.strptime(kwargs["created_at"],
                                                        date)
                elif "updated_at" == key:
                    self.updated_at = datetime.strptime(kwargs["updated_at"],
                                                        date)
                elif "__class__" == key:
                    pass
                else:
                    setattr(self, key, value)
        else:
            self.id = str(uuid4())
            self.created_at = datetime.now()
            self.updated_at = datetime.now()
            models.storage.new(self)

    def __str__(self):
        """
        Generates a string representation of the instance.
        Returns:
            str: A string representation of the instance.
        """
        return ('[{}] ({}) {}'.
                format(self.__class__.__name__, self.id, self.__dict__))

    def save(self):
        """
        Updates the updated_at timestamp and saves the instance
        to the storage system.
        """
        self.updated_at = datetime.now()
        models.storage.save()

    def to_dict(self):
        """
        Return dictionary of BaseModel with string formats of times
        """
        dic = self.__dict__.copy()
        dic["created_at"] = self.created_at.isoformat()
        dic["updated_at"] = self.updated_at.isoformat()
        dic["__class__"] = self.__class__.__name__
        return dic

    def __repr__(self):
        """
        returns string repr
        """
        return (self.__str__())


import logging

import pandas as pd
from dash.dependencies import Input, Output
import dash_html_components as html
import dash_core_components as dcc

# local imports
from app import app
from load_data import load_csv_table
from visualize import create_bar_chart, create_scatter_plot
from wrangling import (
    subset_year_age_sex_geo
)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.basicConfig(format=formatter)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

@app.callback(
    [Output('income-distribution', 'figure'),
    Output('cumulative-distribution', 'figure')],
    [Input('page2-year', 'value'),
    Input('page2-age', 'value'),
    Input('page2-geo', 'value')])
def update_income_distribution_plots(year, age, geo):
    sex = ["Males", "Females"]
    df = pd.read_csv(r"data/processed/11100008.csv")
    logger.debug("year {}, age {}, sex geo {}".format(year, age, sex, geo))
    df = subset_year_age_sex_geo(df, year, age, sex, geo)
    hist, cumulative_plot = create_bar_chart(df, year, age, sex, geo)
    logger.debug("type(hist)")
    logger.debug(type(hist))
    return hist, cumulative_plot

@app.callback(
    Output('median-income', 'figure'),
    [Input('page3-age', 'value'),
    Input('page3-sex', 'value'),
    Input('page3-geo', 'value')])
def update_income_distribution_plots(age, sex, geo):
    logger.debug("Age: {}, Sex: {}, geo: {}".format(age, sex, geo))
    plot_data = pd.read_csv(r"data/processed/11100239.csv")
    plot = create_scatter_plot(plot_data, sex, age, geo)
    return plot