import logging

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPUnprocessableEntity, HTTPNotFound

from ziggurat_foundations.models.services.user import UserService
from appenlight.lib.utils import permission_tuple_to_dict
from appenlight.models.services.config import ConfigService
from appenlight.models.group import Group
from appenlight.models.services.group import GroupService
from appenlight.models.user import User
from appenlight.models import DBSession
from appenlight import forms
from webob.multidict import MultiDict

log = logging.getLogger(__name__)

_ = str

@view_config(
    route_name="groups_no_id",
    renderer="json",
    request_method="GET",
    permission="authenticated",
)
def groups_list(request):
    """
    Returns groups list
    """
    groups = Group.all().order_by(Group.group_name)
    list_groups = ConfigService.by_key_and_section(
        "list_groups_to_non_admins", "global"
    )
    if list_groups.value or request.has_permission("root_administration"):
        return [g.get_dict() for g in groups]
    else:
        return []

@view_config(
    route_name="groups_no_id",
    renderer="json",
    request_method="POST",
    permission="root_administration",
)
def groups_create(request):
    """
    Returns groups list
    """
    form = forms.GroupCreateForm(
        MultiDict(request.safe_json_body or {}), csrf_context=request
    )
    if form.validate():
        log.info("registering group")
        group = Group()
        # insert new group here
        DBSession.add(group)
        form.populate_obj(group)
        request.session.flash(_("Group created"))
        DBSession.flush()
        return group.get_dict(include_perms=True)
    else:
        return HTTPUnprocessableEntity(body=form.errors_json)

@view_config(
    route_name="groups",
    renderer="json",
    request_method="DELETE",
    permission="root_administration",
)
def groups_DELETE(request):
    """
    Removes a groups permanently from db
    """
    msg = _("You cannot remove administrator group from the system")
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if group:
        if group.id == 1:
            request.session.flash(msg, "warning")
        else:
            DBSession.delete(group)
            request.session.flash(_("Group removed"))
            return True
    request.response.status = 422
    return False

@view_config(
    route_name="groups",
    renderer="json",
    request_method="GET",
    permission="root_administration",
)
@view_config(
    route_name="groups",
    renderer="json",
    request_method="PATCH",
    permission="root_administration",
)
def group_update(request):
    """
    Updates group object
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()

    if request.method == "PATCH":
        form = forms.GroupCreateForm(
            MultiDict(request.unsafe_json_body), csrf_context=request
        )
        form._modified_group = group
        if form.validate():
            form.populate_obj(group)
        else:
            return HTTPUnprocessableEntity(body=form.errors_json)
    return group.get_dict(include_perms=True)

@view_config(
    route_name="groups_property",
    match_param="key=resource_permissions",
    renderer="json",
    permission="root_administration",
)
def groups_resource_permissions_list(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()
    return [
        permission_tuple_to_dict(perm)
        for perm in GroupService.resources_with_possible_perms(group)
    ]

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="GET",
    renderer="json",
    permission="root_administration",
)
def groups_users_list(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()
    props = [
        "user_name",
        "id",
        "first_name",
        "last_name",
        "email",
        "last_login_date",
        "status",
    ]
    users_dicts = []
    for user in group.users:
        u_dict = user.get_dict(include_keys=props)
        u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
        users_dicts.append(u_dict)
    return users_dicts

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="DELETE",
    renderer="json",
    permission="root_administration",
)
def groups_users_remove(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    user = UserService.by_user_name(request.GET.get("user_name"))
    if not group or not user:
        return HTTPNotFound()
    if len(group.users) > 1:
        group.users.remove(user)
        msg = "User removed from group"
        request.session.flash(msg)
        group.member_count = group.users_dynamic.count()
        return True
    msg = "Administrator group needs to contain at least one user"
    request.session.flash(msg, "warning")
    return False

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="POST",
    renderer="json",
    permission="root_administration",
)
def groups_users_add(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    user = UserService.by_user_name(request.unsafe_json_body.get("user_name"))
    if not user:
        user = UserService.by_email(request.unsafe_json_body.get("user_name"))

    if not group or not user:
        return HTTPNotFound()
    if user not in group.users:
        group.users.append(user)
        group.member_count = group.users_dynamic.count()
    props = [
        "user_name",
        "id",
        "first_name",
        "last_name",
        "email",
        "last_login_date",
        "status",
    ]
    u_dict = user.get_dict(include_keys=props)
    u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
    return u_dict


import logging
import netCDF4 as nc
import numpy as np
import pandas as pd
import pygrib
import xarray as xr

from abc import ABC, abstractmethod
from datetime import datetime, timedelta
from os import rename
from pathlib import Path
from typing import Optional, Union
from urllib.error import URLError

from globsim.download.GenericDownload import GenericDownload
from globsim.download.JRA3Q_helpers import GribSubsetter, download_daily_gribs, download_constant
from globsim.download.JRAdownload import getDate
from globsim.download.JRA3Q_dl import GetAccessor
from globsim.download.JRAdownload import getPressureLevels

logger = logging.getLogger('globsim.download')

CHUNK_LAT = 4
CHUNK_LON = 4

class J3QD(GenericDownload):

    _tunits = "hours since 1947-01-01"
    _tcal = "standard"
    REANALYSIS = 'jra3q'

    def __init__(self, pfile):
        super().__init__(pfile)
        self.retry_delay_min = 1
        par = self.par
        self._set_input_directory("jra3q")
        levels = getPressureLevels(GribSubsetter.DEFAULT_LEV_HPA, self.elevation['min'], self.elevation['max'])
        self._subsetter = GribSubsetter(round((float(par['bbW']) - 1.25) / 1.25) * 1.25,
                                        round(float(par['bbE']) / 1.25) * 1.25,
                                        round((float(par['bbS']) - 1.25) / 1.25) * 1.25,
                                        round(float(par['bbN']) / 1.25) * 1.25,
                                        levels=levels)

        # time bounds
        self.date  = getDate(par)

        # Connect
        self.credential = Path(par['credentials_directory'], ".netrc")
        self.connect()

        # chunk size for downloading and storing data [days]
        logger.info("Chunk size ignored for JRA3Q download. Using monthly")
        # self.chunk_size = par['chunk_size'] * 2000

        # variables
        logger.info(f"Variable list ignored for {self.REANALYSIS}")

        if par.get("download_only") is not None:
            self.download_only = par.get("download_only")
        else:
            self.download_only = False

    def file_in_progress(self, filename):
        return str(Path(filename).with_suffix(".inp"))

    def complete(self, filename):
        f = self.file_in_progress(filename)
        rename(f, filename)

    def connect(self):
        try:
            self.access = GetAccessor(netrc_file=self.credential)
        except ValueError:
            logger.error("Could not find credentials. Aborting")
            raise ValueError(f"Could not find credentials file {self.credential}")
        except URLError:
            logger.error("Could not connect to JRA server. This aint gonna work.")

    @staticmethod
    def date2num(dt:datetime) -> float:
        t = nc.date2num(dt, J3QD._tunits, J3QD._tcal)
        t = np.round(t, 5)  # catch any floating-point errors
        return t

    def nc_filename(self, year, month, kind="sa") -> str:
        return f"{self.REANALYSIS}_{kind}_{year}{month}01_to_{year}{month}31.nc"

    def retrieve(self):
        days = pd.date_range(self.date['beg'], self.date['end'] + timedelta(days=1))  # add extra day but end when we get to it

        # Constant surface data
        to_grib = [download_constant(self.access, self.directory)]
        t = days[0]
        to_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_to_{t.strftime(r"%Y%m%d")}_to_{t.strftime(r"%Y%m%d")}.nc')
        gribs_to_netcdf(*to_grib,
                        netcdf_file=to_ncfile,
                        tstart=self.date2num(t),
                        tstop=self.date2num(days[1]),
                        kind="to",
                        subsetter=self._subsetter)

        # Temporal data
        running_sa, running_sf, running_pl, running_dates = [], [], [], []

        for date in days:
            end_of_days = (date == days[-1])

            # Aggregation subroutine
            if (not self.download_only) and ((date.day == 1 and len(running_dates) > 1) or end_of_days):
                t1 = running_dates[0]
                t2 = running_dates[-1]
                tstart = J3QD.date2num(t1)
                tstop = J3QD.date2num(date)  # t2 + timedelta(days=1)

                sa_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sa_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
                sf_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sf_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
                pl_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_pl_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')

                # aggregate gribs then delete them
                logger.info(f"Aggregating gribs to netcdf: {t1} to {t2}")

                if not sa_ncfile.is_file():
                    gribs_to_netcdf(*running_sa, netcdf_file=self.file_in_progress(sa_ncfile),
                                    tstart=tstart, tstop=tstop, kind="sa", subsetter=self._subsetter, engine="pygrib")
                    self.complete(sa_ncfile)

                if not sf_ncfile.is_file():
                    gribs_to_netcdf(*running_sf, netcdf_file=self.file_in_progress(sf_ncfile),
                                    tstart=tstart, tstop=tstop, kind="sf", subsetter=self._subsetter, engine="pygrib")
                    self.complete(sf_ncfile)

                if not pl_ncfile.is_file():
                    gribs_to_netcdf(*running_pl, netcdf_file=self.file_in_progress(pl_ncfile),
                                    tstart=tstart, tstop=tstop, kind="pl", subsetter=self._subsetter, engine="xarray")
                    self.complete(pl_ncfile)

                #  delete grib files
                logger.info("Not deleting grib files")
                #for grib in running_sa + running_sf + running_pl:
                    #Path(grib).unlink()

                # clear running
                running_sa, running_sf, running_pl, running_dates = [], [], [], []

            if end_of_days:
                break

            # Download
            logger.info(f"Downloading gribs for {date}")
            sa, sf, pl = download_daily_gribs(self.access, self.directory, date.year, date.month, date.day)
            running_sa += sa
            running_sf += sf
            running_pl += pl
            running_dates.append(date)

class VariableError(ValueError):
    pass

class ConversionHandler(ABC):
    """
    Handles the translation of grib files into a netcdf file
    """
    _include = []
    _dims = ()

    def __init__(self,
                 tstart: float,
                 tstop: float,
                 subsetter:Optional[GribSubsetter] = None):

        self._tstart = tstart
        self._tstop = tstop
        self.subsetter = subsetter if subsetter else GribSubsetter(0, 360, -90, 90)

    @property
    def tstep(self):
        raise NotImplementedError("implement in child class")

    @property
    def times(self):
        t = np.arange(self._tstart, self._tstop, self.tstep)
        return t

    @abstractmethod
    def empty_file(self, *args, **kwargs):
        raise NotImplementedError("implement in child class")

    def create_variable(self, shortName, ncd, ncvar_dict):
        var = ncd.createVariable(shortName, "f8", self._dims, chunksizes=self.chunk_sizes(ncd))
        for attr in ncvar_dict:
            var.setncattr(attr, ncvar_dict[attr])

    def chunk_sizes(self, ncd):
        return None

    def valid_record(self, record):
        is_valid = record.shortName in self._include
        return is_valid

    def get_valid_vars(self, dataset) -> "list[str]":
        return [v for v in dataset.variables if v in self._include.keys()]

    def xread_grib(self, gribfile) -> "xr.Dataset":
        d = xr.open_dataset(gribfile, engine='cfgrib')
        return d

    def xgrib_to_nc(self, ncfile, gribfile):
        logger.debug(f"de-gribbing {Path(gribfile).name} with xarray")
        dataset = self.xread_grib(gribfile)
        for v in self.get_valid_vars(dataset):
            var = dataset[v]
            self.xvar_to_nc(var, ncfile)

    def xvar_to_nc(self, var, ncfile):
        if self.subsetter is not None:
            vals, lats, lons = self.subsetter.xsubset(var)
        else:
            vals, lats, lons = var.values, var.latitude.values, var.longitude.values

        # check if ncfile exists
        if not Path(ncfile).exists():
            logger.warning(f"missing file {ncfile}. Creating it")
            self.empty_file(ncfile, lats, lons, self.times)

        with nc.Dataset(ncfile, 'a') as ncd:
            # check if variable is in ncfile
            # if not, add it
            ncvar_dict = {"units" : var.units,
                          "long_name" : var.name}

            if var.name not in ncd.variables:
                self.create_variable(var.name, ncd, ncvar_dict)

            # check if timestep is in ncfile
            timestep = J3QD.date2num(pd.Timestamp(var.valid_time.values).to_pydatetime())
            try:
                t_i = np.where(self.times == timestep)[0][0]
            except IndexError:
                t_i = self.handle_missing_timestep(var.valid_time.values, ncfile)

            # write data
            if t_i is not None:
                self.xwrite_record(ncd=ncd,
                                   varname=var.name,
                                   t_i=t_i,
                                   vals=vals,
                                   var=var)

    def grib_to_nc(self, ncfile, record):
        varname = record.shortName

        if not self.valid_record(record):
            raise VariableError(f"{varname}")

        if self.subsetter is not None:
            vals, lats, lons = self.subsetter.subset(record)
        else:
            vals, lats, lons = record.data()

        # check if ncfile exists
        if not Path(ncfile).exists():
            logger.warning(f"missing file {ncfile}. Creating it")
            lats = lats[:,0]
            lons = lons[0,:]

            self.empty_file(ncfile, lats, lons, self.times)  # if not, create it

        with nc.Dataset(ncfile, 'a') as ncd:
            # check if variable is in ncfile
            # if not, add it
            ncvar_dict = {"units" : record.units,
                          "long_name" : record.name,
                          "comment": record.__repr__()}

            if varname not in ncd.variables:
                self.create_variable(record.shortName, ncd, ncvar_dict)

            # check if timestep is in ncfile
            timestep = J3QD.date2num(record.validDate)
            try:
                t_i = np.where(self.times == timestep)[0][0]
            except IndexError:
                t_i = self.handle_missing_timestep(record, ncfile)

            # write temporary array
            # self.write_mem(ncd, t_i, vals, record)

            # write data
            if t_i is not None:
                self.write_record(ncd=ncd,
                                  varname=varname,
                                  t_i=t_i,
                                  vals=vals,
                                  record=record)

    def handle_missing_timestep(self, date, ncfile):
        print(f"err: timestamp {date} not in ncfile {ncfile}")
        return None

    def write_record(self, ncd, varname, t_i, vals, record):
        ncd[varname][t_i, :, :] = vals

    def xwrite_record(self, ncd, varname, t_i, vals, var):
        ncd[varname][t_i, :, :] = vals

    def valid_messages(self):
        m = [d["messagenumber"] for d in list(self._include.values()) if "messagenumber" in d]
        return m

class ToConverter(ConversionHandler):
    _include = {"z": None}
    _dims = ("time", "latitude", "longitude",)

    @property
    def tstep(self):
        return 24

    def empty_file(self, filename:str, lats, lons, times):
        empty_surface_file(filename, lats, lons, times)

    def write_record(self, ncd, varname, t_i, vals, record):
        ncd[varname][0, :, :] = vals

    def xwrite_record(self, ncd, varname, t_i, vals, var):
        ncd[varname][0, :, :] = vals

    def handle_missing_timestep(self, date, ncfile):
        return 0

class SaConverter(ConversionHandler):
    _include = {"sp": None,
                "2t": None,
                "2sh": None,
                "2r": None,
                "10u": None,
                "10v": None}
    _dims = ("time", "latitude", "longitude",)

    @property
    def tstep(self):
        # 6-hourly, calculated on the fly in case we change time units in the future
        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
        return t2 - t1

    def chunk_sizes(self, ncd):
        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)

    def xread_grib(self, gribfile) -> xr.Dataset:
        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
        return d

    def empty_file(self, filename:str, lats, lons, times):
        empty_surface_file(filename, lats, lons, times)

class FcstConverter(ConversionHandler):
    _include = {"tprate": {"messagenumber": 6},
                "sp": {"messagenumber": 12},
                'dswrf': {"messagenumber": 13},
                "dlwrf": {"messagenumber": 17}
                }
    _dims = ("time", "latitude", "longitude",)

    @property
    def tstep(self):
        # 6-hourly, calculated on the fly in case we change time units in the future
        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
        t2 = J3QD.date2num(datetime(1980, 3, 1, 1))
        return t2 - t1

    def valid_record(self, record):
        return record.messagenumber in self.valid_messages()

    def empty_file(self, filename:str, lats, lons, times):
        empty_surface_file(filename, lats, lons, times)

    def xread_grib(self, gribfile) -> xr.Dataset:
        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
        return d

    def chunk_sizes(self, ncd):
        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)

class PlConverter(ConversionHandler):
    _dims = ("time", "level", "latitude", "longitude",)
    _include = {"t":None,  # temperature
                "u":None,  # u-wind
                "v":None,  # v-wind
                "q":None,  # specific humidity
                "r":None,   # relative humidity
                "gh":None    # geopotential
                }

    def __init__(self, tstart, tstop, subsetter):
        super().__init__(tstart, tstop, subsetter)
        # self.mem_array = np.empty(len(self.times), len(self.subsetter.levels), len(self.subsetter.lats), len(self.subsetter.lons))

    def valid_record(self, record):
        good_variable = record.shortName in self._include
        lv_hPa = record.level if record.pressureUnits == "hPa" else record.level / 10
        good_level = lv_hPa in self.subsetter.levels

        return (good_variable and good_level)

    def xread_grib(self, gribfile) -> xr.Dataset:
        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'isobaricInhPa'}})
        return d

    @property
    def tstep(self):
        # 6-hourly, calculated on the fly in case we change time units in the future
        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
        return t2 - t1

    def empty_file(self, filename:str, lats, lons, times):
        levels = self.subsetter.levels
        empty_pressure_levels_file(filename, lats, lons, times, levels)

    def write_record(self, ncd, varname, t_i, vals, record):
        lev = record.level if record.pressureUnits == "hPa" else record.level / 10

        lev_i = np.where(ncd['level'][:] == lev)[0][0]

        ncd[varname][t_i, lev_i, :, :] = vals

    def xwrite_record(self, ncd, varname, t_i, vals, var):
        ncd[varname][t_i, :, :, :] = vals[::-1, :, :]  # put levels in backwards to match JRA55

    def chunk_sizes(self, ncd):
        return (ncd['time'].shape[0], ncd['level'].shape[0], CHUNK_LAT, CHUNK_LON,)

def empty_constant_file(filename: str, lats, lons):
    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
        _ = ncd.createDimension('latitude', len(lats))
        _ = ncd.createDimension('longitude', len(lons))

        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
        var_lat.units = "degrees_north"
        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
        var_lon.units = "degrees_east"

        var_lat[:] = lats
        var_lon[:] = lons

def empty_surface_file(filename:str, lats, lons, times):
    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
        _ = ncd.createDimension('time', len(times))
        _ = ncd.createDimension('latitude', len(lats))
        _ = ncd.createDimension('longitude', len(lons))

        var_time = ncd.createVariable('time', 'f8', ('time',))
        var_time.units = J3QD._tunits
        var_time.calendar = J3QD._tcal
        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
        var_lat.units = "degrees_north"
        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
        var_lon.units = "degrees_east"

        var_time[:] = times
        var_lat[:] = lats
        var_lon[:] = lons

def empty_pressure_levels_file(filename:str, lats, lons, times, levs):
    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
        _ = ncd.createDimension('level', len(levs))
        _ = ncd.createDimension('time', len(times))
        _ = ncd.createDimension('latitude', len(lats))
        _ = ncd.createDimension('longitude', len(lons))

        var_time = ncd.createVariable('time', 'f8', ('time',))
        var_time.units = J3QD._tunits
        var_time.calendar = J3QD._tcal
        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
        var_lat.units = "degrees_north"
        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
        var_lon.units = "degrees_east"
        var_lev = ncd.createVariable('level', 'f8', ('level',))
        var_lev.units = "hPa"

        var_time[:] = times
        var_lat[:] = lats
        var_lon[:] = lons
        var_lev[:] = levs

def gribs_to_netcdf(*grib_files:"Union[str,Path]", netcdf_file:"Union[str,Path]", tstart:float, tstop:float, kind:str, subsetter=None, engine="xarray"):
    logger.info(f"creating {netcdf_file}")
    notified = []
    Converter = get_converter(kind)
    converter = Converter(tstart, tstop, subsetter)

    for gribfile in grib_files:
        if engine == "xarray":
            converter.xgrib_to_nc(netcdf_file, gribfile)

        elif engine == "pygrib":
            logger.debug(f"de-gribbing {Path(gribfile).name} with pygrib")

            with pygrib.open(str(gribfile)) as f:  # pygrib doesn't like Path objects
                for record in f:
                    if not converter.valid_record(record):
                        continue
                    try:
                        converter.grib_to_nc(netcdf_file, record)
                    except VariableError as e:
                        if e.args[0] not in notified:
                            logger.warning(f"warning: skip variable {e}")
                            notified.append(e.args[0])

        else:
            raise ValueError("engine must be 'pygrib' or 'xarray'")

def get_converter(kind:str) -> "type[ConversionHandler]":
    if kind == "sf":
        return FcstConverter
    elif kind == "sa":
        return SaConverter
    elif kind == "pl":
        return PlConverter
    elif kind == "to":
        return ToConverter
    else:
        raise ValueError(f"unknown kind {kind}")


# coding=utf-8

import json
import urllib
import urllib2
import os

class ShouHuiJianZhu(object):
    def __init__(self):
        pass

    def gain_response(self, base_url, headers):
        # url = 'http://image.so.com/j?q=%E6%89%8B%E7%BB%98%E5%BB%BA%E7%AD%91&src=srp' \
        #       '&correct=%E6%89%8B%E7%BB%98%E5%BB%BA%E7%AD%91&pn=60&ch=&sn=96&cn=0&gn=0&kn=36'
        page = 0
        for i in range(10):
            page += 60
            params = {
                'q': '手绘建筑',
                'src': 'srp',
                'correct': '手绘建筑',
                'pn': page,
                'ch': '',
                'cn': '0',
                'gn': '0',
                'kn': '36'
            }
            data = urllib.urlencode(params)
            url = base_url + data
            request = urllib2.Request(url, headers=headers)
            response = urllib2.urlopen(request).read()
            json_data = json.loads(response)
            # print(json_data)
            list = json_data.get('list')
            for image in list:
                image_url = image.get('thumb')
                self.download_image(image_url)

    def download_image(self, image_url):
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'}
        request = urllib2.Request(image_url, headers=headers)
        response = urllib2.urlopen(request).read()
        filename = image_url[-10:]
        # os.makedirs('images')
        # folder = '/images'
        # if not os.path.exists(folder):
        #     os.makedirs(folder)
        with open(filename, 'wb')as f:
            f.write(response)

    def main(self):
        base_url = 'http://image.so.com/j?'
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'}
        self.gain_response(base_url, headers)

if __name__ == '__main__':
    jz = ShouHuiJianZhu()
    jz.main()


import random

from sklearn.linear_model import LogisticRegression

from metal.utils import convert_labels
from metal.metrics import metric_score

class LogisticRegressionWrapper(object):
    """A wrapper around scikit-learn's LogisticRegression class

    The wrapper is necessary both to convert labels from categorical to one-zero
    and to match the interface expected by snorkel-metal's ModelTuners.
    """
    def __init__(self, C=1.0, penalty='l2', seed=None):
        if seed:
            random.seed(seed)
        self.model = LogisticRegression(C=C, penalty='l2')

    def train(self, X, Y, X_dev=None, Y_dev=None, **kwargs):
        Y_bin = convert_labels(Y, 'categorical', 'onezero')
        self.model.fit(X, Y_bin)

    def predict(self, X):
        return self.model.predict(X)

    def score(self, X, Y, metric='f1', verbose=True):
        Y = convert_labels(Y, 'categorical', 'onezero')
        Y_p = self.predict(X)

        metric_list = metric if isinstance(metric, list) else [metric]
        scores = []
        for metric in metric_list:
            score = metric_score(Y, Y_p, metric)
            scores.append(score)
            if verbose:
                print(f"{metric.capitalize()}: {score:.3f}")

        if isinstance(scores, list) and len(scores) == 1:
            return scores[0]
        else:
            return scores


#!/usr/bin/env python
# coding: utf-8

# In[3]:

import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from string import punctuation
from heapq import nlargest

# In[4]:

from newspaper import Article   #뉴스기사 text크롤링 해주는 라이브러리

# In[97]:

url = 'https://www.nytimes.com/2023/04/09/world/europe/europe-shells-ukraine-ammunition.html'

# In[98]:

article = Article(url)

# In[99]:

article.download()

# In[100]:

article.parse()

# In[101]:

newsTitle = article.title

# In[102]:

newstext= article.text

# In[115]:

# summarize code

def summarize(text,per):
    nlp = spacy.load('en_core_web_sm')
    doc = nlp(text)
    tokens = [token.text for token in doc]
    word_frequencies = {}
    for word in doc:
        if word.text.lower() not in list(STOP_WORDS):
            if word.text.lower() not in punctuation:
                if word.text not in word_frequencies.keys():
                    word_frequencies[word.text]=1
                else:
                    word_frequencies[word.text] += 1
    max_frequency = max(word_frequencies.values())
    for word in word_frequencies.keys():
        word_frequencies[word]= word_frequencies[word]/max_frequency
    sentence_tokens = [sent for sent in doc.sents]
    sentence_scores = {}
    for sent in sentence_tokens:
        for word in sent:
            if word.text.lower() in word_frequencies.keys():
                sentence_scores[sent]=word_frequencies[word.text.lower()]
            else:
                sentence_scores[sent]+=word_frequencies[word.text.lower()]

    select_length = int(len(sentence_tokens)*per)
    summary = nlargest(select_length,sentence_scores,key=sentence_scores.get)
    final_summary =[word.text for word in summary]
    summary = ''.join(final_summary)
    return summary

# In[116]:

# 번역 코드

import googletrans

translator = googletrans.Translator()

instr = '안녕하세요'

outstr = translator.translate(instr,dest='en', src = 'auto')

print(f"{instr} => {outstr.text}")

# In[117]:

summarized_text = summarize(newstext,0.05)
summarized_text

# In[ ]:


import lightning as pl
import torch
import transformers
from transformers import AdamW, T5ForConditionalGeneration, T5TokenizerFast as T5Tokenizer
from model import SummaryModel

tokenizer = T5Tokenizer.from_pretrained("t5-base")

checkpoint="base-checkpoint.ckpt"
model = SummaryModel()
model=model.load_from_checkpoint(checkpoint,map_location="cpu")

def generate_summary(question):

    inputs_encoding =  tokenizer(
        question,
        add_special_tokens=True,
        max_length= 512,
        padding = 'max_length',
        truncation='only_first',
        return_attention_mask=True,
        return_tensors="pt"
        )

    generate_ids = model.model.generate(
        input_ids = inputs_encoding["input_ids"],
        attention_mask = inputs_encoding["attention_mask"],
        max_length = 128,
        num_beams = 4,
        num_return_sequences = 1,
        no_repeat_ngram_size=2,
        early_stopping=True,
        )

    preds = [
        tokenizer.decode(gen_id,
        skip_special_tokens=True,
        clean_up_tokenization_spaces=True)
        for gen_id in generate_ids
    ]

    return "".join(preds)


from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def formatar_preco(preco):
    preco = preco[:-3]
    preco = preco[:-2] + ',' + preco[-2:]
    return preco

register.filter('formatar_preco', formatar_preco)


import pandas as pd
import numpy as np
import datetime as dt
from divvy.ml_logic.data_import import get_station_data
import pygeohash as gh
import os

precision_level= int(os.environ.get("PRECISION_LEVEL"))

def compute_geohash_stations(precision: int = 4) -> np.ndarray:
    """
    Add a geohash (ex: “dr5rx”) of len “precision” = 5 by default
    corresponding to each (lon,lat) tuple, for pick-up, and drop-off
    """

    df_stations=get_station_data()
    assert isinstance(df_stations, pd.DataFrame)
    df_stations["geohash"] = df_stations.apply(lambda x: gh.encode(x.lat,
                                                                   x.lon,
                                                                   precision=precision),
                                               axis=1)
    df_stations_reduced=df_stations[["name","geohash"]]
    df_stations_reduced.rename(columns={"name":"station_name"}, inplace=True)

    return df_stations_reduced

def weather_cleaning(df):
    '''
    This function cleans the weather data by removing characters from the dt_iso variable
    to allow the creation of a datetime variable

    It also removes the duplicates and selects the relevant features

    '''

    features = ["dt_iso","temp","pressure","humidity","wind_speed","wind_deg","clouds_all"]

    df_clean_tmp=df[features]

    df_clean_tmp = df_clean_tmp.drop_duplicates()

    df_clean_tmp["dt_iso"] = df_clean_tmp["dt_iso"].apply(lambda x: x.replace("+0000 UTC", ""))

    df_clean_tmp["dt_iso"] = pd.to_datetime(df_clean_tmp["dt_iso"])
    df_clean_tmp['hourly_data']=df_clean_tmp['dt_iso']

    return df_clean_tmp

def cleaning_divvy_gen_agg(df):

    df['started_at']=pd.to_datetime(df['started_at'])
    df['ended_at']=pd.to_datetime(df['ended_at'])
    df['hourly_data_started'] = df.started_at.dt.round('60min')
    df['hourly_data_ended'] = df.ended_at.dt.round('60min')

    df_departures=df[[
                    "start_station_name",
                    "start_station_id",
                    "hourly_data_started"]]

    df_departures=df_departures.rename(columns={'hourly_data_started':'hourly_data',
                                                "start_station_name":"station_name",
                                                "start_station_id": "station_id"})

    df_departures["nb_departures"]=1

    df_dep_agg=df_departures.groupby(by=["station_name",
                                        "station_id",
                                        'hourly_data']).count().reset_index()

    station_df = compute_geohash_stations(precision = precision_level)
    df_dep_agg_geohash = df_dep_agg.merge(station_df, how="left", on="station_name")
    df_dep_agg_geohash = df_dep_agg_geohash.drop(columns=["station_name","station_id"])
    df_dep_final=df_dep_agg_geohash.groupby(by=["geohash",
                                        'hourly_data']).mean().reset_index()

    df_arrivals=df[["end_station_name",
                 "end_station_id",
                 "hourly_data_ended"]]

    df_arrivals=df_arrivals.rename(columns={'hourly_data_ended':'hourly_data',
                                            "end_station_name":"station_name",
                                            "end_station_id": "station_id"})
    df_arrivals["nb_arrivals"]=1

    df_arr_agg=df_arrivals.groupby(by=["station_name",
                                            "station_id",
                                            'hourly_data']).count().reset_index()

    df_arr_agg_geohash = df_arr_agg.merge(station_df, how="left", on="station_name")
    df_arr_agg_geohash = df_arr_agg_geohash.drop(columns=["station_name","station_id"])
    df_arr_final=df_arr_agg_geohash.groupby(by=["geohash",
                                        'hourly_data']).mean().reset_index()

    merge_ratio=pd.merge(
    df_dep_final,
    df_arr_final,
    how="outer",
    on=['hourly_data',"geohash"])

    merge_ratio["nb_departures"] = merge_ratio["nb_departures"].replace(np.nan, 0)
    merge_ratio["nb_arrivals"] = merge_ratio["nb_arrivals"].replace(np.nan, 0)

    merge_ratio['ratio']=merge_ratio['nb_departures']/merge_ratio['nb_arrivals']

    return merge_ratio

def cleaning_divvy_gen(df):

    df['started_at']=pd.to_datetime(df['started_at'])
    df['ended_at']=pd.to_datetime(df['ended_at'])
    df['hourly_data_started'] = df.started_at.dt.round('60min')
    df['hourly_data_ended'] = df.ended_at.dt.round('60min')

    df_departures=df[[
                    "start_station_name",
                    "start_station_id",
                    "hourly_data_started"]]

    df_departures=df_departures.rename(columns={'hourly_data_started':'hourly_data',
                                                "start_station_name":"station_name",
                                                "start_station_id": "station_id"})

    df_departures["nb_departures"]=1

    df_dep_agg=df_departures.groupby(by=["station_name",
                                        "station_id",
                                        'hourly_data']).count().reset_index()

    df_arrivals=df[["end_station_name",
                 "end_station_id",
                 "hourly_data_ended"]]

    df_arrivals=df_arrivals.rename(columns={'hourly_data_ended':'hourly_data',
                                            "end_station_name":"station_name",
                                            "end_station_id": "station_id"})
    df_arrivals["nb_arrivals"]=1

    df_arr_agg=df_arrivals.groupby(by=["station_name",
                                            "station_id",
                                            'hourly_data']).count().reset_index()

    merge_ratio=pd.merge(
    df_dep_agg,
    df_arr_agg,
    how="outer",
    on=['hourly_data',"station_name","station_id"])

    merge_ratio["nb_departures"] = merge_ratio["nb_departures"].replace(np.nan, 0)
    merge_ratio["nb_arrivals"] = merge_ratio["nb_arrivals"].replace(np.nan, 0)

    merge_ratio['ratio']=merge_ratio['nb_departures']/merge_ratio['nb_arrivals']

    return merge_ratio

def merge_divvy_weather(df_bikes , df_weather):

    '''
    This function merges the Divvy data aggregated at the station and hourly level
    with the hourly weather cleaned data
    '''

    merged_df = df_bikes.merge(df_weather,
    how="left",
    on='hourly_data')

    return merged_df

def features_target(df, target):
    '''
    This function creates the features and target dataframes, the latter
    being created by identifying the target (nb_departures, nb_arrivals, ratio of the two)
    as an argument
    '''

    features_df = df.drop(columns=["ratio","nb_departures","nb_arrivals"])
    target_df =  df[target]

    return features_df, target_df

def get_retained_geohash(df):

    geohash_df = df[["geohash"]]
    geohash_df = geohash_df.drop_duplicates()

    return geohash_df


from new_constraints import constraints
from mysolver_interface import call_my_solver
from lmisr_interface import call_solver
import numpy as np
import torch.nn.functional as F
import torch
from verify import aux_array_as_hex, parseaux
from tqdm import tqdm

np.set_printoptions(precision = 2, suppress = True)

# an interesting 3x3 aux consisting only of input and aux-based hyperplanes
#["ffff0003ffffffff", "10007000f550f770", "ecffccccccffccc"]

#aux_hex = ["1033ff675f3fff7f", "0f0b2f02cf8bff2f", "ffffffefcd8b5544"]
aux_hex = ["3333ffff2200bbbb3333ffff2220bbbb",
 "0000000000bb57770000000000035577",
 "3fffdfff0eff0fff0fff0fff00ff01ff"]
aux_hex = None

if aux_hex is not None:
    aux_array = parseaux(aux_hex)
    aux = torch.tensor(aux_array)
else:
    aux = None

n1, n2 = 3,3
A = len(aux_hex) if aux_hex is not None else 0
desired = (0,1,2,3,4)
included = (5,)
num_inputs = n1 + n2 + A +  (len(included) if included is not None else 0)
num_outputs = (len(desired) if desired is not None else n1 + n2)
num_vars = num_inputs + num_outputs
original_vars = num_vars
hyperplanes = []
radius = None
search_iterations = 50
num_samples = 150
search_sigma = .2
cache = {}
for i in range(12):
    candidates = []
    loop = tqdm(range(search_iterations), leave = True)
    #current_weight = torch.randn(num_vars)
    #current_weight[num_inputs:original_vars] = 0
    for j in range(num_samples):
        current_weight = torch.randn(num_inputs)
        current_weight = F.normalize(current_weight, dim=0)
        current_bias = torch.randn(1).clamp(0,1).item()
        current_plane = (current_weight, current_bias)
        M, terms, correct = constraints(n1, n2, aux = aux, radius = radius, desired = desired, included = included, hyperplanes = hyperplanes + [current_plane], auxfix = True)
        M_hash = hash(M.numpy().data.tobytes())
        if M_hash in cache:
            objective = cache[M_hash]
        else:
            objective = call_my_solver(M.to_sparse_csc(), tolerance = 1e-8, fullreturn = False)
            cache[M_hash] = objective
        candidates.append({'plane': current_plane, 'val': objective})
    best = min(candidates, key = lambda item: item['val'])
    current_plane = best['plane']
    current_weight, current_bias = current_plane
    for j in loop:
        search_sigma = 0.05 * (1 - j/search_iterations)#np.exp(-3 * j/search_iterations)
        samples = []
        for k in range(num_samples):
            #new_weight = search_sigma * torch.randn(num_vars) + current_weight
            new_weight = search_sigma * torch.randn(num_inputs) + current_weight
            new_weight[num_inputs:original_vars] = 0
            new_weight = F.normalize(new_weight, dim=0)
            #print(new_weight)

            new_bias = (search_sigma * torch.randn(1) + current_bias).clamp(0,1).item()
            new_plane = (new_weight, new_bias)
            M, terms, correct = constraints(n1, n2, aux = aux, radius = radius, desired = desired, included = included, hyperplanes = hyperplanes + [new_plane], auxfix = True)
            M_hash = hash(M.numpy().data.tobytes())
            if M_hash in cache:
                objective = cache[M_hash]
            else:
                objective = call_my_solver(M.to_sparse_csc(), tolerance = 1e-8, fullreturn = False)
                cache[M_hash] = objective
            if objective < 1e-2:
                print("done")
                aux_array = torch.t(correct[...,original_vars:]).numpy().astype(np.int8)
                if aux is not None:
                    aux_array = np.concatenate([aux_array, aux])
                print(aux_array_as_hex(aux_array))
                objective = call_solver(n1, n2, aux_array)
                print(objective)
                exit()

            candidates.append({'plane': new_plane, 'val': objective})
        best = min(candidates, key = lambda item: item['val'])
        current_plane = best['plane']
        current_weight, current_bias = current_plane
        loop.set_postfix(objective = best['val'])
    else:
        plane_dict = min(candidates, key = lambda item: item['val'])
        print(f'current min rho: {plane_dict["val"]}')
        if plane_dict["val"] < 50:
            radius = None
        hyperplanes.append(plane_dict['plane'])
        num_vars += 1
        continue
    break


from django.contrib import admin
from .models import *

# Register your models here.
class CustomUserAdmin(admin.ModelAdmin):
    list_display = ('phone_number', 'joined', 'email', 'is_superuser')
    date_hierarchy = 'joined'
    list_filter = ('joined',    )
    search_fields = ('email', 'phone_number')

    fieldsets = (
        (None, {
            'fields': (
                'email', 'password'
            ),
        }),
        ('User status', {
            'fields': (
                'is_staff', 'is_superuser', 'is_active'
            ),
        }),
    )

admin.site.register(CustomUser, CustomUserAdmin)
admin.site.register(UserOTP)


from vinfraclient.cmd.base import Lister, ShowOne, TaskCommand
from vinfraclient.cmd.node.disk import DiskOption
from vinfraclient.utils import get_cluster, find_resource

def node_arg(parser):
    parser.add_argument(
        "node",
        metavar="<node>",
        help="Node ID or hostname"
    )

class ListNode(Lister):
    _description = "List storage nodes."
    _default_fields = ['id', 'host', 'is_primary', 'is_online', 'is_assigned',
                       'is_in_ha']

    @staticmethod
    def find_network_name(networks, network_id):
        for network in networks:
            if network.id == network_id:
                return network.name
        return ''

    def do_action(self, parsed_args):
        nodes = self.app.vinfra.nodes.list()
        return nodes

class ShowNode(ShowOne):
    _description = "Show storage node details."

    def configure_parser(self, parser):
        node_arg(parser)

    def do_action(self, parsed_args):
        return find_resource(self.app.vinfra.nodes, parsed_args.node)

class ForgetNode(TaskCommand):
    _description = "Remove a node from the storage cluster."

    def configure_parser(self, parser):
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)
        task = node.delete_async()
        return task

class JoinNode(TaskCommand):
    _description = "Join a node to the storage cluster"

    def configure_parser(self, parser):
        parser.add_argument(
            "--disk",
            dest="disks",
            action="append",
            metavar=DiskOption.metavar,
            type=DiskOption.from_string,
            help=DiskOption.help,
        )
        parser.add_argument(
            "node",
            metavar="<node>",
            help="Node ID or hostname"
        )

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)

        disks = None
        if parsed_args.disks:
            disks = []
            for disk in parsed_args.disks:
                disk.disk = find_resource(node.disks_manager, disk.disk).id
                if disk.params.get('journal_disk_id'):
                    disk.params['journal_disk_id'] = find_resource(
                        node.disks_manager, disk.params['journal_disk_id']).id
                disks.append(disk.to_dict())

        cluster = get_cluster(self.app.vinfra)
        return cluster.join_node_async(node.id, disks=disks)

class ReleaseNode(TaskCommand):
    _description = ("Release a node from the storage cluster.\n"
                    "Start data migration from the node as well as cluster "
                    "replication and rebalancing to meet the configured "
                    "redundancy level.")

    def configure_parser(self, parser):
        parser.add_argument(
            "--force",
            action="store_true",
            default=None,
            help="Release node without data migration."
        )
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)
        return node.release_async(force=parsed_args.force)

class MaintenanceNodeStatus(ShowOne):
    _description = "Show node maintenance details."

    def configure_parser(self, parser):
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)
        return node.maintenance_status()

class MaintenanceNodeStart(TaskCommand):
    _description = "Start node maintenance."

    def configure_parser(self, parser):
        parser.add_argument(
            "--iscsi-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore ISCSI evacuation during maintenance"
        )
        parser.add_argument(
            "--compute-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore compute evacuation during maintenance"
        )
        parser.add_argument(
            "--s3-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore S3 evacuation during maintenance"
        )
        parser.add_argument(
            "--storage-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore storage evacuation during maintenance"
        )
        parser.add_argument(
            "--alua-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore Block Storage target groups during maintenance"
        )
        parser.add_argument(
            "--nfs-mode",
            metavar='<mode>',
            choices=["ignore"],
            help="Ignore NFS evacuation during maintenance"
        )
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)

        return node.maintenance_start(
            iscsi_mode=parsed_args.iscsi_mode,
            compute_mode=parsed_args.compute_mode,
            s3_mode=parsed_args.s3_mode,
            storage_mode=parsed_args.storage_mode,
            alua_mode=parsed_args.alua_mode,
            nfs_mode=parsed_args.nfs_mode,
        )

class MaintenanceNodeStop(TaskCommand):
    _description = "Return node to operation."

    def configure_parser(self, parser):
        parser.add_argument(
            "--ignore-compute",
            action='store_true',
            default=False,
            help="Ignore compute resources while returning a node to operation"
        )
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)
        return node.maintenance_stop(
            ignore_compute=parsed_args.ignore_compute
        )

class MaintenanceNodePrecheck(TaskCommand):
    _description = "Start node maintenance precheck."

    def configure_parser(self, parser):
        node_arg(parser)

    def do_action(self, parsed_args):
        node = find_resource(self.app.vinfra.nodes, parsed_args.node)
        return node.maintenance_precheck()


# Run-length encoding is a fast and simple method of encoding strings.
# The basic idea is to represent repeated successive characters as a single count
#  and character. For example, the string "AAAABBBCCDAA" would be encoded as "4A3B2C1D2A".

# Implement run-length encoding and decoding.
# You can assume the string to be encoded have no digits
# and consists solely of alphabetic characters.
# You can assume the string to be decoded is valid.

def runEncode(string):

	testletter = ""
	prev = string[0]
	count = 0
	encoded = ""
	for letter in string:
		# print(letter)
		testletter = letter
		if testletter != prev:
			encoded += (str(count) + prev)
			count = 1
			# print(encoded)
		else:
			count += 1
		prev = testletter
	# have to add the last one
	encoded += (str(count) + string[-1])
	return encoded

def runDecode(encoded):

	number = 0
	letter = ""
	decoded = ""
	# all even indices [0..2..4..] are numbers
	# all odd indices [1..3..5..] are the letters
	# all encoded strings are even length
	for i in range(0,len(encoded), 2):
		for j in range(int(encoded[i])):
			decoded += encoded[i + 1]
	return decoded

# print(runEncode("AAAABBBCCDAA"))
# print(runEncode("teststring"))

# algebraic law: runDecode(runEncode(anystring)) == anystring
assert runDecode(runEncode("teststring")) == "teststring"
assert runDecode(runEncode("AAAABBBCCDAA")) == "AAAABBBCCDAA"

# print(runEncode("AABBCCDDDDAABBCCEEEEEDGGJZVBNNSSLLL"))
assert runDecode(runEncode("AABBCCDDDDAABBCCEEEEEDGGJZVBNNSSLLL")) == "AABBCCDDDDAABBCCEEEEEDGGJZVBNNSSLLL"


from log import *
from telegram import ReplyKeyboardMarkup, ReplyKeyboardRemove
from telegram.ext import ConversationHandler

GET_NUM, RATIONAL_1, RATIONAL_2, COMPLEX_1, COMPLEX_2, RATIONAL_OPER, COMPLEX_OPER = range(
    7)

async def start(update, context):
    reply_keyboard = [['Рациональные', 'Комплексные', 'Выход']]
    markup_key = ReplyKeyboardMarkup(reply_keyboard, one_time_keyboard=True)
    await update.message.reply_text(
        'Добро пожаловать в калькулятор рациональных и комплексных чисел. '
        'Выберите с какими числами выполнить вычисления. Или нажмите "Выход", чтобы закончить разговор. \n',
        reply_markup=markup_key)
    return GET_NUM

async def get_num(update, context):
    choice = update.message.text
    if choice == 'Рациональные':
        await update.message.reply_text('Введите первое рациональное число (через точку):', reply_markup=ReplyKeyboardRemove())
        return RATIONAL_1
    elif choice == 'Комплексные':
        await update.message.reply_text('Введите действительную и мнимую часть числа (через пробел): ', reply_markup=ReplyKeyboardRemove())
        return COMPLEX_1
    else:
        return cancel(update, context)

async def rational_1(update, context):
    rational_first = update.message.text
    if rational_first == str(float(rational_first)):
        rational_first = float(rational_first)
        context.user_data['rational_first'] = rational_first
        await update.message.reply_text('Введите второе рациональное число (через точку): ')
        return RATIONAL_2
    else:
        await update.message.reply_text('Неправильный ввод! Попробуйте еще раз: ')
        return RATIONAL_1

async def rational_2(update, context):
    rational_second = update.message.text
    if rational_second == str(float(rational_second)):
        rational_second = float(rational_second)
        context.user_data['rational_second'] = rational_second
        reply_keyboard = [['+', '-', 'х', '/']]
        markup_key = ReplyKeyboardMarkup(
            reply_keyboard, one_time_keyboard=True)
        await update.message.reply_text('Нажмите, какая будет операция.', reply_markup=markup_key)
        return RATIONAL_OPER
    else:
        await update.message.reply_text('Неправильный ввод! Попробуйте еще раз: ')
        return RATIONAL_2

async def rational_oper(update, context):
    rational_first = context.user_data.get('rational_first')
    rational_second = context.user_data.get('rational_second')
    choice = update.message.text
    if choice == '+':
        result = rational_first + rational_second
    if choice == '-':
        result = rational_first - rational_second
    if choice == 'х':
        result = rational_first * rational_second
    if choice == '/':
        try:
            result = rational_first / rational_second
        except:
            await update.message.reply_text('Делить на ноль нельзя.')
            await update.message.reply_text('Нажмите /start для запуска Калькулятора.')
            return ConversationHandler.END
    await update.message.reply_text(
        f'Результат: {rational_first} {choice} {rational_second} = {result} \n'
        'Если хотите еще что-то посчитать, нажмите /start')
    return ConversationHandler.END

async def complex_1(update, context):
    complex_num = update.message.text
    complex_num = complex_num.split(' ')
    complex_first = complex(int(complex_num[0]), int(complex_num[1]))
    context.user_data['complex_first'] = complex_first
    await update.message.reply_text('Введите действительную и мнимую часть второго числа (через пробел):')
    return COMPLEX_2

async def complex_2(update, context):
    complex_num = update.message.text
    complex_num = complex_num.split(' ')
    complex_second = complex(int(complex_num[0]), int(complex_num[1]))
    context.user_data['complex_second'] = complex_second
    reply_keyboard = [['+', '-', 'х', '/']]
    markup_key = ReplyKeyboardMarkup(reply_keyboard, one_time_keyboard=True)
    await update.message.reply_text('Нажмите, какая будет операция.', reply_markup=markup_key)
    return COMPLEX_OPER

async def complex_oper(update, context):
    complex_first = context.user_data.get('complex_first')
    complex_second = context.user_data.get('complex_second')
    choice = update.message.text
    if choice == '+':
        result = complex_first + complex_second
    if choice == '-':
        result = complex_first - complex_second
    if choice == 'х':
        result = complex_first * complex_second
    if choice == '/':
        try:
            result = complex_first / complex_second
        except:
            await update.message.reply_text('Делить на 0 нельзя.')
            await update.message.reply_text('Нажмите /start для запуска Калькулятора.')
            return ConversationHandler.END
    await update.message.reply_text(
        f'Результат: {complex_first} {choice} {complex_second} = {result} \n'
        'Если хотите еще что-то посчитать, нажмите /start')
    return ConversationHandler.END

async def cancel(update, context):
    await update.message.reply_text('Хорошо, пока!', reply_markup=ReplyKeyboardRemove())
    return ConversationHandler.END


def solution(x):
    s = sum([int(i) for i in str(x)])
    if x % s == 0:
        return True
    else:
        return False

# 더 간단하게 작성하기
# def Harshad(n):
#     return n%sum(int(x) for x in str(n)) == 0     # 0이면 자동으로 True, 0이 아닐 경우에는 자동으로 False 반환


# Advent of Code - Day 9
# Part 2

import time
import pygame

def parseInput():
    full_map = []
    for line in open("input.txt"):
        line_map = []
        for string in line.strip():
            for number in string:
                line_map.append(int(number))
        full_map.append(line_map)
    return full_map

def isLowPoint(basin_map, row, col):
    n_cols = len(basin_map[0])-1
    n_rows = len(basin_map)-1
    if row != 0:
        if basin_map[row][col] >= basin_map[row-1][col]:
            return False
    if row != n_rows:
        if basin_map[row][col] >= basin_map[row+1][col]:
            return False
    if col != 0:
        if basin_map[row][col] >= basin_map[row][col-1]:
            return False
    if col != n_cols:
        if basin_map[row][col] >= basin_map[row][col+1]:
            return False
    return True

def updateBasin(basin_map, row, col, unique_positions):
    n_cols = len(basin_map[0])-1
    n_rows = len(basin_map)-1
    if row != 0:
        if basin_map[row-1][col] != 9 and [row-1, col] not in unique_positions:
            unique_positions.append([row-1, col])
    if row != n_rows:
        if basin_map[row+1][col] != 9 and [row+1, col] not in unique_positions:
            unique_positions.append([row+1, col])
    if col != 0:
        if basin_map[row][col-1] != 9 and [row, col-1] not in unique_positions:
            unique_positions.append([row, col-1])
    if col != n_cols:
        if basin_map[row][col+1] != 9 and [row, col+1] not in unique_positions:
            unique_positions.append([row, col+1])

    return unique_positions

def calcBasinSize(basin_map, row, col):
    unique_positions = [[row, col]]

    for pos in unique_positions:
        unique_positions = updateBasin(basin_map, pos[0], pos[1], unique_positions)
        value = basin_map[pos[0]][pos[1]]
        pygame.draw.rect(win, (value*20, value*20, 255), (pos[0]*8, pos[1]*8, 8, 8))
        pygame.time.delay(2)
        pygame.display.update()
        pygame.event.get()

    return len(unique_positions)

#### Visualize day9 with pygame ####
full_map = parseInput()

# Initiate pygame window
win = pygame.display.set_mode((800, 800))
pygame.display.set_caption("Advent of Code day 9")
win.fill((255, 255, 255))
pygame.display.update()
pygame.event.get()

# Show all borders
for row_id, row in enumerate(full_map):
    for col_id, value in enumerate(row):
        if value == 9:
            pygame.draw.rect(win, (0, 0, 0), (row_id*8, col_id*8, 8, 8))

pygame.display.update()
pygame.event.get()

# Calculate basins and show them in window
basin_size_list = []
for row_id, row in enumerate(full_map):
    for col_id, value in enumerate(row):
        if isLowPoint(full_map, row_id, col_id):
            basin_size = calcBasinSize(full_map, row_id, col_id)
            basin_size_list.append(basin_size)
            pygame.draw.rect(win, (150, 0, 150), (row_id*8, col_id*8, 8, 8))

# Wait 10s before closing
time.sleep(10)


from django.shortcuts import render, reverse
from django.http import HttpResponse, HttpResponseRedirect
from .models import MyModel

def myview(request):
    myinstances = MyModel.objects.all()
    context = {
        'myinstances': myinstances
    }
    return render(request, 'myapp/mytemplate.html', context)

def mycreate(request):
    post_title = request.POST['post_title']
    post_body = request.POST['post_body']
    mymodel = MyModel(post_title=post_title, post_body=post_body)
    mymodel.save()
    return HttpResponseRedirect(reverse('myapp:myview'))

# def index(request):
#     return HttpResponse('you are at the index')

def about(request):
    return HttpResponse('you are at the About page')


#!/usr/bin/env python3

"""
This is a script preparing data for the post-editing task. It encodes the
sentences to be post-edited as  a sequence of delete, keep and insert
operations from the target sentence.

Example:
    source:   Good afternoon , John ! !
    target:   Good evening , John !

    result:   <keep> <delete> evening <keep> <keep> <delete>

The inverse to this script is 'postedit_reconstruct_data.py'.
"""

# tests: lint

import argparse
import re
import numpy as np
from neuralmonkey.processors.german import GermanPreprocessor

def load_tokenized(text_file, preprocess=None):
    if not preprocess:
        preprocess = lambda x: x
    return [preprocess(re.split(r"[ ]", l.rstrip())) for l in text_file]

def convert_to_edits(source, target):
    keep = '<keep>'
    delete = '<delete>'

    lev = np.zeros([len(source) + 1, len(target) + 1])
    edits = [[[] for _ in range(len(target) + 1)]
             for _ in range(len(source) + 1)]

    for i in range(len(source) + 1):
        lev[i, 0] = i
        edits[i][0] = [delete for _ in range(i)]

    for j in range(len(target) + 1):
        lev[0, j] = j
        edits[0][j] = target[:j]

    for j in range(1, len(target) + 1):
        for i in range(1, len(source) + 1):

            if source[i - 1] == target[j - 1]:
                keep_cost = lev[i - 1, j - 1]
            else:
                keep_cost = np.inf

            delete_cost = lev[i - 1, j] + 1
            insert_cost = lev[i, j - 1] + 1

            lev[i, j] = min(keep_cost, delete_cost, insert_cost)

            if lev[i, j] == keep_cost:
                edits[i][j] = edits[i - 1][j - 1] + [keep]

            elif lev[i, j] == delete_cost:
                edits[i][j] = edits[i - 1][j] + [delete]

            else:
                edits[i][j] = edits[i][j - 1] + [target[j - 1]]

    return edits[-1][-1]

def main():
    # print convert_to_edits(["hello", "john"], ["hi", "john", "how"])

    parser = argparse.ArgumentParser(
        description="Convert postediting target data to sequence of edits")
    parser.add_argument("--translated-sentences",
                        type=argparse.FileType('r'), required=True)
    parser.add_argument("--target-sentences",
                        type=argparse.FileType('r'), required=True)
    parser.add_argument("--target-german", type=bool, default=False)

    args = parser.parse_args()

    preprocess = None
    if args.target_german:
        preprocess = GermanPreprocessor()

    trans_sentences = load_tokenized(
        args.translated_sentences, preprocess=preprocess)
    tgt_sentences = load_tokenized(
        args.target_sentences, preprocess=preprocess)

    for trans, tgt in zip(trans_sentences, tgt_sentences):
        edits = convert_to_edits(trans, tgt)
        print(" ".join(edits))

if __name__ == '__main__':
    main()


import sys
import numpy as np
import scipy.linalg as la
import scipy.sparse.linalg as las

def cg(A,k,x0,niter,tol):
    A=np.array(A.split(',')).astype(int)
    m=np.sqrt(A.shape[0]).astype(int)
    A=A.reshape(-1,m)
    k=np.array(k.split(',')).astype(int)
    n=int(niter)
    x=np.zeros((m,n))
    x[:,0]=np.array(x0.split(',')).astype(int)
    p=np.zeros((m,n))
    r=np.zeros((m,n))
    a=np.zeros(n)
    b=np.zeros(n)
    tol=float(tol)

    print("x[:,0]:",x[:,0])
    p[:,0]=r[:,0]=k-A@x[:,0]
    print("p[:,0]:",p[:,0])
    print("r[:,0]:",r[:,0])
    for i in np.arange(0,n-1):
        print("ronde:",i)
        a[i]=(r[:,i] @ r[:,i])/( p[:,i] @ A @ p[:,i] )
        print("a[i]:",a[i])
        x[:,i+1]=x[:,i]+a[i]*p[:,i]
        print("x[:,i+1]:",x[:,i+1])
        sol=x[:,i+1]
        if la.norm(k-A@x[:,i+1])<tol:#*la.norm(k):
            print("break")
            break
        r[:,i+1]=r[:,i]-a[i]*A @ p[:,i]
        print("r[:,i+1]:",r[:,i+1])
        b[i]=( r[:,i+1] @ r[:,i+1] )/( r[:,i] @ r[:,i] )
        print("b[i]:",b[i])
        p[:,i+1]=r[:,i+1] + b[i]*p[:,i]
        print("p[:,i+1]:",p[:,i+1])
    print("i:",i)
    print("sol:",sol)

cg("2,1,0,1,2,1,0,1,2","3,4,3","0,0,0","5","1e-5")

#A=np.array([2,-1,-1,2]).reshape(-1,2)
#lu,piv=la.lu_factor(A)
#x=la.lu_solve((lu,piv),grad)


import torchvision.models as models
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import transforms
from transformers import DeiTForImageClassification
import database.dataset as dataset
import numpy as np
import time
from database.loss import MarginLoss, ProxyNCA_prob, NormSoftmax

from argparse import ArgumentParser, ArgumentTypeError

class Model(nn.Module):
    def __init__(self, model='densenet', eval=True, batch_size=32, num_features=128,
                 name='weights', use_dr=True, device='cuda:0', freeze=False):
        super(Model, self).__init__()
        if model == 'densenet':
            self.forward_function = self.forward_conv
            self.conv_net = models.densenet121(pretrained=True).to(device=device)
        elif model == 'resnet':
            self.forward_function = self.forward_conv
            self.conv_net = models.resnet50(pretrained=True).to(device=device)
        elif model == 'transformer':
            self.forward_function = self.forward_transformer
            self.model = DeiTForImageClassification.from_pretrained('facebook/deit-base-distilled-patch16-224').to(device=device)

        if freeze and model != 'transformer':
            for param in self.conv_net.parameters():
                param.requires_grad = False
        elif freeze:
            for param in self.model.parameters():
                param.requires_grad = False

        if use_dr and model != 'transformer':
            out_features = 4096
            if model == 'densenet':
                self.conv_net.classifier = nn.Linear(1024, out_features).to(device=device)
            elif model == 'resnet':
                self.conv_net.fc = nn.Linear(2048, out_features).to(device=device)

            self.first_conv1 = nn.Conv2d(3, 96, kernel_size=8, padding=1, stride=16).to(device=device)
            self.first_conv2 = nn.MaxPool2d(3, 4, 1).to(device=device)

            self.second_conv1 = nn.Conv2d(3, 96, kernel_size=7, padding=4, stride=32).to(device=device)
            self.second_conv2 = nn.MaxPool2d(7, 2, 3).to(device=device)

            self.linear = nn.Linear(7168, num_features).to(device=device)
            self.use_dr = True
        else:
            if model == 'densenet':
                self.conv_net.classifier = nn.Linear(1024, num_features).to(device=device)
            elif model == 'resnet':
                self.conv_net.fc = nn.Linear(2048, num_features).to(device=device)
            elif model == 'transformer':
                self.model.classifier = torch.nn.Linear(768, num_features).to(device=device)
                for module in filter(lambda m: type(m) == nn.LayerNorm, self.model.modules()):
                    module.eval()
                    module.train = lambda _: None
            self.use_dr = False

        self.num_features = num_features
        self.norm = nn.functional.normalize

        self.transformer = model == 'transformer'

        self.name = name
        self.device = device

        if eval == True:
            self.load_state_dict(torch.load(name))
            self.eval()
            self.eval = True
        else:
            self.train()
            self.eval = False
            self.batch_size = batch_size

    def forward_conv(self, input):
        tensor1 = self.conv_net(input)

        tensor1 = self.norm(tensor1)

        if self.use_dr:
            tensor2 = self.first_conv1(input)
            tensor2 = self.first_conv2(tensor2)
            tensor2 = torch.flatten(tensor2, start_dim=1)

            tensor3 = self.second_conv1(input)
            tensor3 = self.second_conv2(tensor3)
            tensor3 = torch.flatten(tensor3, start_dim=1)

            tensor4 = self.norm(torch.cat((tensor2, tensor3), 1))

            return self.norm(self.linear(torch.cat((tensor1, tensor4), 1)))

        return tensor1

    def forward_transformer(self, input):
        return self.norm(self.model(input).logits, 1)

    def forward(self, input):
        return self.forward_function(input)

    def train_epochs(self, dir, epochs, sched, loss, generalise, lr, decay, beta_lr, gamma, lr_proxies):
        data = dataset.TrainingDataset(dir, 2, generalise, self.transformer)
        print('Size of dataset', data.__len__())

        if loss == 'margin':
            loss_function = MarginLoss(n_classes=len(data.classes))

            to_optim = [{'params':self.parameters(),'lr':lr,'weight_decay':decay},
                        {'params':loss_function.parameters(), 'lr':beta_lr, 'weight_decay':0}]

            optimizer = torch.optim.Adam(to_optim)
        elif loss == 'proxy_nca_pp':
            loss_function = ProxyNCA_prob(len(data.classes), self.num_features, 3, device)

            to_optim = [
                {'params':self.parameters(), 'weight_decay':0},
                {'params':loss_function.parameters(), 'lr': lr_proxies},
            ]

            optimizer = torch.optim.Adam(to_optim, lr=lr, eps=1)
        elif loss == 'softmax':
            loss_function = NormSoftmax(0.05, len(data.classes), self.num_features, lr_proxies, self.device)

            to_optim = [
                {'params':self.parameters(),'lr':lr,'weight_decay':decay},
                {'params':loss_function.parameters(),'lr':lr_proxies,'weight_decay':decay}
            ]

            optimizer = torch.optim.Adam(to_optim)

        if sched == 'exponential':
            scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)
        elif sched == 'step':
            scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[epochs//2, epochs],
                                                            gamma=gamma)

        loader = torch.utils.data.DataLoader(data, batch_size=self.batch_size,
                                             shuffle=True, num_workers=12,
                                             pin_memory=True)

        loss_list = []
        try:
            for epoch in range(epochs):
                start_time = time.time()

                for i, (labels, images) in enumerate(loader):
                    images_gpu = images.to(device=self.device)
                    labels = labels.to(device=self.device)

                    if not self.transformer:
                        out = self.forward(images_gpu)
                    else:
                        out = self.forward(images_gpu.view(-1, 3, 224, 224))

                    loss = loss_function(out, labels)

                    optimizer.zero_grad(set_to_none=True)
                    loss.backward()
                    optimizer.step()

                    loss_list.append(loss.item())

                print("epoch {}, loss = {}, time {}".format(epoch, np.mean(loss_list),
                                                            time.time() - start_time))

                print("\n----------------------------------------------------------------\n")
                if sched != None:
                    scheduler.step()

                torch.save(self.state_dict(), self.name)

        except KeyboardInterrupt:
            print("Interrupted")

    def train_dr(self, data, num_epochs, lr):
        data = dataset.DRDataset(data)
        print('Size of dataset', data.__len__())

        loader = torch.utils.data.DataLoader(data, batch_size=self.batch_size,
                                             shuffle=True, num_workers=12,
                                             pin_memory=True)
        loss_function = torch.nn.TripletMarginLoss()
        optimizer = torch.optim.Adam(self.parameters(), lr=lr)
        loss_list = []
        try:
            for epoch in range(num_epochs):
                start_time = time.time()

                for i, (image0, image1, image2) in enumerate(loader):
                    image0 = image0.to(device='cuda:0')
                    image1 = image1.to(device='cuda:0')
                    image2 = image2.to(device='cuda:0')

                    out0 = self.forward(image0).cpu()
                    out1 = self.forward(image1).cpu()
                    out2 = self.forward(image2).cpu()

                    loss = loss_function(out0, out1, out2)

                    optimizer.zero_grad(set_to_none=True)
                    loss.backward()
                    optimizer.step()

                    loss_list.append(loss.item())

                print("epoch {}, batch {}, loss = {}".format(epoch, i,
                                                             np.mean(loss_list)))
                loss_list.clear()
                print("time for epoch {}".format(time.time()- start_time))

                torch.save(self.state_dict(), self.name)

                if (epoch + 1) % 4:
                    lr /= 2
                    for param in optimizer.param_groups:
                        param['lr'] = lr
        except KeyboardInterrupt:
            print("Interrupted")

if __name__ == "__main__":
    parser = ArgumentParser()

    parser.add_argument(
        '--num_features',
        type=int,
        help='number of features to use',
        default=128
    )

    parser.add_argument(
        '--batch_size',
        type=int,
        default=32
    )

    parser.add_argument(
        '--model',
        help='feature extractor to use',
        default='densenet'
    )

    parser.add_argument(
        '--weights',
        default='weights'
    )

    parser.add_argument(
        '--training_data',
    )

    parser.add_argument(
        '--dr_model',
        action="store_true"
    )

    parser.add_argument(
        '--num_epochs',
        type=int,
        default=5
    )

    parser.add_argument(
        '--scheduler',
        default=None,
        help='<exponential, step>'
    )

    parser.add_argument(
        '--gpu_id',
        default=0,
        type=int
    )

    parser.add_argument(
        '--loss',
        default='margin',
        help='<margin, proxy_nca_pp, softmax, deep_ranking>'
    )

    parser.add_argument(
        '--freeze',
        action='store_true'
    )

    parser.add_argument(
        '--generalise',
        action='store_true',
        help='train on only half the classes of images'
    )

    parser.add_argument(
        '--lr',
        default=0.0001,
        type=float
    )

    parser.add_argument(
        '--decay',
        default=0.0004,
        type=float
    )

    parser.add_argument(
        '--beta_lr',
        default=0.0005,
        type=float
    )

    parser.add_argument(
        '--gamma',
        default=0.3,
        type=float
    )

    parser.add_argument(
        '--lr_proxies',
        default=0.00001,
        type=float
    )

    args = parser.parse_args()

    if args.gpu_id >= 0:
        device = 'cuda:' + str(args.gpu_id)
    else:
        device = 'cpu'

    m = Model(model=args.model, eval=False, batch_size=args.batch_size,
              num_features=args.num_features, name=args.weights,
              use_dr=args.dr_model, device=device, freeze=args.freeze)

    if args.loss == 'deep_ranking':
        m.train_dr(args.training_data, args.num_epochs, args.lr)
    else:
        m.train_epochs(args.training_data, args.num_epochs, args.scheduler, args.loss, args.generalise,
                       args.lr, args.decay, args.beta_lr, args.gamma, args.lr_proxies)


from django.conf.urls import patterns, include, url
from django.conf import settings

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'hackmum.views.home', name='home'),
    # url(r'^hackmum/', include('hackmum.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

    # User Views
    url(r'^$', 'hsmreg.views.index'),
    url(r'^event/(?P<event_id>\d+)/$', 'hsmreg.views.event_detail'),
    url(r'^event/(?P<event_id>\d+)/user/register/$','hsmreg.views.user_register'),
    url(r'^event/(?P<event_id>\d+)/user/(?P<user_id>\d+)/$','hsmreg.views.users_detail'),
    url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT,}),
    url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT,}),
    url(r'^about/$', 'hsmreg.views.about'),
)


import cv2
import numpy as np
import matplotlib.pyplot as plt

image = cv2.imread('img/sample9.jpg', 0)

hist, bin_edges = np.histogram(image, bins=256)

plt.figure(figsize=(14, 5))
plt.subplot(121)
plt.imshow(image, cmap='gray')
plt.title('original image', fontsize=15)
plt.xticks([])
plt.yticks([])

plt.subplot(122), plt.plot(hist), plt.title('Histogram', fontsize=15)

weight1 = np.cumsum(hist)
weight2 = np.cumsum(hist[::-1])[::-1]

bin_mids = (bin_edges[:-1] + bin_edges[1:]) / 2.

mean1 = np.cumsum(hist * bin_mids) / weight1
mean2 = (np.cumsum((hist * bin_mids)[::-1]) / weight2[::-1])[::-1]

inter_class_variance = weight1[:-1] * \
    weight2[1:] * (mean1[:-1] - mean2[1:]) ** 2
index_of_max_val = np.argmax(inter_class_variance)

threshold = bin_mids[:-1][index_of_max_val]

print("Otsu's algorithm implementation thresholding result: ", threshold)

# Plotting weight and between-class variance
plt.figure(figsize=(14, 5))
plt.subplot(121)
plt.plot(weight1)
plt.plot(weight2)
plt.title('Weight profiles', fontsize=15)
plt.xlabel('pixel values'), plt.ylabel('weight')

plt.subplot(122)
plt.plot(inter_class_variance)
plt.title('Between-class variance', fontsize=15)
plt.xlabel('pixel values'), plt.ylabel('variance')

otsu_threshold, image_result = cv2.threshold(
    image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
print(otsu_threshold)

maxValue = 255
th, dst = cv2.threshold(image, otsu_threshold, maxValue, cv2.THRESH_BINARY)

plt.figure(figsize=(14, 5))
plt.subplot(121)
plt.imshow(image, cmap='gray')
plt.title('original image', fontsize=15)
plt.xticks([]), plt.yticks([])

plt.subplot(122)
plt.imshow(dst, cmap='gray')
plt.title('filtered image', fontsize=15)
plt.xticks([]), plt.yticks([])

plt.show()


from tweepipe import settings
from tweepipe.db import db_schema, db_client

def get_follower_retrieval_parallel_kwargs(
    twitter_apis, uids, issue, env_file, include_relations, include_users, cap_count
):
    chunk_size = int(len(uids) / len(twitter_apis))
    kwargs_list = []
    for i in range(len(twitter_apis)):
        kwargs = {
            "twitter_api": twitter_apis[i],
            "include_users": include_users,
            "include_relations": include_relations,
            "cap_count": cap_count,
            "env_file": env_file,
            "issue": issue,
            "uids": uids[i * chunk_size : (i + 1) * chunk_size],
        }
        kwargs_list.append(kwargs)

    return kwargs_list

def fetch_followers(
    twitter_api, uids, include_users, include_relations, cap_count, env_file, issue
):
    if env_file:
        settings.load_config(env_file)

    db_conn = db_client.DBClient(
        issue=issue,
        include_relations=include_relations,
        include_users=include_users,
        schema=db_schema.INDEX_V3,
    )

    for uid in uids:
        pass


from django.core.exceptions import ImproperlyConfigured
from getpaid.utils import get_backend_settings

class PaymentProcessorBase(object):
    """
    Base for all payment processors. It should at least be able to:
     * redirect to a gateway based on Payment object
     * manage all necessary logic to accept payment from gateway, e.g. expose
       a View for incoming transaction notification status changes
    """

    #Each backend need to define this values
    BACKEND = None
    """
    This constant should be set to fully qualified python path to the module. This is also
    a name that will be used to identify and enabling this backend in django-getpaid
    """
    BACKEND_NAME = None
    """
    This constant should be set to human readable backend name. Consider using lazy translation strings
    for i18n support
    """
    BACKEND_ACCEPTED_CURRENCY = tuple()
    """
    This constant should be any type of iterable that defines accepted currencies by the backend. Currencies should be
    set as three letters ISO code strings (eg. 'USD', 'EUR')
    """

    def __init__(self, payment):

        if payment.currency not in self.BACKEND_ACCEPTED_CURRENCY:
            raise ValueError("Backend '%s' cannot process '%s' payments." % self.BACKEND, payment.currency)
        self.payment = payment

    def get_gateway_url(self, request):
        """
        Should return an URL that redirects to payment Gateway. Request context need to be given
        because various payment engines requires information about client (e.g. a client IP).
        """
        raise NotImplementedError('Must be implemented in PaymentProcessor')

    @classmethod
    def get_backend_setting(cls, name, default=None):
        """
        Reads ``name`` setting from backend settings dictionary.

        If `default` value is omitted, raises ``ImproperlyConfigured`` when
        setting ``name`` is not available.
        """
        backend_settings = get_backend_settings(cls.BACKEND)
        if default is not None:
            return backend_settings.get(name, default)
        else:
            try:
                return backend_settings[name]
            except KeyError:
                raise ImproperlyConfigured("getpaid '%s' requires backend '%s' setting" % (cls.BACKEND, name))


def _read_word():
    """
    **要求：`2008`年复试上机题**
    * 用`IE`从`FTP`上下载`org.dat`，并保存在`D`盘的根目录中.
    * 此文件中按文本方式存放了一段其他文章，其中有若干长度小于`15`的英文单词，
    单词之间用空格分开，无其他符号.
    * 顺序读取这段文章的不同的单词(大小写敏感)，
    同时在读取的过程中排除所有的单词`THE`以及变形，即这些单词不能出现在读取的结果中.
    * 将读取的所有单词的首字母转大写后，输出`D`根目录下`new.txt`，每个单词一行.
    **程序：**
    """
    source = open('org.dat', encoding='utf-8')
    words = [x.title() for x in ' '.join(source.readlines()).split()]
    source.close()
    word_times = {}
    file = open('new.txt', 'a', encoding='utf-8')
    for word in words:
        if word == 'The':
            continue
        if word_times.get(word):
            continue
        word_times[word] = word_times.get(word, 0) + 1
        file.writelines(word + '\n')
    file.close()
_read_word()


import unittest
from test_common import *
import task6

class TestTask6(TestCase):
    "Every where I have used 'the' to check the rarity because the is highest occured word."
    def test_add_files_1(self):
        "Here I have just taken a words_empty.txt file to find out the frequency"
        X = task6.Freq()
        X.add_file('words_empty.txt')

        val = X.rarity('and')
        self.assertEqual(val, 0)
        self.assertEqual(X.word_frequency.count, 9)
        with self.assertRaises(KeyError) as context:
            val = X.rarity('rocks')
        self.assertTrue("Wrong key" in str(context.exception))

    "These are the test on all the ebooks that are given"
    def test_add_files_2(self):
        X = task6.Freq()
        X.add_file('84-0.txt')

        # rocks is rare word
        val = X.rarity('rocks')
        self.assertEqual(val, 2)

    def test_add_files_3(self):
        X = task6.Freq()
        X.add_file('98-0.txt')

        # mail is  Uncomman word
        val = X.rarity('mail')
        self.assertEqual(val, 1)

    def test_add_files_4(self):
        X = task6.Freq()
        X.add_file('1342-0.txt')

        val = X.rarity('the')
        self.assertEqual(val, 0)

    def test_add_files_5(self):
        X = task6.Freq()
        X.add_file('2600-0.txt')

        val = X.rarity('the')
        self.assertEqual(val, 0)

if __name__ == '__main__':
  unittest.main()


class Car(object):

    def __init__(self, brand, model, year, engines, transmissions):
        self.brand = brand
        self.model = model
        self.year = year
        self.engines = engines
        self.transmissions = transmissions

    def __str__(self):
        print("brand =", self.brand, ",", "model =", self.model, ",", "year =", self.year, ",",
              "engines =", self.engines, ",", "transmissions =", self.transmissions)


import youtube_dl
import os

ydl_opts = {
    'format': 'bestaudio/best',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '192',
    }],
    'outtmpl': 'music/%(title)s.mp3',
    'quiet': False
}

def dl_file(url):
    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        ydl.download([url])
        result = ydl.extract_info(url, download=False)
        outfile = ydl.prepare_filename(result)
    return outfile


while True:
    s1, s2 = str(input('Ведите первую строку: ')), str(input('Введите вторую строку: '))
    set1, set2 = set(), set()

    for i in s1:
        if i != ' ' and i.isalpha():
            set1.add(i.lower())
    for j in s2:
        if j != ' ' and j.isalpha():
            set2.add(j.lower())
    if len(set1) == 0 and len(set2) == 0:
        print('Вы должны ввести не только цифрв')
        continue
    flag = True
    for i in set2:
        if i in set1:
            continue
        else:
            print("Невозможно сформировать первую строку из второй")
            flag = False
            break
    if flag:
        print('Возможно создать новую строку')
    r = input('Если хотите попробовать еще введите "1", если нет то введите что угодно другое: ')
    if r == '1':
        continue
    else:
        break


import meradb
db = meradb.load("table.db")
db.set("key", "value")
db.get('key')
db.get_all()
db.rem()
db.exists()
db.total_keys()
db.del_db()
db.random_insert(5)


# -*- coding:utf-8 -*-
import random
import time

from utils.config import config
from utils.es_client import ElasticsearchClient
from utils.mq_client import RocketProducer, RocketPushConsumer

rocket_producer = RocketProducer({"host": "172.31.236.21", "port": 9876})
rocket_producer.start()
for i in range(10):
    data = {
        "id": random.randint(1, 10000),
        "trance_id": random.randint(1, 1000),
        "order_id": random.randint(1, 1000),
        "biz_job_id": "aaaa",
        "transfer_job_id": "bbbb",
        "move_job_id": "cccc",
        "agv_code": "dddd",
        "container_code": 9001001002,
        "log_level": "ERROR",
        "message": "abcdefg222222222222222",
        "created_time": int(round(time.time() * 1000)),
        "created_app": "wcs"
    }
    rocket_producer.push("log", data)
rocket_producer.shutdown()

# rocket_consumer = RocketPushConsumer({"host": "172.31.236.21", "port": 9876, "topic": "log"})
# es_instance = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# rocket_consumer.start(es_instance)
# from utils.config import config
# from utils.es_client import ElasticsearchClient
#
# es1 = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# print(es1)
# es2 = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# print(es2)
# print(id(es1))
# print(id(es2))


import xml.etree.ElementTree as ET
import io as io
from lxml import etree

class BaseResponse:
    """
    Base class for which all Responses will inherit
    """
    def __init__(self, response):
        self.response = response
        try:
            self.root = ET.fromstring(response.text)
        except ET.ParseError as parse_error:
            raise ValueError from parse_error
        self.namespace = None
        self.dtd = None

    def set_namespace(self, response_tag: str):
        """
        Sets the Namespace to use for each response
        """
        for elem in self.root.iter():
            if response_tag in elem.tag:
                self.namespace = elem.tag.split("}")[0].lstrip("{")
                break

    def get_qualified_tag(self, desired_tag: str, namespace: str =None):
        """
        Returns a qualified tag for the given tag string using the set
            namespace for the response class
        """
        if namespace is None:
            namespace = self.namespace
        return f"{{{namespace}}}{desired_tag}"

    def verify_by_dtd(self):
        """
        Verifies the XML response by the self.dtd string for this response
        """
        utf8_parser = etree.XMLParser(encoding='utf-8')

        unicode_str = self.response.text
        s = unicode_str.encode('utf-8')
        print(s)
        tree = etree.fromstring(s, parser=utf8_parser)

        # Parse the DTD string
        try:
            dtd = etree.DTD(io.StringIO(self.dtd))
        except etree.DTDParseError as parse_error:
            raise ValueError from parse_error

        # Validate XML against DTD
        if not dtd.validate(tree):
            error_messages = [error.message for error in dtd.error_log]
            raise ValueError(f"DTD validation failed:\n{chr(10).join(error_messages)}")
        return True


import os
import sys
import subprocess
import sys
from contextlib import contextmanager
from io import StringIO

@contextmanager
def captured_output():
    new_out, new_err = StringIO(), StringIO()
    old_out, old_err = sys.stdout, sys.stderr
    try:
        sys.stdout, sys.stderr = new_out, new_err
        yield sys.stdout, sys.stderr
    finally:
        sys.stdout, sys.stderr = old_out, old_err

def execute(args, timeout=3):
    cmd = [sys.executable, '-m', 'gitlabber']
    cmd.extend(args)
    with subprocess.Popen(cmd, stdout=subprocess.PIPE, env=os.environ.copy()) as process:
        outs, err = process.communicate(timeout=timeout)
        process.wait()
        return outs.decode('utf-8')


import cv2
import mediapipe as mp
import numpy as np
from flask import Flask, render_template, Response

mp_drawing = mp.solutions.drawing_utils
mp_pose = mp.solutions.pose

counterhands = 0
counterlegs = 0
stagelegs = None
stagehands = None
pose_det = None
app = Flask(__name__)

def calculate_angle(a, b, c):
    a = np.array(a) # First
    b = np.array(b) # Mid
    c = np.array(c) # End

    radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0])
    angle = np.abs(radians * 180.0 / np.pi)

    if angle > 180.0:
        angle = 360 - angle

    return angle

def generate_frames():
    cap = cv2.VideoCapture(0)

    with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:
        while True:
            ret, frame = cap.read()

            # Recolor image to RGB
            image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            image.flags.writeable = False

            # Make detection
            results = pose.process(image)

            # Recolor back to BGR
            image.flags.writeable = True
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

            # Extract landmarks
            try:
                landmarks = results.pose_landmarks.landmark

                # Get coordinates
                shoulder = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y]
                elbow = [landmarks[mp_pose.PoseLandmark.LEFT_ELBOW.value].x, landmarks[mp_pose.PoseLandmark.LEFT_ELBOW.value].y]
                wrist = [landmarks[mp_pose.PoseLandmark.LEFT_WRIST.value].x, landmarks[mp_pose.PoseLandmark.LEFT_WRIST.value].y]
                hip = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x, landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y]
                knee = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y]
                ankle = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x, landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y]

                angleupper = calculate_angle(wrist, shoulder, hip)

                # Calculate angle
                anglehands = calculate_angle(shoulder, elbow, wrist)
                anglelegs = calculate_angle(hip, knee, ankle)

                # Visualize angle
                cv2.putText(image, str(anglehands),
                            tuple(np.multiply(elbow, [640, 480]).astype(int)),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA)

                cv2.putText(image, str(anglelegs),
                            tuple(np.multiply(knee, [640, 480]).astype(int)),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA)

                cv2.putText(image, str(angleupper),
                            tuple(np.multiply(wrist, [640, 480]).astype(int)),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA)

                # Curl counter logic
                if anglehands > 160:
                    stagehands = "down"
                if anglehands < 30 and stagehands =='down':
                    stagehands="up"
                    counterhands +=1
                    print(counterhands)

                if anglelegs > 160:
                    stagelegs = "down"
                if anglelegs < 30 and stagelegs =='down':
                    stagelegs="up"
                    counterlegs +=1
                    print(counterhands)

            except:
                pass

            # Render detections
            mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,
                                    mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2),
                                    mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)
                                    )

            ret, buffer = cv2.imencode('.jpg', image)
            frame = buffer.tobytes()
            yield (b'--frame\r\n'
                   b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')

    cap.release()
    cv2.destroyAllWindows()

@app.route('/')
def index():
    # Replace these with the actual values you want to display
    anglehands = 0
    anglelegs = 0
    angleupper = 0
    counterhands = 0
    counterlegs = 0

    return render_template('index.html', anglehands=anglehands, anglelegs=anglelegs,
                           angleupper=angleupper, counterhands=counterhands, counterlegs=counterlegs)

@app.route('/video_feed')
def video_feed():
    return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')

if __name__ == "__main__":
    app.run(debug=True)


import argparse

class Line:
    def __init__(self, line):
        self.text = line
        self.changes = 0
        self.new_text = ''

    def __repr__(self):
        return self.new_text

    def change_line(self, configuration):
        number_of_symbols = len(self.text)
        for i in range(number_of_symbols):
            if self.text[i] in configuration:
                symbol_for_change = self.text[i]
                self.new_text += configuration[symbol_for_change]
                self.changes += 1
            else:
                self.new_text += self.text[i]

def import_configuration(configuration = {}):
    with open(args.conf, 'r') as file:
        for line in file:
            key, value = line.strip().split("=")
            configuration[key] = value
    return configuration

def import_test_data(configuration, lines = []):
    with open(args.input, 'r') as file:
        for line in file:
            line = Line(line.strip())
            line.change_line(configuration)
            lines.append(line)
    return lines

parser = argparse.ArgumentParser(description='Parser to define configuration files and test data')
parser.add_argument(
    '--conf',
    type=str,
    default='configuration.txt',
    help='path to configuration file (default: configuration.txt)'
)
parser.add_argument(
    '--input',
    type=str,
    default='test_data.txt',
    help='path to test data file (default: test_data.txt)'
)
args = parser.parse_args()

configuration = import_configuration()
lines = import_test_data(configuration)
lines = sorted(lines, key=lambda line: line.changes, reverse=True)
for line in lines:
    print(line)