import urllib.request
import json

class JamendoUtil:

    def __init__(self, client_id, base_url):
        if not client_id:
            raise Exception("You must provide a client_id setting")

        self.client_id = client_id

        self.base_url = base_url

    def _createUrlParameters(self, **kwargs):
        params = dict()
        params["client_id"] = self.client_id
        for key in kwargs:
            params[key] = kwargs[key]

        return urllib.parse.urlencode(params)

    def _parseResponse(self, response):

        jsonResp = json.loads(response.decode("UTF-8"))
        if(jsonResp == None or jsonResp["headers"] == None):
            raise JamendoError(12000, "Error when load JSON response")

        if(jsonResp["headers"]["status"] != "success"):
            raise JamendoError(jsonResp["headers"]["code"], jsonResp["headers"]["error_message"])

        return jsonResp

    def getUserId(self, userName):

        if(userName == None or len(userName) <= 0):
            raise Exception("User name should not be empty")

        requestUrl = self.base_url + "/users/?%s" % self._createUrlParameters(name=userName)

        request = urllib.request.Request(requestUrl)
        response = urllib.request.urlopen(request)

        jsonResponse=response.read()
        obj = self._parseResponse(jsonResponse)

        if(len(obj["results"]) < 1):
            raise JamendoError(12001, "Empty result when trying to get user id")

        return obj["results"][0]["id"]

    def getPublicPlaylists(self, userId, limit = 10, offset = 0):

        if(userId == None or int(userId) <= 0):
            raise Exception("User id must be set")

        requestUrl = self.base_url + "/playlists/?%s" % self._createUrlParameters(user_id=userId, limit=limit, offset = offset)
        request = urllib.request.Request(requestUrl)
        response = urllib.request.urlopen(request)

        jsonResponse=response.read()
        obj = self._parseResponse(jsonResponse)

        return obj["results"]

    def getPlaylistTracks(self, playlistId):
        if(playlistId == None or int(playlistId) <= 0):
            raise Exception("Playlist id must be set")

        requestUrl = self.base_url + "/playlists/tracks/?%s" % self._createUrlParameters(id=playlistId)
        request = urllib.request.Request(requestUrl)
        response = urllib.request.urlopen(request)

        jsonResponse=response.read()
        obj = self._parseResponse(jsonResponse)

        if len(obj["results"]) <= 0 or len(obj["results"][0]) <= 0:
            return []

        return obj["results"][0]["tracks"]

class JamendoError(Exception):

    def __init__(self, code, message):
        self.code = code
        self.message = message


from __future__ import annotations
from abc import ABC, abstractmethod
from typing import List, Tuple
from argparse import ArgumentParser
from rstcloth import RstCloth
import os
from io import StringIO

from ..east import EnrichedAST
from ..astline import ASTLine

class Component(ABC):
    """
    Abstract class representing a component of the documentation.

    :param builder: The builder object that is creating the documentation.
    :param parameters: A dictionary of parameters for the component.
    """

    parse_group_description = "DEFAULT PARSER GROUP DESCRIPTION"
    name = "DEFAULT_NAME"

    def __init__(self, builder:'Builder', parameters) -> None:
        self.builder = builder
        self._sio = StringIO()
        self.document = RstCloth(self._sio)

        self.parameters = parameters
        self.check_parameters()

    @classmethod
    def cmdline_documentation(cls, parser: ArgumentParser) -> ArgumentParser:
        """
        Add command line arguments for this component to the given ArgumentParser.

        :param parser: The ArgumentParser to add the arguments to.
        :return: The created argument group.
        """
        parser_group = parser.add_argument_group(
            cls.parse_group_description
        )

        parser_group.add_argument(
            f'--{cls.name}.exclude',
            action="store_true",
            help='(flag) component will be excluded from the documentation'
        )

        return parser_group

    def check_parameters(self):
        """
        Check if the given parameters are valid and set default values if necessary.
        """
        if not self.name in self.parameters:
            self.parameters[self.name] = {}

        if not 'exclude' in self.parameters[self.name]:
            self.parameters[self.name]['exclude'] = False

    @abstractmethod
    def build_rst_file(self) -> None:
        """
        Build the reStructuredText file for this component.
        """
        pass

    def write_rst_file(self):
        """
        Write the reStructuredText file to disk.
        """
        if self._sio.getvalue():  # Things have been written
            with open(os.path.join(self.parameters['doc_dir'], self.name + ".rst"), 'w') as file:
                file.write(self._sio.getvalue())

    def _get_name(self, astline: ASTLine):
        """
        Get the name of the given AST line, taking into account the presence of multiple EASTs.

        :param astline: The AST line to get the name for.
        :return: The name of the AST line.
        """
        if len(self.builder.easts) > 1:
            return astline.prefix + astline.identifier

        return astline.identifier

    def _get_location(self, astline: ASTLine):
        """
        Get the location of the given AST line in the source code.

        :param astline: The AST line to get the location for.
        :return: The location of the AST line.
        """
        if astline.location.begin.line != astline.location.end.line:
            return f"{astline.location.begin.line}-{astline.location.end.line}"
        else:
            return f"{astline.location.begin.line}"

    def _include_code(self, astline: ASTLine):
        """
        Include the code of the given AST line in the rst file.

        :param astline: The AST line to include the code for.
        """
        self.document.directive('literalinclude',
                                arg=f'/{astline.location.begin.filename}',
                                fields=[('language','prolog'),
                                        ('lines' , f'{self._get_location(astline)}')])

        self.document.newline()

    def _include_comments(self, astline: ASTLine):
        """
        Helper function for including comments of an ASTLine in the documentation.

        :param astline: The ASTLine object whose comments are to be included in the documentation.
        """
        if astline.comments:
            for comment in astline.comments:
                self.document.directive('note',content=f"↳ {comment}")

            self.document.newline()

    def _include_source(self, east: EnrichedAST):
        """
        Include the source code for the current file being processed in the output documentation.

        :param east: An instance of the EnrichedAST class.
        """
        self.document.directive('literalinclude',
                                arg=f'/{east.filename}',
                                fields=[('language','prolog'),
                                        ('linenos' , '')])
        self.document.newline()

class Index(Component):
    name = 'index'

    @classmethod
    def cmdline_documentation(cls, parser: ArgumentParser) -> ArgumentParser:
        return

    def __init__(self, builder, parameters) -> None:
        super().__init__(builder, parameters)

    def check_parameters(self):
        super().check_parameters()

    def _generate_toctree(self):
        self.document._add('.. toctree::')
        self.document._add(' :maxdepth: 2')
        self.document.newline()

        for c in self.builder.components:
            if c.name != self.name:
                self.document._add(" " + c.name)

    def build_rst_file(self) -> None:
        self.document.title(self.parameters['project_name'])
        self.document.newline()
        self.document.content(self.parameters['description'])
        self.document.newline(3)
        self.document.h1("Content")
        self.document.newline()
        self._generate_toctree()

class Source(Component):
    name = "source"
    parse_group_description = "Source files parameters"

    @classmethod
    def cmdline_documentation(cls, parser: ArgumentParser) -> ArgumentParser:
        parser_group = super().cmdline_documentation(parser)
        parser_group.add_argument(
            f'--{cls.name}.exclude-file',
            action="store",
            nargs= '*',
            help='source file(s) that will be excluded from source documentation section'
        )

    def __init__(self, builder, parameters) -> None:
        super().__init__(builder, parameters)

    def check_parameters(self):
        super().check_parameters()
        if not 'exclude_file' in self.parameters[self.name]:
            self.parameters[self.name]['exclude_file']= False

    def build_rst_file(self) -> None:
        self.document.title('Source Code')
        self.document.newline()

        for east in self.builder.easts:
            self.document.h2(
                east.filename[self.parameters['src_dir'].rindex('/')+1:])
            self.document.newline()
            self._include_source(east)
            self.document.newline()


# This script is used to compare precip extreme difference bewteen vrcesm and CORDEX-SEA
# Several steps are inplemented:
# S1-read precip data from vrcesm and CORDEX-SEA
# S2-calculate extreme
# S3-plot contour
#
# Written by Harry Li

# import libraries
import pathmagic  # noqa: F401
from modules.stats.mod_stats_extremes import gpdfit_moment
from modules.datareader.mod_dataread_obs_CPC import readobs_pre_CPC
from modules.datareader.mod_dataread_obs_TRMM import readobs_pre_TRMM_day
from modules.datareader.mod_dataread_obs_pre import read_SAOBS_pre
from modules.plot.mod_plt_lines import plot_lines
from modules.plot.mod_plt_findstns import data_findstns
from modules.datareader.mod_dataread_vrcesm import readvrcesm
from modules.datareader.mod_dataread_cordex_sea import readcordex

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits import basemap
import pickle
import pandas as pd
plt.switch_backend('agg')

############################################################################
# setup directory
############################################################################
outdir = '/scratch/d/dylan/harryli/cesm1/vrcesm/Analysis/vrseasia_AMIP_1979_to_2005/pre/VRseasia_vs_CORDEX_SEA/extremes/'

############################################################################
# set parameters
############################################################################
# variable info
var_longname = 'Total Precip'
varstr = 'prect'
var_unit = 'mm/day'
varname = 'PRECT'

# time bounds
iniyear = 1980
endyear = 2005

# define regions
latbounds = [10, 25]
lonbounds = [100, 110]

# mainland Southeast Asia
reg_lats = [0, 25]
reg_lons = [90, 120]
reg_name = 'mainSEA'

# set data frequency
frequency = 'day'

# set percentile
percentile = 99
obsrate = (100.-percentile)/100.
percents = [50, 70, 80, 90, 95, 97, 99, 99.5]

# return years
return_years = [2, 5, 10, 20, 50, 75,  100, 150]
m = np.array(return_years) * 365.

# create months for plot
months = np.arange(1, 13, 1)
monnames = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']

# create seasons
seasons = [[12, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]
seasnames = ['DJF', 'MAM', 'JJA', 'SON', 'Annual']

# plot legend
legends = ['CESM-vrseasia', 'CESM-ne30', 'CESM-fv0.9x1.25', 'CESM-fv1.9x2.5', 'CORDEX-EC-Earth-RegCM4',
           'CORDEX-IPSL-CM5A-RegCM4', 'CORDEX-MPI-ESM-RegCM4', 'CORDEX-HadGEM2-ES-RCA4']
colors = ['red', 'yellow', 'green', 'blue', 'tomato', 'goldenrod', 'darkcyan', 'darkmagenta']
markers = ['o', '+', '^', 'x', 'D', 's', 'P', 'h']
linetypes = ['-', '-', '-', '-', '--', '--', '--', '--']

cesm_legends = ['CESM-vrseasia', 'CESM-ne30', 'CESM-fv0.9x1.25',
                'CESM-fv1.9x2.5']
cesm_colors = ['red', 'yellow', 'green', 'blue']
cesm_markers = ['o', '+', '^', 'x']
cesm_linetypes = ['-', '-', '-', '-']

############################################################################
# define functions
############################################################################

def rmse(model, reference):
    diff = model - reference
    diff = diff.flatten()
    diff = diff[~np.isnan(diff)]
    res = np.sqrt(np.mean(diff**2))

    return res

def get_vars(time, prect, months):
    if np.isscalar(months):
        res_prect = prect[time.month == months, :, :]
    else:
        # print(time[np.in1d(time.month, months)])
        res_prect = prect[np.in1d(time.month, months), :, :]

    return res_prect

def get_bias(var, ref_var, in_lons, in_lats, out_lons, out_lats):
    lonout, latout = np.meshgrid(out_lons, out_lats)
    temp = basemap.interp(ref_var, in_lons, in_lats, lonout, latout, order=1)
    res = var - temp

    return res

def get_percentiles(vars, percents):
    res = []
    for idata in vars:
        temp_data = idata.flatten()
        temp_data = temp_data[~np.isnan(temp_data)]
        temp = np.percentile(temp_data, percents)
        temp = np.round(temp, decimals=2)
        res.append(temp)

    return res

def get_mean_percentiles(vars, percents):
    res = []
    for idata in vars:
        var_percentiles = []
        for ipercent in percents:
            temp_percentile = np.nanpercentile(idata, ipercent, axis=0)
            var_percentiles.append(np.nanmean(temp_percentile))
        var_percentiles = np.array(var_percentiles)
        var_percentiles = np.round(var_percentiles, decimals=2)
        res.append(var_percentiles)

    return res

def get_percentiles_rmse(vars, ref_var, model_lons, model_lats, ref_lons, ref_lats, percents):
    res = []
    for idx_data, idata in enumerate(vars):
        model_rmse = []
        out_lons = model_lons[idx_data]
        out_lats = model_lats[idx_data]
        lonout, latout = np.meshgrid(out_lons, out_lats)
        for ipercent in percents:
            model_percentile = np.nanpercentile(idata, ipercent, axis=0)
            ref_percentile = np.nanpercentile(ref_var, ipercent, axis=0)
            ref_percentile = basemap.interp(ref_percentile, ref_lons, ref_lats, lonout, latout, order=1)
            model_rmse.append(rmse(model_percentile, ref_percentile))

        model_rmse = np.array(model_rmse)
        model_rmse = np.round(model_rmse, decimals=2)
        model_rmse = np.array(model_rmse)

        res.append(model_rmse)

    return res

def get_percentiles_rmse_SA(vars, ref_var, percents):
    res = []
    for idx_data, idata in enumerate(vars):
        model_rmse = []
        for ipercent in percents:
            model_percentile = np.nanpercentile(idata.values, ipercent, axis=0)
            ref_percentile = np.nanpercentile(ref_var.values, ipercent, axis=0)
            model_rmse.append(rmse(model_percentile, ref_percentile))

        model_rmse = np.array(model_rmse)
        model_rmse = np.round(model_rmse, decimals=2)
        model_rmse = np.array(model_rmse)

        res.append(model_rmse)

    return res

def get_returns(vars, percentile):
    res = []
    for idata in vars:
        temp_data = idata.flatten()
        temp_data = temp_data[~np.isnan(temp_data)]
        temp_percent = np.percentile(temp_data, percentile)
        temp_data_sub = temp_data[temp_data > temp_percent] - temp_percent
        temp_fit = gpdfit_moment(temp_data_sub)
        temp_return = temp_percent + temp_fit[1] / temp_fit[0] * ((m*obsrate) ** temp_fit[0] - 1)
        temp_return = np.array(temp_return)
        temp_return = np.round(temp_return, decimals=2)
        res.append(temp_return)

    return res

def get_returns_biases(returns):
    ref_return = returns[-1]
    res = []
    for ireturn in returns[:-1]:
        temp_bias = ireturn - ref_return
        temp_bias = np.array(temp_bias)
        temp_bias = np.round(temp_bias, decimals=2)
        res.append(temp_bias)

    return res

############################################################################
# plot data
############################################################################

def plot_scatters(plot_xdata, plot_data, colors, markers, line_types, legends, subtitles, title, fname, **kwargs):
    arranges = {1: [1, 1], 2: [2, 1], 3: [1, 3], 4: [2, 2], 5: [3, 2],
                6: [2, 4], 8: [2, 4], 9: [3, 3], 10: [3, 4], 12: [3, 4]}

    nfigs = len(plot_data)
    if nfigs not in arranges:
        print('plot_2Dcontour: Error! Too many Sub-figures, the maximum number is 9!')
        return -1

    plt.clf()
    fig = plt.figure()
    xdata = range(len(plot_xdata))

    for idx in range(len(plot_data)):
        ax = fig.add_subplot(arranges[nfigs][0], arranges[nfigs][1], idx+1)
        ax.set_title(subtitles[idx], fontsize=5, pad=2)

        temp_data = plot_data[idx]
        for idx_data, idata in enumerate(temp_data):
            print(line_types[idx_data])
            ax.plot(xdata, idata, c=colors[idx_data], markersize=3, marker=markers[idx_data], linestyle=line_types[idx_data], linewidth=0.75, alpha=0.85, label=legends[idx_data])

        # set x/y tick label size
        ax.yaxis.set_tick_params(labelsize=5)
        ax.set_xticks(xdata)
        ax.set_xticklabels(plot_xdata, fontsize=5)

        if 'xlabel' in kwargs:
            xlabel = kwargs['xlabel']

            if (len(plot_data)-idx) <= arranges[nfigs][1]:
                if type(xlabel) == str:
                    ax.set_xlabel(xlabel, fontsize=5)
                else:
                    ax.set_xlabel(xlabel[int(idx % (arranges[nfigs][1]))], fontsize=5)

        if 'ylabel' in kwargs:
            ylabel = kwargs['ylabel']
            if (idx % (arranges[nfigs][1]) == 0):
                if type(ylabel) == str:
                    ax.set_ylabel(ylabel, fontsize=5)
                else:
                    ax.set_ylabel(ylabel[int(idx/(arranges[nfigs][1]))], fontsize=5)

        if (idx == len(plot_data)-1):
            ax.legend(loc='upper left', fontsize=4.5, handlelength=3.5)

    # add title
    plt.savefig(fname+'.png', bbox_inches='tight', dpi=1200)
    plt.suptitle(title, fontsize=7, y=0.95)

    # save figure
    plt.savefig(fname+'.pdf', bbox_inches='tight', dpi=1200)
    plt.close(fig)

############################################################################
# read data
############################################################################

# read Observations
# read CPC

print('Reading CPC data...')

varname = 'precip'

obs_var1, obs_time1, obs_lats1, obs_lons1 = readobs_pre_CPC(
    varname, iniyear, endyear, frequency, latbounds, lonbounds)

obs_var1[obs_var1.mask] = np.nan
print(obs_var1[0, :, :])

# read TRMM

print('Reading TRMM data...')

obs_var2, obs_time2, obs_lats2, obs_lons2 = readobs_pre_TRMM_day(
    'precipitation', 1998, endyear, latbounds, lonbounds, oceanmask=0)

# read SA-OBS

print('Reading SA-OBS data...')

# read SA-OBS
version = 'countries'
countries = ['Thailand', 'Vietnam', 'Myanmar', 'Cambodia']
dataset3, obs_var3, stnids, stnnames, countrynames, stnlats, stnlons, stnhgts, stnmiss = read_SAOBS_pre(
    version, 1990, endyear, countries, missing_ratio=10, ignore_years=[1999])
obs_time3 = dataset3.index

############################################################################
# Calculate the percentiles and return levels against CPC
############################################################################
print('Calculating for CPC...')

print('Reading CORDEX-SEA data...')

# read cordex
project = 'SEA-22'
varname = 'pr'
cordex_models = ['ICHEC-EC-EARTH', 'IPSL-IPSL-CM5A-LR', 'MPI-M-MPI-ESM-MR', 'MOHC-HadGEM2-ES']

modelname = 'ICHEC-EC-EARTH'
cordex_var1, cordex_time1, cordex_lats1, cordex_lons1 = readcordex(
    varname, iniyear, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=1)
modelname = 'IPSL-IPSL-CM5A-LR'
cordex_var2, cordex_time2, cordex_lats2, cordex_lons2 = readcordex(
    varname, iniyear, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=1)
modelname = 'MPI-M-MPI-ESM-MR'
cordex_var3, cordex_time3, cordex_lats3, cordex_lons3 = readcordex(
    varname, iniyear, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=1)
modelname = 'MOHC-HadGEM2-ES'
cordex_var4, cordex_time4, cordex_lats4, cordex_lons4 = readcordex(
    varname, iniyear, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=1)

cordex_var1[cordex_var1.mask] = np.nan
cordex_var2[cordex_var2.mask] = np.nan
cordex_var3[cordex_var3.mask] = np.nan
cordex_var4[cordex_var4.mask] = np.nan

# convert from kg/(m^2*s) to mm/day
cordex_var1 = cordex_var1 * 86400 * 1000 / 997
cordex_var2 = cordex_var2 * 86400 * 1000 / 997
cordex_var3 = cordex_var3 * 86400 * 1000 / 997
cordex_var4 = cordex_var4 * 86400 * 1000 / 997

# print(cordex_var1[0, :, :])
# print(cordex_var1.shape)
# print(cordex_var4.shape)

print('Reading VRCESM data...')

varname = 'PRECT'

resolution = 'fv02'
varfname = 'PREC'
case = 'vrseasia_AMIP_1979_to_2005'
model_var1, model_time1, model_lats1, model_lons1 = readvrcesm(
    varname, iniyear, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=1)

resolution = 'fv09'
varfname = 'PREC'
case = 'ne30_ne30_AMIP_1979_to_2005'
model_var2, model_time2, model_lats2, model_lons2 = readvrcesm(
    varname, iniyear, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=1)

resolution = 'fv09'
varfname = 'PREC'
case = 'f09_f09_AMIP_1979_to_2005'
model_var3, model_time3, model_lats3, model_lons3 = readvrcesm(
    varname, iniyear, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=1)

resolution = 'fv19'
varfname = 'PREC'
case = 'f19_f19_AMIP_1979_to_2005'
model_var4, model_time4, model_lats4, model_lons4 = readvrcesm(
    varname, iniyear, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=1)

model_var1[model_var1.mask] = np.nan
model_var2[model_var2.mask] = np.nan
model_var3[model_var3.mask] = np.nan
model_var4[model_var4.mask] = np.nan

# convert to mm/day
model_var1 = model_var1 * 86400 * 1000
model_var2 = model_var2 * 86400 * 1000
model_var3 = model_var3 * 86400 * 1000
model_var4 = model_var4 * 86400 * 1000

# print(model_var1[0, :, :])

# calculate percentiles for all grids
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var1]
percentiles_allgrids1 = get_percentiles(vars, percents)
# print(percentiles_allgrids1)

# calculate mean of each grid percentiles
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var1]
percentiles_mean1 = get_mean_percentiles(vars, percents)
# print(percentiles_mean1)

# calculate RMSE of percentiles between model and obs
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4]
var_lons = [model_lons1, model_lons2, model_lons3, model_lons4,
            cordex_lons1, cordex_lons2, cordex_lons3, cordex_lons4]
var_lats = [model_lats1, model_lats2, model_lats3, model_lats4,
            cordex_lats1, cordex_lats2, cordex_lats3, cordex_lats4]
ref_var = obs_var1
ref_lons = obs_lons1
ref_lats = obs_lats1
percentiles_rmse1 = get_percentiles_rmse(vars, ref_var, var_lons, var_lats, ref_lons, ref_lats, percents)
# print(percentiles_rmse1)

# calculate return levels
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var1]
extreme_returns1 = get_returns(vars, percentile)
# print(extreme_returns1)

# calculate return levels biaes
extreme_returns_biases1 = get_returns_biases(extreme_returns1)
# print(extreme_returns_biases1)

############################################################################
# Calculate the percentiles and return levels against TRMM
############################################################################
print('Calculating for TRMM...')

print('Reading CORDEX-SEA data...')

# read cordex
project = 'SEA-22'
varname = 'pr'
cordex_models = ['ICHEC-EC-EARTH', 'IPSL-IPSL-CM5A-LR', 'MPI-M-MPI-ESM-MR', 'MOHC-HadGEM2-ES']

modelname = 'ICHEC-EC-EARTH'
cordex_var1, cordex_time1, cordex_lats1, cordex_lons1 = readcordex(
    varname, 1998, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=0)
modelname = 'IPSL-IPSL-CM5A-LR'
cordex_var2, cordex_time2, cordex_lats2, cordex_lons2 = readcordex(
    varname, 1998, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=0)
modelname = 'MPI-M-MPI-ESM-MR'
cordex_var3, cordex_time3, cordex_lats3, cordex_lons3 = readcordex(
    varname, 1998, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=0)
modelname = 'MOHC-HadGEM2-ES'
cordex_var4, cordex_time4, cordex_lats4, cordex_lons4 = readcordex(
    varname, 1998, endyear, project, modelname, frequency, latbounds, lonbounds, oceanmask=0)

# convert from kg/(m^2*s) to mm/day
cordex_var1 = cordex_var1 * 86400 * 1000 / 997
cordex_var2 = cordex_var2 * 86400 * 1000 / 997
cordex_var3 = cordex_var3 * 86400 * 1000 / 997
cordex_var4 = cordex_var4 * 86400 * 1000 / 997

# print(cordex_var1[0, :, :])
# print(cordex_var1.shape)
# print(cordex_var4.shape)

print('Reading VRCESM data...')

varname = 'PRECT'

resolution = 'fv02'
varfname = 'PREC'
case = 'vrseasia_AMIP_1979_to_2005'
model_var1, model_time1, model_lats1, model_lons1 = readvrcesm(
    varname, 1998, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=0)

resolution = 'fv09'
varfname = 'PREC'
case = 'ne30_ne30_AMIP_1979_to_2005'
model_var2, model_time2, model_lats2, model_lons2 = readvrcesm(
    varname, 1998, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=0)

resolution = 'fv09'
varfname = 'PREC'
case = 'f09_f09_AMIP_1979_to_2005'
model_var3, model_time3, model_lats3, model_lons3 = readvrcesm(
    varname, 1998, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=0)

resolution = 'fv19'
varfname = 'PREC'
case = 'f19_f19_AMIP_1979_to_2005'
model_var4, model_time4, model_lats4, model_lons4 = readvrcesm(
    varname, 1998, endyear, resolution, varfname, case, frequency, latbounds, lonbounds, oceanmask=0)

# convert to mm/day
model_var1 = model_var1 * 86400 * 1000
model_var2 = model_var2 * 86400 * 1000
model_var3 = model_var3 * 86400 * 1000
model_var4 = model_var4 * 86400 * 1000

# print(model_var1[0, :, :])

# calculate percentiles for all grids
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var2]
percentiles_allgrids2 = get_percentiles(vars, percents)
# print(percentiles_allgrids2)

# calculate mean of each grid percentiles
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var2]
percentiles_mean2 = get_mean_percentiles(vars, percents)
# print(percentiles_mean2)

# calculate RMSE of percentiles between model and obs
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4]
var_lons = [model_lons1, model_lons2, model_lons3, model_lons4,
            cordex_lons1, cordex_lons2, cordex_lons3, cordex_lons4]
var_lats = [model_lats1, model_lats2, model_lats3, model_lats4,
            cordex_lats1, cordex_lats2, cordex_lats3, cordex_lats4]
ref_var = obs_var2
ref_lons = obs_lons2
ref_lats = obs_lats2
percentiles_rmse2 = get_percentiles_rmse(vars, ref_var, var_lons, var_lats, ref_lons, ref_lats, percents)
# print(percentiles_rmse2)

# calculate return levels
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4, obs_var2]
extreme_returns2 = get_returns(vars, percentile)
# print(extreme_returns2)

# calculate return levels biaes
extreme_returns_biases2 = get_returns_biases(extreme_returns2)
# print(extreme_returns_biases2)

############################################################################
# Calculate the percentiles and return levels against SA-OBS
############################################################################
print('Calculating for SA-OBS...')

print('Reading CORDEX-SEA data...')

# read cordex
project = 'SEA-22'
varname = 'pr'
cordex_models = ['ICHEC-EC-EARTH', 'IPSL-IPSL-CM5A-LR', 'MPI-M-MPI-ESM-MR', 'MOHC-HadGEM2-ES']

modelname = 'ICHEC-EC-EARTH'
cordex_var1, cordex_time1, cordex_lats1, cordex_lons1 = readcordex(
    varname, 1990, endyear, project, modelname, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])
modelname = 'IPSL-IPSL-CM5A-LR'
cordex_var2, cordex_time2, cordex_lats2, cordex_lons2 = readcordex(
    varname, 1990, endyear, project, modelname, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])
modelname = 'MPI-M-MPI-ESM-MR'
cordex_var3, cordex_time3, cordex_lats3, cordex_lons3 = readcordex(
    varname, 1990, endyear, project, modelname, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])
modelname = 'MOHC-HadGEM2-ES'
cordex_var4, cordex_time4, cordex_lats4, cordex_lons4 = readcordex(
    varname, 1990, endyear, project, modelname, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])

# convert from kg/(m^2*s) to mm/day
cordex_var1 = cordex_var1 * 86400 * 1000 / 997
cordex_var2 = cordex_var2 * 86400 * 1000 / 997
cordex_var3 = cordex_var3 * 86400 * 1000 / 997
cordex_var4 = cordex_var4 * 86400 * 1000 / 997

# print(cordex_var1[0, :, :])
# print(cordex_var1.shape)
# print(cordex_var4.shape)

print('Reading VRCESM data...')

varname = 'PRECT'

resolution = 'fv02'
varfname = 'PREC'
case = 'vrseasia_AMIP_1979_to_2005'
model_var1, model_time1, model_lats1, model_lons1 = readvrcesm(
    varname, 1990, endyear, resolution, varfname, case, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])

resolution = 'fv09'
varfname = 'PREC'
case = 'ne30_ne30_AMIP_1979_to_2005'
model_var2, model_time2, model_lats2, model_lons2 = readvrcesm(
    varname, 1990, endyear, resolution, varfname, case, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])

resolution = 'fv09'
varfname = 'PREC'
case = 'f09_f09_AMIP_1979_to_2005'
model_var3, model_time3, model_lats3, model_lons3 = readvrcesm(
    varname, 1990, endyear, resolution, varfname, case, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])

resolution = 'fv19'
varfname = 'PREC'
case = 'f19_f19_AMIP_1979_to_2005'
model_var4, model_time4, model_lats4, model_lons4 = readvrcesm(
    varname, 1990, endyear, resolution, varfname, case, frequency, reg_lats, reg_lons, oceanmask=0, ignore_years=[1999])

# convert to mm/day
model_var1 = model_var1 * 86400 * 1000
model_var2 = model_var2 * 86400 * 1000
model_var3 = model_var3 * 86400 * 1000
model_var4 = model_var4 * 86400 * 1000

# print(model_var1[0, :, :])

# find stations in gridded data
cordex_var1 = data_findstns(cordex_var1, cordex_time1, cordex_lats1, cordex_lons1, obs_var3, stnlats, stnlons, stnnames)
cordex_var2 = data_findstns(cordex_var2, cordex_time2, cordex_lats2, cordex_lons2, obs_var3, stnlats, stnlons, stnnames)
cordex_var3 = data_findstns(cordex_var3, cordex_time3, cordex_lats3, cordex_lons3, obs_var3, stnlats, stnlons, stnnames)
cordex_var4 = data_findstns(cordex_var4, cordex_time4, cordex_lats4, cordex_lons4, obs_var3, stnlats, stnlons, stnnames)

model_var1 = data_findstns(model_var1, model_time1, model_lats1, model_lons1, obs_var3, stnlats, stnlons, stnnames)
model_var2 = data_findstns(model_var2, model_time2, model_lats2, model_lons2, obs_var3, stnlats, stnlons, stnnames)
model_var3 = data_findstns(model_var3, model_time3, model_lats3, model_lons3, obs_var3, stnlats, stnlons, stnnames)
model_var4 = data_findstns(model_var4, model_time4, model_lats4, model_lons4, obs_var3, stnlats, stnlons, stnnames)

# calculate percentiles for all grids
vars = [model_var1.values, model_var2.values, model_var3.values, model_var4.values,
        cordex_var1.values, cordex_var2.values, cordex_var3.values, cordex_var4.values, dataset3.values]
percentiles_allgrids3 = get_percentiles(vars, percents)
# print(percentiles_allgrids3)

# calculate mean of each grid percentiles
vars = [model_var1.values, model_var2.values, model_var3.values, model_var4.values,
        cordex_var1.values, cordex_var2.values, cordex_var3.values, cordex_var4.values, dataset3.values]
percentiles_mean3 = get_mean_percentiles(vars, percents)
# print(percentiles_mean3)

# calculate RMSE of percentiles between model and obs
vars = [model_var1, model_var2, model_var3, model_var4,
        cordex_var1, cordex_var2, cordex_var3, cordex_var4]
ref_var = dataset3
percentiles_rmse3 = get_percentiles_rmse_SA(vars, ref_var, percents)
# print(percentiles_rmse3)

# calculate return levels
vars = [model_var1.values, model_var2.values, model_var3.values, model_var4.values,
        cordex_var1.values, cordex_var2.values, cordex_var3.values, cordex_var4.values, dataset3.values]
extreme_returns3 = get_returns(vars, percentile)
# print(extreme_returns3)

# calculate return levels biaes
extreme_returns_biases3 = get_returns_biases(extreme_returns3)
# print(extreme_returns_biases3)

############################################################################
# Plot RMSD
############################################################################
print('Plot for RMSD...')

# cesm and cordex
plot_data = [percentiles_rmse1, percentiles_rmse2, percentiles_rmse3]
plot_xdata = percents

subtitles = ['Biases relative to CPC', 'Biases relative to TRMM', 'Biases relative to SACA']
title = 'Biases of precip extemes relative to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_biases_RMSE_references_vs_cordex'
plot_scatters(plot_xdata, plot_data, colors, markers, linetypes, legends, subtitles,
              title, outdir+fname, xlabel='Percentiles', ylabel='RMSD (mm/day)')

# cesm only
plot_data = [percentiles_rmse1[:4], percentiles_rmse2[:4], percentiles_rmse3[:4]]
plot_xdata = percents

subtitles = ['Biases relative to CPC', 'Biases relative to TRMM', 'Biases relative to SACA']
title = 'Biases of precip extemes relative to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_biases_RMSE_references'
plot_scatters(plot_xdata, plot_data, cesm_colors, cesm_markers, cesm_linetypes, cesm_legends, subtitles,
              title, outdir+fname, xlabel='Percentiles', ylabel='RMSD (mm/day)')

############################################################################
# Plot Return levels
############################################################################
print('Plot for return levels...')

# cesm and cordex
plot_data = [extreme_returns_biases1, extreme_returns_biases2, extreme_returns_biases3]
plot_xdata = return_years

subtitles = ['Biases relative to CPC', 'Biases relative to TRMM', 'Biases relative to SACA']
title = 'Biases of precip return levels relative to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_return_levels_references_vs_cordex'
plot_scatters(plot_xdata, plot_data, colors, markers, linetypes, legends, subtitles,
              title, outdir+fname, xlabel='Return years', ylabel='Biases (mm/day)')

# cesm only
plot_data = [extreme_returns_biases1[:4], extreme_returns_biases2[:4], extreme_returns_biases3[:4]]
plot_xdata = return_years

subtitles = ['Biases relative to CPC', 'Biases relative to TRMM', 'Biases relative to SACA']
title = 'Biases of precip return levels relative to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_return_levels_references'
plot_scatters(plot_xdata, plot_data, cesm_colors, cesm_markers,  cesm_linetypes, cesm_legends, subtitles,
              title, outdir+fname, xlabel='Return years', ylabel='Biases (mm/day)')

############################################################################
# Plot percentiles
############################################################################
print('Plot for Percentiles...')

# cesm and cordex
plot_data = [percentiles_allgrids1, percentiles_allgrids2, percentiles_allgrids3]
plot_xdata = percents

colors.append('black')
markers.append('*')
legends.append('Obervation')

subtitles = ['CPC as reference', 'TRMM as reference', 'SACA as reference']
title = 'Total precip percentiles compared to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_percentiles_references_vs_cordex'
plot_scatters(plot_xdata, plot_data, colors, markers,  linetypes, legends, subtitles,
              title, outdir+fname, xlabel='Percentiles', ylabel='Precip (mm/day)')

# cesm only
select_data = [0, 1, 2, 3, 8]
plot_data = [list(percentiles_allgrids1[i] for i in select_data),
             list(percentiles_allgrids2[i] for i in select_data),
             list(percentiles_allgrids3[i] for i in select_data)]
plot_xdata = percents

cesm_colors.append('black')
cesm_markers.append('*')
cesm_legends.append('Obervation')

subtitles = ['CPC as reference', 'TRMM as reference', 'SACA as reference']
title = 'Total precip percentiles compared to observations over'+reg_name
fname = 'vrseasia_prect_extremes_SEA_percentiles_references'
plot_scatters(plot_xdata, plot_data, cesm_colors, cesm_markers,  cesm_linetypes, cesm_legends, subtitles,
              title, outdir+fname, xlabel='Percentiles', ylabel='Precip (mm/day)')

############################################################################
# output the results
############################################################################

result = {}
result['models'] = legends
result['CPC-all-grids-percentiles'] = percentiles_allgrids1
result['CPC-grids-mean-percentiles'] = percentiles_mean1
result['CPC-percentiles-RMSE'] = percentiles_rmse1
result['CPC-return-levels'] = extreme_returns1
result['CPC-return-levels-biases'] = extreme_returns_biases1

result['TRMM-all-grids-percentiles'] = percentiles_allgrids2
result['TRMM-grids-mean-percentiles'] = percentiles_mean2
result['TRMM-percentiles-RMSE'] = percentiles_rmse2
result['TRMM-return-levels'] = extreme_returns2
result['TRMM-return-levels-biases'] = extreme_returns_biases2

result['SAOBS-all-grids-percentiles'] = percentiles_allgrids3
result['SAOBS-grids-mean-percentiles'] = percentiles_mean3
result['SAOBS-percentiles-RMSE'] = percentiles_rmse3
result['SAOBS-return-levels'] = extreme_returns3
result['SAOBS-return-levels-biases'] = extreme_returns_biases3

pickle.dump(result, open(outdir+'vrseasia_prect_extremes_SEA_relatives_to_references_results_vs_cordex.p', "wb"))

# save to csv

# CPC
percentiles_rmse1.append([0., 0., 0., 0., 0., 0., 0., 0.])
extreme_returns_biases1.append([0., 0., 0., 0., 0., 0., 0., 0.])
result_percents = {}
result_percents['Percentiles'] = percents
legends[-1] = 'CPC'
for idx in range(len(legends)):
    result_percents[legends[idx]+'-all-grids'] = percentiles_allgrids1[idx]
    result_percents[legends[idx]+'-grids-mean'] = percentiles_mean1[idx]
    result_percents[legends[idx]+'-RMSE'] = percentiles_rmse1[idx]
result_percents = pd.DataFrame(result_percents)
result_percents = result_percents.set_index('Percentiles')
result_percents.to_csv(outdir+'CPC_'+str(iniyear)+'to'+str(endyear) +
                       '_prect_extremes_percentiles_results.csv', sep=',', index=True)

result_returns = {}
result_returns['Return years'] = percents
for idx in range(len(legends)):
    result_returns[legends[idx]+'-return-levels'] = extreme_returns1[idx]
    result_returns[legends[idx]+'-biases'] = extreme_returns_biases1[idx]
result_returns = pd.DataFrame(result_returns)
result_returns = result_returns.set_index('Return years')
result_returns.to_csv(outdir+'CPC_'+str(iniyear)+'to'+str(endyear) +
                      '_prect_extremes_return_levels_results.csv', sep=',', index=True)

# TRMM
percentiles_rmse2.append([0., 0., 0., 0., 0., 0., 0., 0.])
extreme_returns_biases2.append([0., 0., 0., 0., 0., 0., 0., 0.])
result_percents = {}
result_percents['Percentiles'] = percents
legends[-1] = 'TRMM'
for idx in range(len(legends)):
    result_percents[legends[idx]+'-all-grids'] = percentiles_allgrids2[idx]
    result_percents[legends[idx]+'-grids-mean'] = percentiles_mean2[idx]
    result_percents[legends[idx]+'-RMSE'] = percentiles_rmse2[idx]
result_percents = pd.DataFrame(result_percents)
result_percents = result_percents.set_index('Percentiles')
result_percents.to_csv(outdir+'TRMM_1998to'+str(endyear) +
                       '_prect_extremes_percentiles_results.csv', sep=',', index=True)

result_returns = {}
result_returns['Return years'] = percents
for idx in range(len(legends)):
    result_returns[legends[idx]+'-return-levels'] = extreme_returns2[idx]
    result_returns[legends[idx]+'-biases'] = extreme_returns_biases2[idx]
result_returns = pd.DataFrame(result_returns)
result_returns = result_returns.set_index('Return years')
result_returns.to_csv(outdir+'TRMM_1998to'+str(endyear) +
                      '_prect_extremes_return_levels_results.csv', sep=',', index=True)

# SA-OBS
percentiles_rmse3.append([0., 0., 0., 0., 0., 0., 0., 0.])
extreme_returns_biases3.append([0., 0., 0., 0., 0., 0., 0., 0.])
result_percents = {}
result_percents['Percentiles'] = percents
legends[-1] = 'SA-OBS'
for idx in range(len(legends)):
    result_percents[legends[idx]+'-all-grids'] = percentiles_allgrids3[idx]
    result_percents[legends[idx]+'-grids-mean'] = percentiles_mean3[idx]
    result_percents[legends[idx]+'-RMSE'] = percentiles_rmse3[idx]
result_percents = pd.DataFrame(result_percents)
result_percents = result_percents.set_index('Percentiles')
result_percents.to_csv(outdir+'SAOBS_1990to'+str(endyear) +
                       '_prect_extremes_percentiles_results.csv', sep=',', index=True)

result_returns = {}
result_returns['Return years'] = return_years
for idx in range(len(legends)):
    result_returns[legends[idx]+'-return-levels'] = extreme_returns3[idx]
    result_returns[legends[idx]+'-biases'] = extreme_returns_biases3[idx]
result_returns = pd.DataFrame(result_returns)
result_returns = result_returns.set_index('Return years')
result_returns.to_csv(outdir+'SAOBS_1990to'+str(endyear) +
                      '_prect_extremes_return_levels_results.csv', sep=',', index=True)


from abc import abstractmethod

from django.conf import settings
from django.db import models
from django.urls import reverse
from mdeditor.fields import MDTextField
from uuslug import slugify

# Create your models here.
class BaseModel(models.Model):
    id = models.AutoField(primary_key=True)
    created_time = models.DateTimeField('创建时间', auto_now_add=True)
    last_mod_time = models.DateTimeField('修改时间', auto_now=True)

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        # is_update_views = isinstance(
        #     self,
        #     Article) and 'update_fields' in kwargs and kwargs['update_fields'] == ['views']
        # if is_update_views:
        #     Article.objects.filter(pk=self.pk).update(views=self.views)
        # else:
        if 'slug' in self.__dict__:
            slug = getattr(
                self, 'title') if 'title' in self.__dict__ else getattr(
                self, 'name')
            setattr(self, 'slug', slugify(slug))
        super().save(*args, **kwargs)

    def get_full_url(self):
        # site = get_current_site().domain
        site = settings.SITE
        url = "https://{site}{path}".format(site=site,
                                            path=self.get_absolute_url())
        return url

    @abstractmethod
    def get_absolute_url(self):
        pass

class Article(BaseModel):
    """文章"""
    STATUS_CHOICES = (
        ('d', '草稿'),
        ('p', '发表'),
    )
    COMMENT_STATUS = (
        ('o', '打开'),
        ('c', '关闭'),
    )
    TYPE = (
        ('a', '文章'),
        ('p', '页面'),
    )
    title = models.CharField('标题', max_length=200, unique=True)
    body = MDTextField('正文')
    pub_time = models.DateTimeField(
        '发布时间', blank=False, null=False, auto_now_add=True)
    status = models.CharField(
        '文章状态',
        max_length=1,
        choices=STATUS_CHOICES,
        default='p')
    comment_status = models.CharField(
        '评论状态',
        max_length=1,
        choices=COMMENT_STATUS,
        default='o')
    type = models.CharField('类型', max_length=1, choices=TYPE, default='a')
    views = models.PositiveIntegerField('浏览量', default=0)
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name='作者',
        blank=False,
        null=False,
        on_delete=models.CASCADE)
    article_order = models.IntegerField(
        '排序,数字越大越靠前', blank=False, null=False, default=0)
    category = models.ForeignKey(
        'Category',
        verbose_name='分类',
        on_delete=models.CASCADE,
        blank=False,
        null=False)
    tags = models.ManyToManyField('Tag', verbose_name='标签集合', blank=True)

    def get_body(self):
        return f"{self.title} {self.author.username} {self.body}"

    def body_to_string(self):
        return self.body

    def __str__(self):
        return self.title

    class Meta:
        ordering = ['-article_order', '-pub_time']
        verbose_name = "文章"
        verbose_name_plural = verbose_name
        get_latest_by = 'id'

    def viewed(self):
        self.views += 1
        self.save()

    def get_absolute_url(self):
        return reverse('blog:detailbyid', kwargs={
            'article_id': self.id,
            'year': self.created_time.year,
            'month': self.created_time.month,
            'day': self.created_time.day
        })

    def get_tags_str(self):
        tag_name = []
        print(self.tags)
        print(self.tags.name)
        if self.tags.name:
            for tag in self.tags.name:
                print(tag)
                tag_name.append(tag)
        return "，".join(tag_name)

class Category(BaseModel):
    """文章分类"""
    name = models.CharField('分类名', max_length=30, unique=True)
    parent_category = models.ForeignKey(
        'self',
        verbose_name="父级分类",
        blank=True,
        null=True,
        on_delete=models.CASCADE)
    slug = models.SlugField(default='no-slug', max_length=60, blank=True)

    class Meta:
        ordering = ['name']
        verbose_name = "分类"
        verbose_name_plural = verbose_name

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('blog:categorybyslug', kwargs={
            'category_name': self.slug
        })

    def get_category_tree(self):
        """
        递归获得分类目录的父级
        :return:
        """
        categorys = []

        def parse(category):
            categorys.append(category)
            if category.parent_category:
                parse(category.parent_category)

        parse(self)
        return categorys

    def get_sub_categorys(self):
        """
        获得当前分类目录所有子集
        :return:
        """
        categorys = []
        all_categorys = Category.objects.all()

        def parse(category):
            if category not in categorys:
                categorys.append(category)
            childs = all_categorys.filter(parent_category=category)
            for child in childs:
                if category not in categorys:
                    categorys.append(child)
                parse(child)

        parse(self)
        return categorys

class Tag(BaseModel):
    """文章标签"""
    name = models.CharField('标签名', max_length=30, unique=True)
    slug = models.SlugField(default='no-slug', max_length=60, blank=True)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['name']
        verbose_name = "标签"
        verbose_name_plural = verbose_name

    def get_absolute_url(self):
        return reverse('blog:tagbyslug', kwargs={'tag_name': self.slug})

class Comment(models.Model):
    body = models.TextField('正文', max_length=300)
    created_time = models.DateTimeField('创建时间', auto_now_add=True)
    last_mod_time = models.DateTimeField('修改时间', auto_now=True)
    author = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name='作者',
        on_delete=models.CASCADE)
    article = models.ForeignKey(
        Article,
        verbose_name='文章',
        on_delete=models.CASCADE)
    parent_comment = models.ForeignKey(
        'self',
        verbose_name="上级评论",
        blank=True,
        null=True,
        on_delete=models.CASCADE)
    is_enable = models.BooleanField(
        '是否显示', default=True, blank=False, null=False)

    class Meta:
        ordering = ['id']
        verbose_name = "评论"
        verbose_name_plural = verbose_name
        get_latest_by = 'id'

    def __str__(self):
        return self.body

    def get_sub_categorys(self):
        """
        获得当前分类目录所有子集
        :return:
        """
        comments = []
        all_comments = Comment.objects.all()

        def parse(comment):
            if comment not in comments:
                comments.append(comment)
            childs = all_comments.filter(parent_comment=comment)
            for child in childs:
                if comment not in comments:
                    comments.append(child)
                parse(child)

        parse(self)
        return comments


import functools
import threading
import time
import sys

def calc_time(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start = time.time()
        result = func(*args, **kwargs)
        finish = time.time()
        print(f'elapsed time: {finish-start}')
        return result
    return wrapper

@calc_time
def func(n):
    print(f'start : {n}')
    for i in range(10):
        print(f'{n}: {i}')
    print(f'finish: {n}')

@calc_time
def main():
    # define threads
    threads = []
    for i in range(5):
        thread = threading.Thread(target=func, name=f'thread-{i}', args=([i]))
        threads.append(thread)

    # start threads
    for thread in threads:
        thread.start()

    # wait until finished
    for thread in threads:
        thread.join()

    return 0

if __name__ == '__main__':
    sys.exit(main())


from PIL import Image, ImageOps
import numpy as np
from matplotlib import pylab as plt

for k in range(10000):
    img = np.array(Image.open('./image/depth_logarithmic_gray/' + str(k+3000) + '.jpg').convert('I'), 'float32')
    def calc_double(n):
        return n + 40

    img2 = list(map(calc_double, img))

    for i in range(600):
        for j in range(800):
            if img2[i][j] > 255:
                img2[i][j] = 255

    np.save('./image/depth_gray/' + str(k+3000) + '.npy',img2)
    print(str(k+3000) + '.jpg saved')


import argparse
import sys

# We need to import pywrap_tensorflow prior to the toco wrapper.
# pylint: disable=invalid-import-order,g-bad-import-order
from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
from tensorflow.python import _pywrap_toco_api
from absl import app

FLAGS = None

def execute(unused_args):
  """Runs the converter."""
  with open(FLAGS.model_proto_file, "rb") as model_file:
    model_str = model_file.read()

  with open(FLAGS.toco_proto_file, "rb") as toco_file:
    toco_str = toco_file.read()

  with open(FLAGS.model_input_file, "rb") as input_file:
    input_str = input_file.read()

  debug_info_str = None
  if FLAGS.debug_proto_file:
    with open(FLAGS.debug_proto_file, "rb") as debug_info_file:
      debug_info_str = debug_info_file.read()

  enable_mlir_converter = FLAGS.enable_mlir_converter

  output_str = _pywrap_toco_api.TocoConvert(
      model_str,
      toco_str,
      input_str,
      False,  # extended_return
      debug_info_str,
      enable_mlir_converter)
  open(FLAGS.model_output_file, "wb").write(output_str)
  sys.exit(0)

def main():
  global FLAGS
  parser = argparse.ArgumentParser(
      description="Invoke toco using protos as input.")
  parser.add_argument(
      "model_proto_file",
      type=str,
      help="File containing serialized proto that describes the model.")
  parser.add_argument(
      "toco_proto_file",
      type=str,
      help="File containing serialized proto describing how TOCO should run.")
  parser.add_argument(
      "model_input_file", type=str, help="Input model is read from this file.")
  parser.add_argument(
      "model_output_file",
      type=str,
      help="Result of applying TOCO conversion is written here.")
  parser.add_argument(
      "--debug_proto_file",
      type=str,
      default="",
      help=("File containing serialized `GraphDebugInfo` proto that describes "
            "logging information."))
  parser.add_argument(
      "--enable_mlir_converter",
      action="store_true",
      help=("Boolean indicating whether to enable MLIR-based conversion "
            "instead of TOCO conversion. (default False)"))

  FLAGS, unparsed = parser.parse_known_args()

  app.run(main=execute, argv=[sys.argv[0]] + unparsed)

if __name__ == "__main__":
  main()


from pymongo import MongoClient

HOST = "localhost"
PORT = 2222

client = MongoClient(HOST, PORT)
db = client['BouncerEventsDB']
evtColl = db['eventCollection']

def insertItem(item):
    ts = item.pop('time_stamp')
    found = evtColl.find_one(item)
    if not found:
        item['time_stamp'] = ts
        found = item
    else:
        found['time_stamp'] = list(set(found['time_stamp'] + ts))

    evtColl.save(found)

def insertItems(items):
    names = evtColl.find_one({'meta':'names'})
    for i in items:
        i.pop('num')

        #i['time_stamp'] = list(i['time_stamp'])
        ts = list(i.pop('time_stamp'))
        found = evtColl.find_one(i)
        if not found:
            i['time_stamp'] = ts
            found = i
        else:
            found['time_stamp'] = list(set(found['time_stamp'] + ts))

        if not found.get('completed'):
            found['completed'] = []

        e = i['evt_name']
        en = names.get(e)
        if not en:
            names[e] = {
                'fired':0,
                'completed':0,
                i['siminar_id'] :{
                    'fired':0,
                    'completed':0,
                    }
                }

        names[e]['fired'] = len(found['time_stamp'])
        si = names[e].get(i['siminar_id'])
        if not si:
            si = names[e][i['siminar_id']]
            si = 0
        si['fired'] = len(found['time_stamp'])
        evtColl.save(found)

    evtColl.save(names)


import sys
import math
import torch
from torch import nn
from torch.nn import functional as F
from torch.nn.parameter import Parameter
from torchvision import models
#from efficientnet_pytorch import EfficientNet

import sys
sys.path.append("../input/pretrained-models/pretrained-models/pretrained-models.pytorch-master/")
from pretrainedmodels import se_resnext101_32x4d, senet154
from pretrainedmodels import inceptionresnetv2
from pretrainedmodels import pnasnet5large
from pretrainedmodels import xception

class ArcMarginProduct(nn.Module):
    r"""Implement of large margin arc distance: :
        Args:
            in_features: size of each input sample
            out_features: size of each output sample
            s: norm of input feature
            m: margin
            cos(theta + m)
        """

    def __init__(self, in_features, out_features=5004):
        super(ArcMarginProduct, self).__init__()
        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))
        self.reset_parameters()
        # nn.init.kaiming_uniform_(self.weight)

    def reset_parameters(self):
        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)

    def forward(self, features):
        # x=self.head(features)
        # print(self.weight.shape)
        # self.fc1.weight=nn.Parameter(F.normalize(self.fc1.weight)).cuda()
        cosine = F.linear(F.normalize(features), F.normalize(self.weight.cuda()))
        # cosine = cosine.clamp(-1, 1)
        # self.fc1(F.normalize(x))
        # F.linear(F.normalize(x), F.normalize(self.weight.cuda()))
        return cosine

class AvgPool(nn.Module):
    def forward(self, x):
        return F.avg_pool2d(x, x.shape[2:])

class MaxPool(nn.Module):
    def forward(self, x):
        return F.max_pool2d(x, x.shape[2:])

class AdaptiveConcatPool2d(nn.Module):
    def __init__(self, sz=None):
        super().__init__()
        sz = sz or (1,1)
        self.ap = nn.AdaptiveAvgPool2d(sz)
        self.mp = nn.AdaptiveMaxPool2d(sz)
    def forward(self, x):
        return torch.cat([self.mp(x), self.ap(x)], 1)

def gem(x, p=3, eps=1e-6):
    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)

class GeM(nn.Module):
    def __init__(self, p=3, eps=1e-6):
        super(GeM,self).__init__()
        self.p = Parameter(torch.ones(1)*p)
        self.eps = eps

    def forward(self, x):
        return gem(x, p=self.p, eps=self.eps)

    def __repr__(self):
        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'

class CSE(nn.Module):
    def __init__(self, in_ch, r):
        super(CSE, self).__init__()

        self.linear_1 = nn.Linear(in_ch, in_ch//r)
        self.linear_2 = nn.Linear(in_ch//r, in_ch)

    def forward(self, x):
        input_x = x

        x = x.view(*(x.shape[:-2]),-1).mean(-1)
        x = F.relu(self.linear_1(x), inplace=True)
        x = self.linear_2(x)
        x = x.unsqueeze(-1).unsqueeze(-1)
        x = F.sigmoid(x)

        x = input_x * x

        return x

class SSE(nn.Module):
    def __init__(self, in_ch):
        super(SSE, self).__init__()

        self.conv = nn.Conv2d(in_ch, in_ch, kernel_size=1, stride=1)

    def forward(self, x):
        input_x = x

        x = self.conv(x)
        x = F.sigmoid(x)

        x = input_x * x

        return x

class SCSE(nn.Module):
    def __init__(self, in_ch, r=8):
        super(SCSE, self).__init__()

        self.cSE = CSE(in_ch, r)
        self.sSE = SSE(in_ch)

    def forward(self, x):
        cSE = self.cSE(x)
        sSE = self.sSE(x)

        x = cSE + sSE

        return x

class SEBlock(nn.Module):
    def __init__(self, in_ch, r=8):
        super(SEBlock, self).__init__()

        self.linear_1 = nn.Linear(in_ch, in_ch//r)
        self.linear_2 = nn.Linear(in_ch//r, in_ch)

    def forward(self, x):
        input_x = x

        x = F.relu(self.linear_1(x), inplace=True)
        x = self.linear_2(x)
        x = F.sigmoid(x)

        x = input_x * x

        return x

class Flatten(nn.Module):
    def forward(self, input):
        return input.view(input.size(0), -1)

def create_net(net_cls, pretrained: bool):
    #net = net_cls()
    #if pretrained is not None:
    #    net.load_state_dict(torch.load(pretrained))
    net = net_cls(pretrained=pretrained)
    return net

class ResNet(nn.Module):
    def __init__(self, num_classes,
                 pretrained=False, net_cls=models.resnet50):
        super().__init__()
        self.net = create_net(net_cls, pretrained=pretrained)
        self.net.avgpool = AdaptiveConcatPool2d()
        self.net.fc = nn.Sequential(
            Flatten(),
            #SEBlock(2048*2),
            nn.Dropout(),
            nn.Linear(2048*2, num_classes)
        )

    def fresh_params(self):
        return self.net.fc.parameters()

    def forward(self, x):
        return self.net(x)

class DenseNet(nn.Module):
    def __init__(self, num_classes,
                 pretrained=False, net_cls=models.densenet121):
        super().__init__()
        self.net = net_cls(pretrained=pretrained)
        self.avg_pool = AdaptiveConcatPool2d()
        self.net.classifier = nn.Sequential(
            Flatten(),
            SEBlock(1024*2),
            nn.Dropout(),
            nn.Linear(1024*2, num_classes)
        )

    def fresh_params(self):
        return self.net.classifier.parameters()

    def forward(self, x):
        out = self.net.features(x)
        out = F.relu(out, inplace=True)
        out = self.avg_pool(out)
        out = self.net.classifier(out)
        return out

class SEResNext(nn.Module):
    def __init__(self, num_classes, pretrained="imagenet", pool_type="concat"):
        super().__init__()
        self.net = se_resnext101_32x4d(pretrained=pretrained)
        if pool_type == "concat":
            self.net.avg_pool = AdaptiveConcatPool2d()
            last_channel = 2048*2
        elif pool_type == "gem":
            self.net.avg_pool = GeM()
            last_channel = 2048
        self.net.last_linear = nn.Sequential(
            Flatten(),
            SEBlock(last_channel),
            nn.Dropout(),
            nn.Linear(last_channel, num_classes)
        )

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class Efficient(nn.Module):
    def __init__(self, num_classes, encoder='efficientnet-b7'):
        super().__init__()
        n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,
                           'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,
                           'efficientnet-b6': 2304, 'efficientnet-b7': 2560}
        self.net = EfficientNet.from_pretrained(encoder)
        self.avg_pool = AdaptiveConcatPool2d()
        self.classifier = nn.Sequential(
            Flatten(),
            SEBlock(n_channels_dict[encoder] * 2),
            nn.Dropout(),
            nn.Linear(n_channels_dict[encoder] * 2, num_classes)
        )

    def forward(self, x):
        x = self.net.extract_features(x)
        x = self.avg_pool(x)
        out = self.classifier(x)

        return out

class InceptionResNetV2(nn.Module):
    def __init__(self, num_classes, pretrained="imagenet"):
        super().__init__()
        self.net = inceptionresnetv2(pretrained=pretrained)
        self.net.avgpool_1a = AdaptiveConcatPool2d()
        self.net.last_linear = nn.Sequential(
            Flatten(),
            SEBlock(1536*2),
            nn.Linear(1536*2, num_classes)
        )

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class SENet(nn.Module):
    def __init__(self, num_classes, pretrained="senet154-c7b49a05.pth", dropout=False, arcface=False):
        super().__init__()
        self.net = senet154(pretrained=pretrained)
        self.net.last_linear = nn.Linear(2048, num_classes)

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class SENetV2(nn.Module):
    def __init__(self, num_classes, pretrained="senet154-c7b49a05.pth", dropout=False, arcface=False):
        super().__init__()
        self.net = senet154(pretrained=pretrained)
        self.net.avg_pool = AvgPool()
        self.net.last_linear = nn.Linear(2048, num_classes)

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class SENetV3(nn.Module):
    def __init__(self, num_classes, pretrained="senet154-c7b49a05.pth", dropout=False, arcface=False):
        super().__init__()
        self.net = senet154(pretrained=pretrained)
        self.net.avg_pool = AvgPool()
        self.net.last_linear = nn.Linear(2048, num_classes)

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class SENetV4(nn.Module):
    def __init__(self, num_classes, pretrained="senet154-c7b49a05.pth", dropout=False, arcface=False):
        super().__init__()
        self.net = senet154(pretrained=pretrained)
        self.net.avg_pool = AdaptiveConcatPool2d()
        self.net.last_linear = nn.Sequential(
            Flatten(),
            SEBlock(2048*2),
            nn.Linear(2048*2, num_classes)
        )

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class PNASNet5Large(nn.Module):
    def __init__(self, num_classes, pretrained="pnasnet5large-bf079911.pth", dropout=False, arcface=False):
        super().__init__()
        self.net = pnasnet5large(pretrained=pretrained)
        self.net.avg_pool = AdaptiveConcatPool2d()
        self.net.last_linear = nn.Sequential(
            Flatten(),
            SEBlock(4320 * 2),
            nn.Linear(4320 * 2, num_classes)
        )

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        return self.net(x)

class Xception(nn.Module):
    def __init__(self, num_classes, pretrained="xception-43020ad28.pth"):
        super().__init__()
        self.net = xception(pretrained=pretrained)
        self.avg_pool = AdaptiveConcatPool2d()
        self.net.last_linear = nn.Sequential(
            Flatten(),
            SEBlock(2048 * 2),
            nn.Linear(2048 * 2, num_classes)
        )

    def fresh_params(self):
        return self.net.last_linear.parameters()

    def forward(self, x):
        out = self.net.features(x)
        out = F.relu(out, inplace=True)
        out = self.avg_pool(out)
        out = self.net.last_linear(out)
        return out


from django.conf.urls import url, include, patterns
from freight.views import *
from freight.models import *

app_name = 'freight'
urlpatterns = [
    # ex: /freight/
    url(r'^$', index, name='index'),
    # ex: /freight/5
    url(r'^(?P<carrier_id>[0-9]+)/$', carrier_detail, name='carrier_detail'),
    # ex: /freight/locations
    url(r'^locations/$', location_all, name='location_all'),
    #ex: /freight/fuel
    url(r'^fuel/$', fuel_surcharge, name='fuel_surcharge'),
    #ex: /freight/history
    url(r'^history/$', rate_history, name='rate_history')
]


import torch

import settingsModule
from settingsModule import Settings, np, random
from population import Population

# Parent of all algorithms
class Algorithm:
	iteration_num: int
	settings: Settings

	def __init__(self, settings: Settings):
		self.settings = settings
		self.iteration_num = 0

	def iterate(self) -> None:
		pass

	def get_algorithm_name(self) -> str:
		pass

	def get_picture_title(self) -> str:
		pass

	def get_progress_status(self) -> str:
		pass

	def get_file_progress(self) -> str:
		pass

	def get_best_picture(self) -> np.array:
		pass

	def get_best_fitness(self) -> float:
		pass

	def termination_condition(self) -> bool:
		pass

# Genetic Algorithm
class GeneticAlgorithm(Algorithm):
	population: Population
	mutation_probability: float

	def __init__(self, settings: Settings):
		super().__init__(settings)
		self.mutation_probability = self.settings.mutation_probability
		self.population = Population(settings)

	def iterate(self):
		if self.settings.cross_over == settingsModule.CROSS_OVERS.ONE_POINT:
			offsprings = self.one_point_crossover()
		elif self.settings.cross_over == settingsModule.CROSS_OVERS.TWO_POINT:
			offsprings = self.two_points_crossover()
		elif self.settings.cross_over == settingsModule.CROSS_OVERS.UNIFORM:
			offsprings = self.uniform_crossover()
		else:
			print("Illegal Cross Over!")
			exit()

		self.population.steady_state_replace(self.random_mutations(offsprings))

	def one_point_crossover(self):
		parent1, parent2 = self.population.roulette_wheel_select_2_parents()

		rand_col = random.randint(0, parent1.shape[1] - 1)

		offspring1 = np.copy(parent1)
		offspring1[:, :, :rand_col] = parent2[:, :, :rand_col]

		offspring2 = np.copy(parent2)
		offspring2[:, :, rand_col:] = parent1[:, :, rand_col:]

		return np.array([offspring1, offspring2])

	def two_points_crossover(self):
		parent1, parent2 = self.population.roulette_wheel_select_2_parents()

		rand_row1 = random.randint(0, parent1.shape[0] - 1)
		rand_col1 = random.randint(0, parent1.shape[1] - 1)

		rand_row2 = random.randint(0, parent1.shape[0] - 1)
		rand_col2 = random.randint(0, parent1.shape[1] - 1)

		offspring1 = np.copy(parent1)
		offspring1[:, rand_row1:rand_row2, rand_col1:rand_col2] = parent2[:, rand_row1:rand_row2, rand_col1:rand_col2]

		offspring2 = np.copy(parent2)
		offspring2[:, rand_row1:rand_row2, rand_col1:rand_col2] = parent1[:, rand_row1:rand_row2, rand_col1:rand_col2]

		return np.array([offspring1, offspring2])

	def uniform_crossover(self):
		parent1, parent2 = self.population.roulette_wheel_select_2_parents()

		random_matching = np.random.random_integers(0, 1, (parent1.shape[1], parent1.shape[2]))

		offspring1 = random_matching * parent1[:] + (1-random_matching) * parent2[:]
		offspring2 = (1-random_matching) * parent1[:] + random_matching * parent2[:]

		return np.array([offspring1, offspring2])

	def random_mutations(self, offsprings: np.array):
		for offspring_idx in range(len(offsprings)):
			if self.settings.mutation == settingsModule.MUTATIONS.UNIFORMAL:
				offsprings[offspring_idx] = self.uniformal_mutation(offsprings[offspring_idx])
			elif self.settings.mutation == settingsModule.MUTATIONS.INDIVIDUAL:
				offsprings[offspring_idx] = self.individual_mutation(offsprings[offspring_idx])
			else:
				print("Illegal Mutation!")
				exit()

		return offsprings

	def uniformal_mutation(self, offspring: np.array):
		# Make a mutation for each gene in a specific probability
		mutations_activations = np.random.rand(offspring.shape[0], offspring.shape[1], offspring.shape[2]) < self.mutation_probability
		return mutations_activations * np.random.rand(offspring.shape[0], offspring.shape[1], offspring.shape[2]) \
			   + (1 - mutations_activations) * offspring

	def individual_mutation(self, offspring: np.array):
		# Make a mutation in a specific probability
		if random.random() < self.mutation_probability:
			# Change the value in all the layers of the picture
			for i in range(self.settings.num_of_layers):
				gen_index = np.random.random_integers(0, offspring.shape[1] - 1, 2)
				mutation = random.randint(0, self.settings.num_of_shades-1) / (self.settings.num_of_shades-1)
				offspring[:, gen_index[0], gen_index[1]] = mutation

		return offspring

	def get_population_best(self):
		return self.population.get_population_best()

	def get_population_wrost(self):
		return self.population.get_population_wrost()

	def get_algorithm_name(self):
		return "Genetic Algorithm"

	def get_picture_title(self):
		title = f"Generation:{self.iteration_num}, Label:{self.settings.label}\n"

		if self.settings.penalty_factor != 0:
			return title + "Certainty:{0:.2f}%".format(round(self.get_population_best()['certainty']*100, 2)) +\
				   ", Penalty:{0:.4f}".format(round(self.get_population_best()['penalty'], 4)) + \
				   ", Fitness:{0:.4f}".format(round(self.get_population_best()['fitness'], 4))
		else:
			return title + "Fitness:{0:.2f}%".format(round(self.get_population_best()['fitness']*100, 2))

	def get_progress_status(self):
		status = f"Algorithm:GA, Model:{self.settings.model.name}, Generation:{self.iteration_num}, Label:{self.settings.label} | BEST("

		if self.settings.penalty_factor != 0:
			status += "Certainty:{0:.4f}".format(round(self.get_population_best()['certainty'], 4)) + \
				   ", Penalty:{0:.4f}".format(round(self.get_population_best()['penalty'], 4)) + \
				   ", Fitness:{0:.4f}".format(round(self.get_population_best()['fitness'], 4))
		else:
			status += "Fitness:{0:.4f}".format(round(self.get_population_best()['fitness'], 4))

		status += ") | Worst(Fitness:{0:.4f})".format(round(self.get_population_wrost()['fitness'], 4))
		return status

	def get_file_progress(self):
		return f"Iteration:{self.iteration_num}" + \
			   ", Certainty:{0:.4f}".format(round(self.get_population_best()['certainty'], 4)) + \
			   ", Penalty:{0:.4f}".format(round(self.get_population_best()['penalty'], 4)) + \
			   ", Fitness:{0:.4f}".format(round(self.get_population_best()['fitness'], 4))

	def get_best_picture(self):
		return self.get_population_best()['pic']

	def get_best_fitness(self):
		return self.get_population_best()['fitness']

	def termination_condition(self):
		return self.population.get_stuck_evolution_counter() >= 10 * self.settings.status_period or self.get_population_best()['fitness'] >= 0.99

# Greedy Algorithm
class GreedyAlgorithm(Algorithm):
	pic: np.array
	max_fitness: float
	last_max_fitness: float
	curr_i = 0
	curr_j = 0

	def __init__(self, settings: Settings):
		super().__init__(settings)
		self.pic = np.random.random_integers(0, self.settings.num_of_shades - 1,
										(self.settings.num_of_layers, self.settings.picture_size, self.settings.picture_size)) \
				   / (self.settings.num_of_shades - 1)
		self.max_fitness = self.certainty(self.pic)
		self.last_max_fitness = 0.0
		self.penalty_factor = settings.penalty_factor

	def iterate(self):
		if self.curr_i >= self.pic.shape[1]:
			self.curr_i = 0

		if self.curr_j >= self.pic.shape[2]:
			self.curr_j = 0

		max_fitness_shade = self.pic[0][self.curr_i][self.curr_j]

		for shade in range(self.settings.num_of_shades):
			#print(shade)
			self.pic[0][self.curr_i][self.curr_j] = shade / (self.settings.num_of_shades - 1)
			fitness = self.fitness(self.pic)
			if fitness > self.max_fitness:
				self.max_fitness = fitness
				max_fitness_shade = shade / (self.settings.num_of_shades - 1)

		self.pic[0][self.curr_i][self.curr_j] = max_fitness_shade

		self.curr_i += 1
		self.curr_j += 1

	def certainty(self, pic: np.array):
		with torch.no_grad():
			network_input = torch.Tensor(pic).unsqueeze(dim=0)  # Expending the array (one example)
			network_output = self.settings.network(network_input)
			certainty = torch.nn.functional.softmax(network_output, dim=1)

			return float(certainty[0][self.settings.label_num])

	def penalty(self, pic: np.array):
		# The penalty is defined by the distances between every two adjacent pixels in the picture
		penalty = 0

		# Avoiding penalty computation when possible
		if self.penalty_factor != 0:
			row_neighbors = np.copy(pic)
			row_neighbors[:, 1:, :] = pic[:, :-1, :]

			col_neighbors = np.copy(pic)
			col_neighbors[:, :, 1:] = pic[:, :, :-1]

			# penalty = np.sum(np.sqrt(np.abs(offspring - row_neighbors)) + np.sqrt(np.abs(offspring - col_neighbors)))
			penalty = np.sum(np.abs(pic - row_neighbors) + np.abs(pic - col_neighbors))
			penalty /= pow(self.settings.picture_size, 2)

		return penalty

	def fitness(self, pic):
		if self.penalty_factor != 0:
			certainty = self.certainty(pic)
			penalty = self.penalty(pic)

			fitness = (1-self.penalty_factor)*certainty + self.penalty_factor*(1-penalty)
		else:
			fitness = self.certainty(pic)

		return fitness

	def get_algorithm_name(self):
		return "Greedy Algorithm"

	def get_picture_title(self):
		title = f"Iteration:{self.iteration_num}, Label:{self.settings.label}\n"
		return title + "Fitness:{0:.2f}%".format(round(self.max_fitness*100, 2))

	def get_progress_status(self):
		status = f"Algorithm:Greedy, Model:{self.settings.model.name}, Iteration:{self.iteration_num}, Label:{self.settings.label}" + \
				 ", Fitness:{0:.4f}".format(round(self.max_fitness, 4))
		return status

	def get_file_progress(self):
		return f"Iteration:{self.iteration_num}" + \
			   ", Certainty:{0:.4f}".format(round(self.max_fitness, 4))

	def get_best_picture(self):
		return self.pic

	def get_best_fitness(self):
		return self.max_fitness

	def termination_condition(self):
		return self.last_max_fitness == self.max_fitness or self.max_fitness >= 0.99


'''
UPI: HCAM630
NAME: HEATH LOGAN CAMPBELL

'''
import sys
from functools import cmp_to_key
from datetime import datetime

def distX(a, b):
  return a[0] - b[0]

def distY(a, b):
  return a[1] - b[1]

def dist(a, b):
  return distX(a, b) * distX(a, b) + distY(a, b) * distY(a, b)

'''
takes a min point
'''
def min(a, b):
  if (a < b):
    return a
  return b

'''
Brute forces all the points to find the closest
'''
def brute(points):
    minDist = dist(points[0], points[1])
    pointsCount = len(points)
    if pointsCount == 2:
        return minDist
    for i in range(pointsCount-1):
        for j in range(i + 1, pointsCount):
            if i != 0 and j != 1:
              continue
            curDist = dist(points[i], points[j])
            if curDist < minDist:
                minDist = curDist
    return minDist

def bruteForce(points):
    min = sys.maxsize

    for i in range(len(points)):
      for j in range(i+1, len(points)):
            if (dist(points[i], points[j]) < min):
                min = dist(points[i], points[j]);

    return min

def stripClosest(listX, listY, minDist):
    pointXCount = len(listX)
    pointXX = listX[pointXCount // 2][0]
    listOfYs = [x for x in listY if pointXX - minDist <= x[0] <= pointXX + minDist]
    best = minDist
    pointYCount = len(listOfYs)
    for i in range(pointYCount - 1):
        for j in range(i + 1, min(i + 7, pointYCount)):
            dst = dist(listOfYs[i], listOfYs[j])
            if dst < best:
                best = dst
    return best

def closestUtil(sortedByX, ay):
    rangeDiff = len(sortedByX)
    if(rangeDiff <= 3):
        return bruteForce(sortedByX)
    mid = (rangeDiff) // 2;

    leftX = sortedByX[:mid]
    rightX = sortedByX[mid:]

    midpoint = sortedByX[mid][0]

    leftY = list()
    rightY = list()
    for x in ay:
        if x[0] <= midpoint:
          leftY.append(x)
        else:
          rightY.append(x)

    out1 = closestUtil(leftX, leftY)
    out2 = closestUtil(rightX, rightY)

    if out1 <= out2:
        minDist = out1
    else:
        minDist = out2

    stripDist = stripClosest(sortedByX, ay, minDist)
    if minDist <= stripDist:
        return minDist
    else:
        return stripDist

points = []

while(True):
    predecessor = (sys.stdin.readline().rstrip("\n"))
    # if line is blank, we leave
    if(predecessor == ''):
        break
    cordPair = list(map(int, predecessor.split()))
    points.append(cordPair)

start_time = datetime.now()
# Takes about 1.2 second to sort
sortedByX = sorted(points, key=lambda x: x[0])
sortedByY = sorted(points, key=lambda x: x[1])
time_elapsed = datetime.now() - start_time
print('Time elapsed (ms) {}'.format(time_elapsed))
print("sorted")
print(closestUtil(sortedByX, sortedByY))


import json
import sys
sys.path.append('../src')
import uuid
import unittest
import yaml
import requests
import os
from pprint import pprint as pp

import aws_lambda_logging
import logging

logger = logging.getLogger()
loglevel = "INFO"
logging.basicConfig(level=logging.ERROR)

Branch = "Prod"

Host = os.environ.get('PORPER_ENDPOINT')
authToken = os.environ.get('AUTH_TOKEN')
region = os.environ.get('AWS_DEFAULT_REGION')

test_group = {
            "customer_id": "64b3bbf4-1f3d-446f-b697-06c2a68f2053",
            "name": "Testing321"
}

def genUrl(Host, Branch, Path):
    return "https://{}/{}{}".format(Host, Branch, Path)

def run_lambda(
    method='post',
    path='/sla',
    headers={'Authorization': authToken},
    body=None,
    env=""
):

    if env == "local":
        import api_handler
        event = {
            'httpMethod': method.upper(),
            'path': path,
            'headers': headers,
        }
        if body:
            event['body'] = body
        context = {}
        ret = api_handler.lambda_handler(event, context)
        ret['body'] = json.loads(ret['body'])
        return ret

    elif env == "remote":
        headers = {
            "Content-Type": "application/json",
            "Authorization": "{}".format(authToken)
        }
        url = genUrl(Host, Branch, path)
        if method == "delete":
            ret = requests.delete(url, headers=headers)
        elif method == "put":
            ret = requests.put(url, headers=headers, data=body)
        elif method == "get":
            ret = requests.get(url, headers=headers)
        else:
            ret = requests.post(url, headers=headers, data=body)
        body = ret.json()
        return {
            "body": body,
            "statusCode": ret.status_code
        }

def get_content(filepath, filename):
    with open(filepath + filename) as data:
        return data.read()

class ProxyUserTest(unittest.TestCase):
    def __init__(self, testName, env):
        super(ProxyUserTest, self).__init__(testName)
        self.env = env

    def test_user_get(self):
        ret = run_lambda(
            path=f"/user",
            method="get",
            env=self.env
        )
        from pprint import pprint as pp
        pp(ret)
        assert len(ret['body']) > 0 and type(ret['body']) == list

class ProxyGroupTest(unittest.TestCase):
    def __init__(self, testName, env):
        super(ProxyGroupTest, self).__init__(testName)
        self.env = env

    def test_group_post(self):
        ret = run_lambda(
            path=f"/group",
            method="post",
            body=json.dumps(test_group),
            env=self.env
        )
        from pprint import pprint as pp
        print("response====================")
        pp(ret)
        try:
            id = ret["body"]["id"]
        except:
            id = ret["body"]
        run_lambda(method="delete", path=f'/group', body=json.dumps({"id": id}), env=self.env)
        assert ret["statusCode"] == 200
        assert ret["body"]["name"] == "Testing321"

    def test_group_put(self):
        ret = run_lambda(
            path=f"/group",
            method="post",
            body=json.dumps(test_group),
            env=self.env
        )

        from pprint import pprint as pp
        print("response====================")
        pp(ret)
        try:
            id = ret["body"]["id"]
        except:
            id = ret["body"]

        ret = run_lambda(
            path=f"/group",
            method="put",
            body=json.dumps({"id": id, "name": "Testing654321"}),
            env=self.env
        )

        run_lambda(method="delete", path=f'/group', body=json.dumps({"id": id}), env=self.env)
        assert ret["statusCode"] == 200
        assert ret["body"]["name"] == "Testing654321"

if __name__ == '__main__':

    if sys.argv[1] == "local":
        env = "local"
    elif sys.argv[1] == "remote":
        env = "remote"
    else:
        print("only local or remote are allowed")
        sys.exit()

    suite = unittest.TestSuite()

    suite.addTest(ProxyUserTest("test_user_get", env))

    suite.addTest(ProxyGroupTest("test_group_post", env))
    suite.addTest(ProxyGroupTest("test_group_put", env))

    test_runner = unittest.TextTestRunner(verbosity=2, resultclass=unittest.TextTestResult) # .run(suite)
    result = test_runner.run(suite)
    sys.exit(not result.wasSuccessful())


 # -*- coding: UTF-8 -*-

from dnslib import *
import gevent
from gevent.server import DatagramServer
from gevent import socket
from random import randint
from time import sleep

always_respond_ip = '1.2.3.4'

class DNSServer(DatagramServer):

    def handle_dns_request(self, data, address):

        req = DNSRecord.parse(data)
        qname = str(req.q.qname)
        qid = req.header.id

        response = DNSRecord(DNSHeader(qr=1, aa=1, ra=1),
                                     q=DNSQuestion(qname),
                                     a=RR(qname, rdata=A(always_respond_ip)))

        response.header.id = qid
        sleep(1.5)
        self.socket.sendto(response.pack(), address)

    def handle(self, data, address):

        self.handle_dns_request(data, address)

def main():
    DNSServer(':53').serve_forever()

if __name__ == '__main__':
    main()


import pygame

class print_screen(): # kwadrat 50 pix , 10x14
    def main(tab_main,WIDTH,HEIGHT,screen,BORDER_MAIN,square,tab_moving):
        w=0.625*WIDTH # 500
        h=20*square
        start_x=WIDTH//20
        start_y=HEIGHT//100
        pygame.draw.rect(screen,(50, 168, 82),(WIDTH//20-BORDER_MAIN,HEIGHT//100-BORDER_MAIN,w+2*BORDER_MAIN,h+2*BORDER_MAIN),BORDER_MAIN)
        pygame.draw.rect(screen,(0,0,0),(start_x,start_y,w,h))

        n=10
        m=20
        line=2
        for i in range(n+1):
            pygame.draw.line(screen,(55,51,111),(i*square+start_x-line//2,start_y),(i*square+start_x-line//2,start_y+h-line//2),line)

        for i in range(m+1):
            pygame.draw.line(screen,(55,51,111),(start_x,i*square+start_y-line//2),(w+start_x-line//2,i*square+start_y-line//2),line)

        for i in range(m):
            for j in range(n):
                if tab_moving[i][j]==1:
                    pygame.draw.rect(screen,(245, 66, 81),(start_x+line//2+j*square,start_y+line//2+i*square,square-line,square-line))

        for i in range(m):
            for j in range(n):
                if tab_main[i][j]==1:
                    pygame.draw.rect(screen,(58, 133, 119),(start_x+line//2+j*square,start_y+line//2+i*square,square-line,square-line))

    def menu(tab_next,WIDTH,HEIGHT,BORDER_MAIN,screen,score):
        start_x=0.625*WIDTH+BORDER_MAIN+40
        start_y=0
        # score
        pygame.draw.rect(screen,(135, 89, 73),(start_x+0.12*(WIDTH-start_x),0.215*HEIGHT,200,80))
        font_score = pygame.font.SysFont("monospace", 35, True)
        score_spec = font_score.render("Score: "+str(score),500,(145, 158, 25))
        score_pos = score_spec.get_rect(center=((start_x+WIDTH)//2, 0.25*HEIGHT))
        screen.blit(score_spec,score_pos)

        # next block
        w=WIDTH-start_x
        square_next=40

        font_next = pygame.font.SysFont("monospace", 30)
        next_spec = font_next.render("Next: ",500,(0,0,0))
        next_pos = next_spec.get_rect(center=(w*0.4+start_x, 0.4*HEIGHT))
        screen.blit(next_spec,next_pos)

        pygame.draw.rect(screen,(0,0,0),(start_x+w//2-2*square_next,HEIGHT*0.42,4*square_next,4*square_next))
        line=2
        for i in range(5):
            pygame.draw.line(screen,(55,51,111),(i*square_next+start_x+w//2-2*square_next,HEIGHT*0.42),(i*square_next+start_x+w//2-2*square_next,HEIGHT*0.42+4*square_next),line)

        for i in range(5):
            pygame.draw.line(screen,(55,51,111),(start_x+w//2-2*square_next,i*square_next+HEIGHT*0.42),(start_x+w//2+2*square_next,i*square_next+HEIGHT*0.42),line)

        for i in range(4):
            for j in range(4):
                if tab_next[i][j]==1:
                    pygame.draw.rect(screen,(58, 133, 119),(start_x+w//2-2*square_next+j*square_next+line,HEIGHT*0.42+line+i*square_next,square_next-line,square_next-line))

        # key
        key=["Controls: ",
        "Press space to start",
        "UP - Rotate block ",
        "LEFT - Move left ",
        "RIGHT - Move right ",
        "DOWN - Move down"]
        n=len(key)
        space=20
        font_key = pygame.font.SysFont("monospace", 20)
        for i in range(n):
            key_spec = font_key.render(key[i],500,(145, 158, 25))
            # key_pos = key_spec.get_rect((WIDTH-start_x)*0.3+start_x, i*space+0.8*HEIGHT)
            screen.blit(key_spec,((WIDTH-start_x)*0.05+start_x, i*space+0.7*HEIGHT))
        # pygame.draw(key, (10, 10), (0,0,0))

    def print_all(tab_main,tab_next,WIDTH,HEIGHT,screen,BORDER_MAIN,square,score,tab_moving):
        print_screen.main(tab_main,WIDTH,HEIGHT,screen,BORDER_MAIN,square,tab_moving)
        print_screen.menu(tab_next,WIDTH,HEIGHT,BORDER_MAIN,screen,score)


from PIL import Image

ALPHA = {(255, 255, 0, 255), (255, 255, 0)}

PALETTE_MAP = {
    (1, 1, 1, 255): 1,
    (62, 184, 73, 255): 2,
    (116, 208, 125, 255): 3,
    (89, 85, 224, 255): 4,
    (128, 118, 241, 255): 5,
    (185, 94, 81, 255): 6,
    (101, 219, 239, 255): 7,
    (219, 101, 89, 255): 8,
    (255, 137, 125, 255): 9,
    (204, 195, 94, 255): 10,
    (222, 208, 135, 255): 11,
    (58, 162, 65, 255): 12,
    (183, 102, 181, 255): 13,
    (204, 204, 204, 255): 14,
    (255, 255, 255, 255): 15
}

PALETTE_MAP.update(dict((tuple(k[:3]), v) for k, v in PALETTE_MAP.items()))

def split_char(char_entry):
    """
    Splits the char_entry as in defined_chars returned by create_charset()
    to pattern and colour tables.
    :param char_entry:
    :return:
    """
    pattern = []
    colours = []
    for b in range(0, 8 * 8, 8):
        row = char_entry[b:(b + 8)]
        row = [PALETTE_MAP[px] for px in row]
        cs = set(row)
        if len(cs) == 1:
            cols = [0, cs.pop()]
        else:
            cols = list(cs)
            cols.sort()
        bits = []
        for px in row:
            if px == cols[1]:
                bits.append("0")
            else:
                bits.append("1")
        pattern.append(int("".join(bits), 2))
        colours.append(16 * cols[0] + cols[1])
    return tuple(pattern), tuple(colours)

def create_charset(imgname, defined_chars=None, orientation="horizontal", redundancy=False):
    """
    Read a PNG file and convert it.
    """
    img = Image.open(imgname)
    w, h = img.size
    print("Read image size:", w, h)
    if defined_chars is None:
        defined_chars = {}

    n_key = len(defined_chars)
    actual_output = {}

    if orientation == "horizontal":
        for y0 in range(0, h, 8):
            for x0 in range(0, w, 8):
                ch = []
                for y in range(y0, y0 + 8):
                    for x in range(x0, x0 + 8):
                        px = img.getpixel((x, y))
                        if px in ALPHA:
                            break
                        ch.append(px)
                    if px in ALPHA:
                        break
                if px in ALPHA:
                    continue
                ch = tuple(ch)
                if ch not in defined_chars or redundancy:
                    key = n_key
                    n_key += 1
                    defined_chars[ch] = key
                actual_output[(x0 / 8, y0 / 8)] = defined_chars[ch]
    else:
        for x0 in range(0, w, 8):
            for y0 in range(0, h, 8):
                ch = []
                for y in range(y0, y0 + 8):
                    for x in range(x0, x0 + 8):
                        px = img.getpixel((x, y))
                        if px in ALPHA:
                            break
                        ch.append(px)
                    if px in ALPHA:
                        break
                if px in ALPHA:
                    continue
                ch = tuple(ch)
                if ch not in defined_chars or redundancy:
                    key = n_key
                    n_key += 1
                    defined_chars[ch] = key
                actual_output[(x0 / 8, y0 / 8)] = defined_chars[ch]
    print("Actual output size:", len(actual_output))
    return defined_chars, actual_output

def rle_encode_sequence(seq):
    colourcoding = []
    current_batch = None
    final_output = []
    for r in seq:
        final_output.append(r)
        if current_batch is None or current_batch[1] != r:
            if current_batch is not None:
                colourcoding.extend(current_batch)
            current_batch = [0, r]
        current_batch[0] += 1
        if current_batch[0] == 255:
            colourcoding.extend(current_batch)
            current_batch = None
    if current_batch is not None:
        colourcoding.extend(current_batch)

    colourcoding.append(0)
    return colourcoding

def convert_to_palette(defined_char, colours):
    """
    Take in a defined character and the colour encoding for each
    pixel row; return the character converted to binary using the
    colour mapping defined by colours.
    """

    pxs = []
    for i in range(0, 64, 8):
        pxs.append(defined_char[i:(i+8)])

    patterns = []
    for row, clr in zip(pxs, colours):
        as_bits = []
        for i in range(8):
            c = row[i]
            if PALETTE_MAP[c] == clr & 15:
                as_bits.append("0")
            else:
                as_bits.append("1")
        patterns.append(int("".join(as_bits), 2))
    return patterns

def batch_convert(defined_chars, clr):
    patterns = []
    invmap = dict((v, k) for k, v in defined_chars.items())

    c1 = clr >> 4

    for k, vall in sorted(invmap.items()):
        as_bits = []
        for i in range(0, 64, 8):
            v = vall[i:(i + 8)]
            for c in v:
                c = PALETTE_MAP[c]
                if c == c1:
                    as_bits.append("1")
                else:
                    as_bits.append("0")
                if len(as_bits) == 8:
                    patterns.append(int("".join(as_bits), 2))
                    as_bits = []
    return patterns

def rle_encode_graphics(defined_chars, first_color=(0, 0)):
    patterns = []
    colors = []
    invmap = dict((v, k) for k, v in defined_chars.items())

    prev_col = first_color
    for k, vall in sorted(invmap.items()):
        as_bits = []
        for i in range(0, 64, 8):
            v = vall[i:(i + 8)]
            c0 = PALETTE_MAP[min(v)]
            c1 = PALETTE_MAP[max(v)]
            c0, c1 = min(c0, c1), max(c0, c1)
            if c0 == c1 and c0 in prev_col:
                c0, c1 = prev_col
            for c in v:
                c = PALETTE_MAP[c]
                if c == c0:
                    as_bits.append("0")
                else:
                    as_bits.append("1")
                if len(as_bits) == 8:
                    patterns.append(int("".join(as_bits), 2))
                    as_bits = []
            colors.append(c0 + c1 * 16)
            prev_col = (c0, c1)
        colourcoding = rle_encode_sequence(colors)
    return colourcoding, patterns


#!/usr/bin/env python

from __future__ import print_function

import os
import cgi
from subprocess import Popen, PIPE, STDOUT

# # Haskell
SCRIPTDIR = 'haskell'

# To run NON compiled code
# SCRIPT = ['/usr/bin/runhaskell', 'Shrdlite.hs']

# Run compiled code with garbage collection off
# compile with this command: ghc -O2 -rtsopts Shrdlite.hs
SCRIPT = ['./Shrdlite',' +RTS -A2G']

################################################################################

print('Content-type:text/plain')
print()

try:
    while not os.path.isdir(SCRIPTDIR):
        SCRIPTDIR = os.path.join("..", SCRIPTDIR)

    form = cgi.FieldStorage()
    data = form.getfirst('data')
    script = Popen(SCRIPT, cwd=SCRIPTDIR, stdin=PIPE, stdout=PIPE, stderr=PIPE)
    out, err = script.communicate(data)

    print(out)
    if err:
        raise Exception(err)

except:
    import sys, traceback
    print(traceback.format_exc())
    sys.exit(1)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
import StringIO
import sqlite3
import urllib
import pycurl
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

# 读tocken文件获取用户Tocken
token_file = open('tocken')
try:
	my_token = token_file.read().strip()
finally:
	token_file.close()
# 组合 API URL
pinurl = 'https://api.pinterest.com/v1/pins/?access_token=' + my_token

# 指定发布Pin的 Board ID
board_id = '195765983741497086'

# 定义Pin发布函数
def Pin(pinurl,board_id,note,url,image):
	b=StringIO.StringIO()
	c=pycurl.Curl()
	c.setopt(pycurl.URL, pinurl)
	c.setopt(pycurl.WRITEFUNCTION, b.write)
	c.setopt(pycurl.FOLLOWLOCATION, 1)
	c.setopt(pycurl.MAXREDIRS, 5)
	c.setopt(pycurl.CUSTOMREQUEST,'POST')
	post_data_dic = {"board":board_id,"note":note,"link":url,"image_url":image}
	c.setopt(c.POSTFIELDS, urllib.urlencode(post_data_dic))
	c.perform()
	return c.getinfo(c.HTTP_CODE)
	#print b.getvalue()

# 连接SQLite数据库
conn = sqlite3.connect('products.sqlite')
cursor = conn.cursor()

# 从数据库中取出产品信息（默认1条；可以是多条，修改limit）
cursor.execute('select * from products where status=0 limit 0,1')
values = cursor.fetchall()

# 发送Pin
for value in values:
	pid = value[0]
	note = value[3]
	url = value[1]
	image = value[2]
	status = value[4]
	# 更新status为1，标识为已发布
	cursor.execute('update products set status=? where id=?', ('1',pid))
	conn.commit()
	# 执行发布
	pinreturn = Pin(pinurl,board_id,note,url,image)
	# 写日志，如果成功写入到succes.log
	if pinreturn == 201:
		logfile = open('succes.log','a')
		logfile.write(str(pid) + ',' + url + '\n')
		logfile.close()
	# 如果失败，写入到error.log
	else:
		logfile = open('error.log','a')
		logfile.write(str(pid) + ',' + str(pinreturn) + ',' + url + '\n')
		logfile.close()

# 关闭数据库连接
cursor.close()
conn.close()