from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import logging
from ezdxf.lldxf import validator, const
from ezdxf.lldxf.attributes import (
    DXFAttributes,
    DefSubclass,
    DXFAttr,
    RETURN_DEFAULT,
    XType,
    group_code_mapping,
)
from ezdxf.math import NULLVEC, Z_AXIS
from .dxfentity import base_class
from .dxfgfx import acdb_entity
from .factory import register_entity
from .polygon import DXFPolygon
from .gradient import Gradient, GradientType

if TYPE_CHECKING:
    from ezdxf.lldxf.tagwriter import AbstractTagWriter
    from ezdxf.colors import RGB

__all__ = ["MPolygon"]
logger = logging.getLogger("ezdxf")

acdb_mpolygon = DefSubclass(
    "AcDbMPolygon",
    {
        # MPolygon: version
        "version": DXFAttr(
            70,
            default=1,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # x- and y-axis always equal 0, z-axis represents the elevation:
        "elevation": DXFAttr(10, xtype=XType.point3d, default=NULLVEC),
        "extrusion": DXFAttr(
            210,
            xtype=XType.point3d,
            default=Z_AXIS,
            validator=validator.is_not_null_vector,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern name:
        "pattern_name": DXFAttr(2, default=""),
        # Solid fill color as ACI
        "fill_color": DXFAttr(
            63,
            default=const.BYLAYER,
            optional=True,
            validator=validator.is_valid_aci_color,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: Solid-fill flag:
        # 0 = lacks solid fill
        # 1 = has solid fill
        "solid_fill": DXFAttr(
            71,
            default=0,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch style tag is not supported for MPOLYGON!?
        # I don't know in which order this tag has to be exported.
        # TrueView does not accept this tag in any place!
        # BricsCAD supports this tag!
        "hatch_style": DXFAttr(
            75,
            default=const.HATCH_STYLE_NESTED,
            validator=validator.is_in_integer_range(0, 3),
            fixer=RETURN_DEFAULT,
            optional=True,
        ),
        # Hatch pattern type ... see HATCH
        "pattern_type": DXFAttr(
            76,
            default=const.HATCH_TYPE_PREDEFINED,
            validator=validator.is_in_integer_range(0, 3),
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern angle (pattern fill only) in degrees:
        "pattern_angle": DXFAttr(52, default=0),
        # Hatch pattern scale or spacing (pattern fill only):
        "pattern_scale": DXFAttr(
            41,
            default=1,
            validator=validator.is_not_zero,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: Boundary annotation flag:
        # 0 = boundary is not an annotated boundary
        # 1 = boundary is an annotated boundary
        "annotated_boundary": DXFAttr(
            73,
            default=0,
            optional=True,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern double flag (pattern fill only) .. see HATCH
        "pattern_double": DXFAttr(
            77,
            default=0,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # see ... HATCH
        "pixel_size": DXFAttr(47, optional=True),
        "n_seed_points": DXFAttr(
            98,
            default=0,
            validator=validator.is_greater_or_equal_zero,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: offset vector in OCS ???
        "offset_vector": DXFAttr(11, xtype=XType.point2d, default=(0, 0)),
        # MPolygon: number of degenerate boundary paths (loops), where a
        # degenerate boundary path is a border that is ignored by the hatch:
        "degenerated_loops": DXFAttr(99, default=0),
    },
)
acdb_mpolygon_group_code = group_code_mapping(acdb_mpolygon)

@register_entity
class MPolygon(DXFPolygon):
    """DXF MPOLYGON entity

    The MPOLYGON is not a core DXF entity, and requires a CLASS definition.

    """

    DXFTYPE = "MPOLYGON"
    DXFATTRIBS = DXFAttributes(base_class, acdb_entity, acdb_mpolygon)
    MIN_DXF_VERSION_FOR_EXPORT = const.DXF2000
    LOAD_GROUP_CODES = acdb_mpolygon_group_code

    def preprocess_export(self, tagwriter: AbstractTagWriter) -> bool:
        if self.paths.has_edge_paths:
            logger.warning(
                "MPOLYGON including edge paths are not exported by ezdxf!"
            )
            return False
        return True

    def export_entity(self, tagwriter: AbstractTagWriter) -> None:
        """Export entity specific data as DXF tags."""
        super().export_entity(tagwriter)
        tagwriter.write_tag2(const.SUBCLASS_MARKER, acdb_mpolygon.name)
        dxf = self.dxf
        dxf.export_dxf_attribs(
            tagwriter,
            [  # tag order is important!
                "version",
                "elevation",
                "extrusion",
                "pattern_name",
                "solid_fill",
            ],
        )
        self.paths.export_dxf(tagwriter, self.dxftype())
        # hatch_style not supported?
        dxf.export_dxf_attribs(
            tagwriter,
            [
                # "hatch_style",  # not supported by MPolygon ???
                "pattern_type",
            ],
        )
        if dxf.solid_fill == 0:  # export pattern
            dxf.export_dxf_attribs(
                tagwriter, ["pattern_angle", "pattern_scale", "pattern_double"]
            )

        dxf.export_dxf_attribs(
            tagwriter,
            [
                "annotated_boundary",
                "pixel_size",
            ],
        )
        if dxf.solid_fill == 0:  # export pattern
            if self.pattern:
                self.pattern.export_dxf(tagwriter, force=True)
            else:  # required pattern length tag!
                tagwriter.write_tag2(78, 0)
        if tagwriter.dxfversion > const.DXF2000:
            dxf.export_dxf_attribs(tagwriter, "fill_color")
        dxf.export_dxf_attribs(tagwriter, "offset_vector")
        self.export_degenerated_loops(tagwriter)
        self.export_gradient(tagwriter)

    def export_degenerated_loops(self, tagwriter: AbstractTagWriter):
        self.dxf.export_dxf_attribs(tagwriter, "degenerated_loops")

    def export_gradient(self, tagwriter: AbstractTagWriter):
        if tagwriter.dxfversion < const.DXF2004:
            return
        if self.gradient is None:
            self.gradient = Gradient(kind=0, num=0, type=0)
        self.gradient.export_dxf(tagwriter)

    def set_solid_fill(
        self, color: int = 7, style: int = 1, rgb: Optional[RGB] = None
    ):
        """Set :class:`MPolygon` to solid fill mode and removes all gradient and
        pattern fill related data.

        Args:
            color: :ref:`ACI`, (0 = BYBLOCK; 256 = BYLAYER)
            style: hatch style is not supported by MPOLYGON, just for symmetry
                to HATCH
            rgb: true color value as (r, g, b)-tuple - has higher priority
                than `color`. True color support requires DXF R2004+

        """
        # remove existing gradient and pattern fill
        self.gradient = None
        self.pattern = None
        self.dxf.solid_fill = 1

        # if a gradient is present, the solid fill_color is ignored
        self.dxf.fill_color = color
        self.dxf.pattern_name = "SOLID"
        self.dxf.pattern_type = const.HATCH_TYPE_PREDEFINED
        if rgb is not None:
            self.set_solid_rgb_gradient(rgb)

    def set_solid_rgb_gradient(self, rgb: RGB):
        """Set solid fill color as gradient of a single RGB color.
        This disables pattern fill!

        (internal API)
        """
        self.gradient = Gradient(kind=1, num=2, type=GradientType.LINEAR)
        self.gradient.color1 = rgb
        self.gradient.color2 = rgb


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 17 16:09:33 2018

@author: zili & haolinl
"""

import numpy as np
import torch
import torchvision
import torch.nn as nn
import torchvision.transforms as transforms
import scipy.io
import time

class Net1(nn.Module):
    def __init__(self):
        super(Net1, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(1,10,13),
#            nn.Dropout2d(0.5),
            nn.BatchNorm2d(10),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(10,20,7),
#            nn.Dropout2d(0.5),
            nn.BatchNorm2d(20),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(20,40,7),
            nn.BatchNorm2d(40),
#            nn.Dropout2d(0.5),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv4 = nn.Sequential(
            nn.Conv2d(40,80,5),
            nn.BatchNorm2d(80),
#            nn.Dropout2d(0.5),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.fc1 = nn.Sequential(
            nn.Linear(720,400),
            nn.ReLU(),
            nn.Dropout(0.5)
        )
        self.fc2 = nn.Linear(400,1)

    def forward(self, x):
        f1 = self.conv1(x)
        f2 = self.conv2(f1)
        f3 = self.conv3(f2)
        f4 = self.conv4(f3)
        f4_flat = f4.view(f4.size(0), -1)
        f5 = self.fc1(f4_flat)
        output = self.fc2(f5)
        return output

if __name__ == "__main__":
    train_data_path = 'all_data_1000.mat'
    neural_net_savePath = 'CNN.pkl' # Path to save CNN's parameters

    data = scipy.io.loadmat(train_data_path)
    data_x, data_y, rNorm = data['data_x'], data['data_y'], data['Normalization_Factor']
    num_data = np.shape(data_x)[0]
    train_x, train_y = data_x[:int(0.9*num_data)], data_y[:int(0.9*num_data)]
    valid_x,valid_y = data_x[int(0.9*num_data):], data_y[int(0.9*num_data):]
    #test_x, test_y = data_x[int(0.9*num_data):], data_y[int(0.9*num_data):]

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #set up parameters
    batch_size = 32
    learning_rate = 0.00005
    num_epochs = 200

    train_x = torch.from_numpy(train_x).float()
    train_y = torch.from_numpy(train_y).float()
    valid_x = torch.from_numpy(valid_x).float()
    valid_y = torch.from_numpy(valid_y).float()
    #test_x = torch.from_numpy(test_x).float()
    #test_y = torch.from_numpy(test_y).float()

    train_dataset = torch.utils.data.TensorDataset(train_x, train_y)
    valid_dataset = torch.utils.data.TensorDataset(valid_x, valid_y)
    #test_dataset = torch.utils.data.TensorDataset(test_x, test_y)

    train_dataloader = torch.utils.data.DataLoader(
        dataset=train_dataset,
        batch_size=batch_size,
        shuffle=True
    )
    valid_dataloader = torch.utils.data.DataLoader(
        dataset=valid_dataset,
        batch_size=batch_size,
        shuffle=True
    )
    #test_dataloader = torch.utils.data.DataLoader(
    #    dataset=test_dataset
    #)
    neural_net = Net1().to(device)

    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(neural_net.parameters(),learning_rate)
    lossList = []
    accList = []
    valid_lossList = []
    valid_accList = []

    for epoch in range(num_epochs):
        loss_sum_train = 0
        loss_sum_valid = 0
        acc_sum = 0

        for iteration, (images, labels) in enumerate(train_dataloader):
            x_batch = torch.autograd.Variable(images)
            x_batch = x_batch.reshape(-1,1,128,128)
            y_batch = torch.autograd.Variable(labels)
            x_batch = x_batch.to(device)
            y_batch = y_batch.to(device)

            output = neural_net(x_batch)
            loss = criterion(output,y_batch)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            acc = abs((output.cpu() - labels)/labels).mean()
            acc_sum += acc
            loss_sum_train += loss.cpu().data.numpy()
        loss_sum_train = batch_size * loss_sum_train /len(train_dataloader.dataset)
        lossList.append(loss_sum_train*rNorm[0][0])

        for iteration, (images, labels) in enumerate(valid_dataloader):
            x_batch = torch.autograd.Variable(images)
            x_batch = x_batch.reshape(-1,1,128,128)
            y_batch = torch.autograd.Variable(labels)
            x_batch = x_batch.to(device)
            y_batch = y_batch.to(device)

            output_valid = neural_net(x_batch)
            loss = criterion(output_valid,y_batch)
            loss_sum_valid += loss.cpu().data.numpy()

        loss_sum_valid = batch_size * loss_sum_valid/len(valid_dataloader.dataset)
        valid_lossList.append(loss_sum_valid*rNorm[0][0])
        print('Epoch: ', epoch, '| train loss: %.6f | valid loss: %.6f  ' % (loss_sum_train*rNorm[0][0], loss_sum_valid*rNorm[0][0]))

        if ((epoch+1)%20 == 0):
            CNN_savePath_temp = 'CNN_' + str(int((epoch+1)/20)) + '.pkl'
            torch.save(neural_net.state_dict(), CNN_savePath_temp) # Save model every 20 epochs

    np.save('valid_loss.npy', valid_lossList)
    np.save('train_loss.npy', lossList)
    torch.save(neural_net.state_dict(), neural_net_savePath) # Save the final CNN model parameters

#__name__ = "__main__"


import matplotlib.pyplot as plt
import numpy as np

def plot_chains(chain, fileout=None, tracers=0, labels=None, delay=0, ymax=200000, thin=100, num_xticks=7, truths=None):

    if chain.ndim < 3:
        print("You must include a multiple chains")
        return

    n_chains, length, n_var = chain.shape
    print(n_chains, length, n_var)

    if (labels is not None) and (len(labels) != n_var):
        print("You must provide the correct number of variable labels.")
        return

    if (truths is not None) and (len(truths) != n_var):
        print("You must provide the correct number of truths.")
        return

    fig, ax = plt.subplots(int(n_var/2) + n_var%2, 2, figsize=(8, 0.8*n_var))
    plt.subplots_adjust(left=0.09, bottom=0.07, right=0.96, top=0.96, hspace=0)

    color = np.empty(n_chains, dtype=str)
    color[:] = 'k'
    alpha = 0.01 * np.ones(n_chains)
    zorder = np.ones(n_chains)
    if tracers > 0:
        idx = np.random.choice(n_chains, tracers, replace=False)
        color[idx] = 'r'
        alpha[idx] = 1.0
        zorder[idx] = 2.0

    for i in range(n_var):
        ix = int(i/2)
        iy = i%2

        for j in range(n_chains):

            xvals = (np.arange(length)*thin - delay) / 1000.0
            ax[ix,iy].plot(xvals, chain[j,:,i], color=color[j], alpha=alpha[j], rasterized=True, zorder=zorder[j])

        if ymax is None: ymax = (length*thin-delay)

        ax[ix,iy].set_xlim(-delay/1000.0, ymax/1000.0)
        ax[ix,iy].set_xticks(np.linspace(-delay/1000.0,ymax/1000.0,num_xticks))
        ax[ix,iy].set_xticklabels([])

        # Add y-axis labels if provided by use
        if labels is not None: ax[ix,iy].set_ylabel(labels[i])

        if delay != 0: ax[ix,iy].axvline(0, color='k', linestyle='dashed', linewidth=2.0, zorder=9)

        if truths is not None: ax[ix,iy].axhline(truths[i], color='C0', linestyle='dashed', linewidth=2.0, zorder=10)

    # plt.tight_layout()

    ax[-1,0].set_xticklabels(np.linspace(-delay/1000.0,ymax/1000.0,num_xticks).astype('i8').astype('U'))
    ax[-1,1].set_xticklabels(np.linspace(-delay/1000.0,ymax/1000.0,num_xticks).astype('i8').astype('U'))

    ax[-1,0].set_xlabel(r'Steps ($\times$1000)')
    ax[-1,1].set_xlabel(r'Steps ($\times$1000)')

    if fileout is None:
        plt.show()
    else:
        plt.savefig(fileout, rasterized=True)

    return

# from dart_board import plotting
# import numpy as np
# import pickle
# chains = pickle.load(open("../data/HMXB_chain.obj", "rb"))
# plotting.plot_chains(chains)


from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import pytest
from versioningit.basics import basic_write
from versioningit.errors import ConfigError

@pytest.mark.parametrize(
    "filename,params,content",
    [
        ("foo/bar.txt", {}, "1.2.3\n"),
        ("foo/bar.py", {}, '__version__ = "1.2.3"\n'),
        ("foo/bar", {}, "1.2.3\n"),
        (
            "foo/bar.py",
            {"template": "__version__ = {version!r}"},
            "__version__ = '1.2.3'\n",
        ),
        ("foo/bar.tex", {"template": r"$v = {version}$\bye"}, "$v = 1.2.3$\\bye\n"),
        (
            "foo/bar.py",
            {
                "template": (
                    "__version__ = {version!r}\n"
                    '__build_date__ = "{build_date:%Y-%m-%dT%H:%M:%SZ}"'
                )
            },
            "__version__ = '1.2.3'\n__build_date__ = \"2038-01-19T03:14:07Z\"\n",
        ),
    ],
)
def test_basic_write(
    filename: str, params: dict[str, Any], content: str, tmp_path: Path
) -> None:
    basic_write(
        project_dir=tmp_path,
        template_fields={
            "version": "1.2.3",
            "build_date": datetime(2038, 1, 19, 3, 14, 7, tzinfo=timezone.utc),
        },
        params={"file": filename, **params},
    )
    assert (tmp_path / filename).read_text(encoding="utf-8") == content

def test_basic_write_no_file(tmp_path: Path) -> None:
    with pytest.raises(ConfigError) as excinfo:
        basic_write(
            project_dir=tmp_path, template_fields={"version": "1.2.3"}, params={}
        )
    assert str(excinfo.value) == "tool.versioningit.write.file must be set to a string"

def test_basic_write_bad_ext(tmp_path: Path) -> None:
    with pytest.raises(ConfigError) as excinfo:
        basic_write(
            project_dir=tmp_path,
            template_fields={"version": "1.2.3"},
            params={"file": "foo/bar.tex"},
        )
    assert str(excinfo.value) == (
        "tool.versioningit.write.template not specified and file has unknown"
        " suffix '.tex'"
    )
    assert list(tmp_path.iterdir()) == []


import _io
import logging

import numpy as np
import pandas as pd

from ordered_set import OrderedSet

from socket import gethostname
from argparse import ArgumentParser, Namespace

# this can be removed, it was used to distinguish between the local machine and
# the (remote) machine where we run the final experiments
if gethostname() != 'BigMedilytics':
    from ctgan import CTGANSynthesizer

from purify.dataset.metadata import Metadata
from purify.dataset.profiling import profiler
from purify.dataset.processors import PreProcessor
from purify.encoders import label_encoders_fit_transform, label_encoders_inverse_transform
from purify.encoders import get_dummies_fit_transform, get_dummies_inverse_transform
from purify.generation.tabulator import TabularDataGenerator

from typing import List, Dict, Tuple, Union

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', None)

DATASETS: Dict[str, int] = {
    'adult': 30162, 'breast': 569, 'cover': 581012, 'credit': 30000, 'eeg': 14980, 'iris': 150, 'iris_sample': 12,
    'letter': 20000, 'mushroom': 8124, 'news': 39644, 'spam': 4601, 'wine-red': 1599, 'wine-white': 4898, 'yeast': 1484
}

def run_CTGAN(dataset: str = 'adult',
              n_epochs: int = 10,
              n_samples: int = 100,
              in_folder: str = './datasets',
              out_folder: str = './experiments',
              verbose: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]:
    # df_raw: pd.DataFrame = load_demo() if dataset == 'adult' else pd.read_csv(
    #     filepath_or_buffer=f"./datasets/{dataset}.csv", na_values='?')
    df_raw: pd.DataFrame = pd.read_csv(
        filepath_or_buffer=f"{in_folder}/{dataset}.csv", skipinitialspace=True, na_values='?', skip_blank_lines=True)
    df_pre: pd.DataFrame  # pandas DataFrame to hold preprocessed data
    synthesizer: CTGANSynthesizer  # the CTGAN data synthesizer
    df_sam: pd.DataFrame  # to store the samples (i.e., the synthetic data)
    filename: str = f"{dataset}_CTGAN_{n_epochs}"

    # data preprocessing
    df_pre = PreProcessor.drop_vars(dataset=dataset, df=df_raw)
    df_pre = PreProcessor.replace_miss_values_by_nans(df=df_pre, dataset=dataset)
    df_pre = PreProcessor.drop_nans(df=df_pre)
    # create the synthesizer
    synthesizer = CTGANSynthesizer(epochs=n_epochs)
    # learn from the data distribution
    synthesizer.fit(train_data=df_pre, discrete_columns=Metadata.discrete_vars(dataset=dataset, df=df_pre))
    # generate synthetic data
    df_sam = synthesizer.sample(n=n_samples)
    if verbose:
        log: _io.TextIOWrapper = open(file=f"{out_folder}/{filename}.txt", mode='w+')

        log.write(f"{'--- CTGAN ---' * 3}\n")
        log.write(f"dataset: {dataset}\n")
        log.write(f"raw shape: {df_raw.shape}\n")
        log.write(f"preprocessing shape: {df_pre.shape}\n")
        log.write(f"n_epochs: {n_epochs}\n")
        log.write(f"n_samples: {n_samples}\n")
        # log.write("samples:\n")
        # log.write(df_sam.head())
        # log.write("\n")
        # log.write("...\n")
        # log.write(df_sam.tail())
        # log.write("\n")
        log.write(f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
        log.write("\n")
        # log.write(f"are the pandas dataframes equals? {df_pre.equals(other=df_sam)}\n")
    # open(file=f"{out_folder}/{filename}_pre.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))}")
    # open(file=f"{out_folder}/{filename}_sam.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
    df_pre.to_csv(path_or_buf=f"{out_folder}/{filename}_pre.csv", index=False)
    df_sam.to_csv(path_or_buf=f"{out_folder}/{filename}_sam.csv", index=False)
    return df_pre, df_sam

def run_tabulator(dataset: str = 'adult',
                  ampu_rate: float = 0.2,
                  encoder_type: str = 'label',
                  algo: str = 'tabulator',
                  loss: str = 'mse',
                  batch_size: int = 128,
                  n_iterations: int = 1000,
                  n_samples: int = 100,
                  in_folder: str = './datasets',
                  out_folder: str = './experiments',
                  verbose: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]:
    # df_raw: pd.DataFrame = load_demo() if dataset == 'adult' else pd.read_csv(
    #     filepath_or_buffer=f"./datasets/{dataset}.csv")
    df_raw: pd.DataFrame = pd.read_csv(
        filepath_or_buffer=f"{in_folder}/{dataset}.csv", skipinitialspace=True, na_values='?', skip_blank_lines=True)
    df_pre: pd.DataFrame  # pandas DataFrame to hold preprocessed data
    df_enc: pd.DataFrame  # pandas DataFrame to hold the encoded and the non-encoded data
    df_sam: pd.DataFrame  # to store the samples (i.e., the synthetic data) in a pandas DataFrame
    samples: np.ndarray  # the samples (i.e., the synthetic data)
    generator: TabularDataGenerator
    new_discrete_vars: Union[List[str], List[int]]
    filename: str = f"{dataset}_{ampu_rate}_{encoder_type}_{algo}_{batch_size}_{loss}_{n_iterations}"

    # data preprocessing
    df_pre = PreProcessor.drop_vars(dataset=dataset, df=df_raw)
    df_pre = PreProcessor.replace_miss_values_by_nans(df=df_pre, dataset=dataset)
    df_pre = PreProcessor.drop_nans(df=df_pre)
    # encoding the discrete variables
    if encoder_type == 'one-hot':
        # data transformation that looks like one-hot encoding
        df_enc = get_dummies_fit_transform(
            data=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))
        # list of the new discrete variables, which came from the original discrete variables
        new_discrete_vars = [var for var in df_enc if var not in df_pre.columns]
        # replace each zero of one-hot encoding with minus one
        df_enc = PreProcessor.replace_values(
            df=df_enc, to_replace={new_discrete_var: {0: -1} for new_discrete_var in new_discrete_vars})
    else:  # 'label' --> default encoder
        df_enc, label_encoders = label_encoders_fit_transform(
            data=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))
    # create an instance of the generator
    generator = TabularDataGenerator(
        data=df_enc.to_numpy(),
        algo=algo,
        algo_parameters={'miss_rate': ampu_rate, 'batch_size': batch_size, 'loss': loss, 'n_iterations': n_iterations})
    # logging some execution info
    if verbose:
        logging.basicConfig(filename=f"{out_folder}/{filename}.txt", level=logging.INFO)
        logging.info(f"{'--- tabulatorSGAIN ---' * 3}")
        logging.info(f"dataset: {dataset}")
        logging.info(f"amputation rate: {ampu_rate}")
        logging.info(f"encoder type: {encoder_type}")
        logging.info(f"raw shape: {df_raw.shape}")
        logging.info(f"preprocessing shape: {df_pre.shape}")
        logging.info(f"encoded shape: {df_enc.shape}")
        logging.info(f"algorithm: {algo}")
        logging.info(f"batch size: {batch_size}")
        logging.info(f"loss: {loss}")
        logging.info(f"n_iterations: {n_iterations}")
        logging.info(f"n_samples: {n_samples}")
    # sampling (i.e., get the samples)
    samples = generator.sampler(n_samples=n_samples)
    # decoding the discrete variables
    if encoder_type == 'one-hot':
        df_sam = pd.DataFrame(data=samples, columns=df_enc.columns)
        # invert the replacement of each zero of one-hot encoding with minus one
        df_sam = PreProcessor.replace_values(
            df=df_sam, to_replace={new_discrete_var: {-1: 0} for new_discrete_var in new_discrete_vars})
        # data transformation to invert (i.e., to revert) the one that looks line one-hot encoding
        df_sam = get_dummies_inverse_transform(
            dataset=dataset,
            # data=pd.DataFrame(data=samples, columns=df_enc.columns),
            data=df_sam,
            discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre),
            vars_order=df_pre.columns)
    else:  # 'label' --> default encoder
        df_sam = label_encoders_inverse_transform(
            dataset=dataset,
            data=pd.DataFrame(data=samples, columns=df_enc.columns),
            label_encoders=label_encoders)
    if verbose:
        # logging.info("samples:")
        # logging.info(df_sam.head())
        # logging.info("...")
        # logging.info(df_sam.tail())
        logging.info(
            f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
        # logging.info(f"are the pandas dataframes equals? {df_pre.equals(other=df_sam)}")
    # open(file=f"{out_folder}/{filename}_pre.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))}")
    # open(file=f"{out_folder}/{filename}_sam.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
    df_pre.to_csv(path_or_buf=f"{out_folder}/{filename}_pre.csv", index=False)
    df_sam.to_csv(path_or_buf=f"{out_folder}/{filename}_sam.csv", index=False)
    return df_pre, df_sam

def main(args: Namespace) -> None:
    algos: OrderedSet[str] = OrderedSet([algo.strip() for algo in args.algos.split(',')])
    # TODO: GET RID OF HARDCODED
    algos_set: OrderedSet[str] = OrderedSet(['CTGAN', 'tabulator', 'tabulator-CP', 'tabulator-GP'])
    datasets: OrderedSet[str] = OrderedSet([dataset.strip() for dataset in args.datasets.split(',')])
    # TODO: GET RID OF HARDCODED
    datasets_set: OrderedSet[str] = OrderedSet(['breast', 'credit', 'eeg', 'iris', 'letter', 'news',
                                                'spam', 'wine-red', 'wine-white', 'yeast'])
    ampu_rates: OrderedSet[str] = OrderedSet([ampu_rate.strip() for ampu_rate in args.ampu_rates.split(',')])
    dataset: str
    algo: str
    ampu_rate: str
    ampu_rate_tmp: float
    df_pre: pd.DataFrame
    df_sam: pd.DataFrame

    if not algos.issubset(algos_set):
        raise ValueError(f"In terms of algorithms, expecting a subset of {algos_set} but got: {algos}.")
    if not datasets.issubset(datasets_set):
        raise ValueError(f"In terms of datasets, expecting a subset of {datasets_set} but got: {datasets}.")
    for dataset in datasets:
        for ampu_rate in ampu_rates:
            try:
                ampu_rate_tmp = float(ampu_rate)
                if ampu_rate_tmp < 0.00 or ampu_rate_tmp > 1.00:
                    raise ValueError(
                        f"Expecting an amputation rate within the interval [0.00, 1.00] but got: {ampu_rate_tmp}.")
            except ValueError:
                print(f"Expecting an amputation rate within the interval [0.00, 1.00] but got: {ampu_rate}.")
                exit(1)
            for algo in algos:
                print(f"{dataset} :: {ampu_rate} :: {algo}")
                df_pre, df_sam = run_tabulator(dataset=dataset,
                                               ampu_rate=ampu_rate_tmp,
                                               encoder_type='one-hot',
                                               algo=algo,
                                               batch_size=128,
                                               n_iterations=1000,
                                               n_samples=DATASETS[dataset],
                                               verbose=False)

if __name__ == "__main__":
    parser: ArgumentParser = ArgumentParser()

    parser.add_argument(
        '--algos',
        help="a csv list of the algorithms to run (e.g., 'CTGAN, tabulator, tabulator-CP, tabulator-GP')",
        # choices=['CTGAN', 'tabulator', 'tabulator-CP', 'tabulator-GP'],
        default='tabulator',
        type=str)
    parser.add_argument(
        '--datasets',
        help="a csv list of datasets short names",
        # choices=['breast', 'cover-type', 'credit', 'eeg', 'iris', 'letter', 'mushroom',
        #          'news', 'spam', 'wine-red', 'wine-white', 'yeast'],
        default='letter',
        type=str)
    parser.add_argument(
        '--ampu_rates',
        help="a csv list of amputation rates ([0.00, 1.00])",
        default='0.20',
        type=str)
    parser.add_argument(
        '--batch_size',
        help="number of samples in mini-batch",
        default=128,
        type=int)
    parser.add_argument(
        '--hint_rate',  # NOTE: the algorithms SGAIN, WSGAIN-CP, and WSGAIN-GP do NOT use this parameter,
        help='hint probability',  # it is here just because the GAIN algorithm requires the `hint_rate` parameter
        default=0.9,
        type=float)
    parser.add_argument(
        '--alpha',
        help="hyper-parameter to compute generator's loss",
        default=100,
        type=float)
    parser.add_argument(
        '--lambd',
        help="hyper-parameter to compute critic's loss",
        default=10,
        type=float)
    parser.add_argument(
        '--clip_value',
        help="clip (penalty) value",
        default=0.01,
        type=float)
    parser.add_argument(
        '--optimizer',
        help="solvers' optimizer",
        choices=['Adam', 'GDA', 'RMSProp'],
        default='Adam',
        type=str)
    parser.add_argument(
        '--learn_rate',
        help="optimizer's learning rate",
        default=1e-3,
        type=float)
    parser.add_argument(
        '--beta_1',
        help="Adam optimizer's hyper-parameter (1st moment estimates)",
        default=0.900,
        type=float)
    parser.add_argument(
        '--beta_2',
        help="Adam optimizer's hyper-parameter (2nd moment estimates)",
        default=0.999,
        type=float)
    parser.add_argument(
        '--decay',
        help="RMSProp optimizer's hyper-parameter (discounting factor for the history/coming gradient)",
        default=0.900,
        type=float)
    parser.add_argument(
        '--momentum',
        help="RMSProp optimizer's hyper-parameter (a scalar tensor)",
        default=0.000,
        type=float)
    parser.add_argument(
        '--epsilon',
        help="Adam hyper-parameter to ensure numerical stability or RMSProp hyper-parameter to avoid zero denominator",
        default=1e-08,
        type=float)
    parser.add_argument(
        '--n_iterations',
        help="number of training iterations",
        default=10000,
        type=int)
    parser.add_argument(
        '--n_critic',
        help="number of additional iterations to train the critic",
        default=5,
        type=int)
    parser.add_argument(
        '--n_runs',
        help="number of runs",
        default=3,
        type=int)
    parser.add_argument(
        '--verbose',
        help="to control verbosity",
        choices=['False', 'True'],  # `bool` type does NOT work as expected
        default='False',  # `bool` type does NOT work as expected
        type=str)  # `bool` type does NOT work as expected

    main(args=parser.parse_args())  # rock 'n roll

# python main.py --algos="CTGAN,tabulator,tabulator-CP,tabulator-GP" --datasets="iris,yeast" --ampu_rates="0.20" --optimizer=GDA --learn_rate=0.001 --n_iterations=1000 --n_runs=3


from django.http import HttpResponse
import datetime

def welcome(request):
    return HttpResponse("
Welcome To Django
")

def wish(request):
    # check whether name parameter is passed
    if 'name' in request.GET:
        name = request.GET['name']
    else:
        name = "Guest"

    hour = datetime.datetime.now().hour
    if hour < 12:
        msg = "Good Morning"
    elif hour < 17:
        msg = "Good Afternoon"
    else:
        msg = "Good Evening"

    return HttpResponse(f"
{msg} {name}
")


""" ━━━━━━神兽出没━━━━━━
 　　　┏┓　　　┏┓
 　　┃　　　　　　　┃
 　　┃　　　━　　　┃
 　　┃　┳┛　┗┳　┃
 　　┃　　　　　　　┃
 　　┃　　　┻　　　┃
 　　┃　　　　　　　┃
 　　┗━┓　　　┏━┛Code is far away from bug with the animal rotecting
 　　　　┃　　　┃ 神兽保佑,代码无bug
 　　　　┃　　　┃
 　　　　┃　　　┗━━━┓
 　　　　┃　　　　　　　┣┓
 　　　　┃　　　　　　　┏┛
 　　  　┗┓┓┏━┳┓┏┛
　　　　　┃┫┫　┃┫┫
　　　　　┗┻┛　┗┻┛
"""
import cv2
import numpy as np

# 读取 YUV 图像
yuv_img = cv2.imread('D:\\jylz_qrs\\UsePic_1920_1080_420.yuv', cv2.IMREAD_UNCHANGED)
height, width = yuv_img.shape[:2]

# 将 YUV 图像分离成 Y、U、V 三个通道
y_channel, u_channel, v_channel = cv2.split(yuv_img)

# 对 U、V 通道进行上采样
u_channel = cv2.resize(u_channel, (width, height), interpolation=cv2.INTER_CUBIC)
v_channel = cv2.resize(v_channel, (width, height), interpolation=cv2.INTER_CUBIC)

# 将 Y、U、V 三个通道合并成一个图像
yuv_img = cv2.merge([y_channel, u_channel, v_channel])

# 将 YUV 图像转换为 RGB 图像
rgb_img = cv2.cvtColor(yuv_img, cv2.COLOR_YUV2RGB_I420)

# 显示 RGB 图像
cv2.imshow('RGB Image', rgb_img)
cv2.waitKey(0)
cv2.destroyAllWindows()


#
# @lc app=leetcode.cn id=167 lang=python3
#
# [167] 两数之和 II - 输入有序数组
#
from typing import List
# @lc code=start

class Solution:
    def twoSum(self, numbers: List[int], target: int) -> List[int]:
        dic = {}
        for i, num in enumerate(numbers):
            if(target-num) in dic:
                return [dic[target-num]+1, i+1]
            dic[num] = i
        # @lc code=end
so = Solution()

print(so.twoSum([2, 5, 7, 8], 9))


from django.urls import path
from . import views

urlpatterns = [
    path('', views.start_page, name='start_page'),
    path('gallery', views.start_page, name='start_page_redirect'),
    path('gallery/images', views.images_festival_page, name='images_festival_page'),
    path('gallery/stories', views.stories_festival_page, name='stories_festival_page'),
    path('gallery/add_comment/<str:TYPE>_<int:ID>', views.new_comment_page,
         name='images_new_comment'),
    path('gallery/view_comments/<str:TYPE>_<int:ID>', views.view_comments_page,
         name='images_new_comment')
]


"""Functions to parse the formula terms and prepare arrays."""

from collections import OrderedDict

import netCDF4

def get_formula_terms(var):
    """
    Return a `formula_terms` dict mapping var_names to variables.

    The input can be a netCDF variable (`var`) holding the `formula_terms`
    attribute or the attribute itself.

    """
    formula_terms = OrderedDict()
    if isinstance(var, netCDF4.Variable):
        var = var.formula_terms
    terms = [x.strip(":") for x in var.split()]
    for k, v in zip(terms[::2], terms[1::2]):
        formula_terms.update({k: v})
    return formula_terms

def get_formula_terms_variables(nc):
    """Return a list with all variables that holds the formula_terms attribute."""

    def func(v):
        return v is not None

    var = nc.get_variables_by_attributes(formula_terms=func)
    if not var:
        msg = (
            "Could not find the attribute `formula_terms` in any of the "
            "{!r} variables."
        ).format
        raise ValueError(msg(nc))
    return var

def get_formula_terms_dims(nc, formula_terms):
    """Return an OrderedDict object `dims` holding the `formula_terms` dimensions."""
    dims = OrderedDict()
    for k, v in formula_terms.items():
        dims.update({k: nc[v].dimensions})
    return dims

def z_shape(nc, dims):
    """Return the vertical coordinate `shape` based on the formula_terms `dims`."""
    all_dims = (
        dims.get("eta"),
        dims.get("depth"),
        dims.get("sigma"),
        dims.get("s"),
    )
    all_dims = _filter_none(all_dims)
    all_dims = _flatten(all_dims)
    all_dims = _remove_duplicate(all_dims)
    if len(all_dims) > 1:
        z_dims = all_dims[:]
        z_dims.insert(1, z_dims.pop(-1))

    shape = []
    for dim in z_dims:
        try:
            shape.append(len(nc.dimensions.get(dim)))
        except TypeError:
            msg = "Could not get dimension size for dim {!r}".format
            raise ValueError(msg(dim))

    return tuple(shape)

def _reshape(arr, new_shape):
    """Reshape arrays according to the expected dimensions."""
    shape = arr.shape
    dims = [k for k in shape]
    slicer = slice(None, None, None)

    def select(dim):
        return slicer if dim in dims else None

    keys = tuple(select(k) for k in new_shape)

    return arr.__getitem__(keys)

def prepare_arrays(nc, formula_terms, new_shape):
    """Prepare arrays for formulas."""
    import dask.array as da

    arrays = {}
    for term, var in formula_terms.items():
        var = nc[var]
        if var.ndim == 0:
            arr = var[:]
        else:
            if var.ndim > 2:
                chunks = (1,) + var.shape[1:]
            else:
                chunks = var.shape
            if term == "sigma" and var.ndim == 2:
                chunks = var.shape
            if term == "eta" and var.ndim == 2:
                chunks = (1,) + var.shape[1:]
            arr = da.from_array(var, chunks=chunks)
            arr = _reshape(arr, new_shape)
        arrays.update({term: arr})
    return arrays

def _filter_none(seq):
    """Filter out None in a list."""
    return [x for x in seq if x is not None]

def _flatten(seq):
    """Flatten a list."""
    return [item for sublista in seq for item in sublista]

def _remove_duplicate(seq):
    """Drop duplicated from a list."""
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]


#!/usr/bin/env python3
"""
Author: Ken Chen
Email: chenkenbio@gmail.com
"""

import os
import torch
import warnings
import numpy as np
import gzip
from tqdm import tqdm
from collections import defaultdict
from transformers import BertForMaskedLM, BertTokenizer
from torch.utils.data import DataLoader, Dataset
from torch.cuda.amp.autocast_mode import autocast

from utils import get_reverse_strand, load_fasta, distance, fast_seq_to_kmer_list
from config import hg19 as HG19_FASTA
from config import VEXSEQ, MFASS

CHROM2INT = {
    "chr1": 0,
    "chr2": 1,
    "chr3": 2,
    "chr4": 3,
    "chr5": 4,
    "chr6": 5,
    "chr7": 6,
    "chr8": 7,
    "chr9": 8,
    "chr10": 9,
    "chr11": 10,
    "chr12": 11,
    "chr13": 12,
    "chr14": 13,
    "chr15": 14,
    "chr16": 15,
    "chr17": 16,
    "chr18": 17,
    "chr19": 18,
    "chr20": 19,
    "chr21": 20,
    "chr22": 21,
    "chrX": 22,
    "chrY": 23
}
INT2CHROM = {v:k for k, v in CHROM2INT.items()}

def kl_divergence(ref, alt):
    return (alt * np.log(alt / ref)).sum(axis=-1)

class ExonCenterVariant(Dataset):
    def __init__(self, vcf, size: int, genome: str, tokenizer: BertTokenizer, k=1, exon_zero_start=True):
        super().__init__()
        self.k = k
        self.exon_zero_start = exon_zero_start
        self.size = size
        self.genome = load_fasta(genome, cache=True)
        self.samples = list()
        self.tokenizer = tokenizer
        self.distances = list()
        self.mutation_ids = list()
        self.groups = list()
        with gzip.open(vcf, 'rt') as infile:
            for l in infile:
                if l.startswith('#'):
                    continue
                chrom, pos, name, ref, alt = l.strip().split('\t')[:5]
                if ref == '.' or alt == '.':
                    warnings.warn("unsupported allele '.'")
                    continue
                if len(ref) > 1 or len(alt) > 1:
                    warnings.warn("skip indels")
                    continue
                pos = int(pos) - 1
                mut_id, _, e_start, e_end, strand, ref_psi, dpsi, label = name.split('|')
                if not self.exon_zero_start:
                    e_start = int(e_start) - 1
                self.mutation_ids.append(mut_id)
                label = 1 if label == "True" else 0
                self.groups.append(mut_id.split('_')[0])
                self.samples.append((
                    chrom, pos, ref, alt, e_start, e_end, strand, ref_psi, dpsi, label
                ))
                self.distances.append(
                    min(
                        distance(int(e_start) - 2, int(e_start), pos, pos + 1),
                        distance(int(e_end), int(e_end) + 2, pos, pos + 1)
                    )
                )
        self.groups = np.asarray(self.groups)
        self.samples = np.asarray(self.samples)
        self.distances = np.asarray(self.distances)
        self.labels = np.asarray([int(x[-1]) for x in self.samples])
        self.refpsi = np.asarray([float(x[-3]) for x in self.samples])
        self.dpsi = np.asarray([float(x[-2]) for x in self.samples])

        self.is_exonic = list()
        for x in self.samples:
            _, pos, _, _, e1, e2 = x[:6]
            pos, e1, e2 = int(pos), int(e1), int(e2)
            if pos >= e1 and pos < e2:
                self.is_exonic.append(1)
            else:
                self.is_exonic.append(0)
        self.is_exonic = np.asarray(self.is_exonic)

        self.repr_string = "{}(vcf={},genome={}, seq_size={},vocab_size/k={}/{})".format(
            self.__class__.__name__,
            vcf,
            genome,
            size,
            self.tokenizer.vocab_size,
            k
        )

    def __repr__(self) -> str:
        return self.repr_string

    def __getitem__(self, index, get_seq: bool=False):
        chrom, position, ref, alt, e_start, e_end, strand, _, _, label = self.samples[index]
        position, e_start, e_end, label = int(position), int(e_start), int(e_end), int(label)
        left = (e_start + e_end) // 2 - self.size // 2
        seq_wt = self.genome[chrom][left:left + self.size].upper()
        e_start, e_end = e_start - left, e_end - left
        d_start, d_end = e_end, e_end + 2
        a_start, a_end = e_start - 2, e_start

        pos = position - left
        assert seq_wt[pos] == ref, "{}".format((index))
        seq_wt = list(seq_wt)
        seq_mt = seq_wt.copy()
        seq_mt[pos] = alt
        if strand == '-':
            seq_wt = get_reverse_strand(seq_wt, join=False)
            seq_mt = get_reverse_strand(seq_mt, join=False)

            e_start, e_end = len(seq_wt) - e_end, len(seq_wt) - e_start
            d_start, d_end = e_end, e_end + 2
            a_start, a_end = e_start - 2, e_start
            pos = len(seq_wt) - 1 - pos
        if self.k == 1:
            ids_wt = self.tokenizer.encode(' '.join(seq_wt))
            ids_mt = self.tokenizer.encode(' '.join(seq_mt))
        else:
            seq_wt = ' '.join(fast_seq_to_kmer_list(''.join(seq_wt), k=self.k, pad=True))
            seq_mt = ' '.join(fast_seq_to_kmer_list(''.join(seq_mt), k=self.k, pad=True))
            ids_wt = self.tokenizer.encode(seq_wt)
            ids_mt = self.tokenizer.encode(seq_mt)
        ids_wt = torch.as_tensor(ids_wt).long()
        ids_mt = torch.as_tensor(ids_mt).long()
        if get_seq:
            return ids_wt, ids_mt, pos, d_start, d_end, a_start, a_end, int(label), seq_wt, seq_mt
        else:
            return ids_wt, ids_mt, pos, d_start, d_end, a_start, a_end, int(label), CHROM2INT[chrom], position
    def __len__(self):
        return len(self.samples)

    def check_dataset(self):
        d_counts = defaultdict(int)
        a_counts = defaultdict(int)
        noncanonical = list()
        unmatched = list()
        for idx in tqdm(range(len(self.samples))):
            ids_wt, ids_mt, pos, d_start, d_end, a_start, a_end, label, seq_wt, seq_mt = self.__getitem__(idx, get_seq=True)
            d_counts["{}{}".format(*ids_wt[d_start + 1:d_end + 1].numpy())] += 1
            d_counts[''.join(seq_wt[d_start:d_end])] += 1
            a_counts["{}{}".format(*ids_wt[a_start + 1:a_end + 1].numpy())] += 1
            a_counts[''.join(seq_wt[a_start:a_end])] += 1
            if seq_wt[d_start:d_end] != "GT" or seq_wt[a_start:a_end] != "AG":
                noncanonical.append((idx, self.mutation_ids[idx], seq_wt[d_start:d_end], seq_wt[a_start:a_end], self.samples[idx]))
            if ids_wt[pos + 1] == ids_mt[pos + 1] or seq_wt[pos] == seq_mt[pos]:
                unmatched.append(idx)
        return dict(d_counts=d_counts, a_counts=a_counts, unmatched=unmatched, noncanonical=noncanonical)

    @classmethod
    def load_mfass(self, tokenizer, size=510, k=1):
        return self(vcf=MFASS, size=size, genome=HG19_FASTA, tokenizer=tokenizer, k=k)

    @classmethod
    def load_vexseq(self, tokenizer, size=510, k=1):
        return self(vcf=VEXSEQ, size=size, genome=HG19_FASTA, tokenizer=tokenizer, k=k, exon_zero_start=False)

@autocast()
@torch.no_grad()
def evaluate_mutation(model: BertForMaskedLM, ds: Dataset, batch_size: int=16, num_workers=8):
    loader = DataLoader(ds, batch_size=batch_size, num_workers=num_workers)
    all_wt_logits = list()
    all_mt_logits = list()
    all_positions = list()
    all_labels = list()
    all_donors = list()
    all_acceptors = list()
    device = next(model.parameters()).device
    for ids_wt, ids_mt, pos, d_start, d_end, a_start, a_end, label in tqdm(loader):
        ids_wt, ids_mt = ids_wt.to(device), ids_mt.to(device)
        logits_wt = model.forward(ids_wt).logits[:, 1:-1, 5:].detach()
        logits_mt = model.forward(ids_mt).logits[:, 1:-1, 5:].detach()
        logits_wt = torch.softmax(logits_wt, dim=-1).cpu().numpy()
        logits_mt = torch.softmax(logits_mt, dim=-1).cpu().numpy()
        all_wt_logits.append(logits_wt)
        all_mt_logits.append(logits_mt)
        all_labels.append(label.numpy())
        all_positions.append(pos.numpy())
        all_donors.append(np.asarray([(a, b) for a, b in zip(d_start.numpy(), d_end.numpy())]))
        all_acceptors.append(np.asarray([(a, b) for a, b in zip(a_start.numpy(), a_end.numpy())]))
    all_wt_logits = np.concatenate(all_wt_logits)
    all_mt_logits = np.concatenate(all_mt_logits)
    all_positions = np.concatenate(all_positions)
    all_donors = np.concatenate(all_donors)
    all_acceptors = np.concatenate(all_acceptors)
    all_labels = np.concatenate(all_labels)
    return all_wt_logits, all_mt_logits, all_positions, all_donors, all_acceptors, all_labels

# def get_args():
#     p = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     p.add_argument("--model-path", default="../pretrained/mlm_on_tx", required=True)
#     p.add_argument("--mask-ss", action="store_true")
#     p.add_argument("--mask-mut", action="store_true")
#     p.add_argument('-o', "--outdir", required=True)
#     # p.add_argument('--seed', type=int, default=2020)
#     return p

if __name__ == "__main__":
    pass

#     args = get_args().parse_args()
#     args.outdir = make_directory(args.outdir)
#     logger = make_logger(filename=os.path.join(args.outdir, "train.log"))
#     logger.info(get_run_info(argv=sys.argv, args=args))

#     ds = ExonCenterVariant(
#         vcf="../data/mfass/MFASS_mmsplice.vcf",
#         size=510,
#         genome=HG19_FASTA,
#         # mask_mut=args.mask_mut,
#         # mask_ss=args.mask_ss
#     )
#     pickle.dump(ds.labels, open("{}/labels.pkl".format(args.outdir), 'wb'), protocol=pickle.HIGHEST_PROTOCOL)

#     NN2INT = {
#         'N': 0,
#         'A': 1,
#         'C': 2,
#         'G': 3,
#         'T': 4
#     }

#     fn_logits = "{}/logits.pt".format(args.outdir)
#     if os.path.exists(fn_logits):
#         logger.info("- use {}".format(fn_logits))
#         all_wt_logits, all_mt_logits, all_positions, all_donors, all_acceptors, all_labels = pickle.load(open(fn_logits, 'rb'))
#     else:
#         device = torch.device("cuda")
#         bert = BertForMaskedLM.from_pretrained(args.model_path).to(device)

#         all_wt_logits, all_mt_logits, all_positions, all_donors, all_acceptors, all_labels = evaluate_mutation(bert.to(device), ds, batch_size=24, num_workers=12)

#         pickle.dump(
#             (all_wt_logits, all_mt_logits, all_positions, all_donors, all_acceptors, all_labels),
#             open(fn_logits, 'wb'),
#             protocol=pickle.HIGHEST_PROTOCOL
#         )

#     if args.mask_mut:
#         all_mut_scores = list()
#         for idx in tqdm(range(all_wt_logits.shape[0])):
#             ref, alt = ds.samples[idx][2:4]
#             p = all_positions[idx]
#             ar = all_wt_logits[idx, p]
#             mut_score = min(max(ar[NN2INT[alt]], 1E-6), 1 - 1E-6) / min(max(ar[NN2INT[ref]], 1E-6), 1 - 1E-6)
#             all_mut_scores.append(mut_score)
#         all_mut_scores = np.asarray(all_mut_scores)
#         pickle.dump(all_mut_scores, open("{}/all_mut_scores.pkl".format(args.outdir), 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
#     else:
#         all_ss_scores = list()
#         for idx in tqdm(range(all_wt_logits.shape[0])):
#             d1, d2 = all_donors[idx]
#             a1, a2 = all_acceptors[idx]
#             ss_score = max(
#                 kl_divergence(all_wt_logits[idx, d1], all_mt_logits[idx, d1]) + \
#                     kl_divergence(all_wt_logits[idx, d1 + 1], all_mt_logits[idx, d1 + 1]),
#                 kl_divergence(all_wt_logits[idx, a1], all_mt_logits[idx, a1]) + \
#                     kl_divergence(all_wt_logits[idx, a1 + 1], all_mt_logits[idx, a1 + 1])
#             )
#             all_ss_scores.append(ss_score)
#         all_ss_scores = np.asarray(all_ss_scores)
#         if args.mask_ss:
#             pickle.dump(all_ss_scores, open("{}/all_ss_scores.mask_ss.pkl".format(args.outdir), 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
#         else:
#             pickle.dump(all_ss_scores, open("{}/all_ss_scores.pkl".format(args.outdir), 'wb'), protocol=pickle.HIGHEST_PROTOCOL)

# #         average_precision_score(all_labels, all_scores), roc_auc_score(all_labels, all_scores)
# #
# #
# #
# #
# #         average_precision_score(all_labels, all_ss_scores_mask_ss), roc_auc_score(all_labels, all_ss_scores_mask_ss)
# #
# #
# #
# #
# #         p, r, _ = precision_recall_curve(all_labels, all_scores)
# #         f, t, _ = roc_curve(all_labels, all_scores)
# #
# #
# #
# #
# #         ax = plt.subplot(121)
# #         ax.plot(r, p)
# #         ax = plt.subplot(122)
# #         ax.plot(f, t)
# #
# #
# #
# #
# #         all_dpsi = np.asarray([float(x[-2]) for x in ds.samples])
# #         all_refpsi = np.asarray([float(x[-3]) for x in ds.samples])
# #
# #
# #
# #
# #         average_precision_score(np.abs(all_dpsi) > 0.4, all_scores),     average_precision_score(all_labels, all_scores),     average_precision_score(all_dpsi < 0, all_scores)
# #
# #
# #
# #
# #         import pyBigWig
# #
# #
# #
# #
# #         bw.values('chrX', 48837917, 48837917+1)
# #
# #
# #
# #
# #         phast46_scores = list()
# #         bw = pyBigWig.open("/bigdat1/pub/UCSC/hg19/vertebrate.phastCons46way.bw")
# #         for s in tqdm(ds.samples):
# #             chrom, position = s[:2]
# #             phast46_scores.append(
# #                 bw.values(chrom, int(position), int(position) + 1)[0]
# #             )
# #         bw.close()
# #
# #         phast100_scores = list()
# #         bw = pyBigWig.open("/bigdat1/pub/UCSC/hg19/hg19.100way.phastCons.bw")
# #         for s in tqdm(ds.samples):
# #             chrom, position = s[:2]
# #             phast100_scores.append(
# #                 bw.values(chrom, int(position), int(position) + 1)[0]
# #             )
# #         bw.close()
# #
# #
# #
# #
# #         # phast46_scores = np.asarray(phast46_scores)
# #         phast100_scores = np.asarray(phast100_scores)
# #
# #
# #
# #
# #         average_precision_score(all_labels, phast46_scores), roc_auc_score(all_labels, phast46_scores)
# #
# #
# #
# #
# #         average_precision_score(all_labels, phast100_scores), roc_auc_score(all_labels, phast100_scores)
# #
# #
# #
# #
# #         torch.save(phast46_scores, "phast46_scores.pt")
# #         torch.save(phast100_scores, "phast100_scores.pt")
# #
# #
# #
# #
# #         phast100_scores = torch.load("./phast100_scores.pt")
# #
# #
# #
# #
# #         phast100_scores
# #
# #
# #
# #
# #         X = np.stack([phast100_scores, all_scores, ds.distances]).T
# #
# #
# #
# #
# #         lr.fit(X, y=ds.labels)
# #
# #
# #
# #
# #         y = lr.predict_log_proba(X).T[1]
# #
# #
# #
# #
# #         average_precision_score(ds.labels, all_scores), roc_auc_score(ds.labels, all_scores)
# #
# #
# #
# #
# #         average_precision_score(ds.labels, y), roc_auc_score(ds.labels, y)
# #
# #
# #
# #
# #         average_precision_score(ds.labels, y), roc_auc_score(ds.labels, y)
# #
# #
# #
# #
# #
# #
# #


import random, math
from utils import *

class Tile:

	def __init__(self, passable, symbol, stair=False):
		self.passable = passable
		assert len(symbol) == 1, "Symbol must be exactly one character"
		self.symbol = symbol
		self.revealed = False
		self.walked = False
		self.stair = stair
		self.items = []

class Board:

	def __init__(self, g, cols, rows):
		self.g = g
		self.cols = cols
		self.rows = rows
		self.data = [[Tile(True, " ") for x in range(cols)] for y in range(rows)]
		self.clear_cache()

	def clear_cache(self):
		self.mons_cache = [[None for x in range(self.cols)] for y in range(self.cols)]

	def line_between(self, pos1, pos2, skipfirst=False, skiplast=False):
		x1, y1 = pos1
		x2, y2 = pos2
		dx = abs(x2 - x1)
		sx = 1 if x1 < x2 else -1
		dy = -abs(y2 - y1)
		sy = 1 if y1 < y2 else -1
		error = dx + dy
		while True:
			doyield = True
			if (skipfirst and (x1, y1) == pos1) or (skiplast and (x1, y1) == pos2):
				doyield = False
			if doyield:
				yield (x1, y1)
			if (x1, y1) == (x2, y2):
				return
			e2 = 2 * error
			if e2 >= dy:
				if x1 == x2:
					return
				error += dy
				x1 += sx
			if e2 <= dx:
				if y1 == y2:
					return
				error += dx
				y1 += sy

	def in_bounds(self, x, y):
		return 0 <= x < self.cols and 0 <= y < self.rows

	def line_of_sight(self, pos1, pos2):
		for point in self.line_between(pos1, pos2, skiplast=True):
			if self.blocks_sight(*point):
				return False
		return True

	def is_clear_path(self, pos1, pos2):
		for point in self.line_between(pos1, pos2, skipfirst=True, skiplast=True):
			if not self.is_passable(*point):
				return False
		return True

	def get_in_radius(self, pos, radius):
		x, y = pos
		for px in range(x-radius, x+radius+1):
			for py in range(y-radius, y+radius+1):
				if (px, py) != pos and self.in_bounds(x, y):
					yield (px, py)

	def get_in_circle(self, pos, radius):
		cx, cy = pos
		for x, y in self.get_in_radius(pos, radius):
			dx = x - cx
			dy = y - cy
			dist = math.sqrt(dx**2 + dy**2)
			if round(dist) > radius or not self.in_bounds(x, y):
				continue
			yield x, y

	def get_in_cone(self, pos, radius, angle, widthdeg=45):
		cx, cy = pos
		angle %= 360
		for x, y in self.get_in_radius(pos, radius):
			dx = x - cx
			dy = y - cy
			dist = math.sqrt(dx**2 + dy**2)
			if round(dist) > radius or not self.in_bounds(x, y):
				continue
			dir = math.degrees(math.atan2(dy, dx))
			half = widthdeg/2
			if dir < 0:
				dir += 360

			if abs(angle - dir) <= half:
				yield x, y
			elif angle + half >= 360 and dir <= (angle + half) % 360:
				yield x, y
			elif angle - half < 0 and dir >= angle-half+360:
				yield x, y

	#A monster collision cache is used to improve the performance of detecting collisions with monsters
	#This way, checking if there's a monster at a position can be O(1) instead of O(m)

	def set_cache(self, x, y, mon):
		self.mons_cache[y][x] = mon

	def unset_cache(self, x, y):
		self.mons_cache[y][x] = None

	def get_mon_cache(self, x, y):
		return self.mons_cache[y][x]

	def swap_cache(self, pos1, pos2):
		if pos1 == pos2:
			return
		x1, y1 = pos1
		x2, y2 = pos2
		tmp = self.mons_cache[y1][x1]
		self.mons_cache[y1][x1] = self.mons_cache[y2][x2]
		self.mons_cache[y2][x2] = tmp

	def blocks_sight(self, col, row):
		if (col, row) == (self.g.player.x, self.g.player.y):
			return False
		return not self.get(col, row).passable

	def is_passable(self, col, row):
		if self.blocks_sight(col, row):
			return False
		return not self.mons_cache[row][col]

	def generate(self):
		self.data = [[Tile(False, "#") for x in range(self.cols)] for y in range(self.rows)]
		self.clear_cache()
		WIDTH_RANGE = (5, 10)
		HEIGHT_RANGE = (3, 5)
		ATTEMPTS = 100
		NUM = random.randint(5, 8)
		rooms = []
		randchance = dice(2, 10)
		if one_in(7):
			randchance = 100
		for i in range(NUM):
			for _ in range(ATTEMPTS):
				width = random.randint(*WIDTH_RANGE)
				height = random.randint(*HEIGHT_RANGE)
				xpos = random.randint(1, self.cols - width - 1)
				ypos = random.randint(1, self.rows - height - 1)
				for x, y, w, h in rooms:
					flag = True
					if x + w < xpos or xpos + width < x:
						flag = False
					elif y + h < ypos or ypos + height < y:
						flag = False
					if flag:
						break
				else:
					for x in range(width):
						for y in range(height):
							self.carve_at(xpos + x, ypos + y)
					if i > 0:
						prev = rooms[-1]
						if random.randint(1, randchance) == 1:
							prev = random.choice(rooms)
						x, y, w, h = prev
						pos1_x = x + random.randint(1, w - 2)
						pos1_y = y + random.randint(1, h - 2)
						pos2_x = xpos + random.randint(1, width - 2)
						pos2_y = ypos + random.randint(1, height - 2)
						dx = 1 if pos1_x < pos2_x else -1
						dy = 1 if pos1_y < pos2_y else -1
						if one_in(2):
							x = pos1_x
							while x != pos2_x:
								self.carve_at(x, pos1_y)
								x += dx
							y = pos1_y
							while y != pos2_y:
								self.carve_at(pos2_x, y)
								y += dy
						else:
							y = pos1_y
							while y != pos2_y:
								self.carve_at(pos1_x, y)
								y += dy
							x = pos1_x
							while x != pos2_x:
								self.carve_at(x, pos2_y)
								x += dx

					rooms.append((xpos, ypos, width, height))
					break

	def carve_at(self, col, row):
		if not (0 <= col < self.cols and 0 <= row < self.rows):
			raise ValueError(f"carve_at coordinate out of range: ({col}, {row})")
		self.data[row][col] = Tile(True, " ")

	def get(self, col, row):
		return self.data[row][col]

###############
#Pathfinding
#Algorithm used is A* Search
from collections import defaultdict

class OpenSet:

	def __init__(self, key=None):
		self._data = []
		self._dup = set()
		self.key = key or (lambda v: v)

	def add(self, value):
		if value in self._dup:
			return
		self._dup.add(value)
		a = self._data
		key = self.key
		i = len(a)
		a.append(value)
		while i > 0:
			parent = i // 2
			if key(a[parent]) < key(a[i]):
				break
			a[parent], a[i] = a[i], a[parent]
			i = parent

	def pop(self):
		if len(self._data) == 0:
			raise IndexError("pop from an empty heap")
		a = self._data
		val = a[0]
		a[0] = a[-1]
		a.pop()
		key = self.key
		i = 0
		while True:
			left = 2 * i + 1
			right = 2 * i + 2
			if left >= len(a):
				break
			node = left
			if right < len(a) and key(a[right]) < key(a[left]):
				node = right
			if key(a[i]) > key(a[node]):
				a[i], a[node] = a[node], a[i]
				i = node
			else:
				break
		self._dup.remove(val)
		return val

	def __contains__(self, value):
		return value in self._dup

	def __bool__(self):
		return len(self._data) > 0

def pathfind(board, start, end, *, rand=False):
	#Actual A* Search algorithm
	def h(a, b):
		return abs(a[0] - b[0]) + abs(a[1] - b[1])
	gScore = defaultdict(lambda: float("inf"))
	gScore[start] = 0
	fScore = defaultdict(lambda: float("inf"))
	fScore[start] = h(start, end)
	open_set = OpenSet(fScore.__getitem__)
	open_set.add(start)
	came_from = {}
	rows = board.rows
	cols = board.cols
	def can_pass(x, y):
		if (x, y) == end:
			return not board.blocks_sight(x, y)
		return board.is_passable(x, y)
	while open_set:
		curr = open_set.pop()
		if curr == end:
			path = [curr]
			while curr in came_from:
				curr = came_from[curr]
				path.append(curr)
			path.reverse()
			return path
		neighbors = []
		x, y = curr
		if x + 1 < cols and can_pass(x + 1, y):
			neighbors.append((x + 1, y))
		if x - 1 >= 0 and can_pass(x - 1, y):
			neighbors.append((x - 1, y))
		if y + 1 < rows and can_pass(x, y + 1):
			neighbors.append((x, y + 1))
		if y - 1 >= 0 and can_pass(x, y - 1):
			neighbors.append((x, y - 1))
		if rand:
			random.shuffle(neighbors)

		for n in neighbors:
			cost = 1
			t = gScore[curr] + cost
			if t < gScore[n]:
				came_from[n] = curr
				gScore[n] = t
				fScore[n] = t + h(n, end)

				if n not in open_set:
					open_set.add(n)
	return []

#End pathfinding
###############


"""
pyspark.sql.DataFrame.freqItems¶
DataFrame.freqItems(cols, support=None)[source]
Finding frequent items for columns, possibly with false positives. Using the frequent element count algorithm described
 in “https://doi.org/10.1145/762471.762473, proposed by Karp, Schenker, and Papadimitriou”. DataFrame.freqItems() and
 DataFrameStatFunctions.freqItems() are aliases.

New in version 1.4.0.

Parameters
cols: list or tuple
Names of the columns to calculate frequent items for as a list or tuple of strings.

support: float, optional
The frequency with which to consider an item ‘frequent’. Default is 1%. The support must be greater than 1e-4.
"""
import findspark
findspark.init()

"""
The freqItems function in PySpark is used to compute frequent items for selected columns in a DataFrame.
It returns a DataFrame containing the frequent items for each specified column.

Here's an example to illustrate the usage of freqItems with different ways of applying it:
"""
from pyspark.sql import SparkSession

# Create a SparkSession
spark = SparkSession.builder.getOrCreate()

# Create a DataFrame
data = [
    (1, "Apple", "Red"),
    (2, "Orange", "Orange"),
    (3, "Banana", "Yellow"),
    (4, "Apple", "Green"),
    (5, "Orange", "Yellow"),
    (6, "Banana", "Green")
]
df = spark.createDataFrame(data, ["ID", "Fruit", "Color"])

# Example 1: Compute frequent items for a single column
freq_items_1 = df.freqItems(["Fruit"], support=0.5)
freq_items_1.show(truncate=False)

# Example 2: Compute frequent items for multiple columns
freq_items_2 = df.freqItems(["Fruit", "Color"], support=0.5)
freq_items_2.show(truncate=False)

# Example 3: Set parameters for frequent items computation
freq_items_3 = df.freqItems(["Fruit"], support=0.5, numPartitions=2)
freq_items_3.show(truncate=False)

"""
# Example 1:
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Orange]  |
+-----------------+

# Example 2:
+-----------------+----------------------+
|Fruit_freqItems  |Color_freqItems       |
+-----------------+----------------------+
|[Apple, Orange]  |[Green, Yellow, Orange]|
+-----------------+----------------------+

# Example 3:
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Orange]  |
+-----------------+

In the first example, we compute the frequent items for the "Fruit" column. The result shows the list of frequent items in that column.

In the second example, we compute the frequent items for both the "Fruit" and "Color" columns. The result shows the frequent items for each column separately.

In the third example, we set additional parameters for the frequent items computation. We specify the number of partitions to control the parallelism of the
computation.

Note that the support parameter represents the minimum fraction of rows that should contain an item to be considered frequent.
It should be a value between 0 and 1.
"""

##########################################################################################
##########################################################################################
"""
Let's go through the concepts of support and numPartitions, and also address the question about the first example.

Support:

The support parameter in freqItems represents the minimum fraction of rows that should contain an item for it to be considered frequent.
It is specified as a value between 0 and 1. For example, a support value of 0.5 means that an item must appear in at least 50% of the rows to be considered frequent.
By setting a higher support value, you can filter out less frequent items and focus on the most common ones.
numPartitions:

The numPartitions parameter in freqItems determines the number of partitions used for parallel computation of frequent items.
It specifies the degree of parallelism and can be useful for improving performance when working with large datasets.
By default, the value is set to None, which means that the number of partitions is determined automatically based on the cluster configuration.
You can set numPartitions to a specific integer value to control the number of partitions used.
Regarding the first example and the absence of "Banana" in the frequent items:

In the first example, the support parameter was not explicitly specified, so the default value is used.
The default value of support is 0.1, which means that an item must appear in at least 10% of the rows to be considered frequent.
If the frequency of "Banana" is less than 10% in the "Fruit" column, it will not be considered a frequent item and will not appear in the result.
To include "Banana" as a frequent item, you can either reduce the support threshold or increase the frequency of "Banana" in the DataFrame.
"""
freq_items_1 = df.freqItems(["Fruit"], support=0.2)
freq_items_1.show(truncate=False)

"""
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Banana, Orange]  |
+-----------------+

"""


# -*- coding: utf-8 -*-
# import necessary libraries
import numpy as np
import matplotlib.pyplot as plt

# make array of all susceptible population
population = np.zeros( (50, 50) )

# random disease outbreak somewhere in the population
outbreak = [0,0]
population[outbreak[0],outbreak[1]] = 1

plt.figure(figsize=(6,4),dpi=150)

plt.imshow(population, cmap='viridis', interpolation='nearest',vmin=0,vmax=2)

from matplotlib.colors import LinearSegmentedColormap
colors=[(0.80,0.80,0.80),(0.64,0.19,0.13),(0.21,0.23,0.24)]
cm = LinearSegmentedColormap.from_list("my_colors", colors, N=3)

# model parameters: beta, gamma
beta = 0.3
gamma = 0.05
 # make array of all susceptible population
population = np.zeros( (50, 50) )

    # random disease outbreak somewhere in the population
outbreak = [0,0]
population[outbreak[0],outbreak[1]] = 1
    # at each time point:
    # infected cell can infect its neighbours

for time in range(100):

        # find infected points
        infectedIndex = np.where(population==1)
        for i in range(len(infectedIndex[0])):
            # get x, y coordinates for each point
            x = infectedIndex[0][i]
            y = infectedIndex[1][i]
            # infect each neighbour with probability beta
            # infect all 8 neighbours:
            for xNeighbour in range(x-1,x+2):
                for yNeighbour in range(y-1,y+2):
                    if (xNeighbour,yNeighbour) != (x,y):
                        # make sure I don't fall off an edge
                        if xNeighbour != -1 and yNeighbour != -1 and xNeighbour!=50 and yNeighbour!=50:
                            if population[xNeighbour,yNeighbour]==0:
                                population[xNeighbour,yNeighbour]=np.random.choice(range(2),1,p=[1-beta,beta])[0]

            # recover
            population[x,y] =1+np.random.choice(range(2),1,p=[1-gamma,gamma])[0]
        sirmap = {0:0, 1:0, 2:0}
        for row in population:
            for cell in row:
                sirmap[cell] += 1

        # plot heatmap of population
        plt.figure(figsize=(6,4),dpi=150)
        # filename=path+"spatial_SIR"+str(time+1)+".png"
        plt.imshow(population, cmap=cm, interpolation='nearest',vmin=0,vmax=2)
        # plt.show()
        # plt.savefig(filename,type="png")


#!/home/zkfr/.local/share/virtualenvs/xf-5EfV3Nly/bin/python
#-*- coding:utf-8 -*-
# @author : MaLei
# @datetime : 2018-11-29 12:05
# @file : dianping.py
# @software : PyCharm

from datetime import datetime as dt
from dianping import *

def create_db():
    cursor, conn = connect_mysql()
    sql = 'CREATE TABLE IF NOT EXISTS food_rank(' \
          'id INT UNSIGNED AUTO_INCREMENT,' \
          'tip VARCHAR(100) NOT NULL,' \
          'tip_id INT(100) NULL,' \
          'classifi VARCHAR(100) NULL,' \
          'rank_num INT(100) NULL,' \
          'shopId VARCHAR(100) NOT NULL,' \
          'shopName VARCHAR(100) NOT NULL,' \
          'mainRegionName VARCHAR(100) NULL,' \
          'taste FLOAT(10) NULL,' \
          'environment FLOAT(10) NULL,' \
          'service FLOAT(10) NULL,' \
          'avgPrice INT(255) NULL,' \
          'city_id INT(100) NULL,' \
          'address VARCHAR(100) NULL,' \
          'update_time DATE NULL,' \
          'PRIMARY KEY (id))ENGINE=InnoDB DEFAULT CHARSET=utf8;'
    cursor.execute(sql)
    conn.commit()
    cursor.close()
    conn.close()

def insert_db(classifi,class_name):
    cursor, conn = connect_mysql()
    update_time = dt.now().date()

    for i in range(0, len(classifi)):
        # print(classifi[i],class_name[i])
        num = 1
        # 获取分类排行榜的信息
        data = requests.get(url=classifi[i], headers=headers).json()
        shopdata = data['shopBeans']
        cityid = data['cityId']
        try:
            # 保存每个商铺的信息
            for each in shopdata:
                mainCategoryName = each['mainCategoryName']
                shopName = each['shopName']
                branchName = each['branchName']
                if len(branchName) != 0:
                    shopName = shopName + '(' + branchName + ')'
                shopId = each['shopId']
                mainRegionName = each['mainRegionName']
                taste = each['refinedScore1']
                environment = each['refinedScore2']
                service = each['refinedScore3']
                avgPrice = each['avgPrice']
                address = each['address']
                sql = 'INSERT INTO food_rank(tip,tip_id,classifi,rank_num,shopId,shopName,mainRegionName,taste,environment,service,avgPrice,city_id,address,update_time) ' \
                      'VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                cursor.execute(sql, (class_name[i],
                                     i,
                                     mainCategoryName,
                                     num,
                                     shopId,
                                     shopName,
                                     mainRegionName,
                                     taste,
                                     environment,
                                     service,
                                     avgPrice,
                                     cityid,
                                     address,
                                     update_time))
                conn.commit()
                num += 1
            print('爬取{}排行榜成功'.format(class_name[i]))
        except:
            print('获取商铺信息出错')
    # 去重
    a = 'delete from food_rank where id in (select id from (select id from food_rank where id not in (select min(id) from food_rank group by class,shopName,rank_num)) as temple)'
    cursor.execute(a)
    conn.commit()
    cursor.close()
    conn.close()

def get_rank_info():
    create_db()
    base_url = 'http://www.dianping.com'
    start_url = 'http://www.dianping.com/beijing/food'
    try:
        res = requests.get(start_url, headers=headers).text
        response = etree.HTML(res)
        hot_url = response.xpath('//div[@class="item news_list current"]/a[@class="more"]/@href')
        # print(hot_url)
        classifi_res = requests.get((base_url + hot_url[0]), headers).text
        response = etree.HTML(classifi_res)
        # 第一行标签
        classifi1 = hot_url + response.xpath('//div[@class="box shopRankNav"]/p[1]//a/@href')
        class_name1 = ['热门'] + response.xpath('//div[@class="box shopRankNav"]/p[1]//a/text()')
        # 第二行标签
        classifi2 = response.xpath('//div[@class="box shopRankNav"]/p[2]//a/@href')
        class_name2 = response.xpath('//div[@class="box shopRankNav"]/p[2]//a/text()')
        # 所有标签和名称
        class_name = class_name1 + class_name2
        classifi = classifi1 + classifi2
        ajax_base_url = 'http://www.dianping.com/mylist/ajax/shoprank?rankId='
        for each in classifi[:]:
            classifi.append(ajax_base_url + each.split('=')[1])
            classifi.remove(each)
        insert_db(classifi,class_name)
    except:
        print('爬取榜单出错')

get_rank_info()


from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.translation import gettext_lazy as _

from feincms.admin.item_editor import FeinCMSInline
from feincms.module.medialibrary.fields import ContentWithMediaFile
from feincms.utils.tuple import AutoRenderTuple

class MediaFileContentInline(FeinCMSInline):
    raw_id_fields = ("mediafile",)
    radio_fields = {"type": admin.VERTICAL}

class MediaFileContent(ContentWithMediaFile):
    """
    Rehashed, backwards-incompatible media file content which does not contain
    the problems from v1 anymore.

    Create a media file content as follows::

        from feincms.content.medialibrary.models import MediaFileContent
        Page.create_content_type(MediaFileContent, TYPE_CHOICES=(
            ('default', _('Default')),
            ('lightbox', _('Lightbox')),
            ('whatever', _('Whatever')),
            ))

    For a media file of type 'image' and type 'lightbox', the following
    templates are tried in order:

    * content/mediafile/image_lightbox.html
    * content/mediafile/image.html
    * content/mediafile/lightbox.html
    * content/mediafile/default.html

    The context contains ``content`` and ``request`` (if available).
    """

    feincms_item_editor_inline = MediaFileContentInline

    class Meta:
        abstract = True
        verbose_name = _("media file")
        verbose_name_plural = _("media files")

    @classmethod
    def initialize_type(cls, TYPE_CHOICES=None):
        if TYPE_CHOICES is None:
            raise ImproperlyConfigured(
                "You have to set TYPE_CHOICES when" " creating a %s" % cls.__name__
            )

        cls.add_to_class(
            "type",
            models.CharField(
                _("type"),
                max_length=20,
                choices=TYPE_CHOICES,
                default=TYPE_CHOICES[0][0],
            ),
        )

    def render(self, **kwargs):
        return AutoRenderTuple(
            (
                [
                    f"content/mediafile/{self.mediafile.type}_{self.type}.html",
                    "content/mediafile/%s.html" % self.mediafile.type,
                    "content/mediafile/%s.html" % self.type,
                    "content/mediafile/default.html",
                ],
                {"content": self},
            )
        )


from sbb_b import sbb_b
from sbb_b.core.managers import edit_delete, edit_or_reply
from sbb_b.helpers.utils import mentionuser
from telethon import functions
from telethon.errors import ChatAdminRequiredError, UserAlreadyInvitedError
from telethon.tl.types import Channel, Chat, User

async def get_group_call(chat):
    if isinstance(chat, Channel):
        result = await catub(functions.channels.GetFullChannelRequest(channel=chat))
    elif isinstance(chat, Chat):
        result = await catub(functions.messages.GetFullChatRequest(chat_id=chat.id))
    return result.full_chat.call

async def chat_vc_checker(event, chat, edits=True):
    if isinstance(chat, User):
        await edit_delete(event, "- المكالمة الصوتية غير متاحة في المجموعة الخاصة")
        return None
    result = await get_group_call(chat)
    if not result:
        if edits:
            await edit_delete(event, "**- لا توجد دردشة صوتية في هذه المجموعة**")
        return None
    return result

async def parse_entity(entity):
    if entity.isnumeric():
        entity = int(entity)
    return await catub.get_entity(entity)

@sbb_b.ar_cmd(pattern="تشغيل المكالمة")
async def start_vc(event):
    vc_chat = await sbb_b.get_entity(event.chat_id)
    gc_call = await chat_vc_checker(event, vc_chat, False)
    if gc_call:
        return await edit_delete(event, "**- المكالمة الصوتية بالأصل موجودة**")
    try:
        await sbb_b(
            functions.phone.CreateGroupCallRequest(
                peer=vc_chat,
                title="مكالمة جمثون",
            )
        )
        await edit_delete(event, "**- تم بنجاح تشغيل المكالمة الصوتية**")
    except ChatAdminRequiredError:
        await edit_delete(event, "- عليك أن تكون مشرف اولا", time=20)

@sbb_b.ar_cmd(pattern="انهاء المكالمة")
async def end_vc(event):
    vc_chat = await sbb_b.get_entity(event.chat_id)
    gc_call = await chat_vc_checker(event, vc_chat)
    if not gc_call:
        return
    try:
        await sbb_b(functions.phone.DiscardGroupCallRequest(call=gc_call))
        await edit_delete(event, "**- تم بنجاح الغاء المكالمة الصوتية**")
    except ChatAdminRequiredError:
        await edit_delete(event, "- عليك أن تكون مشرف اولا", time=20)

@sbb_b.ar_cmd(pattern="دعوة ?(.*)?")
async def inv_vc(event):
    users = event.pattern_match.group(1)
    reply = await event.get_reply_message()
    vc_chat = await sbb_b.get_entity(event.chat_id)
    gc_call = await chat_vc_checker(event, vc_chat)
    if not gc_call:
        return
    if not users:
        if not reply:
            return await edit_delete(
                "**يجب عليك الرد على المستخدم او وضع معرفه مع الامر**"
            )
        users = reply.from_id
    await edit_or_reply(event, "**- تم دعوة المستخدم الى المكالمة**")
    entities = str(users).split(" ")
    user_list = []
    for entity in entities:
        cc = await parse_entity(entity)
        if isinstance(cc, User):
            user_list.append(cc)
    try:
        await sbb_b(
            functions.phone.InviteToGroupCallRequest(call=gc_call, users=user_list)
        )
        await edit_delete(event, "**- تم دعوة المستخدمين الى المكالمة**")
    except UserAlreadyInvitedError:
        return await edit_delete(
            event, "**- المستحدم بالأصل مدعو لهذه المكالمة**", time=20
        )

@sbb_b.ar_cmd(pattern="معلومات المكالمة")
async def info_vc(event):
    vc_chat = await sbb_b.get_entity(event.chat_id)
    gc_call = await chat_vc_checker(event, vc_chat)
    if not gc_call:
        return
    await edit_or_reply(event, "**- جار الحصول على معلومات المكالمة**")
    call_details = await sbb_b(
        functions.phone.GetGroupCallRequest(call=gc_call, limit=1)
    )
    grp_call = "**معلومات المكالمة الصوتية:**\n\n"
    grp_call += f"**العنوان :** {call_details.call.title}\n"
    grp_call += f"**عدد المشاركين :** {call_details.call.participants_count}\n\n"

    if call_details.call.participants_count > 0:
        grp_call += "**المشاركين**\n"
        for user in call_details.users:
            nam = f"{user.first_name or ''} {user.last_name or ''}"
            grp_call += f"  ● {mentionuser(nam,user.id)} - `{user.id}`\n"
    await edit_or_reply(event, grp_call)

@sbb_b.ar_cmd(pattern="عنوان?(.*)?")
async def title_vc(event):
    title = event.pattern_match.group(1)
    vc_chat = await sbb_b.get_entity(event.chat_id)
    gc_call = await chat_vc_checker(event, vc_chat)
    if not gc_call:
        return
    if not title:
        return await edit_delete("**- عليك وضع اسم مع الامر اولا**")
    await sbb_b(functions.phone.EditGroupCallTitleRequest(call=gc_call, title=title))
    await edit_delete(event, f"-تم بنجاح تغيير اسم المكالمة الصوتية الى **{title}**")


from flask import Flask, render_template, request, jsonify

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def main():
    return render_template('index.html')

@app.route('/create-ticket', methods=['GET', 'POST'])
def createTicket():
    if request.method == 'POST':
        content = request.args.get('content')
        position = request.args.get('position')
        time = request.args.get('time')
        name = request.args.get('name')
        if content and position:
            print(f"{content} {position} {time} {name}")
        else:
            pass
    return jsonify({
        "status": "success"
    })

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port="80")


from email.message import EmailMessage
import smtplib

class MailSender:
    def __init__(self):
        self.msg = EmailMessage()

    def form_message_success(self, destination, device_name):

        self.msg['From'] = 'stalozaidimudalinimasis@outlook.com'
        self.msg['To'] = destination
        self.msg['Subject'] = 'The program was tested successfully!'
        self.msg.set_content('''
        <!DOCTYPE html>

                    The tested device {0} finished without interruptions.

        '''.format(device_name), subtype='html')

    def form_message_failed(self, destination, device_name):
        self.msg['From'] = 'stalozaidimudalinimasis@outlook.com'
        self.msg['To'] = destination
        self.msg['Subject'] = 'The program has failed!'
        self.msg.set_content('''
        <!DOCTYPE html>

                    The tested device {0} failed while testing.

        '''.format(device_name), subtype='html')

    def send_mail(self):
        try:

            mailserver = smtplib.SMTP('smtp.office365.com', 587)
            mailserver.ehlo()
            mailserver.starttls()

            try:
                mailserver.login('stalozaidimudalinimasis@outlook.com',
                                 'gqjcfxntqitshslo')
                mailserver.send_message(self.msg)
            finally:
                mailserver.quit()
        except:
            pass


import argparse
import os
import torch
import torch.nn.functional as F
import datetime

from torch.utils.data import DataLoader
from datasets.samplers import CategoriesSampler
from datasets.mini_imagenet import MiniImageNet
from datasets.tiered_imagenet import TieredImageNet
from datasets.cifarfs import CIFAR_FS
from resnet import resnet12
from util import str2bool, set_gpu, count_acc, seed_torch, compute_confidence_interval
from model import SFTAttn

def get_dataset(args):
    if args.dataset == 'mini':
        testset = MiniImageNet('test', args.size)
        n_cls = 64
        print("=> MiniImageNet...")
    elif args.dataset == 'tiered':
        testset = TieredImageNet('test', args.size)
        n_cls = 351
        print("=> TieredImageNet...")
    elif args.dataset == 'cifarfs':
        testset = CIFAR_FS('test', args.size)
        n_cls = 64
        print("=> CIFAR-FS...")
    else:
        print("Invalid dataset...")
        exit()

    test_sampler = CategoriesSampler(testset.label, args.test_batch,
                                    args.test_way, args.shot + args.test_query)
    test_loader = DataLoader(dataset=testset, batch_sampler=test_sampler,
                            num_workers=args.worker, pin_memory=True)
    return test_loader, n_cls

def main(args):
    loader, n_cls = get_dataset(args)

    if args.dataset in ['mini', 'tiered']:
        encoder1 = resnet12(avg_pool=True, drop_rate=0.1, dropblock_size=5, num_classes=n_cls).cuda()
        encoder2 = resnet12(avg_pool=True, drop_rate=0.1, dropblock_size=5, num_classes=n_cls).cuda()
    elif args.dataset in ['cifarfs']:
        encoder1 = resnet12(avg_pool=True, drop_rate=0.1, dropblock_size=2, num_classes=n_cls).cuda()
        encoder2 = resnet12(avg_pool=True, drop_rate=0.1, dropblock_size=2, num_classes=n_cls).cuda()
    else:
        print("Invalid dataset...")
        exit()

    if args.pretrain_path_1:
        checkpoint_file = os.path.join(args.pretrain_path_1, 'max-acc.pth.tar')
        if not os.path.isfile(checkpoint_file):
            print("=> Pretrain model 1 not found...")
            exit()
        checkpoint = torch.load(checkpoint_file)
        encoder1.load_state_dict(checkpoint['model'])
        print("=> Pretrain encoder 1 loaded...")

    if args.pretrain_path_2:
        checkpoint_file = os.path.join(args.pretrain_path_2, 'max-acc.pth.tar')
        if not os.path.isfile(checkpoint_file):
            print("=> Pretrain model 2 not found...")
            exit()
        checkpoint = torch.load(checkpoint_file)
        encoder2.load_state_dict(checkpoint['model'])
        print("=> Pretrain encoder 2 loaded...")

    model = SFTAttn(args).cuda()

    # check save point
    checkpoint_file = os.path.join(args.save_path, 'max-acc.pth.tar')
    if not os.path.isfile(checkpoint_file):
        print("=> Model not found...")
        exit()
    checkpoint = torch.load(checkpoint_file)
    model.load_state_dict(checkpoint['model'])
    print("=> Model loaded from best epoch", checkpoint['best_epoch'], "...")

    encoder1.eval()
    encoder2.eval()
    model.eval()

    acc_list = []

    with torch.no_grad():
        for _, batch in enumerate(loader, 1):
            data, _ = [_.cuda() for _ in batch]
            label = torch.arange(args.test_way).repeat(args.test_query)
            label = label.type(torch.cuda.LongTensor)

            with torch.no_grad():
                data1 = encoder1(data, is_feat=args.is_feat)
                data2 = encoder2(data, is_feat=args.is_feat)

            logits = model(data1, data2)
            acc = count_acc(logits, label)
            acc_list.append(acc*100)

    a, b = compute_confidence_interval(acc_list)
    print("{}-way {}-shot accuracy with 95% interval : {:.2f}±{:.2f}".format(args.test_way, args.shot, a, b))

if __name__ == '__main__':
    start_time = datetime.datetime.now()

    # settings
    parser = argparse.ArgumentParser()
    parser.add_argument('--save-path', default='./save/mini_final')
    parser.add_argument('--pretrain-path-1', default='./save/mini_rot')
    parser.add_argument('--pretrain-path-2', default='./save/mini_dist')
    parser.add_argument('--gpu', default='0')
    parser.add_argument('--seed', type=int, default=1)
    # fusion
    parser.add_argument('--beta', type=float, default=0.5)
    parser.add_argument('--gamma', type=float, default=0.5)
    # network
    parser.add_argument('--h-dim', type=int, default=640)
    parser.add_argument('--dropout', type=float, default=0.4)
    parser.add_argument('--temperature', type=float, default=1.0)
    # dataset
    parser.add_argument('--dataset', default='mini', choices=['mini','tiered','cifarfs'])
    parser.add_argument('--size', type=int, default=84)
    parser.add_argument('--worker', type=int, default=8)
    # few-shot
    parser.add_argument('--shot', type=int, default=1)
    parser.add_argument('--test-way', type=int, default=5)
    parser.add_argument('--test-query', type=int, default=15)
    parser.add_argument('--test-batch', type=int, default=2000)
    parser.add_argument('--norm', type=str2bool, nargs='?', default=True)
    parser.add_argument('--is-feat', type=str2bool, nargs='?', default=True)
    parser.add_argument('--distance', type=str, default='euc', choices=['cos', 'euc'])
    args = parser.parse_args()

    if args.dataset in ['mini', 'tiered']:
        args.size = 84
    elif args.dataset in ['cifarfs']:
        args.size = 32
        args.worker = 0
    else:
        args.size = 28
    # fix seed
    seed_torch(args.seed)
    set_gpu(args.gpu)

    main(args)

    end_time = datetime.datetime.now()
    print("End time :{} total ({})".format(end_time, end_time - start_time))


"""
@brief      test log(time=14s)
"""

import sys
import os
import unittest
import warnings
import numpy as np

from pyquickhelper.loghelper import fLOG
from pyquickhelper.ipythonhelper import StaticInteract, RangeWidget, RadioWidget, DropDownWidget
from pyquickhelper.pycode import fix_tkinter_issues_virtualenv, ExtTestCase

class TestInteractive(ExtTestCase):

    def test_interactive_StaticInteract(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        def show_fib(N):
            sequence = ""
            a, b = 0, 1
            for i in range(N):
                sequence += f"{a} "
                a, b = b, a + b
            return sequence

        res = StaticInteract(show_fib,
                             N=RangeWidget(1, 100, default=10))
        self.assertNotEmpty(res)
        ht = res.html()
        self.assertNotEmpty(ht)

    def test_interactive2_RadioWidget(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        fix_tkinter_issues_virtualenv(fLOG=fLOG)
        import matplotlib.pyplot as plt

        def plot(amplitude, color, sele):
            fig, ax = plt.subplots(figsize=(4, 3),
                                   subplot_kw={'axisbelow': True})
            ax.grid(color='w', linewidth=2, linestyle='solid')
            x = np.linspace(0, 10, 1000)
            ax.plot(x, amplitude * np.sin(x), color=color,
                    lw=5, alpha=0.4)
            try:
                ax.set_xlim(0, 10)
                ax.set_ylim(-1.1, 1.1)
            except TypeError as e:
                warnings.warn(e)
            return fig

        res = StaticInteract(plot,
                             amplitude=RangeWidget(0.1, 0.3, 0.1, default=0.2),
                             color=RadioWidget(
                                 ['blue', 'green'], default='blue'),
                             sele=DropDownWidget(['a', 'b']))
        self.assertNotEmpty(res)
        ht = res.html()
        self.assertNotEmpty(ht)
        plt.close('all')
        fLOG("end")

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-
#
#
# Author: alex
# Created Time: 2017年12月01日 星期五 22时14分24秒
import jieba
import logging
import gensim
from sklearn.cluster import KMeans
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                    level=logging.INFO)

# get input file,text format
source_filename = 'source.txt'
seq_filename = 'output.seq'
if False:
    with open(source_filename, 'r') as f, open(seq_filename, 'w') as output:
        input_lines = f.readlines()
        count = len(input_lines)
        print('源文件行数为：' + str(count))

        # read file and seperate words
        for line in input_lines:
            line = line.strip('\n')
            seg_list = jieba.cut(line)
            output.write(' '.join(seg_list)+'\n')

# 训练并保存模型
model_filename = "output.model"
sentences = gensim.models.doc2vec.TaggedLineDocument(seq_filename)
model = gensim.models.Doc2Vec(sentences, size=100, window=3)
model.train(sentences)
model.save(model_filename)

# k-means预测
num_clusters = 20    # 设定聚类个数
km = KMeans(n_clusters=num_clusters)  # 设置模型参数
result = km.fit_predict(model.docvecs)  # 用doc2vec训练的模型产生的句向量作为k-means的输入

# 将聚类预测结果和具体文档处理对应上
output_filename = 'output.txt'
with open(source_filename, 'r') as fr, open(output_filename, 'w') as fw:
    for i in range(0, num_clusters-1):
        for linenumber, eachline in enumerate(fr.readlines()):
            if linenumber >= len(result):
                break
            if result[linenumber] == i:
                fw.write(eachline+'\n')

        fw.write('------------------ %d -----------------------\n' % i)


## 5014 스타트링크 (DFS/BFS)

import sys
input = sys.stdin.readline
from collections import deque

f, start, end, up, down = map(int, input().split())
## 백만층까지 ㄱㄴ

## start => end 까지의 최소값
visit = [0]*(f+1)
queue = deque()
ud = [up, (-down)]

def circle(start):
    global visit
    queue.append(start)
    while queue:
        idx = queue.popleft()
        if idx == end:
            break

        for i in range(2):
            next = idx + ud[i]
            if i==0 and up==0:
                continue
            if i==1 and down==0:
                continue
            if 0 < next <= f and visit[next] == 0:
                visit[next] = visit[idx]+1
                queue.append(next)

circle(start)

if start == end:
    print(0)
elif visit[end] == 0:
    print("use the stairs")
else:
    print(visit[end])