from termcolor import colored
import sys
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib._png import read_png
from matplotlib.cbook import get_sample_data
import os
import numpy as np
from PIL import Image
import scipy.cluster as cluster
from sklearn.svm import OneClassSVM
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor,NearestNeighbors
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Ridge,HuberRegressor,RANSACRegressor,TheilSenRegressor
from sklearn.preprocessing import PolynomialFeatures
from scipy import interpolate
import math
import warnings
warnings.filterwarnings("ignore")

def main(draw=True, path=False, paths=None):

    if not path:
        # absolute path
        path = "/media/ash/OS/small_obstacle_bag/synced_data/seq_1/"
        label_files = sorted(os.listdir(os.path.join(path,"labels")))
        velodyne_files = sorted(os.listdir(os.path.join(path,"depth")))

        # path files for various interesting data streams
        index_value = 25
        cloud_files = [os.path.join(path+"depth/",x) for x in label_files]
        image_files = [os.path.join(path+"image/",x) for x in label_files]
        ring_files = [os.path.join(path+'rings', x) for x in label_files]
        label_files = [os.path.join(path+"labels/",x) for x in label_files]

        depth_file = cloud_files[index_value]
        image_file = image_files[index_value]
        label_file = label_files[index_value]
        ring_file = ring_files[index_value]
    else:
        depth_file = paths['depth']
        image_file = paths['image']
        label_file = paths['label']
        ring_file = paths['ring']

    # OPEN ALL RELEVANT IMAGES FOR MANIPULATION
    # open a depth image
    depth = Image.open(depth_file)
    # normalize depth values so that they have range between 1 - 255
    depth = np.array(depth)

    # open rgb image
    image = np.asarray(Image.open(image_file))

    # open ground truth image
    gt = Image.open(label_file)
    gt_unconverted = np.asarray(gt)
    gt = gt.convert('L')
    gt = np.array(gt)

    # RETIRIEVE ROAD POLYGONS FROM THE GROUND TRUTH FILE
    # retrieve max and min of these pixels
    road_x,road_y = np.where(gt == 38)
    max_road_x = np.max(road_x)
    min_road_x = np.min(road_x)
    max_road_y = np.max(road_y)
    min_road_y = np.min(road_y)

    # open a ring image
    ring = np.asarray(Image.open(ring_file))
    # CL
    # CONTROL VARIABLES THAT ARE GOING TO BE USED TO ANALYSE LIDAR DATA
    # data fields to be potted
    # x, y self-explanatory
    # Value is the depth value at x,y
    x=[] # a list of all x positions corresponding to lidar positions within the road polygon
    y=[] # list of y positions corresponding to lidar positions within road polygon
    value = [] # vx, y lidar values
    ring_values = [] # ring value of at the position x, y

    # Loop to extract pixels with valid lidar depth that belong within the road
    # polygon
    for i in range(depth.shape[0]):
        for j in range(depth.shape[1]):
            if depth[i,j] == 0:
                continue
            if i<max_road_x and i>min_road_x and j<max_road_y and j>min_road_y and gt[i,j]!=0:
                x.append(i)
                y.append(j)
                value.append(depth[i, j])
                ring_values.append(ring[i, j])

    x = np.asarray(x)
    y = np.asarray(y)
    value = np.asarray(value)
    ring_values = np.asarray(ring_values)
    unique_road_rings = np.unique(ring_values)

    x_features = []
    y_features = []
    value_features = []
    cluster_preds = []

    # degree of linear regression fit
    for ring in unique_road_rings[:len(unique_road_rings)-1]:
        relevant_indices = np.where(ring_values == ring)[0]
        for i in relevant_indices:
            x_features.append(x[i])
            y_features.append(y[i])
            value_features.append(value[i])
        feature_vector = [[x[i],y[i]] for i in relevant_indices]
        ring_depths = value[relevant_indices]
        ring_depths = np.asarray(ring_depths)
        ring_depths = ring_depths[:, np.newaxis]
        # preds = custom_fit(y[relevant_indices][:,np.newaxis],ring_depths)
        # preds = custom_fit(x[relevant_indices][:,np.newaxis],y[relevant_indices][:,np.newaxis])
        preds = spline_fit(x[relevant_indices],y[relevant_indices],value[relevant_indices])
        # clf = IsolationForest(contamination=0.02).fit(feature_vector)
        # preds = clf.predict(feature_vector)
        for pred in preds:
            cluster_preds.append(pred)
        break

    # TRYING OUT VARIOUS CLUSTERING ALGOS
    # build a feature vector to input to various clustering apis
    # lets cluster ring 1

    # depth_ring_1 = np.asarray(depth[x_1_road, y_1_road])

    # ITERATION 1, FEATURE VECTOR USING X,Y,D
    # feature_vector = [[x_1_road[i],y_1_road[i],depth[x_1_road[i], y_1_road[i]]] for i in range(len(x_1_road))]

    # ITERATION 2, USING ONLY DEPTH
    # feature_vector = [[depth_ring_1[i]] for i in range(len(x_1_road))]
    # feature_vector = np.array(feature_vector)
    # norm_feature_vector = cluster.vq.whiten(feature_vector)
    # depth_ring_1 = depth_ring_1[:, np.newaxis]

    # previous clustering code
    # feature_vector_global = [[x[i], y[i], value[i]] for i in range(len(value))]
    # feature_vector_global = np.array(feature_vector_global)
    # norm_feature_vector_global = cluster.vq.whiten(feature_vector_global)
    # value = np.array(value)
    # value = value[:, np.newaxis]

    """k means clustering"""
    # cluster,distortion = cluster.vq.kmeans(norm_feature_vector.transpose(),k_or_guess=2)

    """SVM clustering"""
    # clf = OneClassSVM(gamma='auto',kernel='poly',degree=3).fit(value)
    # pred = clf.predict(value)
    # clf = OneClassSVM(gamma='auto',kernel='poly',degree=3).fit(depth_ring_1)
    # pred = clf.predict(depth_ring_1)

    "Isolation forest clustering"
   #  clf = IsolationForest(contamination=0.1).fit(depth_ring_1)
   #  pred = clf.predict(depth_ring_1)
    # clf = IsolationForest(n_estimators=500, contamination=0.1).fit(norm_feature_vector)
    # pred = clf.predict(norm_feature_vector)

    # clf = LocalOutlierFactor().fit(feature_vector)
    # pred = clf.fit_predict(feature_vector)

    """Nearest Neighbours"""
    # nbrs = NearestNeighbors(n_neighbors=5,radius=100).fit(value)
    # distances, indices = nbrs.kneighbors(value)
    # print(np.argmin(distances,axis=1),distances)

    colors=[]
    for elem in cluster_preds:
        if elem == -1:
            colors.append('red')
        elif elem == 1:
            colors.append('blue')

    if draw:
        fig=plt.figure(dpi=200)
        # ax_ring = fig.add_subplot(111,projection='3d')
        ax_global = fig.add_subplot(111, projection='3d')
        # ax_global.scatter(xs=norm_feature_vector_global[:,1],ys=norm_feature_vector_global[:,2],zs=norm_feature_vector_global[:,0],s=1,c=ring_values)
        ax_global.scatter(xs=y_features,ys=value_features,zs=x_features,s=1,color=colors)
        # ax.scatter(xs=feature_vector[:,0],ys=feature_vector[:,2],zs=feature_vector[:,1],s=1)
        # ax_ring.scatter(xs=norm_feature_vector[:,1],ys=norm_feature_vector[:,2],zs=norm_feature_vector[:,0],s=1, color=colors)
        # ax.scatter(xs=x_1_road,ys=feature_vector[:,0],zs=y_1_road,s=1)

        # print('gt shape: ', gt_unconverted.shape)
        # print('X: ', gt_unconverted.shape[0])
        # print('Y: ', gt_unconverted.shape[1])
        # gt_unconverted = read_png(label_files[25])
        # stepX, stepY = 1000/image.shape[1], 1000/image.shape[0]
        # X1 = np.arange(0, 1000, stepX)
        # Y1 = np.arange(0, 1000, stepY)
        # stepX, stepY = 1, 1
        # X1 = np.arange(0, 1280, 1)
        # Y1 = np.arange(0, 720, 1)
        # X1, Y1 = np.meshgrid(X1, Y1)
        #
        # ax.plot_surface(X1, np.atleast_2d(255), Y1, rstride=10, cstride=10,
        #                 facecolors=image/256.0)
        # ax_ring.set_xlabel('X')
        # ax_ring.set_ylabel('depth')
        # ax_ring.set_zlabel('Z')
        ax_global.set_xlabel('X')
        ax_global.set_ylabel('depth')
        ax_global.set_zlabel('Z')
        # ax.scatter(xs=cluster[:,1],ys=cluster[:,2],zs=cluster[:,0],s=10,color=colors)
        plt.show()

    return np.asarray(x_features), np.asarray(y_features), np.asarray(cluster_preds), np.asarray(value_features)

def custom_fit(x_feat, y_feat):
    result = []
    degree = 3
    model = make_pipeline(PolynomialFeatures(degree),RANSACRegressor())
    # print(colored("DEBUG/ x shape: {}".format(x_feat.shape), 'cyan'))
    # print(colored("DEBUG/ y shape: {}".format(y_feat.shape), 'cyan'))
    model.fit(x_feat, y_feat)
    y_hat = model.predict(x_feat)

    # plt.scatter(x_feat,y_hat,c='r',s=5)
    # plt.scatter(x_feat,y_feat,c='b',s=10)
    # plt.show()
    # print(colored("DEBUG/ y hat shape: {}".format(y_hat.shape), 'cyan'))
    deviation = np.sqrt(np.mean(abs(y_hat - y_feat)**2))
    for i in range(len(y_feat)):
        if abs(y_feat[i] - y_hat[i]) > 2.5*deviation:
            result.append(-1)
        else:
            result.append(1)
    return result

def spline_fit(x,y,z):
    fig = plt.figure()
    ax = fig.add_subplot(111,projection='3d')
    ax.plot(x,y,z)
    plt.show()

def get_breakpoints(pts):
    diff_log = []
    length = pts.shape[0]
    pred = np.zeros(length)
    new_pred = np.zeros(length)
    # road_height = np.mean(pts[:,2])

    for i in range(2,length):
        d_i_1 = np.linalg.norm(pts[i-1,:3])
        d_i_2 = np.linalg.norm(pts[i-2,:3])
        d_i = np.linalg.norm(pts[i,:3])
        gamma_1 = np.dot(pts[i-1,:3],pts[i-2,:3])/(d_i_1*d_i_2)
        gamma_2 = np.dot(pts[i - 1, :3], pts[i, :3]) / (d_i_1 * d_i)
        gamma = (gamma_1 + gamma_2)/2
        d_p = (d_i_1*d_i_2)/(2*d_i_1*gamma-d_i_2)
        diff = d_i-d_p

        if 0.5 < diff < 1:
            pred[i] = 1
        elif -1 < diff < -0.5:
            pred[i] = -1
        diff_log.append(diff)

    min_segment = 1
    segments = []
    for i in range(length):
        if pred[i] == -1:
            obs_start = i
            obs_end = 0
            end_range = i+11 if i+11 < length else length
            for j in range(i+1,end_range):
                if pred[j] == 1 and j-i > min_segment:
                    obs_end = j
                    break
            if obs_start != 0 and obs_end != 0:
                segments.append((obs_start,obs_end))

    for start,end in segments:
        # if np.mean(pts[start:end,:2]) - road_height < 0.5:
        new_pred[start:end] = -1
    return new_pred

if __name__ == '__main__':
    main()


import numpy as np
from matplotlib import pyplot as plt
import math
from scipy import optimize
import utils
#================================== Storing Data ======================================
data = np.array(np.loadtxt('C:/Users/Kartikaeya/Desktop/ex2data1.txt',delimiter=','))
X,y = data[:,(0,1)],data[:,2]
y=np.reshape(y,(100,1))
#================================== Plotting Data =====================================

def plotData(X, y):
    for i in range(0,100):
        if (y[i]==1):
            plt.scatter(X[i,0],X[i,1],s=10,c='black',marker='x')
        else:
            plt.scatter(X[i,0],X[i,1],s=10,c='yellow',marker='o')

plotData(X, y)
plt.xlabel("Exam 1 Score")
plt.ylabel("Exam 2 Score")
plt.show()
#=============================== Sigmoid Function =====================================
def sigmoid(z):
    if type(z)==int:
        s= (1/(1+math.exp((-1)*z)))
    else:
        m=np.size(z)
        s=np.exp(((-1)*z))
        for i in range(0,m):
            s[i]=(1/(1+s[i]))
    return s
#=============================== Cost Function & Gradient ==============================
m=np.shape(X)[0]
X = np.concatenate([np.ones((m, 1)), X], axis=1)
def CostFunction(theta, X, y):
    m=np.shape(X)[0]
    Cost=0
    for i in range(0,m):
        Cost=Cost-(y[i]*(math.log(sigmoid(np.dot(X[i,:],np.reshape(theta,(3,1))))))+(1-y[i])*(math.log(1-sigmoid(np.dot(X[i,:],np.reshape(theta,(3,1)))))))
    Cost=Cost/m

    grad = np.zeros((3,1))
    for i in range(0,3):
        a=0
        for j in range(0,m):
            a=a+((sigmoid(np.dot(X[j,:],np.reshape(theta,(3,1))))-y[j])*X[j,i])
        grad[i]=a/m
    return Cost,grad
#========================== testing CostFunction and Gradient===========================
initial_theta = np.zeros(3)

cost, grad = CostFunction(initial_theta, X, y)

print('Cost at initial theta (zeros): ',cost)
print('Expected cost (approx): 0.693\n')

print('Gradient at initial theta (zeros):')
print(grad)
print('Expected gradients (approx):\n\t[-0.1000, -12.0092, -11.2628]\n')

test_theta = np.array([[-24], [0.2], [0.2]])
cost, grad = CostFunction(test_theta, X, y)

print('Cost at test theta: ',cost)
print('Expected cost (approx): 0.218\n')

print('Gradient at test theta:')
print(grad)
print('Expected gradients (approx):\n\t[0.043, 2.566, 2.647]')

#============================== Gradient Descent =======================================
options= {'maxiter': 400}
res = optimize.minimize(CostFunction,initial_theta,(X, y),jac=True,method='TNC', options=options)
cost = res.fun
theta = res.x

print('Cost at theta found by optimize.minimize: ',cost)
print('Expected cost (approx): 0.203\n')

print('theta:')
print(theta)
print('Expected theta (approx):\n\t[-25.161, 0.206, 0.201]')
utils.plotDecisionBoundary(plotData, theta, X, y)
plt.show()
#=================================== Predict ===========================================
def predict(theta, X):
    m = X.shape[0]
    n=np.size(theta)
    p = np.zeros(m)
    for i in range(0,m):
        hypothesis=sigmoid(np.dot(X[i,:],np.reshape(theta,(n,1))))
        if hypothesis>= 0.5:
            p[i]=1
        elif(hypothesis< 0.5):
            p[i]=0
    return p

prob = sigmoid(np.dot([1, 45, 85], np.reshape(theta,(3,1))))
print('For a student with scores 45 and 85,'
      'we predict an admission probability of ',prob)
print('Expected value: 0.775 +/- 0.002\n')

p = predict(theta, X)
print('Train Accuracy: %'.format(np.mean(p == y) * 100))
print('Expected accuracy (approx): 89.00 %')

#========================= Regularized Logistic Regression ================================

#===================================== Load Data =========================================
data = np.array(np.loadtxt('C:/Users/Kartikaeya/Desktop/ex2data2.txt', delimiter=','))
X = data[:, :2]
y = data[:, 2]
#====================================== Plot Data ========================================
plotData(X, y)
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.show()
# Feature Mapping
X = utils.mapFeature(X[:, 0], X[:, 1])
#================================ Cost Function & Gradient ===============================
def costFunctionReg(theta, X, y,lambda_):
    m = np.size(y)
    n=np.size(theta)
    J = 0
    grad = np.zeros(theta.shape)
    for i in range(0,m):
        J=J-(y[i]*(math.log(sigmoid(np.dot(X[i,:],np.reshape(theta,(28,1))))))+(1-y[i])*(math.log(1-sigmoid(np.dot(X[i,:],np.reshape(theta,(28,1)))))))
    J =(J/m)+((lambda_/(2*m))*(np.dot(theta,np.reshape(theta,(n,1)))-1))

    for i in range(0, 28):
        a = 0
        for j in range(0, m):
            a = a + ((sigmoid(np.dot(X[j, :], np.reshape(theta, (28, 1)))) - y[j]) * X[j, i])
        if i==0:
            grad[i] = a / m
        else:
            grad[i] = a / m+ (lambda_ / m) * theta[i]
    return J, grad
#========================== testing CostFunction and Gradient===========================
initial_theta = np.zeros(X.shape[1])
lambda_ = 1
cost, grad = costFunctionReg(initial_theta, X, y, lambda_)

print('Cost at initial theta (zeros): {:.3f}',cost)
print('Expected cost (approx)       : 0.693\n')

print('Gradient at initial theta (zeros) - first five values only:')
print(grad[:5])
print('Expected gradients (approx) - first five values only:')
print('\t[0.0085, 0.0188, 0.0001, 0.0503, 0.0115]\n')

test_theta = np.ones(X.shape[1])
cost, grad = costFunctionReg(test_theta, X, y, 10)

print('------------------------------\n')
print('Cost at test theta    : ',cost)
print('Expected cost (approx): 3.16\n')

print('Gradient at initial theta (zeros) - first five values only:')
print(grad[:5])
print('Expected gradients (approx) - first five values only:')
print('\t[0.3460, 0.1614, 0.1948, 0.2269, 0.0922]')

#============================== Gradient Descent =======================================
initial_theta = np.zeros(X.shape[1])
lambda_ = 1
options= {'maxiter': 100}
res = optimize.minimize(costFunctionReg,initial_theta,(X, y, lambda_),jac=True, method='TNC',options=options)
cost = res.fun
theta = res.x

utils.plotDecisionBoundary(plotData, theta, X, y)
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.grid(False)
plt.title('lambda = %0.2f' % lambda_)
plt.show()

p = predict(theta, X)

print('Train Accuracy: %.1f %%' % (np.mean(p == y) * 100))
print('Expected accuracy (with lambda = 1): 83.1 % (approx)\n')


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from vilya.libs.template import st
from vilya.models.center.activity import Activity
from vilya.views.hub.center.activity import ActivitiesUI

_q_exports = ['new', 'activities']

activities = ActivitiesUI()

def _q_index(request):
    context = {}
    context['request'] = request
    if request.method == 'POST':
        user = request.user
        if not user:
            return request.redirect('/hub/center')
        title = request.get_form_var('title', '')
        description = request.get_form_var('body', '')
        type = request.get_form_var('type', 0)
        if not title:
            return request.redirect('/hub/center')
        Activity.create(title=title,
                        description=description,
                        type=int(type),
                        creator_id=user.name)
        return request.redirect('/hub/center')
    context['activities'] = Activity.gets(limit=30)
    return st('center/center.html', **context)

def new(request):
    context = {}
    context['request'] = request
    return st('center/new_activity.html', **context)


import socket
import os
import sys
import struct
import tarfile
save_path = "E:\\test"
port = ('192.168.31.190',5354)

def socket_service():
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, 1)
        s.bind(port)
        s.listen(2)
    except socket.error as msg:
        print(msg)
        sys.exit(1)
    print('waiting..............')
    while True:
        sock, addr = s.accept()
        deal_data(sock, addr, save_path)
        extent_tar(save_path)

def deal_data(sock, addr, save_path):
    print("Accept from{0}".format(addr))
    while True:
        fileinfo_size = struct.calcsize('128sq')
        buf = sock.recv(fileinfo_size)
        if buf:
            filename, filesize = struct.unpack('128sq', buf)
            fn = filename.decode().strip('\x00')

            file_new_name = os.path.join(save_path, fn)
            print('newname is {0}, filesize {1}'.format(file_new_name,filesize))
            recvd_size = 0
            fp = open(file_new_name, 'wb')
            print('start receving')

            while not recvd_size == filesize:
                if filesize - recvd_size >1024:
                    data = sock.recv(1024)
                    recvd_size += len(data)
                else:
                    data = sock.recv(1024)
                    recvd_size = filesize
                fp.write(data)
            fp.close()
            print('Files received')
        sock.close()
        break

def extent_tar(save_path):
    tar_path = os.walk(save_path)
    for path, dir_list, file_list in tar_path:
        for dir_name in file_list:
            tarname = os.path.join(path, dir_name)
            if os.path.splitext(tarname)[-1] == '.tar':
                print('TRUE')
                tar = tarfile.open(tarname, 'r:')
                file_names = tar.getnames()
                for file_name in file_names:
                    tar.extract(file_name, save_path)
                tar.close()
                print('tarfile extented')
                os.remove(tarname)
                print('tarfile deleted')

if __name__ == '__main__':
    socket_service()


# https://leetcode.com/problems/best-meeting-point/
# tags: #google, #math, #matrix
#
# Manhattan distance: (p1, p2) = |p2.x - p1.x| + |p2.y - p1.y|
# The optimal point is the mean
# For example, 1-1-0-0-1, the optimal point is at x = 1, and assume the total distance is d
# If we move the point to the right, x = 2, the total distance will be d + 2 - 1,
# since there will be two points on its left and 1 point on its right
#
# Time complexity: O(m * n + nlog(n)), Space complexity: O(m + n)
from typing import List

class Solution:
    def minTotalDistance(self, grid: List[List[int]]) -> int:
        m, n = len(grid), len(grid[0])
        rows, cols, count_ones = [], [], 0

        for i in range(m):
            for j in range(n):
                if grid[i][j] == 1:
                    count_ones += 1
                    rows.append(i)
                    cols.append(j)

        cols.sort()
        return self.calculate_distance(rows) + self.calculate_distance(cols)

    def calculate_distance(self, points: List[int]) -> int:
        reference = points[len(points) // 2]
        return sum([abs(x - reference) for x in points])

if __name__ == "__main__":
    sol = Solution()

    print(sol.minTotalDistance(grid=[[1, 0, 0, 0, 1],
                                     [0, 0, 0, 0, 0],
                                     [0, 0, 1, 0, 0]]))  # 6

    print(sol.minTotalDistance(grid=[[1, 0, 1, 0, 1],
                                     [0, 1, 0, 0, 0],
                                     [0, 1, 1, 0, 0]]))  # 11


import asyncio
from scrapli.driver.core import AsyncIOSXEDriver
from scrapli import Scrapli
from datetime import datetime
from pathlib import Path
from devices import DEVICES
from commands import COMMANDS
import logging

SUCCESS = 55
logging.addLevelName(SUCCESS , 'SUCCESS')

logging.basicConfig(
    filename="process-logging.log",
    filemode="a",
    format="%(asctime)s:%(name)s - %(message)s",
    datefmt= "%d-%m-%Y %H:%M:%S",
    level=logging.ERROR)

logger=logging.getLogger()

today = datetime.today().date()

async def collect_commands(hostname, device):
    conn = AsyncIOSXEDriver(**device)
    try:
        await conn.open()
    except Exception as error:
        logger.error(f"ERROR: {hostname} - Failed to connect device\n{error}")
    else:
        config_dir = f"archive/{today}/{hostname}"
        Path(config_dir).mkdir(parents=True, exist_ok=True)
        for feature,commands in COMMANDS.items():
            response = await conn.send_commands(commands, strip_prompt=False)
            if response.failed:
                logger.error(f"ERROR: {hostname} - Failed to get command(s)\n{response.result}")
            else:
                filename = f"{config_dir}/{feature}.txt"
                with open(filename, "w") as file:
                    file.write(response.result)
                    file.close()
                logger.log(SUCCESS, f"SUCCESS: {hostname} - Commands collected.\n{commands}")

        await conn.close()

async def main():
    coroutines = [collect_commands(hostname, device) for hostname, device in DEVICES.items()]
    results = await asyncio.gather(*coroutines)

if __name__ == "__main__":
    asyncio.get_event_loop().run_until_complete(main())


import discord, json, asyncio, random
from discord.ext import commands
from datetime import datetime

class Games(commands.Cog):

    def __init__(self, client):
        self.client = client

    @commands.group()
    @commands.guild_only()
    async def wtp(self, ctx):
        """Who's that pokemon game."""
        if ctx.invoked_subcommand is None:
            embed = discord.Embed()
            embed.set_author(name="Minigame: Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
            desc = "```This is a minigame where you have to guess the Pokémon name!\nGame includes Pokémon from 8 generations (Currently includes 897 Pokémons).```"
            rules = "```1. You have to guess the Pokémon name within 15 seconds.\n2. For every correct guess, the guessing time reduces by 0.2 seconds. (Game gets harder as you score more points!)\n3. Only English names are allowed. (Case Insensitive)\n4. Failing to Guess or Wrong guess will end the game.\n```"
            embed.description = desc
            embed.add_field(name="Rules:", value=rules)
            embed.add_field(name="Good Luck!", value="\u200b", inline=False)
            embed.set_image(url="https://i.imgur.com/yAz3xCI.jpg")
            embed.set_footer(text= "To Start the Game | wtp start")
            await ctx.send(embed=embed)
            return

    @wtp.command(name = 'start')
    async def wtp_start(self, ctx):
        """Starts who's that pokemon minigame."""
        if ctx.author.id in self.client.wtpList:
            await ctx.send(f"`Hello {ctx.author.name}, You are already playing who's that pokemon! Please wait until it ends.`")
            return

        self.client.wtpList.append(ctx.author.id)
        with open('./files/wtpNames.json','r') as f:
            wtpData = json.load(f)
        start = True
        points = 0
        time_left = 15.0
        count = 0
        while start:
            pokeID = random.randrange(1,897)
            wtpPoke = wtpData.get(f"{pokeID}")
            pokeName = wtpPoke["name"]
            pokeImgOrg = f"https://github.com/EliteB0Y/TestBot/raw/master/WTP/{pokeID:03d}.png"
            pokeImgWtp = f"https://github.com/EliteB0Y/TestBot/raw/master/WTP/{pokeID:03d}x.png"
            embed = discord.Embed()
            embed.set_author(name="Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
            embed.set_image(url=pokeImgWtp)
            embed.set_footer(text=f"{ctx.author.name} | Current Score: {points} points", icon_url=ctx.author.avatar)
            x = await ctx.send(embed=embed)

            def check(message):
                return message.author == ctx.author

            try:
                msg = await self.client.wait_for('message', timeout = time_left, check = check)
            except asyncio.TimeoutError:
                embed = discord.Embed(description = f"{self.client.emotes.get('redtick', '')}  **You failed to guess {pokeName.title()}**")
                embed.set_author(name="Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
                embed.set_image(url=pokeImgOrg)
                embed.set_footer(text=f"{ctx.author.name} | Final Score: {points} points", icon_url=ctx.author.avatar)
                await x.edit(embed=embed)
                start = False
            else:
                if msg.content.lower() == pokeName:
                    count += 1
                    points += 5
                    time_left -= 0.2
                    embed = discord.Embed(description = f"{self.client.emotes.get('greentick', '')} **Correct!! It's {pokeName.title()}**")
                    embed.set_author(name="Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
                    embed.set_image(url=pokeImgOrg)
                    embed.set_footer(text=f"{ctx.author.name} | Current Score: {points} points", icon_url=ctx.author.avatar)
                    await x.edit(embed=embed)
                else:
                    embed = discord.Embed(description = f"{self.client.emotes.get('redtick', '')}  **You failed to guess {pokeName.title()}**")
                    embed.set_author(name="Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
                    embed.set_image(url=pokeImgOrg)
                    embed.set_footer(text=f"{ctx.author.name} | Final Score: {points} points", icon_url=ctx.author.avatar)
                    await x.edit(embed=embed)
                    start = False

        if points > 200:
            txt = 'Marvelous'
        elif points > 150:
            txt = 'Unbelievable'
        elif points > 100:
            txt = 'Fantastic'
        elif points > 50:
            txt = 'Good Job'
        else:
            txt = 'Hello'
        embed = discord.Embed()
        embed.set_author(name="Who's that Pokémon?", icon_url="https://i.imgur.com/MItw5zU.png")
        embed.set_thumbnail(url=ctx.author.avatar)
        embed.description = f"```{txt} {ctx.author.name}! You have guessed {count} pokemon(s) correctly and scored {points} points.```"
        await ctx.send(embed=embed)
        self.client.wtpList.remove(ctx.author.id)

    @commands.command(name = "10s")
    @commands.guild_only()
    async def _10s(self, ctx):
        """Test your timing by reacting at 10 sec exact!"""
        embed = discord.Embed()
        embed.set_author(name= "Reaction Game", icon_url= ctx.me.avatar)
        embed.description = 'React to this message in exact 10 seconds.\n'
        embed.set_footer(text = ctx.author, icon_url = ctx.author.avatar)
        x = await ctx.send(embed=embed)
        await x.add_reaction("\U0001f44d\U0001f3fb")
        t1 = datetime.utcnow()

        def check(reaction, user):
            return user == ctx.author and str(reaction.emoji) == "\U0001f44d\U0001f3fb"
        try:
            reaction, user = await self.client.wait_for('reaction_add', timeout=20, check=check)
        except:
            embed.description += "\nYour seconds counting sucks for real! 😂"
            await x.edit(embed = embed)
        else:
            t2 = datetime.utcnow()
            tm = t2-t1
            embed.description += f"\nYou have reacted in `{round(tm.total_seconds(),2)}` seconds."
            await x.edit(embed=embed)

    @commands.command()
    @commands.guild_only()
    async def guess(self, ctx):
        """Number guessing game"""
        myGuess = random.randrange(1,10)
        embed = discord.Embed()
        embed.set_author(name="Guess the Number?", icon_url=ctx.me.avatar)
        embed.description = f"```Hello {ctx.author.name},\nI have guessed a number between 1 to 10. Let's see if you can guess the same number within 5 turns.```"
        embed.set_footer(text = ctx.author, icon_url = ctx.author.avatar)
        await ctx.send(embed=embed)

        def check(message):
            return message.author == ctx.author

        for i in range(5):
            try:
                message = await self.client.wait_for('message', check=check, timeout=20)
            except asyncio.TimeoutError:
                embed.description = "```You took too long to guess the number!```"
                return await ctx.send(embed=embed)
            else:
                if message.content != str(myGuess):
                    if i == 4:
                        embed.description = f"{self.client.emotes.get('redtick','')}`Oops!! No turns left. My guess was {myGuess}.`"
                        return await ctx.send(embed=embed)
                    embed.description = f"{self.client.emotes.get('redtick','')}`Nope, that's not it. You have {5-(i+1)} turns left!`"
                    await ctx.send(embed=embed)
                else:
                    embed.description = f"{self.client.emotes.get('greentick','')}`YAY!!! {myGuess} it is... You took {i+1} attempt(s) to guess the number!`"
                    return await ctx.send(embed=embed)

    @commands.command()
    @commands.guild_only()
    async def rps(self, ctx):
        """Rock-Paper-Scissors game"""
        reac = [f"{self.client.emotes.get('rpsrock','')}", f"{self.client.emotes.get('rpspaper','')}", f"{self.client.emotes.get('rpsscissors','')}"]
        mine = random.choice(reac)
        embed = discord.Embed()
        embed.set_author(name="Rock Paper Scissors!", icon_url=ctx.me.avatar)
        embed.description = "Okay! Let's see if you can beat me in Rock-Paper-Scissors game!\n\nI have made my move. It's your turn to choose one!"
        embed.set_footer(text = ctx.author, icon_url = ctx.author.avatar)
        x = await ctx.send(embed = embed)
        [await x.add_reaction(r) for r in reac]

        def check(reaction, user):
            return user == ctx.author and str(reaction.emoji) in reac

        try:
            reaction, user = await self.client.wait_for('reaction_add', check=check, timeout=20)
        except asyncio.TimeoutError:
            embed.description = "You've lost the Rock-Paper-Scissors game due to inactivity..."
            return await x.edit(embed=embed)
        else:
            yours = str(reaction.emoji)
            if yours == mine:
                result = "It's a draw!"
            elif mine == reac[0]:
                if yours == reac[1]:
                    result = "Oh Noo!! You have won. I'll get you next time..."
                else:
                    result = "Haha, You have lost this one."
            elif mine == reac[1]:
                if yours == reac[2]:
                    result = "Oh Noo!! You have won. I'll get you next time..."
                else:
                    result = "Haha, You have lost this one."
            else:
                if yours == reac[0]:
                    result = "Oh Noo!! You have won. I'll get you next time..."
                else:
                    result = "Haha, You have lost this one."
            embed.description = f"Your choice: {str(reaction.emoji)} \nMy choice: {mine}\n\n**{result}**"
            await x.edit(embed=embed)

    @commands.command()
    @commands.guild_only()
    async def pkquiz(self, ctx, points_to_win = 5):
        """Guess the pokemon by dex entries."""
        if ctx.guild.id in self.client.activeQuiz:
            await ctx.send("A quiz is already running in this server. End the quiz to start another one!")
            return
        else:
            try:
                points_to_win = int(points_to_win)
            except:
                _ = await ctx.send("Invalid input for `points_to_win` parameter.")
                return

            self.client.activeQuiz.append(ctx.guild.id)
            await ctx.send(f"A quiz will start in few seconds. First to {points_to_win} point wins!")
            await asyncio.sleep(5)

        with open("./files/dex_entries.json","r") as f:
            data = json.load(f)

        start = True
        fail_count = 0
        points_table = {}
        while start:
            c = random.choice(list(data.keys()))
            quiz = f"{data[c][0]}\n{data[c][1]}"
            answer = c
            if answer in quiz:
                quiz = quiz.replace(answer, "_" * len(answer))

            trans = quiz.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "ᴀʙᴄᴅᴇғɢʜɪᴊᴋʟᴍɴᴏᴘǫʀsᴛᴜᴠᴡxʏᴢᴀʙᴄᴅᴇғɢʜɪᴊᴋʟᴍɴᴏᴘǫʀsᴛᴜᴠᴡxʏᴢ")
            quiz = "```" + quiz.translate(trans) + "```"

            _ = await ctx.send(quiz)

            def check(message):
                if message.content.lower() == answer.lower():
                    return True
                elif message.content.lower() == "skip" and message.author == ctx.author:
                    return True
                elif message.content.lower() == "quit" and message.author == ctx.author:
                    return True

            try:
                msg = await self.client.wait_for('message',check=check, timeout=60)
            except asyncio.TimeoutError:
                _ = await ctx.send(f"You have failed to guess the answer: {answer}!")
                fail_count += 1

            else:
                if msg.content.lower() == answer.lower():
                    point = points_table.get(msg.author, 0) + 1
                    points_table[msg.author] = point
                    await msg.add_reaction("✅")

                    if point >= points_to_win:
                        _ = await ctx.send(f"{msg.author} wins with {point} points!!!")
                        start = False
                    else:
                        _ = await ctx.send(f"{msg.author} : +1 [Total: {point} points]")

                elif msg.content.lower() == "skip":
                    _ = await ctx.send(f"You have skipped this question!! The answer was: {answer}")
                elif msg.content.lower() == "quit":
                    _ = await ctx.send("You have ended the quiz!!!")
                    start = False

            if fail_count >= 3:
                start =  False
                _ = await ctx.send("Ending the quiz due to multiple failed guesses!!!")

            if not start:
                self.client.activeQuiz.remove(ctx.guild.id)
                points_table = dict(sorted(points_table.items(), key=lambda item: item[1], reverse=True))
                desc = "--------------------\n"
                desc += "Points Table: \n"
                desc += "--------------------\n"
                desc += "\n".join([f"{k} : {v} points" for k,v in points_table.items()])
                desc += "\n--------------------"
                _ = await ctx.send(f"```{desc}```")

            await asyncio.sleep(7)

async def setup(client):
    await client.add_cog(Games(client))


from creations import canvas
import botClass
import random
import time

def checkWin(matrix):
    win_lines = [[0, 1, 2],  # первая строка
                 [3, 4, 5],  # вторая строка
                 [6, 7, 8],  # третья строка
                 [0, 3, 6],  # первый столбец
                 [1, 4, 7],  # второй столбец
                 [2, 5, 8],  # третий столбец
                 [0, 4, 8],  # диагональ
                 [2, 4, 6]]  # диагональ

    for line in win_lines:

        if len(set(map(lambda x: matrix[x],line))) == 1 and set(map(lambda x: matrix[x],line)) != {''}:
            return set(map(lambda x: matrix[x],line)).pop()

    return None

matrix = [""]*9
canvas = canvas("0")
bots = [botClass.bot("X"), botClass.bot("0")]

# программа обучения

c = 0
wins = {"X":0,"0":0}
while True:
    matrix = [""] * 9
    choice = random.sample([True,False], 1)[0]
    while checkWin(matrix) == None and any(map(lambda x: x == "", matrix)):
        if choice:
            choice = False
            canvas.update(bots[0].makeChoice(matrix))

        else:
            choice = True
            canvas.update(bots[1].makeChoice(matrix))
        matrix = canvas.getMatrix()

        #canvas.root.update()
        #time.sleep(0.5)

    winner = checkWin(matrix)
    if winner == None:
        for b in bots:
            b.export()
       # print("НИЧЬЯ")
    else:
        wins[winner]+=1
        for b in bots:
            if b.type == winner:
                b.win()
            else:
                b.lose()
    c+=1
    if c == 100:
        c = 0
        print(wins)

       # print("ВЫИГРАЛИ", winner)

'''

# программа игры

while checkWin(matrix) == None and any(map(lambda x:x=="",matrix)):
    while canvas.choice:
        canvas.root.update()
        time.sleep(0.1)
    matrix = canvas.getMatrix()
    if checkWin(matrix) != None or all(map(lambda x:x!="",matrix)):
        break
    canvas.update(bots[0].makeChoice(matrix))

    matrix = canvas.getMatrix()
    canvas.choice = True

winner = checkWin(matrix)
if winner == None:
    for b in bots:
        b.export()
    print("НИЧЬЯ")
else:
    for b in bots:
        if b.type == winner:
            b.win()

    print("ВЫИГРАЛИ",winner)
#'''


import socket
import sys

#####
ZERO = 0
ONE = 1
ACK = "ACK"
TIME_OUT = 0.2
####

class ClientUDP(object):
    def __init__(self,host,port, prefix_size):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.server_address = (host, port)
        self.prefix_size = prefix_size

    def send_data_streaming(self, filename):
        total_messages = ZERO
        total_bytes = ZERO
        try:
            # Send data
            with open(filename, 'rb') as f:
                bytesToSend = f.read(self.prefix_size)
                while (bytesToSend):
                    self.sock.sendto(bytesToSend,self.server_address)
                    total_messages += 1
                    total_bytes += len(bytesToSend)
                    bytesToSend = f.read(self.prefix_size)

        finally:
            return (total_messages, total_bytes)
            pass

    def send_data_stop_and_wait(self, filename):
        total_messages = ZERO
        total_bytes = ONE
        self.sock.settimeout(TIME_OUT)
        # Send data
        with open(filename, 'rb') as f:
            bytesToSend = f.read(self.prefix_size)
            while (bytesToSend):
                self.sock.sendto(bytesToSend,self.server_address)
                while 1:
                    self.sock.settimeout(TIME_OUT)
                    try:
                        data,_ = self.sock.recvfrom(3)
                        ack = data.decode("UTF-8")
                        #print(ack)
                        if ack==ACK:
                            pass
                            #print("Received ack from server")
                    except Exception as e:
                        #print("Time out reached, resending ...package")
                        self.sock.sendto(bytesToSend,self.server_address)
                    else:
                        self.sock.settimeout(ZERO)
                        break

                total_messages += ONE
                total_bytes += len(bytesToSend)
                bytesToSend = f.read(self.prefix_size)

            return (total_messages, total_bytes)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.utils.timezone

class Migration(migrations.Migration):

    dependencies = [
        ('MyPlan', '0006_sprints'),
    ]

    operations = [
        migrations.AlterField(
            model_name='singleplan',
            name='plan_date',
            field=models.DateField(default=django.utils.timezone.now),
        ),
    ]


import openai
import chromadb

from dotenv import load_dotenv

from .variables import OPENAI_API_KEY

from langchain.vectorstores import Chroma
from langchain.docstore.document import Document
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter

load_dotenv()

openai.api_key = OPENAI_API_KEY

HOST = "35.86.173.4"
PORT = 8000

client = chromadb.HttpClient(host=HOST, port=PORT)

class MissingOrganization(Exception):
    """
    Exception raised for errors for missing database.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message="Organization name not in database!!!"):
        self.message = message
        super().__init__(self.message)

def create_organization(org_name):
    try:
        client = chromadb.HttpClient(host=HOST, port=PORT)
        collection = client.create_collection(name=org_name)

        return {
            'status': 200,
            'log': "Successful!!!"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def text_to_docs(ids, metadatas, text):
    """Converts a string or list of strings to a list of Documents with metadata."""

    if isinstance(text, str):
        # Take a single string as one page
        text = [text]

    page_docs = [Document(page_content=page) for page in text]

    # Add page numbers as metadata
    for i, doc in enumerate(page_docs):
        doc.metadata["page"] = i + 1

    # Split pages into chunks
    doc_chunks = []
    doc_id = []
    _metadatas = []

    for _id, meta, doc in zip(ids, metadatas, page_docs):
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=700,
            separators=["\n\n", "\n", ".", "!", "?"],
            chunk_overlap=0,
        )

        chunks = text_splitter.split_text(doc.page_content)
        meta['id_source'] = _id
        for i, chunk in enumerate(chunks):
            doc = Document(
                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
            )
            doc.metadata["source"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"

            doc_chunks.append(doc.page_content)
            doc_id.append(_id + f"_{i}")
            _metadatas.append(meta)

    return doc_id, _metadatas, doc_chunks

def add_embedding(input_data):
    """Adds embedding to multiple documents from the database"""

    if (len(input_data['contents']) == len(input_data['ids'])) and (
            len(input_data['ids']) == len(input_data['metadatas'])):
        for i in range(len(input_data['contents'])):
            input_data['contents'][i] += f"\n||||\nMetadata Information: {str(input_data['metadatas'][i])}"

        input_data['ids'], input_data['metadatas'], input_data['contents'] = text_to_docs(
            input_data['ids'], input_data['metadatas'], input_data['contents']
        )
        input_data['embeddings'] = OpenAIEmbeddings().embed_documents(input_data['contents'])

        return input_data
    else:
        raise ValueError("Input mis-match!!!")

def add_data_to_vector_db(data_dict):
    """
    Receives data as input and adds that to the vector database

    Creates a vector database collection for each organization, and
    adds to existing organizations
    """

    try:
        data_dict = add_embedding(data_dict)

        owners = data_dict['org']
        collection = client.get_or_create_collection(name=owners)

        for i, (metadata, ids) in enumerate(zip(
                data_dict['metadatas'],
                data_dict['ids']
        )):
            metadata['source'] = ids

        collection.add(
            embeddings=data_dict['embeddings'],
            documents=data_dict['contents'],
            metadatas=data_dict['metadatas'],
            ids=[owners + i for i in map(str, data_dict['ids'])]
        )

        return {
            'status': 200,
            'log': "Successful!!!"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def get_vectorstore(org_name):
    embedding_function = OpenAIEmbeddings(
        model="text-embedding-ada-002",
        chunk_size=5000
    )

    try:
        client.get_collection(name=org_name)
    except:
        raise MissingOrganization

    return Chroma(client=client, collection_name=org_name, embedding_function=embedding_function)

def delete_organization(organization_name):
    """
    Deletion of organization from database
    """

    try:
        client.delete_collection(name=organization_name)

        return {
            'status': 200,
            'log': "Successful"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def modify_organization(prev_organization_name, new_organization_name):
    """
    Modification of organizations name
    """

    try:
        collection = client.get_collection(name=prev_organization_name)
        collection.modify(name=new_organization_name)
        client.delete_collection(name=prev_organization_name)

        return {
            'status': 200,
            'log': "Successful"
        }
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }

def get_emb_form_vector_db(owner, ids, return_only_emb=True):
    """Retrieve documents from vector database using their ids"""

    try:
        collection = client.get_or_create_collection(name=owner)

        if len(ids) > 1:
            result = collection.get(
                include=['embeddings', 'documents'],
                where={'$or': [{'id_source': _id} for _id in ids]}
            )
        else:
            result = collection.get(
                include=['embeddings', 'documents'],
                where={'id_source': ids[0]}
            )

        if return_only_emb:
            return result['embeddings']
        else:
            return result['documents'], result['embeddings']
    except Exception as e:
        return {
            'status': 403,
            'log': f"Failed with exception {e}"
        }


from comiee import overload
from public.config import data_path
from enum import IntEnum
import os
import logging

LOG_PATH = data_path('log')

class LogLevel(IntEnum):
    CRITICAL = logging.CRITICAL
    FATAL = logging.FATAL
    ERROR = logging.ERROR
    WARNING = logging.WARNING
    WARN = logging.WARN
    INFO = logging.INFO
    DEBUG = logging.DEBUG
    NOTSET = logging.NOTSET

@overload
def get_logger(name, level) -> logging.Logger:
    return get_logger(name, level, name + '.txt', level)

@overload
def get_logger(name, level, file_level) -> logging.Logger:
    return get_logger(name, level, name + '.txt', file_level)

@overload
def get_logger(name, level, file_name, file_level) -> logging.Logger:
    logger = logging.getLogger(name)
    logger.setLevel(LogLevel.DEBUG)
    formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s: %(message)s')
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    fh = logging.FileHandler(os.path.join(LOG_PATH, file_name), encoding='utf-8')
    fh.setLevel(file_level)
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    return logger

# 底层的client和server使用logger
client_logger: logging.Logger = get_logger('client', LogLevel.INFO, LogLevel.DEBUG)
server_logger: logging.Logger = get_logger('server', LogLevel.DEBUG)

# 服务器各组件使用的logger
master_server_logger: logging.Logger = get_logger('masterServer', LogLevel.INFO, LogLevel.DEBUG)
admin_logger: logging.Logger = get_logger('admin', LogLevel.DEBUG)

# bot客户端各组件使用的logger
bot_client_logger: logging.Logger = get_logger('botClient', LogLevel.DEBUG)
bot_logger: logging.Logger = get_logger('bot', LogLevel.DEBUG)

# 公共部分使用的logger
public_logger: logging.Logger = get_logger('public', LogLevel.DEBUG)


"""empty message

Revision ID: 3e7face35638
Revises: e8f7da151d15
Create Date: 2017-04-26 02:22:03.258321

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '3e7face35638'
down_revision = 'e8f7da151d15'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('auth_tokens', sa.Column('agent', sa.String(), nullable=True))
    op.add_column('auth_tokens', sa.Column('last_ip', sa.String(), nullable=True))
    op.add_column('auth_tokens', sa.Column('last_used', sa.DateTime(), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('auth_tokens', 'last_used')
    op.drop_column('auth_tokens', 'last_ip')
    op.drop_column('auth_tokens', 'agent')
    # ### end Alembic commands ###


import numpy as np
import cv2
import torch

from lav.utils.augmenter import augment
from lav.utils import filter_sem
from .basic_dataset import BasicDataset
from .lidar_dataset import transform_ego

class RGBDataset(BasicDataset):
    def __init__(self, config_path):
        super().__init__(config_path)

        self.augmenter = augment(0.5)

    def __getitem__(self, idx):

        lmdb_txn = self.txn_map[idx]
        index = self.idx_map[idx]

        rgb1 = self.__class__.load_img(lmdb_txn, 'rgb_2', index)
        rgb2 = self.__class__.load_img(lmdb_txn, 'rgb_3', index)
        sem1 = self.__class__.load_img(lmdb_txn, 'sem_2', index)
        sem2 = self.__class__.load_img(lmdb_txn, 'sem_3', index)

        # BEV images
        bev = self.__class__.load_bev(lmdb_txn, index, channels=[0,1,2,6])
        bev = (bev>0).astype(np.uint8).transpose(2,0,1)

        rgb = np.concatenate([rgb1, rgb2], axis=1)
        sem = np.concatenate([sem1, sem2], axis=1)

        rgb = self.augmenter(images=rgb[...,::-1][None])[0]
        sem = filter_sem(sem, self.seg_channels)

        # Vehicle locations/orientations
        ego_id, ego_locs, ego_oris, ego_bbox, msks, locs, oris, bbox, typs = self.__class__.filter(
            lmdb_txn, index,
            max_pedestrian_radius=self.max_pedestrian_radius,
            max_vehicle_radius=self.max_vehicle_radius,
            T=self.num_plan)

        # Normalize coordinates to ego frame
        ego_locs, locs, oris, bbox, typs = transform_ego(ego_locs, locs, oris, bbox, typs, ego_oris[0], self.num_plan+1)

        cmd = int(self.__class__.access('cmd', lmdb_txn, index, 1, dtype=np.uint8))
        nxp = self.__class__.access('nxp', lmdb_txn, index, 1).reshape(2)

        return rgb, sem, bev, -(ego_locs[1:]-ego_locs[:1]), cmd, nxp


import logging
import shutil
import os

from flamingo.core.data_model import ContentSet, AND, NOT, OR, Q, F
from flamingo.core.plugins.plugin_manager import PluginManager
from flamingo.core.parser import FileParser, ParsingError
from flamingo.core.plugins.media import add_media
from flamingo.core.utils.imports import acquire
from flamingo.core.types import OverlayObject

class Context(OverlayObject):
    __overlay_ignore_attributes = [
        'content',
        'contents',
    ]

    def __init__(self, settings, contents=None, setup=True):
        super().__init__()

        self.settings = settings
        self.contents = contents
        self.content = None
        self.plugins = None

        if setup:
            self.setup()

    def setup(self):
        self.errors = []

        # setup logging
        self.logger = logging.getLogger('flamingo')
        self.logger.debug('setting up context')

        # setup plugins
        self.plugins = PluginManager(self)
        self.plugins.run_plugin_hook('setup')
        self.plugins.run_plugin_hook('settings_setup')

        # setup parser
        self.parser = FileParser(context=self)
        self.plugins.run_plugin_hook('parser_setup')

        # setup templating engine
        templating_engine_class, path = acquire(
            self.settings.TEMPLATING_ENGINE)

        self.templating_engine = templating_engine_class(self)

        self.plugins.run_plugin_hook('templating_engine_setup',
                                     self.templating_engine)

        # parse contents
        self.contents = self.contents or ContentSet()
        self.parse_all()

        # context ready
        self.plugins.run_plugin_hook('context_setup')

    def resolve_content_path(self, path, content=None):
        """
        supported formats:

            index.rst           relative path (only available if self.content
                                               is set or content is given)

            /index.rst          absolute path
            content/index.rst   absolute path with CONTENT_ROOT given without /
            /content/index.rst  absolute path with CONTENT_ROOT given with /
        """

        current_content = content or self.content

        def _try_relative_path(path):
            path = os.path.join(os.path.dirname(current_content['path']), path)
            content_set = self.contents.filter(path=path)

            if content_set.exists():
                return content_set.get()

        def _try_absolute_path(path):
            if path.startswith('/'):
                path = path[1:]

            if path.startswith(self.settings.CONTENT_ROOT):
                path = os.path.relpath(path, self.settings.CONTENT_ROOT)

            content_set = self.contents.filter(path=path)

            if content_set.exists():
                return content_set.get()

        # relative paths
        if current_content and not path.startswith('/'):
            content = _try_relative_path(path)

            if content:
                return content

        # try absolute
        content = _try_absolute_path(path)

        if content:
            return content

    @property
    def media_contents(self):
        _media_contents = ContentSet()

        if self.contents:
            for content in self.contents:
                if not content['media']:
                    continue

                _media_contents.add(*content['media'])

        return _media_contents

    def parse(self, content):
        previous_content = self.content
        self.content = content
        path = os.path.join(self.settings.CONTENT_ROOT, content['path'])

        self.logger.debug("reading %s ", path)

        try:
            self.parser.parse(path, content)

            self.plugins.run_plugin_hook('content_parsed', content)

        except ParsingError as e:
            content['_parsing_error'] = e
            self.errors.append(e)

            if hasattr(e, 'line'):
                line = e.line

                if content['content_offset']:
                    line += content['content_offset']

                self.logger.error('%s:%s: %s', path, line, e)

            else:
                self.logger.error('%s: %s', path, e)

        except Exception as e:
            content['_parsing_error'] = e
            self.errors.append(e)

            self.logger.error('exception occoured while reading %s',
                              content['path'], exc_info=True)

        finally:
            self.content = previous_content

    def parse_all(self):
        self.content = None

        for path in self.get_source_paths():
            self.contents.add(
                path=os.path.relpath(path, self.settings.CONTENT_ROOT))

        for content in self.contents:
            if content['content_body']:
                continue

            self.parse(content)

        self.content = None

        self.plugins.run_plugin_hook('contents_parsed')

    def get_source_paths(self):
        self.logger.debug('searching for content')

        supported_extensions = self.parser.get_extensions()

        if self.settings.CONTENT_PATHS:
            self.logger.debug('using user defined content paths')

            for path in self.settings.CONTENT_PATHS:
                path = os.path.join(self.settings.CONTENT_ROOT, path)

                if not os.path.exists(path):
                    continue

                extension = os.path.splitext(path)[1][1:]

                if extension not in supported_extensions:
                    self.logger.debug(
                        "skipping '%s'. extension '%s' is not supported",
                        path, extension)

                    continue

                yield path

        else:
            self.logger.debug(
                "searching content with extension %s recursive in %s",
                repr(supported_extensions), self.settings.CONTENT_ROOT,
            )

            for root, dirs, files in os.walk(
                    self.settings.CONTENT_ROOT,
                    followlinks=self.settings.FOLLOW_LINKS):

                for name in files:
                    extension = os.path.splitext(name)[1][1:]

                    if extension not in supported_extensions:
                        continue

                    yield os.path.join(root, name)

    def gen_template_context(self, **extra_context):
        return {
            'context': self,
            'AND': AND,
            'NOT': NOT,
            'OR': OR,
            'Q': Q,
            'F': F,
            **self.settings.EXTRA_CONTEXT,
            **extra_context,
        }

    def pre_render(self, content, template_context=None):
        if (not self.settings.PRE_RENDER_CONTENT or
            not content.get('is_template', True) or
           content['_content_body_rendered']):

            return True, content['content_body']

        self.logger.debug('pre rendering %s', content['path'] or content)

        if template_context is None:
            template_context = self.gen_template_context(content=content)

        try:
            return self.templating_engine.pre_render_content(
                content=content,
                template_context=template_context,
            )

        except Exception as e:
            self.logger.error(
                '%s: exception raised while pre rendering content',
                content['path'],
                exc_info=True,
            )

            self.errors.append(e)

        return False, ''

    def render(self, content, template_name=''):
        template_name = template_name or content['template']

        self.logger.debug('rendering %s using %s', content['path'] or content,
                          template_name)

        if not template_name:
            content['template_context'] = {}

            return content['content_body'] or ''

        template_context = self.gen_template_context(content=content)

        exitcode, output = self.pre_render(
            content=content,
            template_context=template_context,
        )

        if exitcode:
            content['content_body'] = output
            content['_content_body_rendered'] = True

        else:
            return output

        self.plugins.run_hook('template_context_setup',
                              content, template_name, template_context)

        output = self.templating_engine.render(template_name, template_context)
        content['template_context'] = template_context

        return output

    def rm_rf(self, path, force=False):
        if self.settings.SKIP_FILE_OPERATIONS and not force:
            return

        self.logger.debug('rm -rf %s', path)

        if os.path.isdir(path):
            shutil.rmtree(path)

        else:
            os.unlink(path)

    def mkdir_p(self, path, force=False):
        if self.settings.SKIP_FILE_OPERATIONS and not force:
            return

        dirname = os.path.dirname(path)

        if not os.path.exists(dirname):
            self.logger.debug('mkdir -p %s', dirname)
            os.makedirs(dirname)

    def cp(self, source, destination, force=False):
        if self.settings.SKIP_FILE_OPERATIONS and not force:
            return

        self.mkdir_p(destination)
        self.logger.debug('cp %s %s', source, destination)
        shutil.copy(source, destination)

    def write(self, path, text, mode='w+', force=False):
        if self.settings.SKIP_FILE_OPERATIONS and not force:
            return

        self.logger.debug("writing '%s", path)

        with open(path, mode) as f:
            f.write(text)

    def add_media(self, name, content=None, **extra_meta_data):
        content = content or self.content

        return add_media(name=name, context=self, content=content,
                         **extra_meta_data)

    def build(self, clean=True):
        # remove previous artifacts
        if clean and os.path.exists(self.settings.OUTPUT_ROOT):
            self.rm_rf(self.settings.OUTPUT_ROOT)

        # run pre build hooks
        self.plugins.run_plugin_hook('pre_build')

        # render contents
        if self.settings.CONTENT_PATHS:
            contents = self.contents.filter(
                Q(path__in=self.settings.CONTENT_PATHS) |
                Q(i18n_path__in=self.settings.CONTENT_PATHS),
            )

        else:
            contents = self.contents

        for content in contents:
            output_path = os.path.join(self.settings.OUTPUT_ROOT,
                                       content['output'])

            self.mkdir_p(output_path)
            self.write(output_path, self.render(content))

        self.plugins.run_plugin_hook('post_build')


### Importing libraries
import pandas as pd
import numpy as np
from datetime import datetime
from lightgbm import LGBMRegressor
# import gresearch_crypto
import traceback
import time
from datetime import datetime
import matplotlib.pyplot as plt
import plotly.graph_objects as go
from sklearn.model_selection import GridSearchCV
import seaborn as sns
from sklearn.model_selection import train_test_split

def read_g_research(path):
    df_train = pd.read_csv(path + "train.csv")
    df_test = pd.read_csv(path + "example_test.csv")
    df_asset_details = pd.read_csv(path + "asset_details.csv")
    df_supp_train = pd.read_csv(path + "supplemental_train.csv")
    return df_train,df_test,df_asset_details,df_supp_train

# path="F:/g-research-crypto-forecasting/"
# df_train,df_test,df_asset_details,df_supp_train=read_g_research(path)

# print(df_train.describe())
# print(df_train.head())

# 时间戳
# 定义了一个助手函数，该函数将日期格式转换为时间戳，以用于索引。
# auxiliary function, from datetime to timestamp
totimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, "%d/%m/%Y").timetuple()))

## Checking Time Range
def check_time_range(df,asset_id=1):
    data_index=df[df_train["Asset_ID"]==asset_id].set_index("timestamp")
    beg_time = datetime.fromtimestamp(data_index.index[0]).strftime("%A, %B %d, %Y %I:%M:%S")
    end_time = datetime.fromtimestamp(data_index.index[-1]).strftime("%A, %B %d, %Y %I:%M:%S")

    # btc = df_train[df_train["Asset_ID"]==1].set_index("timestamp") # Asset_ID = 1 for Bitcoin
    # eth = df_train[df_train["Asset_ID"]==6].set_index("timestamp") # Asset_ID = 6 for Ethereum
    # bnb = df_train[df_train["Asset_ID"]==0].set_index("timestamp") # Asset_ID = 0 for Binance Coin
    # ada = df_train[df_train["Asset_ID"]==3].set_index("timestamp") # Asset_ID = 3 for Cardano

    # beg_btc = datetime.fromtimestamp(btc.index[0]).strftime("%A, %B %d, %Y %I:%M:%S")
    # end_btc = datetime.fromtimestamp(btc.index[-1]).strftime("%A, %B %d, %Y %I:%M:%S")
    # beg_eth = datetime.fromtimestamp(eth.index[0]).strftime("%A, %B %d, %Y %I:%M:%S")
    # end_eth = datetime.fromtimestamp(eth.index[-1]).strftime("%A, %B %d, %Y %I:%M:%S")
    # beg_bnb = datetime.fromtimestamp(eth.index[0]).strftime("%A, %B %d, %Y %I:%M:%S")
    # end_bnb = datetime.fromtimestamp(eth.index[-1]).strftime("%A, %B %d, %Y %I:%M:%S")
    # beg_ada = datetime.fromtimestamp(eth.index[0]).strftime("%A, %B %d, %Y %I:%M:%S")
    # end_ada = datetime.fromtimestamp(eth.index[-1]).strftime("%A, %B %d, %Y %I:%M:%S")
    # print('Bitcoin data goes from ', beg_btc, ' to ', end_btc)
    # print('Ethereum data goes from ', beg_eth, ' to ', end_eth)
    # print('Binance coin data goes from ', beg_bnb, ' to ', end_bnb)
    # print('Cardano data goes from ', beg_ada, ' to ', end_ada)
    return beg_time,end_time

def show_heatmap(df):
    plt.figure(figsize=(8,6))
    sns.heatmap(df[['Count','Open','High','Low','Close','Volume','VWAP','Target']].corr(),
                vmin=-1.0, vmax=1.0, annot=True, cmap='coolwarm', linewidths=0.1)
    plt.show()

def show_heatmap_(df):
    # Heatmap: Coin Correlation (Last 10000 Minutes
    data =df[-10000:]
    check = pd.DataFrame()
    for i in data.Asset_ID.unique():
        check[i] = data[data.Asset_ID==i]['Target'].reset_index(drop=True)

    plt.figure(figsize=(10,8))
    sns.heatmap(check.dropna().corr(), vmin=-1.0, vmax=1.0, annot=True, cmap='coolwarm', linewidths=0.1)
    plt.show()

def candlesticks_charts(df):

    # Candlesticks Charts for BTC & ETH, Last 200 Minutes
    btc_mini = df.iloc[-200:] # Select recent data rows

    fig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['Close'])])
    fig.update_xaxes(title_text="$")
    fig.update_yaxes(title_text="Index")
    fig.update_layout(title="Bitcoin Price, 200 Last Minutes")
    fig.show()

def show_close_prices(df):

    # Plotting BTC and ETH closing prices
    f = plt.figure(figsize=(15,4))

    # fill NAs for BTC and ETH
    btc = btc.reindex(range(btc.index[0],btc.index[-1]+60,60),method='pad')
    eth = eth.reindex(range(eth.index[0],eth.index[-1]+60,60),method='pad')

    ax = f.add_subplot(121)
    plt.plot(btc['Close'], color='yellow', label='BTC')
    plt.legend()
    plt.xlabel('Time (timestamp)')
    plt.ylabel('Bitcoin')

    ax2 = f.add_subplot(122)
    ax2.plot(eth['Close'], color='purple', label='ETH')
    plt.legend()
    plt.xlabel('Time (timestamp)')
    plt.ylabel('Ethereum')

    plt.tight_layout()
    plt.show()

# Feature Extraction:定义一些函数来添加到用于预测的特性列表中。

def hlco_ratio(df):
    return (df['High'] - df['Low'])/(df['Close']-df['Open'])
def upper_shadow(df):
    return df['High'] - np.maximum(df['Close'], df['Open'])
def lower_shadow(df):
    return np.minimum(df['Close'], df['Open']) - df['Low']

def get_features(df):
    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']]
    df_feat['Upper_Shadow'] = upper_shadow(df_feat)
    df_feat['hlco_ratio'] = hlco_ratio(df_feat)
    df_feat['Lower_Shadow'] = lower_shadow(df_feat)
    return df_feat

def get_Xy_and_model_for_asset(df_train, asset_id):
    df = df_train[df_train["Asset_ID"] == asset_id]

    df = df.sample(frac=0.2)
    df_proc = get_features(df)
    df_proc['y'] = df['Target']
    df_proc.replace([np.inf, -np.inf], np.nan, inplace=True)
    df_proc = df_proc.dropna(how="any")

    X = df_proc.drop("y", axis=1)
    print(X)
    y = df_proc["y"]
    print("===:",y)
    model = LGBMRegressor()
    model.fit(X, y)
    return X, y, model

if __name__=="__main__":

    path="F:/g-research-crypto-forecasting/"
    df_train,df_test,df_asset_details,df_supp_train=read_g_research(path)
    df_feat=get_features(df_train)
    print("=========:",df_feat)

    # Xs = {}
    # ys = {}
    # models = {}
    # # Prediction
    # # train test split df_train into 80% train rows and 20% valid rows
    # train_data = df_train
    # # train_data = df_train.sample(frac = 0.8)
    # # valid_data = df_train.drop(train_data.index)
    # for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):
    #     print(f"Training model for {asset_name:<16} (ID={asset_id:<2})")
    #     X, y, model = get_Xy_and_model_for_asset(train_data, asset_id)
    #     try:
    #         Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model
    #     except:
    #         Xs[asset_id], ys[asset_id], models[asset_id] = None, None, None

    # # Evaluation, Hyperparam Tuning：对14个硬币的LGBM模型执行GridSearch。
    # parameters = {
    #     # 'max_depth': range (2, 10, 1),
    #     'num_leaves': range(21, 161, 10),
    #     'learning_rate': [0.1, 0.01, 0.05]
    # }

    # new_models = {}
    # for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):
    #     print("GridSearchCV for: " + asset_name)
    #     grid_search = GridSearchCV(
    #         estimator=get_Xy_and_model_for_asset(df_train, asset_id)[2], # bitcoin
    #         param_grid=parameters,
    #         n_jobs = -1,
    #         cv = 5,
    #         verbose=True
    #     )
    #     grid_search.fit(Xs[asset_id], ys[asset_id])
    #     new_models[asset_id] = grid_search.best_estimator_
    #     grid_search.best_estimator_

    # for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):
    #     print(f"Tuned model for {asset_name:<1} (ID={asset_id:})")
    #     print(new_models[asset_id])


# https://www.slideshare.net/hcpc_hokudai/advanced-dp-2016 動的計画法の問題の解説がされている 神

# これが比較的わかりやすいかも https://algo-logic.info/bit-dp/

'''
定式化(本は"集める"DPで定義してるが、わかりやすさのため"配る"DPで定式化)

ノーテーション
S ... 頂点集合
| ... 和集合演算子
dp[S][v] ... 重みの総和の最小。頂点0から頂点集合Sを経由してvに到達する。

更新則
dp[S|{v}] = min{dp[S][u]+d(u,v)} ただしv∉S

初期条件
dp[∅][0] = 0 #Vはあらゆる集合
dp[V][u] = INF #ほかはINFで初期化しておく

答え
dp[すべての要素][0] ... 0からスタートしてすべての要素を使って最後に0に戻るための最小コスト
'''

# verify https://onlinejudge.u-aizu.ac.jp/courses/library/7/DPL/all/DPL_2_A
INF = 2 ** 31
from itertools import product

def solve(n, graph):
    '''nは頂点数、graphは隣接行列形式'''
    max_S = 1 << n  # n個のbitを用意するため
    dp = [[INF] * n for _ in range(max_S)]
    dp[0][0] = 0
    for S in range(max_S):
        for u, v in product(range(n), repeat=2):
            if (S >> v) & 1 or u == v:  # vが訪問済みの場合は飛ばす
                continue
            dp[S | (1 << v)][v] = min(dp[S | (1 << v)][v],
                                      dp[S][u] + graph[u][v])

            # # 別解 #集めるDPの発想
            # if u == v or (S >> v) & 1 == 0:  # Sにはvが含まれている必要がある
            #     continue
            # dp[S][v] = min(dp[S][v],
            #                dp[S - (1 << v)][u] + graph[u][v])
    print(dp[-1][0] if dp[-1][0] != INF else -1)

# # 入力例
# n = 5
# graph = [[INF, 3, INF, 4, INF],
#          [INF, INF, 5, INF, INF],
#          [4, INF, INF, 5, INF],
#          [INF, INF, INF, 0, 3],
#          [7, 6, INF, INF, INF]]
# solve(n, graph)

# verify用
n, e = map(int, input().split())
graph = [[INF] * n for _ in range(n)]
for _ in range(e):
    s, t, d = map(int, input().split())
    graph[s][t] = d
solve(n, graph)


import copy
import functools
import os
import time
import glob
from concurrent.futures import ThreadPoolExecutor

import numpy as np

from PySide6.QtCore import QObject, QThreadPool, Signal
from PySide6.QtWidgets import QMessageBox

from hexrd import imageseries
from hexrd.imageseries.omega import OmegaImageSeries

from hexrdgui.async_worker import AsyncWorker
from hexrdgui.hexrd_config import HexrdConfig
from hexrdgui.image_file_manager import ImageFileManager
from hexrdgui.progress_dialog import ProgressDialog
from hexrdgui.constants import *
from hexrdgui.singletons import QSingleton

class NoEmptyFramesException(Exception):
    pass

class ImageLoadManager(QObject, metaclass=QSingleton):

    # Emitted when new images are loaded
    progress_text = Signal(str)
    update_needed = Signal()
    new_images_loaded = Signal()
    images_transformed = Signal()
    live_update_status = Signal(bool)
    state_updated = Signal()
    enable_transforms = Signal()

    def __init__(self):
        super().__init__(None)
        self.transformed_images = False

    @property
    def thread_pool(self):
        return QThreadPool.globalInstance()

    def load_images(self, fnames):
        files = self.explict_selection(fnames)
        manual = False
        if not files:
            files = self.match_files(fnames)
            matched = self.check_success(files)
            if not matched:
                manual = True
                files = [[fname] for fname in fnames]
        return files, manual

    def check_success(self, files):
        detectors = HexrdConfig().detector_names
        # Make sure there are the same number of files for each detector
        # and at least one file per detector
        if (not files[0]
                or len(files) != len(detectors)
                or any(len(files[0]) != len(elem) for elem in files)):
            return False
        # If the files do not contain detector names they will need to
        # be manually matched
        return all(any(d in f for d in detectors) for f in files[0])

    def explict_selection(self, fnames):
        # Assume the user has selected all of the files they would like to load
        dets = HexrdConfig().detector_names
        files = [[] for i in range(len(dets))]
        for fname in fnames:
            path, f = os.path.split(fname)
            matches = [i for i, det in enumerate(dets) if det in f]
            if matches:
                idx = matches[0]
                files[idx].append(fname)
        if self.check_success(files):
            return files
        else:
            return []

    def match_files(self, fnames):
        dets = HexrdConfig().detector_names
        # Look for files that match everything except detector name
        # ex: /home/user/images/Ruby_line_ff_000017_ge1.npz becomes
        # /home/user/images/Ruby_line_ff_000017_*.npz
        search = []
        for f in fnames:
            path, fname = os.path.split(f)
            files = [det for det in dets if det in fname]
            if not files:
                search.append('/'.join([path, fname]))
            else:
                for d in dets:
                    next_file = d.join(fname.rsplit(files[0]))
                    search.append('/'.join([path, next_file]))
        files = self.match_selected_files(fnames, search)
        if not self.check_success(files):
            # Look in sibling directories if the matching files were not
            # found in the current directory.
            revised_search = []
            for f in fnames:
                directory = os.path.basename(os.path.dirname(f))
                for d in dets:
                    revised_search.append(d.join(f.split(directory)))
            files = self.match_selected_files(fnames, revised_search)
        return files

    def match_selected_files(self, fnames, search):
        dets = HexrdConfig().detector_names
        files = [[] for i in range(len(dets))]
        results = [f for f in search if glob.glob(f, recursive=True)]
        results = [glob.glob(f, recursive=True) for f in results]
        if results:
            for fname in [fname for f in results for fname in f]:
                path, f = os.path.split(fname)
                matches = [i for i, det in enumerate(dets) if det in f]
                if not matches:
                    root, dirs = os.path.split(path)
                    matches = [i for i, det in enumerate(dets) if det in dirs]
                if len(matches):
                    idx = matches[0]
                    files[idx].append('/'.join([path, f]))
        return files

    def read_data(self, files=None, data=None, ui_parent=None, **kwargs):
        # Make sure this is reset to zero when data is being read
        HexrdConfig().current_imageseries_idx = 0

        # When this is pressed read in a complete set of data for all detectors.
        # Run the imageseries processing in a background thread and display a
        # loading dialog
        if files:
            self.parent_dir = HexrdConfig().images_dir
            self.files = files
            self.empty_frames = data['empty_frames'] if data else 0
        self.data = {} if data is None else data
        self.ui_parent = ui_parent
        self.set_state(kwargs.get('state', None))
        self.begin_processing(kwargs.get('postprocess', False))

    def begin_processing(self, postprocess=False):
        self.update_status = HexrdConfig().live_update
        self.live_update_status.emit(False)

        # Create threads and loading dialog
        progress_dialog = ProgressDialog(self.ui_parent)
        progress_dialog.setWindowTitle('Loading Processed Imageseries')
        self.progress_text.connect(progress_dialog.setLabelText)
        self.progress_dialog = progress_dialog

        # Start processing in background
        worker = AsyncWorker(self.process_ims, postprocess)
        self.thread_pool.start(worker)

        worker.signals.progress.connect(progress_dialog.setValue)
        # On completion load imageseries nd close loading dialog
        worker.signals.result.connect(self.finish_processing_ims)
        worker.signals.finished.connect(progress_dialog.accept)
        progress_dialog.exec()

    def set_state(self, state=None):
        if state is None:
            self.state = HexrdConfig().load_panel_state
        else:
            self.state = state
        self.state_updated.emit()

    def process_ims(self, postprocess, update_progress):
        self.update_progress = update_progress
        self.update_progress(0)

        if not postprocess:
            # Open selected images as imageseries
            self.parent_dir = HexrdConfig().images_dir
            det_names = HexrdConfig().detector_names

            options = {
                'empty-frames': self.data.get('empty_frames', 0),
                'max-file-frames': self.data.get('max_file_frames', 0),
                'max-total-frames': self.data.get(
                    'max_total_frames', 0),
            }

            if len(self.files[0]) > 1:
                for i, det in enumerate(det_names):
                    dirs = os.path.dirname(self.files[i][0])
                    ims = ImageFileManager().open_directory(dirs, self.files[i], options)
                    HexrdConfig().imageseries_dict[det] = ims
            else:
                ImageFileManager().load_images(det_names, self.files, options)
        HexrdConfig().reset_unagg_imgs(new_imgs=True)

        # Now that self.state is set, setup the progress variables
        self.setup_progress_variables()

        # Process the imageseries
        self.apply_operations(HexrdConfig().imageseries_dict)
        if self.data:
            self.add_omega_metadata(HexrdConfig().imageseries_dict)
            if 'agg' in self.state and self.state['agg']:
                self.display_aggregation(HexrdConfig().imageseries_dict)
            else:
                HexrdConfig().reset_unagg_imgs()

        self.update_progress(100)

    def finish_processing_ims(self):
        # Display processed images on completion
        self.new_images_loaded.emit()
        self.enable_transforms.emit()
        self.live_update_status.emit(self.update_status)
        if not self.update_status:
            self.update_needed.emit()
        if self.transformed_images:
            HexrdConfig().deep_rerender_needed.emit()

    def get_dark_aggr_op(self, ims, idx):
        """
        Returns a tuple of the form (function, frames), where func is the
        function to be applied and frames is the number of frames to aggregate.
        """

        i = self.data['idx'] if 'idx' in self.data else idx
        dark_idx = self.state['dark'][i]
        if dark_idx == UI_DARK_INDEX_FILE:
            ims = ImageFileManager().open_file(self.state['dark_files'][idx])

        # Create or load the dark image if selected
        frames = len(ims)
        if dark_idx != UI_DARK_INDEX_FILE and frames > 120:
            frames = 120

        if dark_idx == UI_DARK_INDEX_MEDIAN:
            f = imageseries.stats.median_iter
        elif dark_idx == UI_DARK_INDEX_EMPTY_FRAMES:
            f = imageseries.stats.average_iter
            frames = self.empty_frames
        elif dark_idx == UI_DARK_INDEX_AVERAGE:
            f = imageseries.stats.average_iter
        elif dark_idx == UI_DARK_INDEX_MAXIMUM:
            f = imageseries.stats.max_iter
        else:
            f = imageseries.stats.median_iter

        return (f, frames, ims)

    def get_dark_aggr_ops(self, ims_dict):
        """
        Returns a dict of tuples of the form (function, frames), where func is the
        function to be applied and frames is the number of frames to aggregate.
        The key is the detector name.
        """
        ops = {}
        for i, key in enumerate(ims_dict.keys()):
            if self.data:
                idx = self.data['idx'] if 'idx' in self.data else i
                if self.state['dark'][idx] != UI_DARK_INDEX_NONE:
                    if (self.state['dark'][idx] == UI_DARK_INDEX_EMPTY_FRAMES
                            and self.empty_frames == 0):
                        msg = ('ERROR: \n No empty frames set. '
                               + 'No dark subtracion will be performed.')
                        QMessageBox.warning(None, 'HEXRD', msg)
                    else:
                        op = self.get_dark_aggr_op(ims_dict[key], i)
                        ops[key] = op

        return ops

    def apply_operations(self, ims_dict):
        # First perform dark aggregation if we need to
        dark_aggr_ops = {}
        if 'dark' in self.state:
            dark_aggr_ops = self.get_dark_aggr_ops(ims_dict)

        # Now run the dark aggregation
        dark_images = {}
        if dark_aggr_ops:
            self.update_progress_text('Aggregating dark images...')
            dark_images = self.aggregate_dark_multithread(dark_aggr_ops)

        # Apply the operations to the imageseries
        for idx, key in enumerate(ims_dict.keys()):
            ops = []
            # Apply dark subtraction
            if key in dark_images:
                self.get_dark_op(ops, dark_images[key])
            if 'trans' in self.state:
                self.get_flip_op(ops, idx)
            if 'rect' in self.state:
                ops.append(('rectangle', self.state['rect'][idx]))

            frames = self.get_range(ims_dict[key])
            if self.state.get('frames_reversed', False):
                frames = frames[::-1]
            ims_dict[key] = imageseries.process.ProcessedImageSeries(
                ims_dict[key], ops, frame_list=frames)

            # Set these directly so no signals get emitted
            det_conf = HexrdConfig().config['instrument']['detectors'][key]
            det_conf['pixels']['columns']['value'] = ims_dict[key].shape[1]
            det_conf['pixels']['rows']['value'] = ims_dict[key].shape[0]

        self.images_transformed.emit()

    def display_aggregation(self, ims_dict):
        self.update_progress_text('Aggregating images...')
        # Remember unaggregated images
        HexrdConfig().set_unagg_images()

        # Make sure this is reset to 0
        HexrdConfig().current_imageseries_idx = 0

        if self.state['agg'] == UI_AGG_INDEX_MAXIMUM:
            agg_func = imageseries.stats.max_iter
        elif self.state['agg'] == UI_AGG_INDEX_MEDIAN:
            agg_func = imageseries.stats.median_iter
        else:
            agg_func = imageseries.stats.average_iter

        f = functools.partial(self.aggregate_images, agg_func=agg_func)

        for (key, aggr_img) in zip(ims_dict.keys(), self.aggregate_images_multithread(f, ims_dict)):
            ims_dict[key] = ImageFileManager().open_file(aggr_img)

    def add_omega_metadata(self, ims_dict, data=None):
        # Add on the omega metadata if there is any
        data = self.data if data is None else data
        files = self.data['yml_files'] if 'yml_files' in self.data else self.files
        for key, ims in ims_dict.items():
            nframes = data.get('nframes', 0)
            # If number of frames is 0 we assume that no value was provided
            # and we should infer that we are using all frames
            nframes = nframes if nframes > 0 else len(ims)
            omw = imageseries.omega.OmegaWedges(nframes)
            if 'wedges' in data:
                for wedge in data['wedges']:
                    start, stop, nsteps = wedge
                    omw.addwedge(start, stop, nsteps)
            else:
                for i in range(len(files[0])):
                    nsteps = data['nsteps'][i]
                    start = data['omega_min'][i]
                    stop = data['omega_max'][i]
                    omw.addwedge(start, stop, nsteps)
            ims_dict[key].metadata['omega'] = omw.omegas
            ims_dict[key] = OmegaImageSeries(ims_dict[key])

    def get_range(self, ims):
        start = 0
        nsteps = sum(self.data.get('nsteps', [len(ims)]))
        if len(ims) != nsteps:
            # In the case of hdf5 and npz files we need to handle empty
            # frames via the ProcessedImageSeries 'frame_list' arg to slice
            # the frame list
            start = self.data.get('empty_frames', 0)
        return np.arange(start, len(ims))

    def get_flip_op(self, oplist, idx):
        if self.data:
            idx = self.data.get('idx', idx)
        # Change the image orientation
        if self.state['trans'][idx] == UI_TRANS_INDEX_NONE:
            return

        self.transformed_images = True
        if self.state['trans'][idx] == UI_TRANS_INDEX_FLIP_VERTICALLY:
            key = 'v'
        elif self.state['trans'][idx] == UI_TRANS_INDEX_FLIP_HORIZONTALLY:
            key = 'h'
        elif self.state['trans'][idx] == UI_TRANS_INDEX_TRANSPOSE:
            key = 't'
        elif self.state['trans'][idx] == UI_TRANS_INDEX_ROTATE_90:
            key = 'r90'
        elif self.state['trans'][idx] == UI_TRANS_INDEX_ROTATE_180:
            key = 'r180'
        else:
            key = 'r270'

        oplist.append(('flip', key))

    def get_dark_op(self, oplist, dark):
        oplist.append(('dark', dark))

    def setup_progress_variables(self):
        self.current_progress_step = 0

        ims_dict = HexrdConfig().imageseries_dict
        num_ims = len(ims_dict)

        progress_macro_steps = 0
        for idx in range(num_ims):
            if self.data and 'idx' in self.data:
                idx = self.data['idx']

            if ('dark' in self.state and
                    self.state['dark'][idx] != UI_DARK_INDEX_NONE):
                progress_macro_steps += 1

        if 'agg' in self.state and self.state['agg']:
            progress_macro_steps += num_ims

        self.progress_macro_steps = progress_macro_steps

    def increment_progress_step(self):
        self.current_progress_step += 1

    def update_progress_text(self, text):
        self.progress_text.emit(text)

    def calculate_nchunk(self, num_ims, frames):
        """
        Calculate the number of chunks

         :param num_ims: The number of image series.
         :param frames: The number of frames being processed.
        """
        step = int(frames * num_ims / 100)
        step = step if step > 2 else 2
        nchunk = int(frames / step)
        if nchunk > frames or nchunk < 1:
            # One last sanity check
            nchunk = frames

        return nchunk

    def aggregate_images(self, key, ims, agg_func, progress_dict):
        frames = len(ims)
        num_ims = len(progress_dict)
        nchunk = self.calculate_nchunk(num_ims, frames)

        for i, img in enumerate(agg_func(ims, nchunk)):
            progress_dict[key] = (i + 1) / nchunk

        return [img]

    def wait_with_progress(self, futures, progress_dict):
        """
        Wait for futures to be resolved and update progress using the progress
        dict.
        """
        n_macro_steps = self.progress_macro_steps
        orig_progress = self.progress_dialog.value()
        while not all([f.done() for f in futures]):
            total = sum([v for v in progress_dict.values()])
            progress = total * 100 / n_macro_steps
            self.update_progress(orig_progress + progress)
            time.sleep(0.1)

    def aggregate_images_multithread(self, f, ims_dict):
        """
        Use ThreadPoolExecutor to aggregate images

        :param f: The aggregation function
        :param ims_dict: The imageseries to aggregate
        """
        max_workers = HexrdConfig().max_cpus

        futures = []
        progress_dict = {key: 0.0 for key in ims_dict.keys()}
        with ThreadPoolExecutor(max_workers=max_workers) as tp:
            for (key, ims) in ims_dict.items():
                futures.append(tp.submit(f, key, ims, progress_dict=progress_dict))

            self.wait_with_progress(futures, progress_dict)

        return [f.result() for f in futures]

    def aggregate_dark(self, key, func, ims, frames, progress_dict):
        """
        Generate aggregated dark image.

        :param key: The detector
        :param func: The aggregation function
        :param ims: The imageseries
        :param frames: The number of frames to use
        :param progress_dict: Dict for progress reporting
        """
        nchunk = self.calculate_nchunk(len(HexrdConfig().imageseries_dict), frames)

        for i, darkimg in enumerate(func(ims, nchunk, nframes=frames)):
            progress_dict[key] = (i + 1) / nchunk

        return (key, darkimg)

    def aggregate_dark_multithread(self, aggr_op_dict):
        """
        Use ThreadPoolExecutor to dark aggregation. Returns a dict mapping the
        detector name to dark image.

        :param aggr_op_dict: A dict mapping the detector name to a tuple of the form
                            (func, frames, ims), where func is the function to use to do
                            the aggregation, frames is number of images to aggregate, and
                            ims is the image series to perform the aggregation on.
        """
        max_workers = HexrdConfig().max_cpus

        futures = []
        progress_dict = {key: 0.0 for key in aggr_op_dict.keys()}
        with ThreadPoolExecutor(max_workers=max_workers) as tp:
            for (key, (op, frames, ims)) in aggr_op_dict.items():
                futures.append(tp.submit(
                    self.aggregate_dark, key, op, ims, frames, progress_dict))

            self.wait_with_progress(futures, progress_dict)

        return {f.result()[0]: f.result()[1] for f in futures}


#核心思想：通过间隙进行分组，然后组内插入,执行多次

import math

def sort(l):
    gap = 1
    while gap < len(l):
        gap = gap*3+1

    while gap > 0:
        for i in range(gap,len(l)):
            temp = l[i]
            j = i - gap
            while l[j] > temp:
               l[j+gap] = l[j]
               j -= gap
            l[j+gap] = temp
        gap = math.floor(gap/3)

    return l

l = [1,3,6,2,4,7,8,3]
print(sort(l))