# Makes a basic python plot of time series position
# Takes in a namedTuple with the data.
# Makes a basic plot.

import matplotlib.pyplot as plt
import collections
import gps_ts_functions
import gps_seasonal_removals
import offsets
import gps_io_functions
import gps_input_pipeline

# Parameters for controlling plotting
Parameters = collections.namedtuple("Parameters", ['station', 'outliers_remove', 'outliers_def',
                                                   'earthquakes_remove', 'offsets_remove', 'seasonals_remove',
                                                   'seasonals_type', 'datasource', 'refframe', 'data_config_file']);

# Types of seasonal options:
#   lssq: fits seasonals and linear trend by least squares inversion.
#  notch: removes the 1-year and 6-month components by notch filter.
#  grace: uses GRACE loading model interpolated between monthly points where available,
#  and linear inversion where not available.
#    stl: uses a pre-computed look-up table for STL time series

def view_single_station(station_name, offsets_remove=1, earthquakes_remove=0, outliers_remove=0, seasonals_remove=0,
                        starttime=None, endtime=None, outliers_def=15,
                        seasonals_type='lssq', datasource='pbo', refframe='NA',
                        data_config_file="data_config.txt", outdir="single_plots/"):
    MyParams = configure(station_name, offsets_remove, earthquakes_remove, outliers_remove, outliers_def,
                         seasonals_remove,
                         seasonals_type, datasource, refframe, data_config_file);
    [myData, offset_obj, eq_obj] = input_data(MyParams.station, MyParams.datasource, MyParams.refframe,
                                              MyParams.data_config_file);
    [updatedData, detrended] = compute(myData, offset_obj, eq_obj, MyParams, starttime, endtime);
    single_ts_plot(updatedData, detrended, MyParams, outdir);

# -------------- CONFIGURE ------------ #
def configure(station, offsets_remove, earthquakes_remove, outliers_remove, outliers_def,
              seasonals_remove, seasonals_type, datasource, refframe, data_config_file):
    # outliers_def : mm away from median filter.
    # refframe : itrf or nam
    # offsets_remove, earthquakes_remove, outliers_remove, seasonals_remove : booleans
    # seasonals type : lssq, nldas, gldas, grace, lsdm, shasta, stl, notch
    # datasource : unr, cwu, pbo, nmt, usgs
    MyParams = Parameters(station=station, outliers_remove=outliers_remove, outliers_def=outliers_def,
                          earthquakes_remove=earthquakes_remove,
                          offsets_remove=offsets_remove, seasonals_remove=seasonals_remove,
                          seasonals_type=seasonals_type,
                          datasource=datasource, refframe=refframe, data_config_file=data_config_file);
    print("------- %s --------" % station);
    print(
        "Viewing station %s, earthquakes_remove=%d, outliers_remove=%d, seasonals_remove=%d, datasource=%s, refframe=%s"
        % (station, earthquakes_remove, outliers_remove, seasonals_remove, datasource, refframe));
    return MyParams;

# ----------- INPUTS ---------------- #
def input_data(st_name, datasource, refframe, data_config_file):
    [myData, offset_obj, eq_obj] = gps_input_pipeline.get_station_data(st_name, datasource, data_config_file, refframe);
    # First, we embed the data with the eq object (always useful)
    myData = gps_io_functions.Timeseries(name=myData.name, coords=myData.coords, dtarray=myData.dtarray, dN=myData.dN,
                                         dE=myData.dE, dU=myData.dU, Sn=myData.Sn, Se=myData.Se, Su=myData.Su,
                                         EQtimes=eq_obj.evdts);
    return [myData, offset_obj, eq_obj];

# -------------- COMPUTE ------------ #
def compute(myData, offset_obj, eq_obj, MyParams, starttime, endtime):
    if starttime is None:
        starttime = myData.dtarray[0];
    if endtime is None:
        endtime = myData.dtarray[-1];
    newData = gps_ts_functions.impose_time_limits(myData, starttime, endtime);
    if MyParams.offsets_remove == 1:  # Remove offsets and antenna changes
        newData = offsets.remove_offsets(newData, offset_obj);
    if MyParams.outliers_remove == 1:  # Remove outliers
        newData = gps_ts_functions.remove_outliers(newData, MyParams.outliers_def);
    if MyParams.earthquakes_remove == 1:  # Remove earthquakes
        newData = offsets.remove_offsets(newData, eq_obj);
    trend_out = gps_seasonal_removals.make_detrended_ts(newData, MyParams.seasonals_remove, MyParams.seasonals_type,
                                                        MyParams.data_config_file);
    return [newData, trend_out];

# -------------- OUTPUTS ------------ #
def single_ts_plot(ts_obj, detrended, MyParams, outdir):
    label_fontsize = 18;

    # The major figure
    dpival = 500;
    [_, axarr] = plt.subplots(3, 1, sharex=True, figsize=(10, 7), dpi=dpival);
    axarr[0].plot_date(ts_obj.dtarray, ts_obj.dE, color='blue', markeredgecolor='black', markersize=1.5);
    axarr[0].grid(linestyle='--', linewidth=0.5);
    axarr[0].set_ylabel('east (mm)', fontsize=label_fontsize);
    bottom, top = axarr[0].get_ylim();
    for i in range(len(ts_obj.EQtimes)):
        axarr[0].plot_date([ts_obj.EQtimes[i], ts_obj.EQtimes[i]], [bottom, top], '--k', linewidth=0.5);
    ax1 = axarr[0].twinx();
    ax1.plot_date(detrended.dtarray, detrended.dE, marker='D', markersize=1.0, color='red');
    ax1.set_ylabel('detrended (mm)', fontsize=label_fontsize - 2, color='red');
    ax1.tick_params(labelcolor='red', labelsize=label_fontsize, axis='both')
    axarr[0].tick_params(labelsize=label_fontsize);

    axarr[1].plot_date(ts_obj.dtarray, ts_obj.dN, color='blue', markeredgecolor='black', markersize=1.5);
    axarr[1].grid(linestyle='--', linewidth=0.5);
    axarr[1].set_ylabel('north (mm)', fontsize=label_fontsize);
    bottom, top = axarr[1].get_ylim();
    for i in range(len(ts_obj.EQtimes)):
        axarr[1].plot_date([ts_obj.EQtimes[i], ts_obj.EQtimes[i]], [bottom, top], '--k', linewidth=0.5);
    ax2 = axarr[1].twinx();
    ax2.plot_date(detrended.dtarray, detrended.dN, marker='D', markersize=1.0, color='red');
    ax2.set_ylabel('detrended (mm)', fontsize=label_fontsize - 2, color='red');
    ax2.tick_params(labelcolor='red', labelsize=label_fontsize, axis='both')
    axarr[1].tick_params(labelsize=label_fontsize);

    axarr[2].plot_date(ts_obj.dtarray, ts_obj.dU, color='blue', markeredgecolor='black', markersize=1.5);
    axarr[2].grid(linestyle='--', linewidth=0.5);
    axarr[2].set_ylabel('vertical (mm)', fontsize=label_fontsize);
    bottom, top = axarr[2].get_ylim();
    for i in range(len(ts_obj.EQtimes)):
        axarr[2].plot_date([ts_obj.EQtimes[i], ts_obj.EQtimes[i]], [bottom, top], '--k', linewidth=0.5);
    ax3 = axarr[2].twinx();
    ax3.plot_date(detrended.dtarray, detrended.dU, marker='D', markersize=1.0, color='red');
    ax3.set_ylabel('detrended (mm)', fontsize=label_fontsize - 2, color='red');
    axarr[2].set_xlim([min(ts_obj.dtarray), max(ts_obj.dtarray)]);
    ax3.tick_params(labelcolor='red', labelsize=label_fontsize, axis='both')
    axarr[2].tick_params(labelsize=label_fontsize);

    title, savename = get_figure_name(MyParams, outdir);
    axarr[0].set_title(title, fontsize=label_fontsize + 2);
    plt.savefig(savename, dpi=dpival);
    print("Saving figure as %s " % savename)
    return;

def get_figure_name(MyParams, outdir):
    # Things that might go into the name:
    # 1. Station
    # 2. Earthquakes removed
    # 3. Outliers removed
    # 4. Seasonals removed
    # 5. Datasource
    # 6. Refframe

    savename = outdir + MyParams.station;
    title = MyParams.station;

    title = title + ', ' + MyParams.datasource + ' ' + MyParams.refframe
    if MyParams.earthquakes_remove == 0 and MyParams.offsets_remove == 0 and MyParams.seasonals_remove == 0:
        title = title + ', unaltered';
    if MyParams.earthquakes_remove:
        savename = savename + "_noeq";
        title = title + ', no earthquakes'
    if MyParams.seasonals_remove:  # If we are removing seasonals:
        savename = savename + "_noseasons";
        title = title + ', no seasonals'
        if MyParams.seasonals_type == "lssq":
            savename = savename + "_lssq"
            title = title + ' by least squares'
        if MyParams.seasonals_type == "notch":
            savename = savename + "_notch"
            title = title + ' by notch filter'
        if MyParams.seasonals_type == "grace":
            savename = savename + "_grace"
            title = title + ' by GRACE model'
        if MyParams.seasonals_type == "stl":
            savename = savename + "_stl"
            title = title + ' by STL'
        if MyParams.seasonals_type == "nldas":
            savename = savename + "_nldas";
            title = title + ' by NLDAS'
        if MyParams.seasonals_type == "gldas":
            savename = savename + "_gldas";
            title = title + ' by GLDAS'
        if MyParams.seasonals_type == "lsdm":
            savename = savename + "_lsdm";
            title = title + ' by LSDM'
        if MyParams.seasonals_type == "shasta":
            savename = savename + "_shasta";
            title = title + ' by Shasta';
        if MyParams.seasonals_type == "oroville":
            savename = savename + "_oroville";
            title = title + ' by Oroville';
    savename = savename + '_' + MyParams.datasource + '_' + MyParams.refframe;
    savename = savename + "_ts.jpg";
    return title, savename;


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author:Speciallan

import cv2
import json
import numpy as np
import os
from xml.etree import ElementTree
import matplotlib.pyplot as plt
import random
from shutil import copyfile
import os
import xml.etree.cElementTree as ET
from xml.dom import minidom

#保存xml格式的
def save_annotations(filepath, filename, image, boxes):
    #保存图像
    cv2.imwrite(os.path.join(os.path.join(filepath, "JPEGImages"), filename + ".jpg"), image)

    #保存画框图像
    for i in range(len(boxes)):
        cv2.rectangle(image,
                      (boxes[i]['xmin'],#xmin
                       boxes[i]['ymin']), #ymin
                      (boxes[i]['xmax'],#xmax
                       boxes[i]['ymax']), #ymax
                      (0,255,0),
                      1)
    cv2.imwrite(os.path.join(os.path.join(filepath, "DrawBoxes"), filename + ".jpg"), image)

    #保存labels
    root = ET.Element("annotation")
    ET.SubElement(root, "folder").text = "images"
    ET.SubElement(root, "filename").text = filename + ".jpg"
    size = ET.SubElement(root, "size")
    ET.SubElement(size, "width").text = str(image.shape[0])
    ET.SubElement(size, "height").text = str(image.shape[1])
    ET.SubElement(size, "depth").text = str(image.shape[2])
    for box in boxes:
        object = ET.SubElement(root, "object")
        ET.SubElement(object, "name").text = box["name"]
        ET.SubElement(object, "pose").text = "Unspecified"
        ET.SubElement(object, "truncated").text = "0"
        ET.SubElement(object, "difficult").text = "0"
        bndbox = ET.SubElement(object, "bndbox")
        ET.SubElement(bndbox, "xmin").text = str(box["xmin"])
        ET.SubElement(bndbox, "ymin").text = str(box["ymin"])
        ET.SubElement(bndbox, "xmax").text = str(box["xmax"])
        ET.SubElement(bndbox, "ymax").text = str(box["ymax"])

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="    ")
    myfile = open(os.path.join(os.path.join(filepath, "Annotations"), filename + ".xml"), "w")
    myfile.write(xmlstr)

#获得全部的图片路径
def get_all_img_path(original_path):
    img_path=[]
    annotation_path=[]
    for files in os.listdir(original_path):
        img_path.append(original_path+files+"/"+files+".jpg")
        annotation_path.append(original_path+files+"/"+"annotation_ship.xlsx")
    return [img_path,annotation_path]

#获得label文件
def get_label(path):
    #获得xls文件
    workbook=xlrd.open_workbook(path)
    worksheet=workbook.sheet_by_name("Work")

    data=[]
    for i in range(1,worksheet.nrows,1):#获取该表总行数
        one_data=[]
        for j in range(1,5,1):#列
            one_data.append(int(worksheet.cell_value(i,j)))
        data.append(one_data)

    return data#返回xmin ymin xmax ymax

#制作目标检测数据集
def make_datasets(all_img_path,annotation_path,save_path,size,overlap):
    #新建需要保存的文件夹
    if os.path.exists(save_path)==False:os.makedirs(save_path)
    if os.path.exists(save_path+"JPEGImages")==False:os.makedirs(save_path+"JPEGImages")
    if os.path.exists(save_path+"Annotations")==False:os.makedirs(save_path+"Annotations")
    if os.path.exists(save_path+"DrawBoxes")==False:os.makedirs(save_path+"DrawBoxes")
    #对每一张图像制作数据集
    for i in range(7,9,1):
        all_get_box=0#累计所有能得到的box
        #获得label
        labels=get_label(annotation_path[i])
        #读入图像
        print("读入"+all_img_path[i])
        img=cv2.imread(all_img_path[i])

        #按步长依次制作
        for y in range(0,img.shape[0]-size,size-overlap):#
            for x in range(0,img.shape[1]-size,size-overlap):
                #针对左上角为x、y的图像进行labels分配
                boxes=[]
                for t in range(len(labels)):
                    #对label进行大小值转换
                    the_xmin=-1
                    the_xmax=-1
                    the_ymin=-1
                    the_ymax=-1
                    #进行大小写整合
                    if labels[t][0]<labels[t][2]:
                        the_xmin=labels[t][0]
                        the_xmax=labels[t][2]
                    elif labels[t][2]<labels[t][0]:
                        the_xmin=labels[t][2]
                        the_xmax=labels[t][0]
                    if labels[t][1]<labels[t][3]:
                        the_ymin=labels[t][1]
                        the_ymax=labels[t][3]
                    elif labels[t][3]<labels[t][1]:
                        the_ymin=labels[t][3]
                        the_ymax=labels[t][1]
                    #判断是否在内部
                    if(the_xmin>=x and
                            the_ymin>=y and
                            the_xmax<=x+size and
                            the_ymax<=y+size and #在这个框内
                            the_xmin!=-1 and
                            the_ymin!=-1 and
                            the_xmax!=-1 and
                            the_ymax!=-1):#满足严格大小
                        label_temp=dict([('xmin',the_xmin-x),
                                         ('ymin',the_ymin-y),
                                         ('xmax',the_xmax-x),
                                         ('ymax',the_ymax-y),
                                         ('name','ship')])
                        boxes.append(label_temp)
                #保存有label的图像
                if len(boxes)!=0:
                    #统计完成保存
                    this_img=img[y:y+size,x:x+size,:]
                    save_annotations(save_path,
                                     all_img_path[i][17:19]+"_"+str(y)+"_"+str(x),
                                     this_img, boxes)

                    #统计相关数据并输出
                    all_get_box=all_get_box+len(boxes)
                    print(str(all_img_path[i][17:19])+" "+str(y)+"_"+str(x)+" box:"+str(all_get_box)+"/"+str(len(labels)))

    #从这里开始
    return True

#统计相关的分布
def show_statistics(all_img_path,annotations_path):
    all_w=[]#所有的宽度
    all_h=[]#所有的高度
    sum_label=0#所有的label个数
    one_line=0#统计只有一根线的个数
    w_h_ratio=[]#高宽比

    #开始统计
    for i in range(0,len(annotations_path),1):
        print(annotations_path[i])
        print("读入图像:"+all_img_path[i])
        #        img=cv2.imread(all_img_path[i])#获得图像
        boxes=get_label(annotations_path[i])#获得label
        sum_label=sum_label+len(boxes)#所有label个数累计求和
        for j in range(len(boxes)):
            xmin=0
            ymin=0
            w=0
            h=0
            #计算高宽
            if boxes[j][2]-boxes[j][0]>0:
                xmin=boxes[j][0]
                w=boxes[j][2]-boxes[j][0]
            else:
                xmin=boxes[j][2]
                w=boxes[j][0]-boxes[j][2]
            if boxes[j][3]-boxes[j][1]>0:
                ymin=boxes[j][1]
                h=boxes[j][3]-boxes[j][1]
            else:
                ymin=boxes[j][3]
                h=boxes[j][1]-boxes[j][3]
            #统计
            all_w.append(w)
            all_h.append(h)
            if h!=0:
                w_h_ratio.append(w/h)#比例
            #一根线
            if(w==0 or h==0):
                one_line=one_line+1

            #根据条件，把其固定要求的画框显示下来
    #            if os.path.exists("temp/")==False:os.makedirs("temp/")
    #            if (w==1 or h==1):
    #                overlap=100
    #                cv2.rectangle(img,(xmin,ymin),(xmin+w,ymin+h),(0,255,0),1)
    #                this_img=img[ymin-overlap:ymin+h+overlap,
    #                             xmin-overlap:xmin+w+overlap,:]
    #                cv2.imwrite("temp/"+str(all_img_path[i][17:19])+" "+str(j)+".jpg",this_img)

    #画出统计图
    def draw_data(data,bins,title):
        all_w=np.array(data)
        plt.title(title)
        plt.hist(all_w,bins=bins,color='green',edgecolor='black',histtype='bar')
        plt.grid(linestyle='--', color="gray")
        plt.show()

    #画图
    draw_data(all_w,50,"all_w")
    draw_data(all_h,50,"all_h")
    draw_data(w_h_ratio,50,"w_h_ratio")
    print("one line num: "+str(one_line))
    print("all label "+str(sum_label))

    return all_w

#-------------------------------上次的---------------------------------

#获得全部文件名list
def get_all_file_name(path):
    name=[]
    for files in os.listdir(path):
        files=files.replace(".jpg","")
        name.append(files)
    return name

#输入label.xml的连接输出boxes的位置
#def get_label(path):
#    tree = ElementTree.parse(path)
#    root = tree.getroot()
#    bounding_boxes = []
#    for object_tree in root.findall('object'):
#        class_name = object_tree.find('name').text
#        for bounding_box in object_tree.iter('bndbox'):
#            # 得到归一化坐标
#            xmin = int(bounding_box.find('xmin').text)
#            ymin = int(bounding_box.find('ymin').text)
#            xmax = int(bounding_box.find('xmax').text)
#            ymax = int(bounding_box.find('ymax').text)
#        bounding_box ={'class':class_name,'xmin':xmin,'ymin':ymin,'xmax':xmax,'ymax':ymax}
#        bounding_boxes.append(bounding_box)
#    return bounding_boxes

#制作分类数据集
def make_classify_datasets(classify_path,img_path,annotations_path,max_size):
    #建立目录
    if os.path.exists(classify_path+"pos/")==False:os.makedirs(classify_path+"pos/")
    if os.path.exists(classify_path+"neg/")==False:os.makedirs(classify_path+"neg/")

    #首先获得所有原始图像的名字
    pos_name_list=get_all_file_name(classify_path+"/from")

    #统计label分布
    show_statistics(pos_name_list,annotations_path)

    #对每一张图像进行处理
    num_use=0#能使用的数量
    for i in range(len(pos_name_list)):
        print(str(i)+"/"+str(len(pos_name_list)))
        boxes=get_label(annotations_path+pos_name_list[i]+".xml")#getbox
        img=cv2.imread(img_path+pos_name_list[i]+".jpg")
        for j in range(len(boxes)):
            #满足要求就统计
            this_w=boxes[j]['xmax']-boxes[j]['xmin']
            this_h=boxes[j]['ymax']-boxes[j]['ymin']
            new_x=int(boxes[j]['xmin']+this_w/2)
            new_y=int(boxes[j]['ymin']+this_h/2)
            if(this_w<=max_size and#宽度小于max_size
                    this_h<=max_size and#高度小于max_size
                    int(new_x-max_size/2)>=0 and#左上角在范围内
                    int(new_y-max_size/2)>=0 and#左上角在范围内
                    int(new_x+max_size/2)<=img.shape[0] and#右下角在范围内
                    int(new_y+max_size/2)<=img.shape[1]#右下角在范围内
            ):
                num_use=num_use+1
                print(num_use)
                #正例保存
                pos_img=img[int(new_y-max_size/2):int(new_y+max_size/2),
                        int(new_x-max_size/2):int(new_x+max_size/2),:]
                cv2.imwrite(classify_path+"pos/"+pos_name_list[i]+"_"+str(j)+".jpg",pos_img)

                #随机选择一个区域负例
                can_get_neg=False
                wait_time=100000
                while can_get_neg==False and wait_time>=0:#循环生成
                    neg_x=int(random.random()*10000)%(img.shape[1]-max_size)
                    neg_y=int(random.random()*10000)%(img.shape[1]-max_size)
                    #判断这个是否满足要求
                    can_use=True
                    for tt in range(len(boxes)):
                        if (neg_x>=boxes[tt]['xmin']-max_size and
                                neg_y>=boxes[tt]['ymin']-max_size and
                                new_x<=boxes[tt]['xmax'] and
                                new_y<=boxes[tt]['ymax']):
                            can_use=False
                            break
                    #满足就保存，不满足就继续
                    if can_use==True:
                        neg_img=img[neg_y:neg_y+max_size,
                                neg_x:neg_x+max_size,:]
                        cv2.imwrite(classify_path+"neg/"+pos_name_list[i]+"_"+str(j)+".jpg",neg_img)
                        can_get_neg=True
                    wait_time=wait_time-1

#训练模型
def train_model(classify_path,num_max,max_size):
    pos_img=np.ones((num_max,max_size,max_size,3))
    neg_img=np.ones((num_max,max_size,max_size,3))
    i=0
    for img in os.listdir(classify_path+"pos"):
        print("read pos "+str(i)+"/"+str(num_max))
        pos_img[i]=cv2.imread(classify_path+"pos/"+img)
        i=i+1
    i=0
    for img in os.listdir(classify_path+"neg"):
        print("read neg "+str(i)+"/"+str(num_max))
        neg_img[i]=cv2.imread(classify_path+"neg/"+img)
        i=i+1

    img_data=np.concatenate((pos_img,neg_img))
    #数据预处理
    img_data=img_data.astype('float32')/255

    #制作标签
    labels=np.concatenate((np.ones((num_max,)),np.zeros((num_max,))))

    #打乱训练集数据
    indices=np.arange(img_data.shape[0])
    np.random.shuffle(indices)
    img_data= img_data[indices]
    labels= labels[indices]

    #搭建网络框架
    model=models.Sequential()
    model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(max_size,max_size,3)))
    model.add(layers.MaxPool2D((2,2)))
    model.add(layers.Conv2D(64,(3,3),activation='relu'))
    model.add(layers.MaxPool2D((2,2)))
    model.add(layers.Conv2D(128,(3,3),activation='relu'))
    model.add(layers.MaxPool2D((2,2)))
    model.add(layers.Flatten())
    model.add(layers.Dense(1,activation='sigmoid'))
    model.summary()

    #编译
    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy',
                  metrics=['accuracy'])

    #拟合

    model.fit(img_data[0:int(2*num_max*0.7)],
              labels[0:int(2*num_max*0.7)],
              epochs=15,
              batch_size=16,
              validation_data=(img_data[int(2*num_max*0.7):2*num_max],labels[int(2*num_max*0.7):2*num_max]))

    #保存模型
    model.save("model.h5")

#错误分析
def wrong_analyse(classify_path,max_size):
    #建立目录
    if os.path.exists(classify_path+"wrong/")==False:os.makedirs(classify_path+"wrong/")

    model=models.load_model("model.h5")
    model.summary()

    if os.path.exists(classify_path+"wrong/is0but1")==False:os.makedirs(classify_path+"wrong/is0but1")
    if os.path.exists(classify_path+"wrong/is1but0")==False:os.makedirs(classify_path+"wrong/is1but0")
    #正例负例依次预测
    sum_wrong=0
    for img in os.listdir(classify_path+"pos"):
        pos_img=cv2.imread(classify_path+"pos/"+img)
        pos_img_use=np.reshape(pos_img,(1,max_size,max_size,3))/255
        if model.predict(pos_img_use)[0][0]<0.5:
            print("true 1 predict "+str(model.predict(pos_img_use)[0][0]))
            sum_wrong=sum_wrong+1
            cv2.imwrite(classify_path+"wrong/is1but0/"+img,pos_img)

    for img in os.listdir(classify_path+"neg"):
        neg_img=cv2.imread(classify_path+"neg/"+img)
        neg_img_use=np.reshape(neg_img,(1,max_size,max_size,3))/255
        if model.predict(neg_img_use)[0][0]>0.5:
            print("true 0 predict "+str(model.predict(neg_img_use)[0][0]))
            sum_wrong=sum_wrong+1
            cv2.imwrite(classify_path+"wrong/is0but1/"+img,neg_img)
    print("sum wrong = "+str(sum_wrong))

#滑动窗口预测
def win_predict(img_path,annotations_path,pre_name,max_size,threshold):
    model=models.load_model("model.h5")
    model.summary()
    #读入labels
    boxes=get_label(annotations_path+pre_name+".xml")
    #读入图像
    img=cv2.imread(img_path+pre_name+".jpg")
    draw_img=cv2.imread(img_path+pre_name+".jpg")
    #循环窗口生成
    for x in range(0,img.shape[0]-max_size,int(max_size/8)):
        for y in range(0,img.shape[0]-max_size,int(max_size/8)):
            #对该模块进行预测
            this_img=np.reshape(img[y:y+max_size,x:x+max_size,:],(1,max_size,max_size,3))/255
            this_label=model.predict(this_img)[0][0]
            print(str(y)+":"+str(x)+" confidence:"+str(this_label))

            if this_label>=threshold:
                print(this_label)
                #画框
                cv2.rectangle(draw_img,(x,y),(x+max_size,y+max_size),(0,255,255),1)
                #添加文本
                cv2.putText(draw_img,format(this_label,'.4f'),(x,y), cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,255,255),1)

    #画出真正的labels框
    for j in range(len(boxes)):
        #画框
        cv2.rectangle(draw_img,
                      (boxes[j]['xmin'],boxes[j]['ymin']),
                      (boxes[j]['xmax'],boxes[j]['ymax']),
                      (255,255,0),
                      1)

    #保存输出
    cv2.imwrite("final.jpg",draw_img)

#按要求制作不同的数据集
def make_different_datasets(pos_name_list):
    all_w=[]#统计全部的w
    all_h=[]#统计全部的h
    boxes_of_one_img=[]#统计每张图像的boxes数量
    white_area=[]#每张图二值化之后的白色区域做统计
    connected=[]#每张图的连通域统计
    for i in range(len(pos_name_list)):
        print(str(i)+"/"+str(len(pos_name_list)))
        boxes=get_label(annotations_path+pos_name_list[i]+".xml")#读入labels
        img=cv2.imread(img_path+pos_name_list[i]+".jpg")#读入图像

        boxes_of_one_img.append(len(boxes))#统计boxes数目
        _,binary = cv2.threshold(img[:,:,0],127,255 ,cv2.THRESH_BINARY)#二值化
        kernel = np.ones((5, 5), np.uint8)#膨胀核
        after_dilation = cv2.dilate(binary, kernel)#膨胀操作
        white_area.append((np.sum((binary/255)/3)))#统计二值图像中1面积
        connected_num,_ = cv2.connectedComponents(after_dilation,connectivity=8)#统计连通域个数
        connected.append(connected_num-1)

        for j in range(len(boxes)):
            all_w.append(boxes[j]['xmax']-boxes[j]['xmin'])#统计w
            all_h.append(boxes[j]['ymax']-boxes[j]['ymin'])#统计h

        #根据条件保存数据集
        save_datasets_path="new_datasets/over_3boxes/"
        if os.path.exists(save_datasets_path)==False:os.makedirs(save_datasets_path)
        if os.path.exists(save_datasets_path+"Annotations/")==False:
            os.makedirs(save_datasets_path+"Annotations/")
        if os.path.exists(save_datasets_path+"JPEGImages/")==False:
            os.makedirs(save_datasets_path+"JPEGImages/")
        if os.path.exists(save_datasets_path+"draw_boxes/")==False:
            os.makedirs(save_datasets_path+"draw_boxes/")

        if boxes_of_one_img[i]>=3:
            #保存xml文件
            copyfile(annotations_path+pos_name_list[i]+".xml",
                     save_datasets_path+"Annotations/"+pos_name_list[i]+".xml")#复制
            #图像保存
            copyfile(img_path+pos_name_list[i]+".jpg",
                     save_datasets_path+"JPEGImages/"+pos_name_list[i]+".jpg")#复制
            #画框保存
            for j in range(len(boxes)):
                cv2.rectangle(img,
                              (boxes[j]['xmin'],boxes[j]['ymin']),
                              (boxes[j]['xmax'],boxes[j]['ymax']),
                              (0,255,255,3))
            cv2.imwrite(save_datasets_path+"draw_boxes/"+pos_name_list[i]+".jpg",img)

    #--------画出统计图
    def draw_data(data,bins,title):
        all_w=np.array(data)
        plt.title(title)
        plt.hist(all_w,bins=bins,color='green',edgecolor='black',histtype='bar')
        plt.grid(linestyle='--', color="gray")
        plt.show()

    draw_data(all_w,50,"all_w")
    draw_data(all_h,50,"all_h")
    draw_data(boxes_of_one_img,14,"boxes_of_one_img")
    draw_data(white_area,50,"white_area")

    return connected


import urllib.request
import urllib.parse
import json
import os
trans = input('Please type in what you want to be checked:')
params = urllib.parse.urlencode({'a': 'getWordMean', 'c': 'search', 'word': trans})
url = "http://www.iciba.com/index.php?%s" % params
response = urllib.request.urlopen(url)
html = response.read().decode('utf-8')
results = json.loads(html)
check=results['baesInfo']
check_result='suggest' in check.keys()
if check_result == True:
    corword=results['baesInfo']['suggest'][0]['key']
    print( 0)
else:
    print( 1)


'''
The MIT License (MIT)

Portions Copyright (c) 2015-2018, The OmniDB Team
Portions Copyright (c) 2017-2018, 2ndQuadrant Limited

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

import os.path
import re
from collections import OrderedDict
from enum import Enum
import OmniDB_app.include.Spartacus as Spartacus
import OmniDB_app.include.Spartacus.Database as Database
import OmniDB_app.include.Spartacus.Utils as Utils
from urllib.parse import urlparse

'''
------------------------------------------------------------------------
Template
------------------------------------------------------------------------
'''
class TemplateType(Enum):
    EXECUTE = 1
    SCRIPT = 2

class Template:
    def __init__(self, p_text, p_type=TemplateType.EXECUTE):
        self.v_text = p_text
        self.v_type = p_type

'''
------------------------------------------------------------------------
MariaDB
------------------------------------------------------------------------
'''
class MariaDB:
    def __init__(self, p_server, p_port, p_service, p_user, p_password, p_conn_id=0, p_alias='', p_conn_string='', p_parse_conn_string = False):
        self.v_alias = p_alias
        self.v_db_type = 'mariadb'
        self.v_conn_string = p_conn_string
        self.v_conn_string_error = ''
        self.v_conn_id = p_conn_id

        self.v_server = p_server
        self.v_active_server = p_server
        self.v_user = p_user
        self.v_active_user = p_user
        self.v_schema = p_service
        self.v_service = p_service
        self.v_active_service = p_service

        self.v_port = p_port
        if p_port is None or p_port == '':
            self.v_active_port = '3306'
        else:
            self.v_active_port = p_port

        #try to get info from connection string
        if p_conn_string!='' and p_parse_conn_string:
            try:
                parsed = urlparse(p_conn_string)
                if parsed.port!=None:
                    self.v_active_port = str(parsed.port)
                if parsed.hostname!=None:
                    self.v_active_server = parsed.hostname
                if parsed.username!=None:
                    self.v_active_user = parsed.username
                if parsed.query!=None:
                    self.v_conn_string_query = parsed.query
                parsed_database = parsed.path
                if len(parsed_database)>1:
                    self.v_active_service = parsed_database[1:]
            except Exception as exc:
                self.v_conn_string_error = 'Syntax error in the connection string.'
                None

        self.v_connection = Spartacus.Database.MariaDB(self.v_active_server, self.v_active_port, self.v_active_service, self.v_active_user, p_password, p_conn_string)

        self.v_has_schema = True
        self.v_has_functions = True
        self.v_has_procedures = True
        self.v_has_sequences = False
        self.v_has_primary_keys = True
        self.v_has_foreign_keys = True
        self.v_has_uniques = True
        self.v_has_indexes = True
        self.v_has_checks = False
        self.v_has_excludes = False
        self.v_has_rules = False
        self.v_has_triggers = False
        self.v_has_partitions = False

        self.v_has_update_rule = True
        self.v_can_rename_table = True
        self.v_rename_table_command = "alter table #p_table_name# rename to #p_new_table_name#"
        self.v_create_pk_command = "constraint #p_constraint_name# primary key (#p_columns#)"
        self.v_create_fk_command = "constraint #p_constraint_name# foreign key (#p_columns#) references #p_r_table_name# (#p_r_columns#) #p_delete_update_rules#"
        self.v_create_unique_command = "constraint #p_constraint_name# unique (#p_columns#)"
        self.v_can_alter_type = True
        self.v_alter_type_command = "alter table #p_table_name# modify #p_column_name# #p_new_data_type#"
        self.v_can_alter_nullable = True
        self.v_set_nullable_command = "alter table #p_table_name# modify #p_column_name# null"
        self.v_drop_nullable_command = "alter table #p_table_name# modify #p_column_name# not null"
        self.v_can_rename_column = True
        self.v_rename_column_command = "alter table #p_table_name# rename column #p_column_name# to #p_new_column_name#"
        self.v_can_add_column = True
        self.v_add_column_command = "alter table #p_table_name# add #p_column_name# #p_data_type# #p_nullable#"
        self.v_can_drop_column = True
        self.v_drop_column_command = "alter table #p_table_name# drop column #p_column_name#"
        self.v_can_add_constraint = True
        self.v_add_pk_command = "alter table #p_table_name# add constraint #p_constraint_name# primary key (#p_columns#)"
        self.v_add_fk_command = "alter table #p_table_name# add constraint #p_constraint_name# foreign key (#p_columns#) references #p_r_table_name# (#p_r_columns#) #p_delete_update_rules#"
        self.v_add_unique_command = "alter table #p_table_name# add constraint #p_constraint_name# unique (#p_columns#)"
        self.v_can_drop_constraint = True
        self.v_drop_pk_command = "alter table #p_table_name# drop constraint #p_constraint_name#"
        self.v_drop_fk_command = "alter table #p_table_name# drop constraint #p_constraint_name#"
        self.v_drop_unique_command = "alter table #p_table_name# drop constraint #p_constraint_name#"
        self.v_create_index_command = "create index #p_index_name# on #p_table_name# (#p_columns#)";
        self.v_create_unique_index_command = "create unique index #p_index_name# on #p_table_name# (#p_columns#)"
        self.v_drop_index_command = "drop index #p_schema_name#.#p_index_name#"
        self.v_update_rules = [
            "NO ACTION",
            "RESTRICT",
			"SET NULL",
			"CASCADE"
        ]
        self.v_delete_rules = [
            "NO ACTION",
            "RESTRICT",
			"SET NULL",
			"CASCADE"
        ]
        self.v_reserved_words = []
        self.v_console_help = "Console tab. Type the commands in the editor below this box. \? to view command list."
        self.v_use_server_cursor = False

    def GetName(self):
        return self.v_service

    def GetVersion(self):
        return 'MariaDB ' + self.v_connection.ExecuteScalar('select version()')

    def GetUserName(self):
        return self.v_user

    def GetUserSuper(self):
        try:
            v_super = self.v_connection.ExecuteScalar('''
                select super_priv
                from mysql.user
                where user = '{0}'
            '''.format(self.v_user))
            if v_super == 'Y':
                return True
            else:
                return False
        except Exception as exc:
            return False

    def PrintDatabaseInfo(self):
        if self.v_conn_string=='':
            return self.v_active_user + '@' + self.v_active_service
        else:
            return self.v_active_user + '@' + self.v_active_service

    def PrintDatabaseDetails(self):
        if self.v_conn_string=='':
            return self.v_active_server + ':' + self.v_active_port
        else:
            return "<i title='{0}' class='fas fa-asterisk icon-conn-string'></i> ".format(self.v_conn_string) + self.v_active_server + ':' + self.v_active_port

    def HandleUpdateDeleteRules(self, p_update_rule, p_delete_rule):
        v_rules = ''
        if p_update_rule.strip() != '':
            v_rules += ' on update ' + p_delete_rule + ' '
        if p_delete_rule.strip() != '':
            v_rules += ' on delete ' + p_delete_rule + ' '
        return v_rules

    def TestConnection(self):
        v_return = ''
        if self.v_conn_string and self.v_conn_string_error!='':
            return self.v_conn_string_error

        try:
            self.v_connection.Open()
            self.v_connection.Close()
            v_return = 'Connection successful.'
        except Exception as exc:
            v_return = str(exc)
        return v_return

    def GetErrorPosition(self, p_error_message):
        vector = str(p_error_message).split('\n')
        v_return = None
        if len(vector) > 1 and vector[1][0:4]=='LINE':
            v_return = {
                'row': vector[1].split(':')[0].split(' ')[1],
                'col': vector[2].index('^') - len(vector[1].split(':')[0])-2
            }
        return v_return

    def QueryRoles(self):
        return self.v_connection.Query("""
            select concat('''',user,'''','@','''',host,'''') as role_name
            from mysql.user
            order by 1
        """, True)

    def QueryDatabases(self):
        return self.v_connection.Query('show databases', True, True)

    def QueryTables(self, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_schema:
                v_filter = "and table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and table_schema = '{0}' ".format(self.v_schema)
        return self.v_connection.Query('''
            select table_name,
                   table_schema
            from information_schema.tables
            where table_type in ('BASE TABLE', 'SYSTEM VIEW')
            {0}
            order by 2, 1
        '''.format(v_filter), True)

    def QueryTablesFields(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct c.table_name as table_name,
                   c.column_name,
                   c.data_type,
                   c.is_nullable as nullable,
                   c.character_maximum_length as data_length,
                   c.numeric_precision as data_precision,
                   c.numeric_scale as data_scale,
                   c.ordinal_position
            from information_schema.columns c,
                 information_schema.tables t
            where t.table_name = c.table_name
              and t.table_type in ('BASE TABLE', 'SYSTEM VIEW')
            {0}
            order by c.table_name,
                     c.ordinal_position
        '''.format(v_filter), True)

    def QueryTablesForeignKeys(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and i.table_schema = '{0}' and i.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and i.table_schema = '{0}' and i.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and i.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and i.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and i.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct i.constraint_name,
                   i.table_name,
                   k.referenced_table_name as r_table_name,
                   k.table_schema,
                   k.referenced_table_schema as r_table_schema,
                   r.update_rule,
                   r.delete_rule
            from information_schema.table_constraints i
            left join information_schema.key_column_usage k on i.constraint_name = k.constraint_name
            left join information_schema.referential_constraints r on i.constraint_name = r.constraint_name
            where i.constraint_type = 'FOREIGN KEY'
            {0}
            order by i.constraint_name,
                     i.table_name
        '''.format(v_filter), True)

    def QueryTablesForeignKeysColumns(self, p_fkey, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and i.table_schema = '{0}' and i.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and i.table_schema = '{0}' and i.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and i.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and i.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and i.table_name = '{0}' ".format(p_table)
        v_filter = v_filter + "and i.constraint_name = '{0}' ".format(p_fkey)
        return self.v_connection.Query('''
            select distinct i.constraint_name,
                   i.table_name,
                   k.referenced_table_name as r_table_name,
                   k.column_name,
                   k.referenced_column_name as r_column_name,
                   k.table_schema,
                   k.referenced_table_schema as r_table_schema,
                   r.update_rule,
                   r.delete_rule,
                   k.ordinal_position
            from information_schema.table_constraints i
            left join information_schema.key_column_usage k on i.constraint_name = k.constraint_name
            left join information_schema.referential_constraints r on i.constraint_name = r.constraint_name
            where i.constraint_type = 'FOREIGN KEY'
            {0}
            order by i.constraint_name,
                     i.table_name,
                     k.ordinal_position
        '''.format(v_filter), True)

    def QueryTablesPrimaryKeys(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct concat('pk_', t.table_name) as constraint_name,
                   t.table_name,
                   t.table_schema
            from information_schema.table_constraints t
            where t.constraint_type = 'PRIMARY KEY'
            {0}
            order by t.table_schema,
                     t.table_name
        '''.format(v_filter), True)

    def QueryTablesPrimaryKeysColumns(self, p_pkey, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        v_filter = "and concat('pk_', t.table_name) = '{0}' ".format(p_pkey)
        return self.v_connection.Query('''
            select distinct k.column_name,
                   k.ordinal_position
            from information_schema.table_constraints t
            join information_schema.key_column_usage k
            using (constraint_name, table_schema, table_name)
            where t.constraint_type = 'PRIMARY KEY'
            {0}
            order by k.ordinal_position
        '''.format(v_filter), True)

    def QueryTablesUniques(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct t.constraint_name,
                   t.table_name,
                   t.table_schema
            from information_schema.table_constraints t
            where t.constraint_type = 'UNIQUE'
            {0}
            order by t.table_schema,
                     t.table_name
        '''.format(v_filter), True)

    def QueryTablesUniquesColumns(self, p_unique, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        v_filter = "and t.constraint_name = '{0}' ".format(p_unique)
        return self.v_connection.Query('''
            select distinct k.column_name,
                   k.ordinal_position
            from information_schema.table_constraints t
            join information_schema.key_column_usage k
            using (constraint_name, table_schema, table_name)
            where t.constraint_type = 'UNIQUE'
            {0}
            order by k.ordinal_position
        '''.format(v_filter), True)

    def QueryTablesIndexes(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct t.table_schema as schema_name,
                   t.table_name,
                   (case when t.index_name = 'PRIMARY' then concat('pk_', t.table_name) else t.index_name end) as index_name,
                   case when t.non_unique = 1 then 'Non Unique' else 'Unique' end as uniqueness
            from information_schema.statistics t
            where 1 = 1
            {0}
            order by 1, 2, 3
        '''.format(v_filter), True)

    def QueryTablesIndexesColumns(self, p_index, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and t.table_schema = '{0}' and t.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and t.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and t.table_name = '{0}' ".format(p_table)
        v_filter = "and (case when t.index_name = 'PRIMARY' then concat('pk_', t.table_name) else t.index_name end) = '{0}' ".format(p_index)
        return self.v_connection.Query('''
            select distinct t.column_name,
                   t.seq_in_index
            from information_schema.statistics t
            where 1 = 1
            {0}
            order by t.seq_in_index
        '''.format(v_filter), True)

    def QueryDataLimited(self, p_query, p_count=-1):
        if p_count != -1:
            try:
                self.v_connection.Open()
                v_data = self.v_connection.QueryBlock('select * from ( {0} ) t limit {1}'.format(p_query, p_count), p_count, True, True)
                self.v_connection.Close()
                return v_data
            except Spartacus.Database.Exception as exc:
                try:
                    self.v_connection.Cancel()
                except:
                    pass
                raise exc
        else:
            return self.v_connection.Query(p_query, True)

    def QueryTableRecords(self, p_column_list, p_table, p_filter, p_count=-1):
        v_limit = ''
        if p_count != -1:
            v_limit = ' limit ' + p_count
        return self.v_connection.Query('''
            select *
            from (
            select {0}
            from {1} t
            {2}
            ) t
            {3}
        '''.format(
                p_column_list,
                p_table,
                p_filter,
                v_limit
            ), True
        )

    def QueryFunctions(self, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_schema:
                v_filter = "and t.routine_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.routine_schema = '{0}' ".format(self.v_schema)
        return self.v_connection.Query('''
            select t.routine_schema as schema_name,
                   t.routine_name as id,
                   t.routine_name as name
            from information_schema.routines t
            where t.routine_type = 'FUNCTION'
            {0}
            order by 2
        '''.format(v_filter), True)

    def QueryFunctionFields(self, p_function, p_schema):
        if p_schema:
            v_schema = p_schema
        else:
            v_schema = self.v_schema
        return self.v_connection.Query('''
            select 'O' as type,
                   concat('returns ', t.data_type) as name,
                   0 as seq
            from information_schema.routines t
            where t.routine_type = 'FUNCTION'
              and t.routine_schema = '{0}'
              and t.specific_name = '{1}'
            union
            select (case t.parameter_mode
                      when 'IN' then 'I'
                      when 'OUT' then 'O'
                      else 'R'
                    end) as type,
                   concat(t.parameter_name, ' ', t.data_type) as name,
                   t.ordinal_position+1 as seq
            from information_schema.parameters t
            where t.ordinal_position > 0
              and t.specific_schema = '{0}'
              and t.specific_name = '{1}'
            order by 3 desc
        '''.format(v_schema, p_function), True)

    def GetFunctionDefinition(self, p_function):
        v_body = '--DROP FUNCTION {0};\n'.format(p_function)
        v_body = v_body + self.v_connection.Query('show create function {0}.{1}'.format(self.v_schema, p_function), True, True).Rows[0][2]
        return v_body

    def QueryProcedures(self, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_schema:
                v_filter = "and t.routine_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and t.routine_schema = '{0}' ".format(self.v_schema)
        return self.v_connection.Query('''
            select t.routine_schema as schema_name,
                   t.routine_name as id,
                   t.routine_name as name
            from information_schema.routines t
            where t.routine_type = 'PROCEDURE'
            {0}
            order by 2
        '''.format(v_filter), True)

    def QueryProcedureFields(self, p_procedure, p_schema):
        if p_schema:
            v_schema = p_schema
        else:
            v_schema = self.v_schema
        return self.v_connection.Query('''
            select (case t.parameter_mode
                      when 'IN' then 'I'
                      when 'OUT' then 'O'
                      else 'R'
                    end) as type,
                   concat(t.parameter_name, ' ', t.data_type) as name,
                   t.ordinal_position+1 as seq
            from information_schema.parameters t
            where t.specific_schema = '{0}'
              and t.specific_name = '{1}'
            order by 3 desc
        '''.format(v_schema, p_procedure), True)

    def GetProcedureDefinition(self, p_procedure):
        v_body = '--DROP PROCEDURE {0};\n'.format(p_procedure)
        v_body = v_body + self.v_connection.Query('show create procedure {0}.{1}'.format(self.v_schema, p_procedure), True, True).Rows[0][2]
        return v_body

    def QuerySequences(self, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_schema:
                v_filter = "and table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and table_schema = '{0}' ".format(self.v_schema)
        return self.v_connection.Query('''
            select table_name as sequence_name,
                   table_schema as sequence_schema
            from information_schema.tables
            where table_type = 'SEQUENCE'
            {0}
            order by 2, 1
        '''.format(v_filter), True)

    def QueryViews(self, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_schema:
                v_filter = "and table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and table_schema = '{0}' ".format(self.v_schema)
        return self.v_connection.Query('''
            select table_name,
                   table_schema
            from information_schema.views
            where 1=1
            {0}
            order by 2, 1
        '''.format(v_filter), True)

    def QueryViewFields(self, p_table=None, p_all_schemas=False, p_schema=None):
        v_filter = ''
        if not p_all_schemas:
            if p_table and p_schema:
                v_filter = "and c.table_schema = '{0}' and c.table_name = '{1}' ".format(p_schema, p_table)
            elif p_table:
                v_filter = "and c.table_schema = '{0}' and c.table_name = '{1}' ".format(self.v_schema, p_table)
            elif p_schema:
                v_filter = "and c.table_schema = '{0}' ".format(p_schema)
            else:
                v_filter = "and c.table_schema = '{0}' ".format(self.v_schema)
        else:
            if p_table:
                v_filter = "and c.table_name = '{0}' ".format(p_table)
        return self.v_connection.Query('''
            select distinct c.table_name as table_name,
                   c.column_name,
                   c.data_type,
                   c.is_nullable as nullable,
                   c.character_maximum_length as data_length,
                   c.numeric_precision as data_precision,
                   c.numeric_scale as data_scale,
                   c.ordinal_position
            from information_schema.columns c,
                 information_schema.tables t
            where t.table_name = c.table_name
              and t.table_type = 'VIEW'
            {0}
            order by c.table_name,
                     c.ordinal_position
        '''.format(v_filter), True)

    def GetViewDefinition(self, p_view, p_schema):
        if p_schema:
            v_schema = p_schema
        else:
            v_schema = self.v_schema
        return self.v_connection.Query('show create view {0}.{1}'.format(v_schema, p_view), True, True).Rows[0][1]

    def TemplateCreateRole(self):
        return Template('''CREATE USER name
-- IDENTIFIED BY password
-- REQUIRE NONE
-- REQUIRE SSL
-- REQUIRE X509
-- REQUIRE CIPHER 'cipher'
-- REQUIRE ISSUER 'issuer'
-- REQUIRE SUBJECT 'subject'
-- WITH MAX_QUERIES_PER_HOUR count
-- WITH MAX_UPDATES_PER_HOUR count
-- WITH MAX_CONNECTIONS_PER_HOUR count
-- WITH MAX_USER_CONNECTIONS count
-- PASSWORD EXPIRE
-- ACCOUNT { LOCK | UNLOCK }
''')

    def TemplateAlterRole(self):
        return Template('''ALTER USER #role_name#
-- IDENTIFIED BY password
-- REQUIRE NONE
-- REQUIRE SSL
-- REQUIRE X509
-- REQUIRE CIPHER 'cipher'
-- REQUIRE ISSUER 'issuer'
-- REQUIRE SUBJECT 'subject'
-- WITH MAX_QUERIES_PER_HOUR count
-- WITH MAX_UPDATES_PER_HOUR count
-- WITH MAX_CONNECTIONS_PER_HOUR count
-- WITH MAX_USER_CONNECTIONS count
-- PASSWORD EXPIRE
-- ACCOUNT { LOCK | UNLOCK }
-- RENAME USER #role_name# TO new_name
-- SET PASSWORD FOR #role_name# = password
''')

    def TemplateDropRole(self):
        return Template('DROP USER #role_name#')

    def TemplateCreateDatabase(self):
        return Template('''CREATE DATABASE name
-- CHARACTER SET charset
-- COLLATE collate
''')

    def TemplateAlterDatabase(self):
        return Template('''ALTER DATABASE #database_name#
-- CHARACTER SET charset
-- COLLATE collate
''')

    def TemplateDropDatabase(self):
        return Template('DROP DATABASE #database_name#')

    def TemplateCreateFunction(self):
        return Template('''CREATE FUNCTION #schema_name#.name
(
-- argname argtype
)
RETURNS rettype
BEGIN
-- DECLARE variables
-- definition
-- RETURN variable | value
END;
''')

    def TemplateDropFunction(self):
        return Template('DROP FUNCTION #function_name#')

    def TemplateCreateProcedure(self):
        return Template('''CREATE PROCEDURE #schema_name#.name
(
-- [argmode] argname argtype
)
BEGIN
-- DECLARE variables
-- definition
END;
''')

    def TemplateDropProcedure(self):
        return Template('DROP PROCEDURE #function_name#')

    def TemplateCreateTable(self):
        return Template('''CREATE
-- TEMPORARY
TABLE #schema_name#.table_name
-- AS query
(
    column_name data_type
    -- NOT NULL
    -- NULL
    -- DEFAULT default_value
    -- AUTO_INCREMENT
    -- UNIQUE
    -- PRIMARY KEY
    -- COMMENT 'string'
    -- COLUMN_FORMAT { FIXED | DYNAMIC | DEFAULT }
    -- STORAGE { DISK | MEMORY | DEFAULT }
    -- [ GENERATED ALWAYS ] AS (expression) [ VIRTUAL | STORED ]
    -- [ CONSTRAINT [ symbol ] ] PRIMARY KEY [ USING { BTREE | HASH } ] ( column_name, ... )
    -- { INDEX | KEY } [ index_name ] [ USING { BTREE | HASH } ] ( column_name, ... )
    -- [ CONSTRAINT [ symbol ] ] UNIQUE [ INDEX | KEY ] [ index_name ] [ USING { BTREE | HASH } ] ( column_name, ... )
    -- { FULLTEXT | SPATIAL } [ INDEX | KEY ] [ index_name ] [ USING { BTREE | HASH } ] ( column_name, ... )
    -- [ CONSTRAINT [ symbol ] ] FOREIGN KEY [ index_name ]  ( column_name, ... ) REFERENCES reftable ( refcolumn, ... ) [MATCH FULL | MATCH PARTIAL | MATCH SIMPLE] [ON DELETE { RESTRICT | CASCADE | SET NULL | NO ACTION | SET DEFAULT }] [ON UPDATE { RESTRICT | CASCADE | SET NULL | NO ACTION | SET DEFAULT }]
    -- CHECK ( expr )
)
-- AUTO_INCREMENT value
-- AVG_ROW_LENGTH value
-- [ DEFAULT ] CHARACTER SET charset_name
-- CHECKSUM { 0 | 1 }
-- [ DEFAULT ] COLLATE collation_name
-- COMMENT 'string'
-- COMPRESSION { 'ZLIB' | 'LZ4' | 'NONE' }
-- CONNECTION 'connect_string'
-- { DATA | INDEX } DIRECTORY 'absolute path to directory'
-- DELAY_KEY_WRITE { 0 | 1 }
-- ENCRYPTION { 'Y' | 'N' }
-- ENGINE engine_name
-- INSERT_METHOD { NO | FIRST | LAST }
-- KEY_BLOCK_SIZE value
-- MAX_ROWS value
-- MIN_ROWS value
-- PACK_KEYS { 0 | 1 | DEFAULT }
-- PASSWORD 'string'
-- ROW_FORMAT { DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT }
-- STATS_AUTO_RECALC { DEFAULT | 0 | 1 }
-- STATS_PERSISTENT { DEFAULT | 0 | 1 }
-- STATS_SAMPLE_PAGES value
-- TABLESPACE tablespace_name [STORAGE { DISK | MEMORY | DEFAULT } ]
''')

    def TemplateAlterTable(self):
        return Template('''ALTER TABLE #table_name#
-- ADD [ COLUMN ] col_name column_definition  [ FIRST | AFTER col_name ]
-- ADD [ COLUMN ] ( col_name column_definition , ... )
-- ADD { INDEX | KEY } [ index_name ] USING { BTREE | HASH } (index_col_name , ... )
-- ADD [ CONSTRAINT [ symbol ] ] PRIMARY KEY USING { BTREE | HASH } ( index_col_name , ... )
-- ADD [ CONSTRAINT [ symbol ] ] UNIQUE [ INDEX | KEY ] [ index_name ] USING { BTREE | HASH } ( index_col_name , ... )
-- ADD FULLTEXT [ INDEX | KEY ] ( index_col_name , ... )
-- ADD SPATIAL [ INDEX | KEY ] [ index_name ] (index_col_name , ... )
-- ADD [ CONSTRAINT [ symbol ] ] FOREIGN KEY [ index_name ] ( index_col_name , ... ) reference_definition
-- ALGORITHM { DEFAULT | INPLACE | COPY }
-- ALTER [ COLUMN ] col_name { SET DEFAULT literal | DROP DEFAULT }
-- CHANGE [ COLUMN ] old_col_name new_col_name column_definition [ FIRST | AFTER col_name ]
-- [DEFAULT] CHARACTER SET charset_name [ COLLATE collation_name ]
-- CONVERT TO CHARACTER SET charset_name [ COLLATE collation_name ]
-- { DISABLE | ENABLE } KEYS
-- { DISCARD | IMPORT } TABLESPACE
-- DROP [ COLUMN ] col_name
-- DROP { INDEX | KEY } index_name
-- DROP PRIMARY KEY
-- DROP FOREIGN KEY fk_symbol
-- FORCE
-- LOCK { DEFAULT | NONE | SHARED | EXCLUSIVE }
-- MODIFY [ COLUMN ] col_name column_definition [ FIRST | AFTER col_name ]
-- ORDER BY col_name [, col_name] ...
-- RENAME { INDEX | KEY } old_index_name TO new_index_name
-- RENAME [ TO | AS ] new_tbl_name
-- { WITHOUT | WITH } VALIDATION
-- ADD PARTITION ( partition_definition )
-- DROP PARTITION partition_names
-- DISCARD PARTITION { partition_names | ALL } TABLESPACE
-- IMPORT PARTITION { partition_names | ALL } TABLESPACE
-- TRUNCATE PARTITION { partition_names | ALL }
-- COALESCE PARTITION number
-- REORGANIZE PARTITION partition_names INTO ( partition_definitions )
-- EXCHANGE PARTITION partition_name WITH TABLE tbl_name [ { WITH | WITHOUT } VALIDATION ]
-- ANALYZE PARTITION { partition_names | ALL }
-- CHECK PARTITION { partition_names | ALL }
-- OPTIMIZE PARTITION { partition_names | ALL }
-- REBUILD PARTITION { partition_names | ALL }
-- REPAIR PARTITION { partition_names | ALL }
-- REMOVE PARTITIONING
-- UPGRADE PARTITIONING
-- AUTO_INCREMENT value
-- AVG_ROW_LENGTH value
-- [ DEFAULT ] CHARACTER SET charset_name
-- CHECKSUM { 0 | 1 }
-- [ DEFAULT ] COLLATE collation_name
-- COMMENT 'string'
-- COMPRESSION { 'ZLIB' | 'LZ4' | 'NONE' }
-- CONNECTION 'connect_string'
-- { DATA | INDEX } DIRECTORY 'absolute path to directory'
-- DELAY_KEY_WRITE { 0 | 1 }
-- ENCRYPTION { 'Y' | 'N' }
-- ENGINE engine_name
-- INSERT_METHOD { NO | FIRST | LAST }
-- KEY_BLOCK_SIZE value
-- MAX_ROWS value
-- MIN_ROWS value
-- PACK_KEYS { 0 | 1 | DEFAULT }
-- PASSWORD 'string'
-- ROW_FORMAT { DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT }
-- STATS_AUTO_RECALC { DEFAULT | 0 | 1 }
-- STATS_PERSISTENT { DEFAULT | 0 | 1 }
-- STATS_SAMPLE_PAGES value
-- TABLESPACE tablespace_name [STORAGE { DISK | MEMORY | DEFAULT } ]
''')

    def TemplateDropTable(self):
        return Template('''DROP TABLE #table_name#
-- RESTRICT
-- CASCADE
''')

    def TemplateCreateColumn(self):
        return Template('''ALTER TABLE #table_name#
ADD name data_type
--DEFAULT expr
--NOT NULL
''')

    def TemplateAlterColumn(self):
        return Template('''ALTER TABLE #table_name#
-- ALTER #column_name# { datatype | DEFAULT expr | [ NULL | NOT NULL ]}
-- CHANGE COLUMN #column_name# TO new_name
'''
)

    def TemplateDropColumn(self):
        return Template('''ALTER TABLE #table_name#
DROP COLUMN #column_name#
''')

    def TemplateCreatePrimaryKey(self):
        return Template('''ALTER TABLE #table_name#
ADD CONSTRAINT name
PRIMARY KEY ( column_name [, ... ] )
''')

    def TemplateDropPrimaryKey(self):
        return Template('''ALTER TABLE #table_name#
DROP PRIMARY KEY #constraint_name#
--CASCADE
''')

    def TemplateCreateUnique(self):
        return Template('''ALTER TABLE #table_name#
ADD CONSTRAINT name
UNIQUE ( column_name [, ... ] )
''')

    def TemplateDropUnique(self):
        return Template('''ALTER TABLE #table_name#
DROP #constraint_name#
''')

    def TemplateCreateForeignKey(self):
        return Template('''ALTER TABLE #table_name#
ADD CONSTRAINT name
FOREIGN KEY ( column_name [, ... ] )
REFERENCES reftable [ ( refcolumn [, ... ] ) ]
''')

    def TemplateDropForeignKey(self):
        return Template('''ALTER TABLE #table_name#
DROP FOREIGN KEY #constraint_name#
''')

    def TemplateCreateIndex(self):
        return Template('''CREATE [ UNIQUE ] INDEX name
ON #table_name#
( { column_name | ( expression ) } [ ASC | DESC ] )
''')

    def TemplateDropIndex(self):
        return Template('DROP INDEX #index_name#')

    def TemplateCreateSequence(self):
        return Template('''CREATE SEQUENCE #schema_name#.name
--INCREMENT BY increment
--MINVALUE minvalue | NOMINVALUE
--MAXVALUE maxvalue | NOMAXVALUE
--START WITH start
--CACHE cache | NOCACHE
--CYCLE | NOCYCLE
''')

    def TemplateAlterSequence(self):
        return Template('''ALTER SEQUENCE #sequence_name#
--INCREMENT BY increment
--MINVALUE minvalue | NOMINVALUE
--MAXVALUE maxvalue | NOMAXVALUE
--START WITH start
--CACHE cache | NOCACHE
--CYCLE | NOCYCLE
--RESTART WITH restart
''')

    def TemplateDropSequence(self):
        return Template('DROP SEQUENCE #sequence_name#')

    def TemplateCreateView(self):
        return Template('''CREATE OR REPLACE VIEW #schema_name#.name AS
SELECT ...
''')

    def TemplateDropView(self):
        return Template('''DROP VIEW #view_name#
-- RESTRICT
-- CASCADE
''')

    def TemplateSelect(self, p_schema, p_table):
        v_sql = 'SELECT t.'
        v_fields = self.QueryTablesFields(p_table, False, p_schema)
        if len(v_fields.Rows) > 0:
            v_sql += '\n     , t.'.join([r['column_name'] for r in v_fields.Rows])
        v_sql += '\nFROM {0}.{1} t'.format(p_schema, p_table)
        v_pk = self.QueryTablesPrimaryKeys(p_table, False, p_schema)
        if len(v_pk.Rows) > 0:
            v_fields = self.QueryTablesPrimaryKeysColumns(v_pk.Rows[0]['constraint_name'], p_table, False, p_schema)
            if len(v_fields.Rows) > 0:
                v_sql += '\nORDER BY t.'
                v_sql += '\n       , t.'.join([r['column_name'] for r in v_fields.Rows])
        return Template(v_sql)

    def TemplateInsert(self, p_schema, p_table):
        v_fields = self.QueryTablesFields(p_table, False, p_schema)
        if len(v_fields.Rows) > 0:
            v_sql = 'INSERT INTO {0}.{1} (\n'.format(p_schema, p_table)
            v_pk = self.QueryTablesPrimaryKeys(p_table, False, p_schema)
            if len(v_pk.Rows) > 0:
                v_table_pk_fields = self.QueryTablesPrimaryKeysColumns(v_pk.Rows[0]['constraint_name'], p_table, False, p_schema)
                v_pk_fields = [r['column_name'] for r in v_table_pk_fields.Rows]
                v_values = []
                v_first = True
                for r in v_fields.Rows:
                    if v_first:
                        v_sql += '      {0}'.format(r['column_name'])
                        if r['column_name'] in v_pk_fields:
                            v_values.append('      ? -- {0} {1} PRIMARY KEY'.format(r['column_name'], r['data_type']))
                        elif r['nullable'] == 'YES':
                            v_values.append('      ? -- {0} {1} NULLABLE'.format(r['column_name'], r['data_type']))
                        else:
                            v_values.append('      ? -- {0} {1}'.format(r['column_name'], r['data_type']))
                        v_first = False
                    else:
                        v_sql += '\n    , {0}'.format(r['column_name'])
                        if r['column_name'] in v_pk_fields:
                            v_values.append('\n    , ? -- {0} {1} PRIMARY KEY'.format(r['column_name'], r['data_type']))
                        elif r['nullable'] == 'YES':
                            v_values.append('\n    , ? -- {0} {1} NULLABLE'.format(r['column_name'], r['data_type']))
                        else:
                            v_values.append('\n    , ? -- {0} {1}'.format(r['column_name'], r['data_type']))
            else:
                v_values = []
                v_first = True
                for r in v_fields.Rows:
                    if v_first:
                        v_sql += '      {0}'.format(r['column_name'])
                        if r['nullable'] == 'YES':
                            v_values.append('      ? -- {0} {1} NULLABLE'.format(r['column_name'], r['data_type']))
                        else:
                            v_values.append('      ? -- {0} {1}'.format(r['column_name'], r['data_type']))
                        v_first = False
                    else:
                        v_sql += '\n    , {0}'.format(r['column_name'])
                        if r['nullable'] == 'YES':
                            v_values.append('\n    , ? -- {0} {1} NULLABLE'.format(r['column_name'], r['data_type']))
                        else:
                            v_values.append('\n    , ? -- {0} {1}'.format(r['column_name'], r['data_type']))
            v_sql += '\n) VALUES (\n'
            for v in v_values:
                v_sql += v
            v_sql += '\n)'
        else:
            v_sql = ''
        return Template(v_sql)

    def TemplateUpdate(self, p_schema, p_table):
        v_fields = self.QueryTablesFields(p_table, False, p_schema)
        if len(v_fields.Rows) > 0:
            v_sql = 'UPDATE {0}.{1}\nSET '.format(p_schema, p_table)
            v_pk = self.QueryTablesPrimaryKeys(p_table, False, p_schema)
            if len(v_pk.Rows) > 0:
                v_table_pk_fields = self.QueryTablesPrimaryKeysColumns(v_pk.Rows[0]['constraint_name'], p_table, False, p_schema)
                v_pk_fields = [r['column_name'] for r in v_table_pk_fields.Rows]
                v_values = []
                v_first = True
                for r in v_fields.Rows:
                    if v_first:
                        if r['column_name'] in v_pk_fields:
                            v_sql += '{0} = ? -- {1} PRIMARY KEY'.format(r['column_name'], r['data_type'])
                        elif r['nullable'] == 'YES':
                            v_sql += '{0} = ? -- {1} NULLABLE'.format(r['column_name'], r['data_type'])
                        else:
                            v_sql += '{0} = ? -- {1}'.format(r['column_name'], r['data_type'])
                        v_first = False
                    else:
                        if r['column_name'] in v_pk_fields:
                            v_sql += '\n    , {0} = ? -- {1} PRIMARY KEY'.format(r['column_name'], r['data_type'])
                        elif r['nullable'] == 'YES':
                            v_sql += '\n    , {0} = ? -- {1} NULLABLE'.format(r['column_name'], r['data_type'])
                        else:
                            v_sql += '\n    , {0} = ? -- {1}'.format(r['column_name'], r['data_type'])
            else:
                v_values = []
                v_first = True
                for r in v_fields.Rows:
                    if v_first:
                        if r['nullable'] == 'YES':
                            v_sql += '{0} = ? -- {1} NULLABLE'.format(r['column_name'], r['data_type'])
                        else:
                            v_sql += '{0} = ? -- {1}'.format(r['column_name'], r['data_type'])
                        v_first = False
                    else:
                        if r['nullable'] == 'YES':
                            v_sql += '\n    , {0} = ? -- {1} NULLABLE'.format(r['column_name'], r['data_type'])
                        else:
                            v_sql += '\n    , {0} = ? -- {1}'.format(r['column_name'], r['data_type'])
            v_sql += '\nWHERE condition'
        else:
            v_sql = ''
        return Template(v_sql)

    def TemplateDelete(self):
        return Template('''DELETE FROM #table_name#
WHERE condition
''')

    def GetProperties(self, p_schema, p_table, p_object, p_type):
        if p_type == 'table':
            return self.v_connection.Query('''
                select table_schema as "Table Schema",
                       table_name as "Table Name",
                       table_type as "Table Type",
                       engine as "Engine",
                       version as "Version",
                       row_format as "Row Format",
                       table_rows as "Table Rows",
                       avg_row_length as "Average Row Length",
                       data_length as "Data Length",
                       max_data_length as "Max Data Length",
                       index_length as "Index Length",
                       data_free as "Data Free",
                       auto_increment as "Auto Increment",
                       create_time as "Create Time",
                       update_time as "Update Time",
                       check_time as "Check Time",
                       table_collation as "Table Collaction",
                       checksum as "Checksum"
                from information_schema.tables
                where table_schema = '{0}'
                  and table_name = '{1}'
            '''.format(p_schema, p_object), True).Transpose('Property', 'Value')
        elif p_type == 'view':
            return self.v_connection.Query('''
                select table_schema as "View Schema",
                       table_name as "View Name",
                       check_option as "Check Option",
                       is_updatable as "Is Updatable",
                       security_type as "Security Type",
                       character_set_client as "Character Set Client",
                       collation_connection as "Collation Connection",
                       algorithm as "Algorithm"
                from information_schema.views
                where table_schema = '{0}'
                  and table_name = '{1}'
            '''.format(p_schema, p_object), True).Transpose('Property', 'Value')
        elif p_type == 'function':
            return self.v_connection.Query('''
                select routine_schema as "Routine Schema",
                       routine_name as "Routine Name",
                       routine_type as "Routine Type",
                       data_type as "Data Type",
                       character_maximum_length as "Character Maximum Length",
                       character_octet_length as "Character Octet Length",
                       numeric_precision as "Numeric Precision",
                       numeric_scale as "Numeric Scale",
                       datetime_precision as "Datetime Precision",
                       character_set_name as "Character Set Name",
                       collation_name as "Collation Name",
                       routine_body as "Routine Body",
                       external_name as "External Name",
                       external_language as "External Language",
                       parameter_style as "Parameter Style",
                       is_deterministic as "Is Deterministic",
                       sql_data_access as "SQL Data Access",
                       sql_path as "SQL Path",
                       security_type as "Security Type",
                       created as "Created",
                       last_altered as "Last Altered",
                       character_set_client as "Character Set Client",
                       collation_connection as "Collation Connection",
                       database_collation as "Database Collation"
                from information_schema.routines
                where routine_type = 'FUNCTION'
                  and routine_schema = '{0}'
                  and routine_name = '{1}'
            '''.format(p_schema, p_object), True).Transpose('Property', 'Value')
        elif p_type == 'procedure':
            return self.v_connection.Query('''
                select routine_schema as "Routine Schema",
                       routine_name as "Routine Name",
                       routine_type as "Routine Type",
                       data_type as "Data Type",
                       character_maximum_length as "Character Maximum Length",
                       character_octet_length as "Character Octet Length",
                       numeric_precision as "Numeric Precision",
                       numeric_scale as "Numeric Scale",
                       datetime_precision as "Datetime Precision",
                       character_set_name as "Character Set Name",
                       collation_name as "Collation Name",
                       routine_body as "Routine Body",
                       external_name as "External Name",
                       external_language as "External Language",
                       parameter_style as "Parameter Style",
                       is_deterministic as "Is Deterministic",
                       sql_data_access as "SQL Data Access",
                       sql_path as "SQL Path",
                       security_type as "Security Type",
                       created as "Created",
                       last_altered as "Last Altered",
                       character_set_client as "Character Set Client",
                       collation_connection as "Collation Connection",
                       database_collation as "Database Collation"
                from information_schema.routines
                where routine_type = 'PROCEDURE'
                  and routine_schema = '{0}'
                  and routine_name = '{1}'
            '''.format(p_schema, p_object), True).Transpose('Property', 'Value')
        elif p_type == 'sequence':
            return self.v_connection.Query('''
                select next_not_cached_value as "Next Not Cached Value",
                       minimum_value as "Min Value",
                       maximum_value as "Max Value",
                       start_value as "Start Value",
                       increment as "Increment By",
                       cache_size as "Cache Size",
                       (case when 0 then 'No Cycle' else 'Cycle' end) as "Cycle Option",
                       cycle_count as "Cycle Count"
                from {0}.{1}
            '''.format(p_schema, p_object), True).Transpose('Property', 'Value')
        else:
            return None

    def GetDDL(self, p_schema, p_table, p_object, p_type):
        if p_type == 'function' or p_type == 'procedure':
            return self.v_connection.Query('show create {0} {1}.{2}'.format(p_type, p_schema, p_object), True, True).Rows[0][2]
        else:
            return self.v_connection.Query('show create {0} {1}.{2}'.format(p_type, p_schema, p_object), True, True).Rows[0][1]

    def GetAutocompleteValues(self, p_columns, p_filter):
        return None


import h5py
import pandas as pd
import numpy as np
from ann import *

if __name__ == '__main__':
    filename = 'numerical_data.h5'

    trainx = pd.read_hdf(filename, key='train_x').as_matrix()
    trainy = pd.read_hdf(filename, key='train_y').as_matrix()
    testx = pd.read_hdf(filename, key='test_x').as_matrix()
    testy = pd.read_hdf(filename, key='test_y').as_matrix()

    # train = pd.read_csv('numerical_train.csv')
    # test = pd.read_csv('numerical_test.csv')
    #
    # trainy = np.atleast_2d(train['re'].as_matrix()).T
    # trainx = train.loc[:, train.columns != 're'].as_matrix()
    # testy = np.atleast_2d(test['re'].as_matrix()).T
    # testx = test.loc[:, test.columns != 're'].as_matrix()

    # buld model
    # num_layers = 15
    # sizes = [150000, 100000, 75000, 50000, 25000, 20000,
    #          15000, 10000, 7000, 4000, 2000, 1000, 500, 100, 20]
    # sizes = [100 for _ in range(num_layers)]
    # model = ANN(input_size=trainx.shape[1], num_hidden_layers=num_layers, hidden_layer_sizes=sizes,
    #               output_size=1, epochs=250, batch_size=32, fit_verbose=1, optimizer='adam')
    # model.build_model()
    # checkpoint = ModelCheckpoint('model-{epoch:03d}.h5', verbose=1, monitor='mean_absolute_error', save_best_only=True, mode='min')
    # tensorboard = TensorBoard(log_dir='./tf_logs', histogram_freq=10, write_graph=True, write_grads=True, write_images=True)
    # model.train(trainx, trainy, validation_data=(testx, testy), callbacks=[checkpoint, tensorboard])

    # load model from file
    model = ANN()
    model.load_model('numerical_all/15x100NN_all/nn15x100_ep195.h5')

    y_hat = model.predict(testx)
    predictions = open("predictions.dat", 'w')
    testy = testy.tolist()
    for i in range(len(testy)):
        predictions.write(str(y_hat[i][0])+','+str(testy[i][0])+'\n')
    predictions.close()


from Nodo import Nodo

class ListaEncadeada:
    def __init__(self):
        self.primeiro = None
        self.ultimo = None
        self.tamanhoLista = 0

    def verificarListaVazia(self):
        if self.primeiro == None:
            return True
        else:
            return False

    def verificarTamanhoLista(self):
        return self.tamanhoLista

    def mostrarLista(self):
        nodoAtual = self.primeiro
        while (nodoAtual != None):
            print(nodoAtual.getNome(), end = " | ")
            nodoAtual = nodoAtual.getProximo()
        print(" ")

    #para inserir a gente passa o: nome do nodo e a ***posicao desejada
    def push(self, nomeNodo, indice):
       if (indice >= 0):
           #crio o primeiro nodo
            nodoCriado = Nodo(nomeNodo)
            #sprint(nodoCriado)
       if self.verificarListaVazia():
           #inserir nodo em uma lista: temos so um nodo e portanto ele é o primeiro e o ultimo
           self.primeiro = nodoCriado
           self.ultimo = nodoCriado
       else:
           #lista nao esta vazia - vamos comercar inserindo no inicio
            if (indice == 0):
                nodoCriado.setProximo(self.primeiro)
                self.primeiro = nodoCriado
            elif (indice >= self.tamanhoLista): #inserir no final
                self.ultimo.setProximo(nodoCriado)
                self.ultimo = nodoCriado
            else: #inserir no meio
                #preciso percorrer a lista até encontrar a posição desejada
               nodoAnterior = self.primeiro #backup do primeiro nodo
               nodoAtual = self.primeiro.getProximo()
               indiceAtual = 1

                #se nao é o primeiro nodo
                #comeco a percorrer a lista até encontrar
               while (nodoAtual != None):
                   if (indiceAtual == indice):
                       #primeiro inseri o nodo no meio da lista
                       nodoCriado.setProximo(nodoAtual)
                       #atualizei o encadeamento
                       nodoAnterior.setProximo(nodoCriado)

                       break
                   nodoAnterior = nodoAtual
                   nodoAtual = nodoAtual.getProximo()
                   indiceAtual +=1

       self.tamanhoLista +=1

    # metodo de remover
    def pop(self, indice):
        if not self.verificarListaVazia() and indice >= 0 and indice < self.tamanhoLista:
                flagRemove = False
                # removendo do inicio - tem apenas um elemento
                if self.primeiro.getProximo() == None:  # o proximo do primeiro nao existe ou seja  se so tem um elemento
                    # possui apenas 1 elemento
                    self.primeiro = None  # removendo o unico elemento
                    self.ultimo = None
                    flagRemove = True
                # removendo do inicio - tem mais de um elemento
                if indice == 0:  # deseja remover o primeiro elemento, mas a lista tem mais de um elemento
                    # remove do inicio, mais de um elemento
                    self.primeiro = self.primeiro.getProximo()  # removendo primeiro elemento, indice 0
                    flagRemove = True
                else:
                    # ** se chegou aqui é porque a lista possui mais de um elemento
                    # ** e a remoção nao é mais do inicio
                    nodoAnterior = self.primeiro  # fazendo backup
                    nodoAtual = self.primeiro.getProximo()
                    indiceAtual = 1
                    while(nodoAtual != None):
                        if (indiceAtual == indice):
                            nodoAnterior.setProximo(nodoAtual.getProximo())
                            nodoAtual.setProximo(None)
                            flagRemove = True
                            break
                        #se eu nao encontrar o indice, eu tenho que seguir procurando
                        nodoAnterior = nodoAtual
                        nodoAtual = nodoAtual.getProximo()
                        indiceAtual += 1

                if flagRemove:
                    # atualiza o tamanho da lista
                    self.tamanhoLista -= 1