#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created on Wed Mar 18 22:26:26 2020

@author: dale
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Main script the defines the neural network architecture
using keras.

Article References:
https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly
https://pypi.org/project/keras-video-generators/

GPU WORKING CONFIGURATION:
Ubuntu 19.10
tensorflow-gpu 2.1
Cuda 10.1
libcudnn7 (7.6)
nvidia-driver-440
RTX 2070 GPU

@author: Dale Kube
"""

import os
import glob
import numpy as np
from sklearn.utils import class_weight
from sklearn.metrics import log_loss
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras import Input, Model
from tensorflow.keras.losses import binary_crossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Dense, Dropout, ConvLSTM2D,\
BatchNormalization, MaxPooling2D, Flatten
from tensorflow.keras.applications import Xception
from tensorflow.keras.preprocessing.image import ImageDataGenerator

# TensorFlow version and GPU availability test
print(tf.__version__)
gpu_devices = tf.config.experimental.list_physical_devices('GPU')
print(gpu_devices)
assert len(gpu_devices) > 0, "GPU Not Found"
tf.config.experimental.set_memory_growth(gpu_devices[0], True)
tf.compat.v1.disable_eager_execution()

# Allow memory growth
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)

# Load the custom VideoFrameGenerator
parent_dir = '/mnt/extHDD/Kaggle/'
os.chdir(parent_dir)
from VideoFrameGenerator import VideoFrameGenerator

# Key elements for training
FRAME_COUNT = 15
FRAME_HEIGHT = 448
FRAME_WIDTH = 448
FRAME_CHANNELS = 1
BATCH_SIZE = 1
EPOCHS = 1
STEPS_PER_EPOCH = 2000
FACE_DETECT = False
N_NETS = 11

# Image Processing
imgGen = ImageDataGenerator(shear_range=0.2, zoom_range=0.2,\
rotation_range=10, horizontal_flip=True, featurewise_center=False,\
featurewise_std_normalization=False, rescale=False, height_shift_range=0.2,\
width_shift_range=0.2, brightness_range=[0.6,1.2])

# Define custom video generators
genTrain = VideoFrameGenerator(phase='training', face_detection=FACE_DETECT,\
batch_size=BATCH_SIZE, nb_frames=FRAME_COUNT, nb_channel=FRAME_CHANNELS,\
target_shape=(FRAME_HEIGHT,FRAME_WIDTH), transformation=imgGen, split=None,\
glob_pattern='./videos/training/{classname}/*.mp4')

genTest = VideoFrameGenerator(phase='training', face_detection=FACE_DETECT,\
batch_size=BATCH_SIZE, nb_frames=FRAME_COUNT, nb_channel=FRAME_CHANNELS,\
target_shape=(FRAME_HEIGHT,FRAME_WIDTH), transformation=None, split=None,\
glob_pattern='./videos/validation/{classname}/*.mp4')

# Calculate class weights for a balanced training data set
REAL = ['REAL' for i in glob.glob('./videos/training/REAL/*.mp4')]
FAKE = ['FAKE' for i in glob.glob('./videos/training/FAKE/*.mp4')]
y_train = REAL+FAKE
class_weights = class_weight.compute_class_weight('balanced',\
list(set(y_train)),y_train)
class_weights = dict(zip([0,1],class_weights))

# Calculate identify class labels for validation videos
REAL = [0 for i in glob.glob('./videos/validation/REAL/*.mp4')]
FAKE = [1 for i in glob.glob('./videos/validation/FAKE/*.mp4')]
y_val = REAL+FAKE

# Define custom metric
# Clip the ends of the predicted probabilities
def clip_loss(y_true, y_pred):
    clip_loss = binary_crossentropy(y_true, K.clip(y_pred, 0.05, 0.95))
    return clip_loss

# Define a new model
input_shape = Input((FRAME_COUNT, FRAME_HEIGHT, FRAME_WIDTH, FRAME_CHANNELS))
tower1 = ConvLSTM2D(3,(3,3), padding='same', activation='relu')(input_shape)
tower1 = BatchNormalization()(tower1)
tower1 = MaxPooling2D((2,2), strides=(1,1), padding='same')(tower1)
tower1 = Xception(include_top=False, weights='imagenet')(tower1)
tower1 = Flatten()(tower1)
out = Dropout(0.65)(tower1)
out = Dense(1, activation='sigmoid')(out)
model = Model(input_shape, out)

# Compile and view model summary
optimizer = Adam(learning_rate=1e-7)
model.compile(optimizer, loss='binary_crossentropy',\
metrics=['accuracy', clip_loss])
model.summary()

# Fit individual models and store predicted probabilities
model_preds = list()
for i in range(N_NETS):

    # Reset the states of all layers in the model
    model.reset_states()

    model.fit(genTrain, use_multiprocessing=True,\
    epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, workers=15,\
    max_queue_size=15, shuffle=True, class_weight=class_weights)

    # Model predicted probabilities
    yhati = model.predict(genTest, workers=15,\
    max_queue_size=15, use_multiprocessing=True, verbose=1)

    # Clip and store probabilities in the dictionary
    yhati = [np.clip(i, 0.05, 0.95) for i in yhati.flatten()]
    model_preds.append(yhati)

    # Print training round validation accuracy
    print('Round %.0f Complete' % (i+1))

# Calculate the cumulative performance boost
# with each added weak neural network
bootstrap = list()
for j in range(N_NETS):

    # Compute bootstrapped accuracy
    x = model_preds[:j+1]
    avgs = np.min(x, axis=0)
    bootstrap.append(log_loss(y_val, avgs))

# Visualize the log loss with each weak learner
plt.plot(bootstrap)
plt.xticks(range(N_NETS))


import matplotlib.pyplot as plt
import scipy
import VoigtFitting.VoigtFitting as VoigtFitting
import time
#import AbsorptionLine, ResolutionConvolution,FitManager

Fit = VoigtFitting.FitManager()

Ion = Fit.CreateIonGroup('FeII',['FeII_1608','FeII_1611'])
Ion.AddComponent('A',3.0e15, 1500000,4.9887)
Ion.AddComponent('B',1.0e14, 500000,4.9887)

#print Fit.CalcLnLikelihood()

Wavelength = scipy.arange(9000.0,10000.0,0.005)
Flux = Fit.ComputeFlux(Wavelength)

plt.plot(Wavelength,Flux,'-',color='black',label=r'$\delta$-function instrument profile',lw=2)

#Gaussian instrument profile:
Flux_convolve =  Fit.Convolve(1.25/2.35482,Wavelength)

plt.plot(Wavelength,Flux_convolve,'-',color='blue',label='Gaussian intrument profile',lw=2)

#Arbitrary instrument profile:
#Flux_convolve1 =  Fit.Convolve('Data/GaussInstrumentProfile.txt',Wavelength)
#plt.plot(Wavelength,Flux_convolve1,':',color='blue',label='Convolved with gaussian',lw=3)

#Arbitrary instrument profile:
Flux_convolve1 =  Fit.Convolve('Data/Prof.txt',Wavelength)
plt.plot(Wavelength,Flux_convolve1,'--',color='red',label='Measured instrument profile',lw=2)

#Arbitrary instrument profile:
#Flux_convolve1 =  Fit.Convolve('Data/Prof1.txt',Wavelength)
#plt.plot(Wavelength,Flux_convolve1,'--',color='green',label='Measured instrument profile',lw=2)

plt.xlim((9627,9657))
plt.ylim((-0.1,1.2))
plt.legend(loc=4)
plt.grid()
plt.xlabel('Wavelength [Angstrom]',fontsize=16)
plt.ylabel('Normalized flux',fontsize=16)
plt.show()


""" Validation of the modelDescription.xml """

from typing import List

from fmpy.model_description import ModelDescription

def validate_fmu(filename: str) -> List[str]:
    """ Validate the following aspects of an FMU

    modelDescription.xml:
    - validation against the XML schema
    - uniqueness and validity of variable names
    - completeness and integrity of the ModelStructure
    - required start values
    - combinations of causality and variability
    - units

    Parameters:
        filename  filename of the FMU, directory with extracted FMU or file like object

    Returns:
        a list of the problems found
    """

    from . import read_model_description
    from .model_description import ValidationError
    import os
    import zipfile

    problems = []

    # check file paths
    if isinstance(filename, str) and os.path.isfile(filename):
        with zipfile.ZipFile(filename, 'r') as zf:
            for file in zf.filelist:
                if '\\' in file.orig_filename:
                    problems.append(f'The file path "{file.orig_filename}" contains a backslash.')

    try:
        read_model_description(filename,
                               validate=True,
                               validate_variable_names=True,
                               validate_model_structure=True)
    except ValidationError as e:
        problems = e.problems
    except Exception as e:
        problems = [str(e)]

    return problems

def validate_model_description(model_description: ModelDescription, validate_variable_names: bool = False, validate_model_structure: bool = False) -> List[str]:

    problems = []

    if validate_variable_names:
        problems += _validate_variable_names(model_description)

    unit_definitions = {}

    for unit in model_description.unitDefinitions:
        unit_definitions[unit.name] = [display_unit.name for display_unit in unit.displayUnits]

    variable_names = set()

    # assert unique variable names (FMI 1.0 spec, p. 34, FMI 2.0 spec, p. 45)
    for v in model_description.modelVariables:
        if v.name in variable_names:
            problems.append(f'The variable name "{v.name}" (line {v.sourceline}) is not unique.')
        variable_names.add(v.name)

    is_fmi2 = model_description.fmiVersion == '2.0'
    is_fmi3 = model_description.fmiVersion.startswith('3.0')

    # assert legal variability
    for variable in model_description.modelVariables:
        if variable.type not in {'Real', 'Float32', 'Float64'} and variable.variability == 'continuous':
            problems.append(f'The variable "{variable.name}" (line {variable.sourceline}) is of type {variable.type}'
                            f' and must have variability != "continuous".')

    if is_fmi2 or is_fmi3:

        # assert required start values (see FMI 2.0 spec, p. 53)
        for v in model_description.modelVariables:
            if v.type != 'Clock' and (v.initial in {'exact', 'approx'} or v.causality == 'input') and v.start is None:
                problems.append(f'Variable "{v.name}" (line {v.sourceline}) has no start value.')

        # legal combinations of causality, variability, and initial
        legal_combinations = {

            ('structuralParameter', 'fixed', 'exact'),
            ('structuralParameter', 'tunable', 'exact'),

            ('parameter', 'fixed', 'exact'),
            ('parameter', 'tunable', 'exact'),

            ('calculatedParameter', 'fixed', 'calculated'),
            ('calculatedParameter', 'fixed', 'approx'),

            ('calculatedParameter', 'tunable', 'calculated'),
            ('calculatedParameter', 'tunable', 'approx'),

            ('input', 'discrete', 'exact'),
            ('input', 'continuous', 'exact'),

            ('output', 'constant', 'exact'),

            ('output', 'discrete', 'calculated'),
            ('output', 'discrete', 'exact'),
            ('output', 'discrete', 'approx'),

            ('output', 'continuous', 'calculated'),
            ('output', 'continuous', 'exact'),
            ('output', 'continuous', 'approx'),

            ('local', 'constant', 'exact'),

            ('local', 'fixed', 'calculated'),
            ('local', 'fixed', 'approx'),

            ('local', 'tunable', 'calculated'),
            ('local', 'tunable', 'approx'),

            ('local', 'discrete', 'calculated'),
            ('local', 'discrete', 'exact'),
            ('local', 'discrete', 'approx'),

            ('local', 'continuous', 'calculated'),
            ('local', 'continuous', 'exact'),
            ('local', 'continuous', 'approx'),

            ('independent', 'continuous', None),
        }

        for v in model_description.modelVariables:
            if (v.causality, v.variability, v.initial) not in legal_combinations:
                problems.append(f'The combination causality="{v.causality}" and variability="{v.variability}" '
                                f'in variable "{v.name}" (line {v.sourceline}) is not allowed.')

        # check for illegal start values (see FMI 2.0.2 spec, p. 49)
        for v in model_description.modelVariables:

            if v.initial == 'calculated' and v.start:
                problems.append(f'The variable "{v.name}" (line {v.sourceline}) has initial="calculated" but provides a start value.')

            if v.causality in 'independent' and v.start:
                problems.append(f'The variable "{v.name}" (line {v.sourceline}) has causality="independent" but provides a start value.')

        # validate units (see FMI 2.0 spec, p. 33ff.)
        for v in model_description.modelVariables:

            unit = v.unit

            if unit is None and v.declaredType is not None:
                unit = v.declaredType.unit

            if unit is not None and unit not in unit_definitions:
                problems.append(f'The unit "{unit}" of variable "{v.name}" (line {v.sourceline}) is not defined.')

            if v.displayUnit is not None and v.displayUnit not in unit_definitions[unit]:
                problems.append(f'The display unit "{v.displayUnit}" of variable "{v.name}" (line {v.sourceline}) is not defined.')

        if validate_model_structure:
            problems += _validate_model_structure(model_description)

    if is_fmi3:

        # assert independent variable
        if sum(v.causality == 'independent' for v in model_description.modelVariables) != 1:
            problems.append("Exactly one independent variable must be defined.")

        # assert unique value references
        variables = dict()

        for v in model_description.modelVariables:
            if v.valueReference in variables:
                p = variables[v.valueReference]
                problems.append(f'Variable "{v.name}" (line {v.sourceline}) has the same value reference as variable "{p.name}" (line {p.sourceline}).')
            else:
                variables[v.valueReference] = v

    return problems

def _validate_model_structure(model_description: ModelDescription) -> List[str]:

    problems = []

    # validate outputs
    expected_outputs = set(v for v in model_description.modelVariables if v.causality == 'output')
    outputs = set(u.variable for u in model_description.outputs)

    if expected_outputs != outputs:
        problems.append('ModelStructure/Outputs must have exactly one entry for each variable with causality="output".')

    # validate derivatives
    derivatives = set(v for v in model_description.modelVariables if v.derivative is not None)
    for i, state_derivative in enumerate(model_description.derivatives):
        if state_derivative.variable not in derivatives:
            problems.append(f'The variable "{state_derivative.variable.name}" (line {state_derivative.variable.sourceline}) '
                            f'referenced by the continuous state derivative {i + 1} (line {state_derivative.sourceline}) '
                            f'must have the attribute "derivative".')

    # validate initial unknowns
    expected_initial_unknowns = set()

    for variable in model_description.modelVariables:

        if variable.causality == 'output' and variable.initial in {'approx', 'calculated'} and not variable.clocks:
            expected_initial_unknowns.add(variable)

        if variable.causality == 'calculatedParameter':
            expected_initial_unknowns.add(variable)

    try:
        for unknown in model_description.derivatives:
            derivative = unknown.variable
            state = derivative.derivative
            for variable in [state, derivative]:
                if variable.initial in {'approx', 'calculated'}:
                    expected_initial_unknowns.add(variable)

        initial_unknowns = set(v.variable for v in model_description.initialUnknowns)

        if initial_unknowns != expected_initial_unknowns:
            expected = ', '.join(sorted(v.name for v in expected_initial_unknowns))
            actual = ', '.join(sorted(v.name for v in initial_unknowns))
            problem = ("ModelStructure/InitialUnknowns does not contain the expected set of variables."
                       f" Expected {{{ expected }}} but was {{{ actual }}}.")
            problems.append(problem)
    except:
        pass  # this check may fail due to inconsistencies detected above

    return problems

def _validate_variable_names(model_description: ModelDescription) -> List[str]:

    problems = []

    if model_description.variableNamingConvention == 'flat':

        for variable in model_description.modelVariables:

            if u'\u000D' in variable.name:
                problems.append(f'Variable "{variable.name}" (line {variable.sourceline}) contains an illegal carriage return character (U+000D).')

            if u'\u000A' in variable.name:
                problems.append(f'Variable "{variable.name}" (line {variable.sourceline}) contains an illegal line feed character (U+000A).')

            if u'\u0009' in variable.name:
                problems.append(f'Variable "{variable.name}" (line {variable.sourceline}) contains an illegal tab character (U+0009).')

    else:  # variableNamingConvention == structured

        from lark import Lark

        grammar = r"""
            name            : identifier | "der(" identifier ("," unsignedinteger)? ")"
            identifier      : bname arrayindices? ("." bname arrayindices?)*
            bname           : nondigit (nondigit|digit)* | qname
            nondigit        : "_" | "a".."z" | "A".."Z"
            digit           : "0".."9"
            qname           : "'" ( qchar | escape )+ "'"
            qchar           : nondigit | digit | "!" | "#" | "$" | "%" | "&" | "(" | ")"
                              | "*" | "+" | "," | "-" | "." | "/" | ":" | ";" | "<" | ">"
                              | "=" | "?" | "@" | "[" | "]" | "^" | "{" | "}" | "|" | "~" | " "
            escape          : "\'" | "\"" | "\?" | "\\" | "\a" | "\b" | "\f" | "\n" | "\r" | "\t" | "\v"
            arrayindices    : "[" unsignedinteger ("," unsignedinteger)* "]"
            unsignedinteger : digit+
            """

        parser = Lark(grammar, start='name')

        for variable in model_description.modelVariables:
            try:
                parser.parse(variable.name)
            except Exception as e:
                problems.append('"%s" (line %s) is not a legal variable name for naming convention "structured". %s'
                                % (variable.name, variable.sourceline, e))

    return problems


import numpy as np
import cv2 as cv

def nothing(x):
    pass

def defineColor():
    global r
    global g
    global b
    img = np.zeros((300, 512, 3), np.uint8)
    cv.namedWindow('image')
    # create trackbars for color change
    cv.createTrackbar('R', 'image', 0, 255, nothing)
    cv.createTrackbar('G', 'image', 0, 255, nothing)
    cv.createTrackbar('B', 'image', 0, 255, nothing)
    # create switch for ON/OFF functionality
    next_step = '0 : OFF \n1 : O'
    cv.createTrackbar(next_step, 'image', 0, 1, nothing)
    while 1:
        cv.imshow('image', img)
        k = cv.waitKey(1) & 0xFF
        if k == 27:
            cv.destroyAllWindows()
            return None

        # get current positions of four trackbars
        r = cv.getTrackbarPos('R', 'image')
        g = cv.getTrackbarPos('G', 'image')
        b = cv.getTrackbarPos('B', 'image')
        turn_on = cv.getTrackbarPos(next_step, 'image')
        if turn_on == 1:
            return [r, g, b]

        img[:] = [b, g, r]

lower_color = np.array(defineColor())
upper_color = np.array(defineColor())

cap = cv.VideoCapture(0)
while 1:
    # Take each frame
    _, frame = cap.read()
    # Convert BGR to HSV
    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)
    # define range of blue color in HSV
    lower_blue = np.array([110, 50, 50])
    upper_blue = np.array([130, 255, 255])
    # Threshold the HSV image to get only blue colors
    mask = cv.inRange(hsv, lower_color, upper_color)
    # Bitwise-AND mask and original image
    res = cv.bitwise_and(frame, frame, mask=mask)
    cv.imshow('frame', frame)
    cv.imshow('mask', mask)
    cv.imshow('res', res)
    k = cv.waitKey(5) & 0xFF
    if k == 27:
        break
cv.destroyAllWindows()


#!/usr/bin/python

'''
WAS  (Whole Air Sample) log as google-earth overlay. The flight track covered during
the time the bottle was filled is represented as a line.

'''

import netCDF4
import numpy as np
import os
import re
import sys
import shutil
import tempfile
import zipfile

import faampy

_KML_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2" xmlns:gx="http://www.google.com/kml/ext/2.2" xmlns:kml="http://www.opengi
s.net/kml/2.2" xmlns:atom="http://www.w3.org/2005/Atom">

<Folder>
<Style id="was">
  <IconStyle>
    <Icon>
      <href>icons/was_bottle_32x32.png</href>
    </Icon>
  </IconStyle>
  <LineStyle>
    <color>#ff0000ff</color>
    <width>8</width>
  </LineStyle>
</Style>

    <name>%s</name>
        <open>0</open>
"""

_KML_FOOTER = """</Folder>
</kml>"""

_KML_WAS_BOTTLE = """<Placemark>
  <name>%s@%i m</name>
  <description>
     %s to %s@%im
  </description>
  <styleUrl>#was</styleUrl>
  <MultiGeometry>
  <Point>
  <altitudeMode>absolute</altitudeMode>
      <coordinates>
        %8.4f,%8.4f,%8.2f
      </coordinates>
    </Point>
    <LineString>
      <altitudeMode>absolute</altitudeMode>
      <coordinates>
      %s
      </coordinates>
    </LineString>
   </MultiGeometry>
   </Placemark>
"""

# global variables
_KMZ_TMP_DIRECTORY = tempfile.mkdtemp()
_FID = ''

def __zip__(outfile):
    files4zipping = []
    for root, subFolders, files in os.walk(_KMZ_TMP_DIRECTORY):
        for f in files:
            files4zipping.append(os.path.join(root,f))
    zip = zipfile.ZipFile(outfile, mode='w')
    for file in files4zipping:
        zipname = file[len(_KMZ_TMP_DIRECTORY) + (len(os.sep)):]
        zip.write(file, zipname)
    return

def __setup_kmz__():
    icon_path = os.path.join(_KMZ_TMP_DIRECTORY, 'icons')
    os.mkdir(icon_path)
    icon_file = os.path.join(os.path.dirname(faampy.__file__),
                             '..', 'files', 'icons',
                             'was_bottle_32x32.png')
    shutil.copy(icon_file, icon_path)

def t2s(s):
    s = s.split(':')
    result = int(s[0]) * 3600 + int(s[1]) * 60 + int(s[2])
    return result

def get_index(ds, secs):
    ix = np.where(ds.variables['Time'][:] == secs)
    return ix[0]

def read_was_log(was_log_file):
    result = []
    f = open(was_log_file, 'r')
    lines = f.readlines()
    f.close()
    for line in lines:
        if (line.strip()[0]).isdigit():
            result.append(str.split(line))
    return result

def parse_was_log(was_log_file, ds):
    was_log_lines = read_was_log(was_log_file)
    plcmark_txt = ""
    for was in was_log_lines:
        if not was:
            continue
        # get coordinates for bottle start
        s_secs = t2s(was[1])
        s_ix = int(get_index(ds, s_secs))
        e_secs = t2s(was[2])
        e_ix = int(get_index(ds, e_secs))
        if len(ds.variables['ALT_GIN'][:].shape) == 2:
            s_lon = ds.variables['LON_GIN'][:][s_ix, 0]
            s_lat = ds.variables['LAT_GIN'][:][s_ix, 0]
            s_alt = ds.variables['ALT_GIN'][:][s_ix, 0]
            # get coordinates for bottle end
            e_lon = ds.variables['LON_GIN'][:][e_ix, 0]
            e_lat = ds.variables['LAT_GIN'][:][e_ix, 0]
            e_alt = ds.variables['ALT_GIN'][:][e_ix, 0]
        else:
            s_lon = ds.variables['LON_GIN'][:][s_ix]
            s_lat = ds.variables['LAT_GIN'][:][s_ix]
            s_alt = ds.variables['ALT_GIN'][:][s_ix]
            # get coordinates for bottle end
            e_lon = ds.variables['LON_GIN'][:][e_ix]
            e_lat = ds.variables['LAT_GIN'][:][e_ix]
            e_alt = ds.variables['ALT_GIN'][:][e_ix]

        linstring_txt = "%f,%f,%f\n%f,%f,%f" % (s_lon, s_lat, s_alt,
                                                e_lon, e_lat, e_alt)
        plcmark_txt += _KML_WAS_BOTTLE % (_FID+': ' +was[0], s_alt, was[1], was[2], s_alt, s_lon, s_lat, s_alt, linstring_txt)
    return plcmark_txt

def process(was_log_file, ncfile, outpath):
    global _FID, _KMZ_TMP_DIRECTORY

    _KMZ_TMP_DIRECTORY = tempfile.mkdtemp()
    __setup_kmz__()
    _FID = re.search('[bBcC]\d{3}', ncfile).group()
    ds = netCDF4.Dataset(ncfile, 'r')
    if hasattr(ds, 'title'):
        flight_desc = str(ds.title).split()[2] + '-' + str(ds.title).split()[4] + '-WAS-Bottles'
    elif hasattr(ds, 'Title'):
        flight_desc = str(ds.Title).split()[2] + '-' + str(ds.Title).split()[4] + '-WAS-Bottles'
    else:
        flight_desc = ''
    was_kml = open(os.path.join(_KMZ_TMP_DIRECTORY, 'doc.kml'), 'w')
    plcmark_txt = parse_was_log(was_log_file, ds)
    ds.close()
    was_kml.write(_KML_HEADER % (flight_desc))
    was_kml.write(plcmark_txt)
    was_kml.write(_KML_FOOTER)
    was_kml.close()
    __zip__(os.path.join(outpath, 'faam-was_%s.kmz' % _FID))

def __get_ncfile__(path, fid):
    pattern = '.*core_faam.*%s_.*.nc' % str.lower(fid)
    for root, subFolders, files in os.walk(path):
        for f in files:
            if re.match(pattern, f):
                return os.path.join(root, f)

def _argparser():
    import argparse
    sys.argv.insert(0, 'faampy ge_was_to_kmz')
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('was_log_file', action="store", type=str,
                        help='Input WAS log file or folder which contains log files')
    parser.add_argument('ncfile', action="store", type=str,
                       help='input netCDF-file or path to netcdf files')
    parser.add_argument('outpath', action="store", type=str,
                        help='outpath for kmz file')
    return parser

def main():
    parser = _argparser()
    args = parser.parse_args()
    if os.path.isdir(args.was_log_file):
        tmp = os.listdir(args.was_log_file)
        new_list = []
        for t in tmp:
            if re.match('[b,B]\d{3}.WAS', t):
                new_list.append(os.path.join(args.was_log_file, t))
        args.was_log_file = new_list
    else:
        args.was_log_file = [args.was_log_file, ]

    for was_log_file in args.was_log_file:
        if os.path.isdir(args.ncfile):
            ncfile = __get_ncfile__(args.ncfile, os.path.basename(os.path.splitext(was_log_file)[0]))
        else:
            ncfile = args.ncfile
        try:
            process(was_log_file, ncfile, args.outpath)
        except:
            sys.stdout.write('Problem processing ... %s\n' % (os.path.basename(was_log_file)))

if __name__ == '__main__':
    main()


from collections import Counter, defaultdict

class Solution:

    def isAnswer(self, counts, tSet, tCounts):
        for alpha in tSet:
            if counts[alpha] < tCounts[alpha]:
                return False
        return True

    def minWindow(self, s: str, t: str) -> str:
        res = ""

        l = r = 0
        tSet = set(t)
        tCounts = Counter(t)

        counts = defaultdict(int)
        counts[s[r]] += 1

        while l <= r and r != len(s):
            if self.isAnswer(counts, tSet, tCounts):
                if res == "" or len(res) > (r - l + 1):
                    res = s[l:r+1]

                counts[s[l]] -= 1
                l += 1
            else:
                r += 1
                if r != len(s):
                    counts[s[r]] += 1

        return res

"""
Given two strings s and t of lengths m and n respectively, return the minimum window substring of s such that every character in t (including duplicates) is included in the window. If there is no such substring, return the empty string "".

The testcases will be generated such that the answer is unique.

Example 1:

Input: s = "ADOBECODEBANC", t = "ABC"
Output: "BANC"
Explanation: The minimum window substring "BANC" includes 'A', 'B', and 'C' from string t.
Example 2:

Input: s = "a", t = "a"
Output: "a"
Explanation: The entire string s is the minimum window.
Example 3:

Input: s = "a", t = "aa"
Output: ""
Explanation: Both 'a's from t must be included in the window.
Since the largest window of s only has one 'a', return empty string.

Constraints:

m == s.length
n == t.length
1 <= m, n <= 105
s and t consist of uppercase and lowercase English letters.

"""


import numpy as np
from pandas import DataFrame

def split_matrix_horizontal(matrix, value):
    upper = []
    lower = []
    for i in range (matrix.shape[0]):
        if i < value:
            upper.append(matrix[i])
        elif i > value:
            lower.append(matrix[i])
    return upper, lower

def split_matrix_vertical(matrix, value):
    left = []
    right = []
    for row in matrix:
        left.append(row[:value])
        right.append(row[value+1:])
    return left, right

def add_horizontals(upper, lower):
    if upper.__len__() == lower.__len__():
        return upper + lower
    else:
        diff = abs(upper.__len__() - lower.__len__())
        print("diff","hori", diff, type(upper), type(lower))
        if upper.__len__() > lower.__len__():
            result = upper[diff:] + lower
        else:
            result = lower[diff:] + upper
        return result

def add_vertical(left, right):
    left = np.array(left)
    right = np.array(right)
    if left.shape[1]== right.shape[1]:
        return left + right
    else:
        diff = abs(left.shape[1] - right.shape[1])
        print("diff","vert", diff)
        if left.__len__() > right.__len__():
            for i in range(left.__len__()):
                result = left[i][diff:] + right[i]
        else:
            for i in range(left.__len__()):
                result = right[i][diff:] + left[i]
        return result

def fold_matrix(matrix, axis, value):
    if axis == "y":
        upper, lower = split_matrix_horizontal(matrix, value)
        lower = np.flip(lower, axis=0)
        result = add_horizontals(upper, lower)
        return result
    if axis == "x":
        left, right = split_matrix_vertical(matrix, value)
        right = np.flip(right, axis=1)
        result = add_vertical(left,right)
        return result

def calc_values_in_arr(matrix):
    rows = matrix.__len__()
    cols = matrix[0].__len__()
    sum = 0
    for row in range(rows):
        for col in range(cols):
            if matrix[row][col] != 0:
                sum +=1
    return sum

def matrix_to_readable(matrix):
    rows = matrix.__len__()
    cols = matrix[0].__len__()
    result = []
    for row in range(rows):
        res_row = []
        for col in range(cols):
            if matrix[row][col] != 0:
                res_row.append("#")
            else:
                res_row.append(".")
        result.append(res_row)
    return result

def run():
    file = open("day13")
    lines = file.read().splitlines()

    #dots = lines[:-3]
    dots = lines[:-13]
    print (dots)
    X = []
    Y = []
    for dot in dots:
        x, y = dot.strip().split(",")
        X.append(int(x))
        Y.append(int(y))

    height = max(Y)+2
    width = max(X)+1

    matrix = np.zeros((height, width))
    for i in range(X.__len__()):
        matrix[Y[i]][X[i]] = 1

    instructions = lines[-12:]
    #instructions = lines[-2:]
    print(instructions)
    I = []
    for i in instructions:
        axis, value = i.split("=")
        axis = axis[-1]
        value = int(value)
        I.append([axis, value])
    print(I)

    for instruction in instructions:
        instruction = instruction.split("=")
        matrix = fold_matrix(matrix, instruction[0][-1], int(instruction[1]))

    test_m = np.matrix([[1,2,3,4,], [5,6,7,8]])
    add_m = np.matrix([[1,1,1],[0,0,0]])
    #test_res = add_vertical(test_m, add_m)
    #print("test", test_res)
#    test_m[1:] += add_m
    print(test_m)
    print(np.flip(test_m, axis=0))
    values = calc_values_in_arr(matrix)
    str_matrix = matrix_to_readable(matrix)
    print(values)
    print(DataFrame(matrix))
    print(DataFrame(str_matrix))
    print(str_matrix)

    output_file = open("output_13", "a")
    for i in range(str_matrix[0].__len__() //5):
        for row in str_matrix:
            print(row[i*5:i*5+5])
        print()
    output_file.close()

run()


import uuid
import faulthandler
from functools import cache
from html import escape as html_escape
import numpy as np
import scipy.stats

from psycopg.rows import dict_row as psycopg_rows_dict_row

from pydantic import BaseModel

faulthandler.enable()  # will catch segfaults and write to STDERR

from lib.db import DB

def rescale_energy_value(value, unit):
    # We only expect values to be mJ for energy!
    if unit != 'mJ' and not unit.startswith('ugCO2e/'):
        raise RuntimeError('Unexpected unit occured for energy rescaling: ', unit)

    unit_type = unit[1:]

    if unit.startswith('ugCO2e'): # bring also to mg
        value = value / (10**3)
        unit = f"m{unit_type}"

    if value > 1_000_000_000: return [value/(10**12), f"G{unit_type}"]
    if value > 1_000_000_000: return [value/(10**9), f"M{unit_type}"]
    if value > 1_000_000: return [value/(10**6), f"k{unit_type}"]
    if value > 1_000: return [value/(10**3), f"{unit_type}"]
    if value < 0.001: return [value*(10**3), f"n{unit_type}"]

    return [value, unit] # default, no change

def is_valid_uuid(val):
    try:
        uuid.UUID(str(val))
        return True
    except ValueError:
        return False

def html_escape_multi(item):
    """Replace special characters "'", "\"", "&", "<" and ">" to HTML-safe sequences."""
    if item is None:
        return None

    if isinstance(item, str):
        return html_escape(item)

    if isinstance(item, list):
        return [html_escape_multi(element) for element in item]

    if isinstance(item, dict):
        for key, value in item.items():
            if isinstance(value, str):
                item[key] = html_escape(value)
            elif isinstance(value, dict):
                item[key] = html_escape_multi(value)
            elif isinstance(value, list):
                item[key] = [
                    html_escape_multi(item)
                    if isinstance(item, dict)
                    else html_escape(item)
                    if isinstance(item, str)
                    else item
                    for item in value
                ]
        return item

    if isinstance(item, BaseModel):
        item_copy = item.model_copy(deep=True)
        # we ignore keys that begin with model_ because pydantic v2 renamed a lot of their fields from __fields to model_fields:
        # https://docs.pydantic.dev/dev-v2/migration/#changes-to-pydanticbasemodel
        # This could cause an error if we ever make a BaseModel that has keys that begin with model_
        keys = [key for key in dir(item_copy) if not key.startswith('_') and not key.startswith('model_') and not callable(getattr(item_copy, key))]
        for key in keys:
            setattr(item_copy, key, html_escape_multi(getattr(item_copy, key)))
        return item_copy

    return item

def get_machine_list():
    query = """
            SELECT id, description, available
            FROM machines
            ORDER BY description ASC
            """
    return DB().fetch_all(query)

def get_run_info(run_id):
    query = """
            SELECT
                id, name, uri, branch, commit_hash,
                (SELECT STRING_AGG(t.name, ', ' ) FROM unnest(runs.categories) as elements
                    LEFT JOIN categories as t on t.id = elements) as categories,
                filename, start_measurement, end_measurement,
                measurement_config, machine_specs, machine_id, usage_scenario,
                created_at, invalid_run, phases, logs
            FROM runs
            WHERE id = %s
            """
    params = (run_id,)
    return DB().fetch_one(query, params=params, row_factory=psycopg_rows_dict_row)

def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_date=None, end_date=None, detail_name=None, limit_365=False, sorting='run'):

    if filename is None or filename.strip() == '':
        filename =  'usage_scenario.yml'

    params = [uri, filename, machine_id, f"%{phase}"]

    branch_condition = 'AND r.branch IS NULL'
    if branch is not None and branch.strip() != '':
        branch_condition = 'AND r.branch = %s'
        params.append(branch)

    metrics_condition = ''
    if metrics is None or metrics.strip() == '' or metrics.strip() == 'key':
        metrics_condition =  "AND (p.metric LIKE '%%_energy_%%' OR metric = 'software_carbon_intensity_global')"
    elif metrics.strip() != 'all':
        metrics_condition =  "AND p.metric = %s"
        params.append(metrics)

    start_date_condition = ''
    if start_date is not None and start_date.strip() != '':
        start_date_condition =  "AND DATE(r.created_at) >= TO_DATE(%s, 'YYYY-MM-DD')"
        params.append(start_date)

    end_date_condition = ''
    if end_date is not None and end_date.strip() != '':
        end_date_condition =  "AND DATE(r.created_at) <= TO_DATE(%s, 'YYYY-MM-DD')"
        params.append(end_date)

    detail_name_condition = ''
    if detail_name is not None and detail_name.strip() != '':
        detail_name_condition =  "AND p.detail_name = %s"
        params.append(detail_name)

    limit_365_condition = ''
    if limit_365:
        limit_365_condition = "AND r.created_at >= CURRENT_DATE - INTERVAL '365 days'"

    sorting_condition = 'r.commit_timestamp ASC, r.created_at ASC'
    if sorting is not None and sorting.strip() == 'run':
        sorting_condition = 'r.created_at ASC, r.commit_timestamp ASC'

    query = f"""
            SELECT
                r.id, r.name, r.created_at, p.metric, p.detail_name, p.phase,
                p.value, p.unit, r.commit_hash, r.commit_timestamp,
                row_number() OVER () AS row_num
            FROM runs as r
            LEFT JOIN phase_stats as p ON
                r.id = p.run_id
            WHERE
                r.uri = %s
                AND r.filename = %s
                AND r.end_measurement IS NOT NULL
                AND r.machine_id = %s
                AND p.phase LIKE %s
                {branch_condition}
                {metrics_condition}
                {start_date_condition}
                {end_date_condition}
                {detail_name_condition}
                {limit_365_condition}
                AND r.commit_timestamp IS NOT NULL
            ORDER BY
                p.metric ASC, p.detail_name ASC,
                p.phase ASC, {sorting_condition}

            """
    return (query, params)

def determine_comparison_case(ids):

    query = '''
            WITH uniques as (
                SELECT uri, filename, machine_id, commit_hash, COALESCE(branch, 'main / master') as branch FROM runs
                WHERE id = ANY(%s::uuid[])
                GROUP BY uri, filename, machine_id, commit_hash, branch
            )
            SELECT
                COUNT(DISTINCT uri ), COUNT(DISTINCT filename), COUNT(DISTINCT machine_id),
                COUNT(DISTINCT commit_hash ), COUNT(DISTINCT branch)
            FROM uniques
    '''

    data = DB().fetch_one(query, (ids, ))
    if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result
        raise RuntimeError('Could not determine compare case')

    [repos, usage_scenarios, machine_ids, commit_hashes, branches] = data

    # If we have one or more measurement in a phase_stat it will currently just be averaged
    # however, when we allow comparing runs we will get same phase_stats but with different repo etc.
    # these cannot be just averaged. But they have to be split and then compared via t-test
    # For the moment I think it makes sense to restrict to two repositories. Comparing three is too much to handle I believe if we do not want to drill down to one specific metric

    # Currently we support six cases:
    # case = 'Repository' # Case D : RequirementsEngineering Case
    # case = 'Branch' # Case C_3 : SoftwareDeveloper Case
    # case = 'Usage Scenario' # Case C_2 : SoftwareDeveloper Case
    # case = 'Machine' # Case C_1 : DataCenter Case
    # case = 'Commit' # Case B: DevOps Case
    # case = 'Repeated Run' # Case A: Blue Angel
    # case = 'Multi-Commit' # Case D: Evolution of repo over time

    if repos == 2: # diff repos
        if usage_scenarios <= 2: # diff repo, diff usage scenarios. diff usage scenarios are NORMAL for now
            if machine_ids == 2: # diff repo, diff usage scenarios, diff machine_ids
                raise RuntimeError('Different repos & machines not supported')
            if machine_ids == 1: # diff repo, diff usage scenarios, same machine_ids
                if branches <= 2:
                    if commit_hashes <= 2: # diff repo, diff usage scenarios, same machine_ids,  same branches, diff/same commits_hashes
                        # for two repos we expect two different hashes, so this is actually a normal case
                        # even if they are identical we do not care, as the repos are different anyway
                        case = 'Repository' # Case D
                    else:
                        raise RuntimeError('Different repos & more than 2 different commits commits not supported')
                else:
                    raise RuntimeError('Different repos & more than 2 branches not supported')
            else:
                raise RuntimeError('Less than 1 or more than 2 Machines and different repos not supported.')
        else:
            raise RuntimeError('Only 2 or less usage scenarios for different repos not supported.')
    elif repos == 1: # same repos
        if usage_scenarios == 2: # same repo, diff usage scenarios
            if machine_ids == 2: # same repo, diff usage scenarios, diff machines
                raise RuntimeError('Different usage scenarios & machines not supported')
            if branches <= 1:
                if commit_hashes == 1: # same repo, diff usage scenarios, same machines, same branches, same commit hashes
                    case = 'Usage Scenario' # Case C_2
                else: # same repo, diff usage scenarios, same machines, same branches, diff commit hashes
                    raise RuntimeError('Different usage scenarios & commits not supported')
            else: # same repo, diff usage scenarios, same machines, diff branches
                raise RuntimeError('Different usage scenarios & branches not supported')
        elif usage_scenarios == 1: # same repo, same usage scenario
            if machine_ids == 2: # same repo, same usage scenarios, diff machines
                if branches <= 1:
                    if commit_hashes == 1: # same repo, same usage scenarios, diff machines, same branches, same commit hashes
                        case = 'Machine' # Case C_1
                    else: # same repo, same usage scenarios, diff machines, same branches, diff commit hashes
                        raise RuntimeError('Different machines & commits not supported')
                else: # same repo, same usage scenarios, diff machines, diff branches
                    raise RuntimeError('Different machines & branches not supported')

            elif machine_ids == 1: # same repo, same usage scenarios, same machines
                if branches <= 1:
                    if commit_hashes == 2: # same repo, same usage scenarios, same machines, diff commit hashes
                        case = 'Commit' # Case B
                    elif commit_hashes > 2: # same repo, same usage scenarios, same machines, many commit hashes
                        raise RuntimeError('Multiple commits comparison not supported. Please switch to Timeline view')
                    else: # same repo, same usage scenarios, same machines, same branches, same commit hashes
                        case = 'Repeated Run' # Case A
                else: # same repo, same usage scenarios, same machines, diff branch
                    # diff branches will have diff commits in most cases. so we allow 2, but no more
                    if commit_hashes <= 2:
                        case = 'Branch' # Case C_3
                    else:
                        raise RuntimeError('Different branches and more than 2 commits not supported')
            else:
                raise RuntimeError('Less than 1 or more than 2 Machines per repo not supported.')
        else:
            raise RuntimeError('Less than 1 or more than 2 Usage scenarios per repo not supported.')

    else:
        # The functionality I imagine here is, because comparing more than two repos is very complex with
        # multiple t-tests / ANOVA etc. and hard to grasp, only a focus on one metric shall be provided.
        raise RuntimeError('Less than 1 or more than 2 repos not supported for overview. Please apply metric filter.')

    return case

def get_phase_stats(ids):
    query = """
            SELECT
                a.phase, a.metric, a.detail_name, a.value, a.type, a.max_value, a.min_value, a.unit,
                b.uri, c.description, b.filename, b.commit_hash, COALESCE(b.branch, 'main / master') as branch
            FROM phase_stats as a
            LEFT JOIN runs as b on b.id = a.run_id
            LEFT JOIN machines as c on c.id = b.machine_id

            WHERE
                a.run_id = ANY(%s::uuid[])
            ORDER BY
                a.phase ASC,
                a.metric ASC,
                a.detail_name ASC,
                b.uri ASC,
                b.machine_id ASC,
                b.filename ASC,
                b.commit_hash ASC,
                branch ASC,
                b.created_at ASC
            """
    data = DB().fetch_all(query, (ids, ))
    if data is None or data == []:
        raise RuntimeError('Data is empty')
    return data

# Would be interesting to know if in an application server like gunicor @cache
# Will also work for subsequent requests ...?
'''  Object structure
    comparison_case: str
    statistics: dict
                                        // MEAN of the detailed-metric over all repos -> non-sense
                                        // MEAN of the detailed-metric over all usage_scenarios -> debateable
                                        // MEAN of the detailed-metric per machines  -> interesting
                                        // MEAN of the detailed-metric per commits -> debateable

                                        // Case A: No T-Test necessary, as only repetition. STDDEV only important
                                        // Case B: comparison as T-test betweeen every commit ???
                                        // => Technically possible, but we will most likely only have 1 commit.
                                        // => So maybe only compare last commit against all before?
                                        // => Yes, that sounds best! Still
                                        // Case C_1: comparison as T-test betweeen the two machines
                                        // => Impossible if we have only one sample
                                        // => Problematic if samples are < 20
                                        // Case C_2: comparison as T-test betweeen the two usage scenarios
                                        // => Impossible if we have only one sample
                                        // => Problematic if samples are < 20
                                        // Case C_3: comparison as T-test betweeen the two usage scenarios
                                        // => Impossible if we have only one sample
                                        // => Problematic if samples are < 20
                                        // Case D: comparison as T-test betweeen the two repos
                                        // => Impossible if we have only one sample
                                        // => Problematic if samples are < 20

                                        // although we can have 2 commits on 2 repos, we do not keep
                                        // track of the multiple commits here as key
                                        // currently the system is limited to compare only two runs until we have
                                        // figured out how big our StdDev is and how many runs we can run per day
                                        // at all (and how many repetitions are needed and possbile time-wise)
                                        repo/usage_scenarios/machine/commit/:
                                            mean: // mean per commit/repo etc.
                                            stddev:
                                            ci:
                                            p_value: // Only case A for last key vs compare to the rest. one-sided t-test
                                            is_significant: // Only case A .... one-sided t-test
                                            values: [] // 1 for stats.html. 1+ for cases
                                            max_values: []

    data: dict -> key: repo/usage_scenarios/machine/commit/branch
        run_1: dict
        run_2: dict
        ...
        run_x : dict  -> key: phase_name
            [BASELINE]: dict
            [INSTALLATION]: dict
            ....
            [PHASE]: dict -> key: metric_name
                -mean: - // will NOT be implemented. See explanation
                    // mean of the phase for all the metrics and their details
                    // this really only makes sense for all the energy values, and then only for selected ones ...
                    // actually only really helpful if we have multiple metrics that report the same value
                    // this shall NOT be supported
                // we reserve this level for phase-level data
                // atm there is only the data key
                core_energy_powermetrics_component: dict
                ...
                ane_energy_powermetrics_component: dict
                    clean_name: str
                    explanation: str
                    source: str
                    ....
                    mean: float
                        // mean of the metric over all details per per repo / usage_scenario etc.
                        // => Interesting for multiple docker containers
                        repo/usage_scenarios/machine/commit/: number
                        repo/usage_scenarios/machine/commit/: number
                        ...
                    stddev: float
                    p-value: float

                    data: dict -> key: detail_name
                        [COMPONENT]: dict
                        [SYSTEM]: dict
                        ...
                        [MACHINE]:
                                mean: float // mean of the metric for this detail for this phase
                                stddev: float
                                ci: float
                                max: float
                                values: list // the actual values
            ...
'''
def get_phase_stats_object(phase_stats, case):

    phase_stats_object = {
        'comparison_case': case,
        'comparison_details': set(),
        'data': {}
    }

    for phase_stat in phase_stats:
        [
            phase, metric_name, detail_name, value, metric_type, max_value, min_value, unit,
            repo, machine_description, filename, commit_hash, branch
        ] = phase_stat # unpack

        phase = phase.split('_', maxsplit=1)[1] # remove the 001_ prepended stuff again, which is only for ordering

        if case == 'Repository':
            key = repo # Case D : RequirementsEngineering Case
        elif case == 'Branch':
            key = branch # Case C_3 : SoftwareDeveloper Case
        elif case == 'Usage Scenario':
            key = filename # Case C_2 : SoftwareDeveloper Case
        elif case == 'Machine':
            key = machine_description # Case C_1 : DataCenter Case
        else:
            key = commit_hash # No comparison case / Case A: Blue Angel / Case B: DevOps Case

        if phase not in phase_stats_object['data']: phase_stats_object['data'][phase] = {}

        if metric_name not in phase_stats_object['data'][phase]:
            phase_stats_object['data'][phase][metric_name] = {
                'type': metric_type,
                'unit': unit,
                #'mean': None, # currently no use for that
                #'stddev': None,  # currently no use for that
                #'ci': None,  # currently no use for that
                #'p_value': None,  # currently no use for that
                #'is_significant': None,  # currently no use for that
                'data': {},
            }

        if detail_name not in phase_stats_object['data'][phase][metric_name]['data']:
            phase_stats_object['data'][phase][metric_name]['data'][detail_name] = {
                'name': detail_name,
                # 'mean': None, # mean for a detail over multiple machines / branches makes no sense
                # 'max': max_value, # max for a detail over multiple machines / branches makes no sense
                # 'min': min_value, # min for a detail over multiple machines / branches makes no sense
                # 'stddev': None, # stddev for a detail over multiple machines / branches makes no sense
                # 'ci': None, # since we only compare two keys atm this  could no be calculated.
                'p_value': None, # comparing the means of two machines, branches etc. Both cases must have multiple values for this to get populated
                'is_significant': None, # comparing the means of two machines, branches etc. Both cases must have multiple values for this to get populated
                'data': {},
            }

        detail_data = phase_stats_object['data'][phase][metric_name]['data'][detail_name]['data']
        if key not in detail_data:
            detail_data[key] = {
                'mean': None, # this is the mean over all repetitions of the detail_name for the key
                'max': max_value,
                'min': min_value,
                'max_mean': None,
                'min_mean': None,
                'stddev': None,
                'ci': None,
                'p_value': None, # only for the last key the list compare to the rest. one-sided t-test
                'is_significant': None, # only for the last key the list compare to the rest. one-sided t-test
                'values': [],
            }
            phase_stats_object['comparison_details'].add(key)

        detail_data[key]['values'].append(value)

        # since we do not save the min/max values we need to to the comparison here in every loop again
        # all other statistics are derived later in add_phase_stats_statistics()
        detail_data[key]['max'] = max((x for x in [max_value, detail_data[key]['max']] if x is not None), default=None)
        detail_data[key]['min'] = min((x for x in [min_value, detail_data[key]['min']] if x is not None), default=None)

    phase_stats_object['comparison_details'] = list(phase_stats_object['comparison_details'])

    return phase_stats_object

'''
    Here we need to traverse the object again and calculate all the averages we need
    This could have also been done while constructing the object through checking when a change
    in phase / detail_name etc. occurs., however this is more efficient
'''
def add_phase_stats_statistics(phase_stats_object):

    for _, phase_data in phase_stats_object['data'].items():
        for _, metric in phase_data.items():
            for _, detail in metric['data'].items():
                for _, key_obj in detail['data'].items():

                    # if a detail has multiple values we calculate a std.dev and the one-sided t-test for the last value

                    key_obj['mean'] = key_obj['values'][0] # default. might be overridden
                    key_obj['max_mean'] = key_obj['values'][0] # default. might be overridden
                    key_obj['min_mean'] = key_obj['values'][0] # default. might be overridden

                    if len(key_obj['values']) > 1:
                        t_stat = get_t_stat(len(key_obj['values']))

                        # JSON does not recognize the numpy data types. Sometimes int64 is returned
                        key_obj['mean'] = float(np.mean(key_obj['values']))
                        key_obj['stddev'] = float(np.std(key_obj['values']))
                        key_obj['max_mean'] = np.max(key_obj['values']) # overwrite with max of list
                        key_obj['min_mean'] = np.min(key_obj['values']) # overwrite with min of list
                        key_obj['ci'] = key_obj['stddev']*t_stat

                        if len(key_obj['values']) > 2:
                            data_c = key_obj['values'].copy()
                            pop_mean = data_c.pop()
                            _, p_value = scipy.stats.ttest_1samp(data_c, pop_mean)
                            if not np.isnan(p_value):
                                key_obj['p_value'] = p_value
                                if key_obj['p_value'] > 0.05:
                                    key_obj['is_significant'] = False
                                else:
                                    key_obj['is_significant'] = True

    ## builds stats between the keys
    if len(phase_stats_object['comparison_details']) == 2:
        # since we currently allow only two comparisons we hardcode this here
        # this is then needed to rebuild if we would allow more
        key1 = phase_stats_object['comparison_details'][0]
        key2 = phase_stats_object['comparison_details'][1]

        # we need to traverse only one branch of the tree like structure, as we only need to compare matching metrics
        for _, phase_data in phase_stats_object['data'].items():
            for _, metric in phase_data.items():
                for _, detail in metric['data'].items():
                    if key1 not in detail['data'] or key2 not in detail['data']:
                        continue

                    # Welch-Test because we cannot assume equal variances
                    _, p_value = scipy.stats.ttest_ind(detail['data'][key1]['values'], detail['data'][key2]['values'], equal_var=False)

                    if not np.isnan(p_value):
                        detail['p_value'] = p_value
                        if detail['p_value'] > 0.05:
                            detail['is_significant'] = False
                        else:
                            detail['is_significant'] = True

    return phase_stats_object

@cache
def get_t_stat(length):
    #alpha = .05
    if length <= 1: return None
    dof = length-1
    t_crit = np.abs(scipy.stats.t.ppf((.05)/2,dof)) # for two sided!
    return t_crit/np.sqrt(length)


"""Meta-analyses regarding flipping of ranks between systems."""

from __future__ import annotations

import itertools

from explainaboard.info import SysOutputInfo
from explainaboard.meta_analyses.meta_analysis import MetaAnalysis
from explainaboard.meta_analyses.utils import report_to_sysout
from explainaboard.utils.typing_utils import unwrap

class RankFlippingMetaAnalysis(MetaAnalysis):
    """A class to perform meta-analysis of how ranks flip between buckets."""

    def __init__(self, model1_report: SysOutputInfo, model2_report: SysOutputInfo):
        """Constructor.

        Args:
            model1_report: The report for the first model.
            model2_report: The report for the second model.
        """
        self.model1_report = model1_report
        self.model2_report = model2_report

    def run_meta_analysis(self) -> dict | list:
        """Return the result of the meta-analysis."""
        # construct the new "metadata", treating each metric as a "feature"
        metadata = self._metrics_to_metadata()

        # construct the new "system outputs", treating each bucket from each
        # report as an "example/observation" to be analyzed/bucketed further
        model1_buckets = report_to_sysout(self.model1_report)
        model2_buckets = report_to_sysout(self.model2_report)

        # get reference info
        reference_dict = self._get_reference_info(metadata)
        self.reference_dict = reference_dict

        # calculate paired metrics. This is what will be bucketed further to
        # obtain the results of meta-analysis.
        aggregated_sysout = self._get_aggregated_sysout(
            model1_buckets, model2_buckets, reference_dict, metadata
        )
        self.aggregated_sysout = (
            aggregated_sysout  # save before bucketing for more fine-grained analysis
        )

        # bucket the paired system outputs
        return self._bucket_aggregated_sysout(aggregated_sysout, metadata)

    @staticmethod
    def _is_flipped(val1: float, val2: float, reference: bool) -> bool:
        """Check whether values are flipped.

        This function checks whether `val1` and `val2` stands in the *opposite*
        relationship as what is expected in `reference`. This is a sort of signal
        for "surprise".

        Args:
            val1: the first value
            val2: the second value
            reference: true if we expect `val1` should be greater than `val2`.

        Returns:
            True if we are surprised by the result
        """
        return (val1 > val2) != reference

    def _metrics_to_metadata(self) -> dict:
        """Turns a `metric_configs` object into a metadata object for system output.

        Uses the metric configs of `self.model1_report` as metadata.

        Returns:
            The metadata dictionary.
        """
        metric_config_dict_iter = (
            level.metric_configs for level in self.model1_report.analysis_levels
        )
        # TODO(odashi): This iterator may generate name collision.
        metric_names_iter = itertools.chain.from_iterable(metric_config_dict_iter)

        metadata = {
            "custom_features": {
                name: {
                    "dtype": "string",  # for rank flipping, True or False
                    "description": name,
                    "num_buckets": 2,  # for rank flipping, True or False
                }
                for name in metric_names_iter
            }
        }
        return metadata

    def _get_reference_info(self, metadata: dict) -> dict:
        """Return whether model1 has a higher score than model2 for each metric.

        Interpretation: this tells us which model (model1 or model2) we should
        expect to outperform the other, for each metric.

        The rank-flipping meta-analysis will then reveal which buckets, and how
        many, subvert this expectation.
        """
        m1_overall = list(
            itertools.chain.from_iterable(unwrap(self.model1_report.results.overall))
        )
        m2_overall = list(
            itertools.chain.from_iterable(unwrap(self.model2_report.results.overall))
        )
        model1_metric_is_greater = {
            metric_name: m1_overall[metric_name].value > m2_overall[metric_name].value
            for metric_name in metadata["custom_features"].keys()
        }
        return model1_metric_is_greater

    def _get_aggregated_sysout(
        self,
        model1_buckets: list[dict],
        model2_buckets: list[dict],
        reference_dict: dict,
        metadata: dict,
    ) -> list[dict]:
        """Find the rank-flipping statistics for each bucket.

        Check whether, for each bucket,
        the relationship between model1's bucket-level metrics and model2's bucket-level
        metrics is the opposite from what is expected.

        Args:
            model1_buckets: The buckets for model 1
            model2_buckets: The buckets for model 2
            reference_dict: Whether we expect model 1 to be better than model 2
            metadata: Contains `custom_features`, the features to analyze

        Returns:
            A list of dictionaries indicating the rank flipping statistics.
        """
        paired_score_examples = []
        for m1_bucket, m2_bucket in zip(model1_buckets, model2_buckets):
            # bucket info (feature name, bucket interval, bucket name, bucket size)
            # should match exactly
            if m1_bucket["feature_name"] != m2_bucket["feature_name"]:
                raise ValueError(
                    f"feature name does not match:\n{m1_bucket} vs {m2_bucket}"
                )
            if m1_bucket["bucket_interval"] != m2_bucket["bucket_interval"]:
                raise ValueError(
                    f"bucket interval does not match:\n{m1_bucket} vs {m2_bucket}"
                )
            if m1_bucket["bucket_name"] != m2_bucket["bucket_name"]:
                raise ValueError(
                    f"bucket name does not match:\n{m1_bucket} vs {m2_bucket}"
                )
            if m1_bucket["bucket_size"] != m2_bucket["bucket_size"]:
                raise ValueError(
                    f"bucket size does not match:\n{m1_bucket} vs {m2_bucket}"
                )

            # calculate difference metrics
            example = {
                "feature_name": m1_bucket["feature_name"],
                "bucket_interval": m1_bucket["bucket_interval"],
                "bucket_name": m1_bucket["bucket_name"],
                "bucket_size": m1_bucket["bucket_size"],
            }
            for feature in metadata["custom_features"].keys():
                reference = reference_dict[feature]
                example[feature] = RankFlippingMetaAnalysis._is_flipped(
                    m1_bucket[feature], m2_bucket[feature], reference
                )
            paired_score_examples.append(example)

        return paired_score_examples

    def _bucket_aggregated_sysout(
        self, aggregated_sysout: list[dict], metadata: dict
    ) -> dict:
        """Taking paired_sysout do bucketing based on the metadata provided.

        Here we write the code which does the bucketing directly in this method,
        but in the future we may consider using the base ExplainaBoard package
        to do it for us, since `paired_sysout` is already in the format of a
        legitimate system output.

        Args:
            aggregated_sysout: The system output aggregated according to
              `_get_aggregated_sysout()`
            metadata: The metadata containing `custom_features` indicating which
              features to analyze.

        Returns:
            A dictionary with counts of how often the rank is flipped.
        """
        # for rank-flipping, it's very easy to bucket, since there are only 2
        # buckets (True or False). Let's just immediately calculate it here.
        rank_flipping_buckets = {
            metric_name: {"ranking_same": 0, "ranking_flipped": 0}
            for metric_name in metadata["custom_features"].keys()
        }

        for example in aggregated_sysout:
            for metric_name in metadata["custom_features"].keys():
                if example[metric_name]:
                    rank_flipping_buckets[metric_name]["ranking_flipped"] += 1
                else:
                    rank_flipping_buckets[metric_name]["ranking_same"] += 1
        return rank_flipping_buckets


from flask import Flask, render_template, request

from enc_dec import encrypt_to_emojis, decrypt_to_text, emoji_mapping, reverse_emoji_mapping

app = Flask(__name__)

# Load the HTML page
@app.route('/')
def index():
    return render_template('index.html')

# Encrypt route
@app.route('/encrypt', methods=['POST'])
def encrypt():
    plain_text = request.form['plain_text']
    emojis_text = encrypt_to_emojis(plain_text)
    return emojis_text

# Decrypt route
@app.route('/decrypt', methods=['POST'])
def decrypt():
    emojis_text = request.form['emojis_text']
    decrypted_text = decrypt_to_text(emojis_text)
    return decrypted_text

if __name__ == "__main__":
    app.run(debug=True)


import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import sys

def read_data(model):
    gene_list = pd.read_csv('./GradientBoostingClassifier.drugnome_ai_predictions_'+model+'.csv', sep=",")
    print(gene_list.head())

    glist = gene_list['Gene_Name'].squeeze().str.strip().tolist()
    top_genes = glist[0:int(len(glist)*0.05)]
    print(top_genes[:10])

    ot_gene_list = pd.read_csv('./ot_clinical_data_'+model+'.tsv', sep="\t")
    print(ot_gene_list.head())

    return top_genes, ot_gene_list

def analyze_phase_molecule_type(top_genes, ot_gene_list):
    ot_list = ot_gene_list['target'].unique()

    no_clinic_genes = len(top_genes) - len(ot_list)
    print(no_clinic_genes)

    clinic = {}
    clinic['sm'] = [0 for x in range(5)]
    clinic['ab'] = [0 for x in range(5)]
    clinic['other'] = [0 for x in range(5)]

    sm_genes = set()

    for x in [4,3,2,1,0]:
        sm_list = ot_gene_list.loc[(ot_gene_list['mol_type'] == 'Small molecule') & (ot_gene_list['phase'] == x)]
        sm_set = set(sm_list['target'].unique()) - sm_genes
        clinic['sm'][x] = len(sm_set)
        sm_genes = sm_genes | sm_set

    ab_genes = set()

    for x in [4,3,2,1,0]:
        ab_list = ot_gene_list.loc[(ot_gene_list['mol_type'] == 'Antibody') & (ot_gene_list['phase'] == x)]
        ab_set = set(ab_list['target'].unique()) - ab_genes
        clinic['ab'][x] = len(ab_set)
        ab_genes = ab_genes | ab_set

    other_genes = set()

    for x in [4,3,2,1,0]:
        other_list = ot_gene_list.loc[(~ot_gene_list['mol_type'].isin(['Small molecule', 'Antibody'])) & (ot_gene_list['phase'] == x)]
        other_set = set(other_list['target'].unique()) - other_genes
        clinic['other'][x] = len(other_set)
        other_genes = other_genes | other_set

    return clinic, no_clinic_genes

def draw_bargraph(model, clinic, no_clinic_genes):
    plt.rcParams.update({'font.size': 18})

    clinic['no_clinic'] = [0,0,0,0,no_clinic_genes]
    labels = ['Small Molecule','Antibody','Other','No clinical data']
    x = np.arange(len(labels))
    width = 0.2

    fig, ax = plt.subplots()
    fig.set_figheight(8)
    fig.set_figwidth(8)

    ax.bar(x - (width*2), [clinic['sm'][4],clinic['ab'][4],clinic['other'][4],clinic['no_clinic'][0]], width, label='Phase IV')
    ax.bar(x - width, [clinic['sm'][3],clinic['ab'][3],clinic['other'][3],clinic['no_clinic'][0]], width, label='Phase III')
    ax.bar(x, [clinic['sm'][2],clinic['ab'][2],clinic['other'][2],clinic['no_clinic'][0]], width, label='Phase II')
    ax.bar(x + width, [clinic['sm'][1],clinic['ab'][1],clinic['other'][1],clinic['no_clinic'][0]], width, label='Phase I')
    ax.bar(x + width, [clinic['sm'][0],clinic['ab'][0],clinic['other'][1],clinic['no_clinic'][4]], width, label='No Clinical Data')

    ax.set_ylabel('Number of genes')
    ax.set_title('Number of top ranked genes per molecule type\nand clinical trial phase (DrugnomeAI-'+model+')')
    ax.set_xticks(x)
    ax.set_xticklabels(labels)
    plt.xticks(rotation=45, ha='right')
    ax.tick_params(axis=u'both', which=u'both',length=0)

    ax.legend()
    fig.tight_layout()

    plt.show()
    plt.savefig('./phase_mol_bar_'+model+'.pdf')

if __name__ == '__main__':
    model = sys.argv[1]

    top_genes, ot_gene_list = read_data(model)
    clinic, no_clinic_genes = analyze_phase_molecule_type(top_genes, ot_gene_list)
    draw_bargraph(model, clinic, no_clinic_genes)


import pandas as pd
from sklearn.feature_selection import VarianceThreshold
import numpy as np

def drop_miss_column(data: pd.DataFrame) -> pd.DataFrame:
    """Remove columns with NaN values"""

    missing_val_cols = data.columns[data.isnull().any()]
    x = data.copy()
    df_ = x.drop(missing_val_cols, axis=1)

    return df_

def low_var(data: pd.DataFrame, threshold=0.01) -> pd.DataFrame:
    """This function removes low variance columns in dataframe
    threshold is minimum allowed variance in data columns"""

    selector = VarianceThreshold(threshold)
    selector.fit(data)
    return data[data.columns[selector.get_support(indices=True)]]

def high_corr(data: pd.DataFrame, threshold=0.95) -> pd.DataFrame:
    """Removes correlated columns in data frame with pearson
    correlation higher then threshold"""

    cor_matrix = data.corr().abs()
    upper_tri = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(np.bool))
    cols = [column for column in upper_tri.columns if any(upper_tri[column] > threshold)]
    x = data.copy()

    return x.drop(cols, axis=1)

def high_skewed(data: pd.DataFrame, threshold=4) -> pd.DataFrame:
    """Retain dataframe columns with skewness lower then threshold"""

    return data[data.columns[abs(data.skew()) < threshold]]

def compose(*function_param):
    def inner(arg):
        for f, param in function_param:
            arg = f(arg, param)
        return arg

    return inner

class Select_descriptors:
    """This class combines the above filtering steps"""

    def __init__(self, var_: float, cor_: float, skew_: float,wrapper:object):
        self.var_ = var_
        self.cor_ = cor_
        self.skew_ = skew_
        self.transforms = [(x, y) for x, y in zip([low_var, high_corr, high_skewed], [var_, cor_, skew_]) if
                           y is not None]

        self.wrapper   = wrapper
        self.variables = None

    def transform(self, *args) :
        """Input descriptors data frame with smiles column"""
        func = compose(*self.transforms)
        x    = args[0].copy()
        descriptors = func(x.drop('SMILES', axis=1))

        if self.wrapper == None:
            self.variables = descriptors.columns

            return pd.concat([x['SMILES'], descriptors], axis=1)
        else:
            y = args[1].reset_index(drop=True)
            desc_sel = self.wrapper.fit_transform(descriptors,y)
            self.variables = descriptors.columns[self.wrapper.get_support()]

            return pd.concat([x['SMILES'],pd.DataFrame(desc_sel, columns=self.variables)],axis=1)


# -*- coding: utf-8 -*-
#https://www.kaggle.com/act444/lb-0-158-xgb-handcrafted-leaky?scriptVersionId=1188812

import argparse
import functools
from collections import defaultdict

import numpy as np
import pandas as pd
import xgboost as xgb

from nltk.corpus import stopwords
from collections import Counter
from sklearn.metrics import log_loss
from sklearn.cross_validation import train_test_split

from xgboost import XGBClassifier

def word_match_share(row, stops=None):
    q1words = {}
    q2words = {}
    for word in row['question1']:
        if word not in stops:
            q1words[word] = 1
    for word in row['question2']:
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0
    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))
    return R

def jaccard(row):
    wic = set(row['question1']).intersection(set(row['question2']))
    uw = set(row['question1']).union(row['question2'])
    if len(uw) == 0:
        uw = [1]
    return (len(wic) / len(uw))

def common_words(row):
    return len(set(row['question1']).intersection(set(row['question2'])))

def total_unique_words(row):
    return len(set(row['question1']).union(row['question2']))

def total_unq_words_stop(row, stops):
    return len([x for x in set(row['question1']).union(row['question2']) if x not in stops])

def wc_diff(row):
    return abs(len(row['question1']) - len(row['question2']))

def wc_ratio(row):
    l1 = len(row['question1'])*1.0
    l2 = len(row['question2'])
    if l2 == 0:
        return np.nan
    if l1 / l2:
        return l2 / l1
    else:
        return l1 / l2

def wc_diff_unique(row):
    return abs(len(set(row['question1'])) - len(set(row['question2'])))

def wc_ratio_unique(row):
    l1 = len(set(row['question1'])) * 1.0
    l2 = len(set(row['question2']))
    if l2 == 0:
        return np.nan
    if l1 / l2:
        return l2 / l1
    else:
        return l1 / l2

def wc_diff_unique_stop(row, stops=None):
    return abs(len([x for x in set(row['question1']) if x not in stops]) - len([x for x in set(row['question2']) if x not in stops]))

def wc_ratio_unique_stop(row, stops=None):
    l1 = len([x for x in set(row['question1']) if x not in stops])*1.0
    l2 = len([x for x in set(row['question2']) if x not in stops])
    if l2 == 0:
        return np.nan
    if l1 / l2:
        return l2 / l1
    else:
        return l1 / l2

def same_start_word(row):
    if not row['question1'] or not row['question2']:
        return np.nan
    return int(row['question1'][0] == row['question2'][0])

def char_diff(row):
    return abs(len(''.join(row['question1'])) - len(''.join(row['question2'])))

def char_ratio(row):
    l1 = len(''.join(row['question1']))
    l2 = len(''.join(row['question2']))
    if l2 == 0:
        return np.nan
    if l1 / l2:
        return l2 / l1
    else:
        return l1 / l2

def char_diff_unique_stop(row, stops=None):
    return abs(len(''.join([x for x in set(row['question1']) if x not in stops])) - len(''.join([x for x in set(row['question2']) if x not in stops])))

def get_weight(count, eps=10000, min_count=2):
    if count < min_count:
        return 0
    else:
        return 1 / (count + eps)

def tfidf_word_match_share_stops(row, stops=None, weights=None):
    q1words = {}
    q2words = {}
    for word in row['question1']:
        if word not in stops:
            q1words[word] = 1
    for word in row['question2']:
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0

    shared_weights = [weights.get(w, 0) for w in q1words.keys() if w in q2words] + [weights.get(w, 0) for w in q2words.keys() if w in q1words]
    total_weights = [weights.get(w, 0) for w in q1words] + [weights.get(w, 0) for w in q2words]

    R = np.sum(shared_weights) / np.sum(total_weights)
    return R

def tfidf_word_match_share(row, weights=None):
    q1words = {}
    q2words = {}
    for word in row['question1']:
        q1words[word] = 1
    for word in row['question2']:
        q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0

    shared_weights = [weights.get(w, 0) for w in q1words.keys() if w in q2words] + [weights.get(w, 0) for w in q2words.keys() if w in q1words]
    total_weights = [weights.get(w, 0) for w in q1words] + [weights.get(w, 0) for w in q2words]

    R = np.sum(shared_weights) / np.sum(total_weights)
    return R

def build_features(data, stops, weights):
    X = pd.DataFrame()
    f = functools.partial(word_match_share, stops=stops)
    print("word_match")
    X['word_match'] = data.apply(f, axis=1, raw=True) #1

    print("tfidf_wm")
    f = functools.partial(tfidf_word_match_share, weights=weights)
    X['tfidf_wm'] = data.apply(f, axis=1, raw=True) #2

    print("tfidf_wm_stops")
    f = functools.partial(tfidf_word_match_share_stops, stops=stops, weights=weights)
    X['tfidf_wm_stops'] = data.apply(f, axis=1, raw=True) #3

    print("jaccard")
    X['jaccard'] = data.apply(jaccard, axis=1, raw=True) #4
    print("wc_diff")
    X['wc_diff'] = data.apply(wc_diff, axis=1, raw=True) #5
    print("wc_ratio")
    X['wc_ratio'] = data.apply(wc_ratio, axis=1, raw=True) #6
    print("wc_diff_unique")
    X['wc_diff_unique'] = data.apply(wc_diff_unique, axis=1, raw=True) #7
    print("wc_ratio_unique")
    X['wc_ratio_unique'] = data.apply(wc_ratio_unique, axis=1, raw=True) #8

    print("wc_diff_unq_stop")
    f = functools.partial(wc_diff_unique_stop, stops=stops)
    X['wc_diff_unq_stop'] = data.apply(f, axis=1, raw=True) #9
    print("wc_ratio_unique_stop")
    f = functools.partial(wc_ratio_unique_stop, stops=stops)
    X['wc_ratio_unique_stop'] = data.apply(f, axis=1, raw=True) #10

    print("same_start")
    X['same_start'] = data.apply(same_start_word, axis=1, raw=True) #11
    print("char_diff")
    X['char_diff'] = data.apply(char_diff, axis=1, raw=True) #12

    print("char_diff_unq_stop")
    f = functools.partial(char_diff_unique_stop, stops=stops)
    X['char_diff_unq_stop'] = data.apply(f, axis=1, raw=True) #13

    print("common_words")
    #X['common_words'] = data.apply(common_words, axis=1, raw=True)  #14
    print("total_unique_words")
    X['total_unique_words'] = data.apply(total_unique_words, axis=1, raw=True)  #15

    print("total_unq_words_stop")
    f = functools.partial(total_unq_words_stop, stops=stops)
    X['total_unq_words_stop'] = data.apply(f, axis=1, raw=True)  #16

    print("char_ratio")
    X['char_ratio'] = data.apply(char_ratio, axis=1, raw=True) #17

    return X

laptop = 0
if laptop == 1:
    drive = "C"
else:
    drive = "F"

preprocessing = "_spacy_cleaned"
#preprocessing = "_MyMagic3_spacyLemma_trigram"

import numpy as np
import pandas as pd
import operator
from sklearn.model_selection import train_test_split
from collections import Counter
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
from pylab import plot, show, subplot, specgram, imshow, savefig
from sklearn.model_selection import KFold
K = 5
kf = KFold(n_splits = K)
#==============================================================================
# print(list(enumerate(kf.split([1,2,3,4,5,6,7,8,9]))))
# X= np.array(["a","b","c","d","e","f","g","h","i"])
# for kth, (train_index, test_index) in enumerate(kf.split(X)):
#     print(test_index)
#     max = np.amax(test_index)+1
#     min = np.amin(test_index)
#     print(X[min:max])
#==============================================================================

np.set_printoptions(threshold=400000)
pd.set_option('display.max_rows', 2000, 'display.max_columns', 2000,  'display.show_dimensions', 'truncate')

RS = 12357
np.random.seed(RS)

print("Loading data")
input_folder = drive + ':/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/'
df_train = pd.read_csv(input_folder + 'train.csv')
df_test  = pd.read_csv(input_folder + 'test.csv')
df_test.drop(["question1", "question2"], axis=1, inplace=True)

#adding final x features
print("Loading x final features")
#x = pd.read_csv(drive + ':/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/x_final_features.csv', header=0)
#x = pd.read_csv(drive + ':/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/x_final_features_expended.csv', header=0)
#x = pd.read_csv(drive + ':/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/x_final_features_lowRemoved.csv', header=0)
x = pd.read_csv(drive + ':/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/x_final_features' + preprocessing  + '.csv', header=0)
#x = x.drop(["rarestWordID1", "rarestWordID2", 'max_kcore', "tfidf_word_match", "wordMatchShare"], axis=1)

#corr = np.corrcoef(x)
print(x.columns)
print(x.describe())

feature_names = list(x.columns.values)
print("Features: {}".format(feature_names))

x_train = x[:df_train.shape[0]].astype("float64").reset_index()
x_test_real  = x[df_train.shape[0]:].astype("float64").reset_index()
labels = df_train['is_duplicate'].values
x_train_real =  x_train
labels_real =  labels
del df_train, x

print("setting parser")
parser = argparse.ArgumentParser(description='XGB with Handcrafted Features')
parser.add_argument('--save', type=str, default='XGB_leaky',
                    help='save_file_names')
args = parser.parse_args()

X_train_ab = x_train
X_train_ab = X_train_ab.drop('euclidean_distance', axis=1)
X_train_ab = X_train_ab.drop('jaccard_distance', axis=1)
X_train_ab = X_train_ab.drop('q2_freq', axis=1)
X_train_ab = X_train_ab.drop('q1_freq', axis=1)
X_train_ab = X_train_ab.drop('q1_q2_intersect', axis=1)
X_train_ab = X_train_ab.drop("word_match", axis=1)

print("load train")
df_train = pd.read_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/train_spacy_cleaned.csv')
df_train = df_train.fillna(' ')

print("load test")
df_test = pd.read_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/test_spacy_cleaned.csv')
ques = pd.concat([df_train[['question1', 'question2']], \
    df_test[['question1', 'question2']]], axis=0).reset_index(drop='index')

print("qdict")
q_dict = defaultdict(set)
for i in range(ques.shape[0]):
        q_dict[ques.question1[i]].add(ques.question2[i])
        q_dict[ques.question2[i]].add(ques.question1[i])

def q1_freq(row):
    return(len(q_dict[row['question1']]))

def q2_freq(row):
    return(len(q_dict[row['question2']]))

def q1_q2_intersect(row):
    return(len(set(q_dict[row['question1']]).intersection(set(q_dict[row['question2']]))))

print("q1_q2_intersect")
df_train['q1_q2_intersect'] = df_train.apply(q1_q2_intersect, axis=1, raw=True)
print("q1_freq")
df_train['q1_freq'] = df_train.apply(q1_freq, axis=1, raw=True)
print("q2_freq")
df_train['q2_freq'] = df_train.apply(q2_freq, axis=1, raw=True)

print("q1_q2_intersect")
df_test['q1_q2_intersect'] = df_test.apply(q1_q2_intersect, axis=1, raw=True)
print("q1_freq")
df_test['q1_freq'] = df_test.apply(q1_freq, axis=1, raw=True)
print("q2_freq")
df_test['q2_freq'] = df_test.apply(q2_freq, axis=1, raw=True)

test_leaky = df_test.loc[:, ['q1_q2_intersect','q1_freq','q2_freq']]
del df_test

train_leaky = df_train.loc[:, ['q1_q2_intersect','q1_freq','q2_freq']]

# explore
stops = set(stopwords.words("english"))

print("lower split 1")
df_train['question1'] = df_train['question1'].map(lambda x: str(x).lower().split())
print("lower split 2")
df_train['question2'] = df_train['question2'].map(lambda x: str(x).lower().split())

print("train_qs")
train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist())

words = [x for y in train_qs for x in y]
counts = Counter(words)
weights = {word: get_weight(count) for word, count in counts.items()}

print('Building Features')
X_train_full = build_features(df_train, stops, weights)
X_train_full = pd.concat((X_train_full, X_train_ab, train_leaky), axis=1)
y_train = df_train['is_duplicate'].values

X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train, test_size=0.1, random_state=4242)

#UPDownSampling
print("updownsampling")
pos_train = X_train[y_train == 1]
neg_train = X_train[y_train == 0]
X_train = pd.concat((neg_train, pos_train.iloc[:int(0.8*len(pos_train))], neg_train))
y_train = np.array([0] * neg_train.shape[0] + [1] * pos_train.iloc[:int(0.8*len(pos_train))].shape[0] + [0] * neg_train.shape[0])
print(np.mean(y_train))
del pos_train, neg_train

pos_valid = X_valid[y_valid == 1]
neg_valid = X_valid[y_valid == 0]
X_valid = pd.concat((neg_valid, pos_valid.iloc[:int(0.8 * len(pos_valid))], neg_valid))
y_valid = np.array([0] * neg_valid.shape[0] + [1] * pos_valid.iloc[:int(0.8 * len(pos_valid))].shape[0] + [0] * neg_valid.shape[0])
print(np.mean(y_valid))
del pos_valid, neg_valid

params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 7
params['subsample'] = 0.6
params['base_score'] = 0.2
#params['scale_pos_weight'] = 0.36

print(list(X_train))

d_train = xgb.DMatrix(X_train, label=y_train)
d_valid = xgb.DMatrix(X_valid, label=y_valid)

watchlist = [(d_train, 'train'), (d_valid, 'valid')]

bst = xgb.train(params, d_train, 2500, watchlist, early_stopping_rounds=50, verbose_eval=10)
print(log_loss(y_valid, bst.predict(d_valid)))
bst.save_model(args.save + '.mdl')

print('Building Test Features')
x_test_real = x_test_real.drop('euclidean_distance', axis=1)
x_test_real = x_test_real.drop('jaccard_distance', axis=1)
x_test_real = x_test_real.drop('q2_freq', axis=1)
x_test_real = x_test_real.drop('q1_freq', axis=1)
x_test_real = x_test_real.drop('q1_q2_intersect', axis=1)
x_test_real = x_test_real.drop("word_match", axis=1)

df_test = pd.read_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/test_spacy_cleaned.csv')
df_test = df_test.fillna(' ')

df_test['question1'] = df_test['question1'].map(lambda x: str(x).lower().split())
df_test['question2'] = df_test['question2'].map(lambda x: str(x).lower().split())

x_test_build = build_features(df_test, stops, weights)
x_test = pd.concat((x_test_build, x_test_real, test_leaky), axis=1)
d_test = xgb.DMatrix(x_test)
p_test = bst.predict(d_test)
sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/layer1Features/handcraftKernel0158_spacy_cleaned.csv', index=False)

#saving variables
X_train_full.to_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/train_4_spacy_cleaned.csv')
x_test.to_csv('F:/DS-main/Kaggle-main/Quora Question Pairs - inputs/data/test_4_spacy_cleaned.csv')


import os
from ccs2.semantic.semantic_exception import SemanticException

class SemanticAnalysis:

    def __init__(self):
        self.symbolTable = []

    def insertSymbol(self, newSymbol, expression = None, isDeclaring = False):

        symbolValue = None
        if (expression):
            try:
                symbolValue = eval(expression)
            except Exception:
                pass

        if (newSymbol.type == 'int' and symbolValue and symbolValue % 1 != 0):
            raise SemanticException('Variable (' + newSymbol.identifier + ') of type int with floating point value (' + str(symbolValue) + ')')

        if (not isDeclaring):
            symbol = self.checkIdentifierExistence(newSymbol.identifier, newSymbol.scope, identifierValueInUse=False)
            symbol.setInitialized(True)
            if (symbol.type == 'int' and (symbolValue and symbolValue % 1 != 0)):
                raise SemanticException('Variable (' + newSymbol.identifier + ') of type int with floating point value (' + str(symbolValue) + ')')
        else:
            for symbol in self.symbolTable:
                if (
                    symbol.identifier == newSymbol.identifier and
                    symbol.scope == newSymbol.scope and isDeclaring
                ):
                    raise SemanticException('Variable (' + str(newSymbol.identifier) + ') already declared')
            self.symbolTable.append(newSymbol)

    def checkIdentifierExistence(self, identifier, scope, identifierValueInUse = True):
        for symbol in self.symbolTable:
            if (
                symbol.identifier == identifier and
                symbol.scope <= scope
            ):
                if (identifierValueInUse and not symbol.initialized):
                    raise SemanticException("Variable (" + symbol.identifier + ') was declared but not initialized')

                return symbol

        raise SemanticException("Variable (" + str(identifier) + ") undeclared")

    def outputSymbolTable(self, code_name): #pragma: no cover
        text_file = open(os.getcwd() + "/output/" + code_name + "_symbol_table", "w")
        for symbol in self.symbolTable:
            text_file.write(symbol.__repr__() + '\n')
        text_file.close()


massive = []
N = int(input("Number of skittles: "))
cointer = 0
for i in range(N):
    skittle = 'I'
    massive.append(skittle)
K = int(input("Number of shots: "))
for j in range(K):
    li = int(input("Enter the lowest number of knocked-down skittles: "))
    ri = int(input("Enter the highest number of knocked-down skittles: "))
    if ri >= li:
        for i in range(li,ri+1):
            massive[i-1] = '.'

for i in range(N):
    print(massive[i], end = ' ')

input()


import unittest
from nine.maryjane import calculator

class MyTestCase(unittest.TestCase):

    def test_something(self):
        result = calculator.addition(9, 4)
        self.assertEqual(result, 13)  # add assertion here

    def test_subtraction(self):
        result = calculator.subtraction(3, 9)
        self.assertEqual(result, -6)

if __name__ == '__main__':
    unittest.main()


import pygame as p
import im
import sta
ds = p.display.set_mode((500, 300))
mx,my=p.mouse.get_pos()
array=[[0,0],[0,0]]
black = (0,0,0)
color2 = (255,255,255)
white = (255,255,255)
color = black
p.font.init()
font = p.font.Font('freesansbold.ttf', 32)
font2 = p.font.Font('freesansbold.ttf', 20)
text = font.render('Pause', True, color, white)
text6 = font2.render('Inventory full', True, color, white)
textRect = text.get_rect()
textRect.center = (250, 50)
eqspells = [1,2,None,None]
learnedspells = [3,5,1]
inv=[im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty,im.Empty]
playerinv=[im.Stick,im.Empty,im.Empty]
pm1inv=[im.Wsword,im.Empty,im.Empty]
pm2inv=[im.Wsword,im.Empty,im.Empty]
currentinv=1
Slot1=0
Slot2=0
Slot3=0
Slot4=0
Slot5=0
Slot6=0
Slot7=0
Slot8 =0
slotitemid ={
  im.Empty:0,
  im.Stick:1,
  im.Wsword:2,
  im.Isword:3,
}
itemid = {
  0:im.Empty,
  1:im.Stick,
  2:im.Wsword,
  3:im.Isword,
}
playerslotitem = [slotitemid[playerinv[0]],slotitemid[playerinv[1]],slotitemid[playerinv[2]]]
pm1slotitem = [slotitemid[pm1inv[0]],slotitemid[pm1inv[1]],slotitemid[pm1inv[2]]]
pm2slotitem = [slotitemid[pm2inv[0]],slotitemid[pm2inv[1]],slotitemid[pm2inv[2]]]
def collision(x,y,x2,y2,w):
    if x + w > x2 > x and y+w > y2 > y:
        return True
    else:
        return False
class Slots(object):
  def __init__(self,x,y):
    self.tx = x + 20
    self.ty = y + 50
  def draw(self,ds):
    global slotItem
        #Draws the slots using the x and y values
    p.draw.rect(ds,(color2),   (self.tx,self.ty,50,50))
    if collision(self.tx,self.ty,mx,my,60):
      p.draw.rect(ds,(128,0,0),(self.tx,self.ty,50,50))

class Slots2(object):
  def __init__(self,x,y):
    self.X = x + 20
    self.Y = y + 150
  def draw2(self,ds):
        #Draws the slots using the x and y values
        p.draw.rect(ds,(color2),   (self.X,self.Y,50,50))
        if collision(self.X,self.Y,mx,my,60):
            p.draw.rect(ds,(128,0,0),(self.X,self.Y,50,50))

slotArray = []
#Slot information
slotCount = 4
while len(slotArray) != slotCount:
  slotArray.append(Slots(30+len(slotArray)*100,50))
slotArray2 = []
#Slot information
slotCount2 = 4
while len(slotArray2) != slotCount2:
  slotArray2.append(Slots2(30+len(slotArray2)*100,50))
color = black
clock = p.time.Clock()
direction = 1
def checkslot():
  global slotItem,playerslotitem,pm1slotitem,pm2slotitem
  playerslotitem = [slotitemid[playerinv[0]],slotitemid[playerinv[1]],slotitemid[playerinv[2]]]
  pm1slotitem = [slotitemid[pm1inv[0]],slotitemid[pm1inv[1]],slotitemid[pm1inv[2]]]
  pm2slotitem = [slotitemid[pm2inv[0]],slotitemid[pm2inv[1]],slotitemid[pm2inv[2]]]
  slotItem = [slotitemid[inv[0]],slotitemid[inv[1]],slotitemid[inv[2]],slotitemid[inv[3]],slotitemid[inv[4]],slotitemid[inv[5]],slotitemid[inv[6]],slotitemid[inv[7]],slotitemid[inv[8]],slotitemid[inv[9]],slotitemid[inv[10]]]
def spellselect():
  mx,my = p.mouse.get_pos()
  test = p.mouse.get_pressed()[0]
def slotselect():
  global SelectedSlot, t,array
  mx,my=p.mouse.get_pos()
  test = p.mouse.get_pressed()[0]
  if test and collision(50,100,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 1
      array[0][1]= slotitemid[inv[0]]
    elif array[1][0]==0:
      array[1][0] = 1
      array[1][1]= slotitemid[inv[0]]
    t=1
  if test and collision(150,100,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 2
      array[0][1]= slotitemid[inv[1]]
    elif array[1][0]==0:
      array[1][0] = 2
      array[1][1]= slotitemid[inv[1]]
    t=1
  if test and collision(250,100,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 3
      array[0][1]= slotitemid[inv[2]]
    elif array[1][0]==0:
      array[1][0] = 3
      array[1][1]= slotitemid[inv[2]]
    t=1
    SelectedSlot = 3
  if test and collision(350,100,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 4
      array[0][1]= slotitemid[inv[3]]
    elif array[1][0]==0:
      array[1][0] = 4
      array[1][1]= slotitemid[inv[3]]
    t=1
  if test and collision(50,200,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 5
      array[0][1]= slotitemid[inv[4]]
    elif array[1][0]==0:
      array[1][0] = 5
      array[1][1]= slotitemid[inv[4]]
    t=1
  if test and collision(150,200,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 6
      array[0][1]= slotitemid[inv[5]]
    elif array[1][0]==0:
      array[1][0] = 6
      array[1][1]= slotitemid[inv[5]]
    t=1
  if test and collision(250,200,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 7
      array[0][1]= slotitemid[inv[6]]
    elif array[1][0]==0:
      array[1][0] = 7
      array[1][1]= slotitemid[inv[6]]
    t=1
  if test and collision(350,200,mx,my,50) and t==0:
    if array[0][0]==0:
      array[0][0] = 8
      array[0][1]= slotitemid[inv[7]]
    elif array[1][0]==0:
      array[1][0] = 8
      array[1][1]= slotitemid[inv[7]]
    t=1
  if test and collision(50,0,mx,my,50) and t==0:
    if array[1][0]==0 and 0<array[0][1]<=3:
      array[1][0] = 9
      array[1][1]= slotitemid[inv[8]]
    t=1
  if test and collision(150,0,mx,my,50) and t==0:
    if array[1][0]==0 and 3<array[0][1]<=10:
      array[1][0] = 10
      array[1][1]= slotitemid[inv[9]]
    t=1
  if test and collision(250,0,mx,my,50) and t==0:
    if array[1][0]==0 and 10<array[0][1]<=13:
      array[1][0] = 10
      array[1][1]= slotitemid[inv[10]]
    t=1
  if not test:
    t=0
  if not array[1][0]==0:
    array[0][1],array[1][1]=array[1][1],array[0][1]
    inv[(array[0][0]-1)] = itemid[array[0][1]]
    inv[(array[1][0]-1)] = itemid[array[1][1]]
    array = [[0,0],[0,0]]
    sta.mc.defense = sta.mcstats[0]+sta.itemstats[slotitemid[inv[8]]][0] +sta.itemstats[slotitemid[inv[9]]][0]
    sta.mc.attack = sta.mcstats[1]+sta.itemstats[slotitemid[inv[8]]][1] +sta.itemstats[slotitemid[inv[9]]][1]
    sta.mc.maxHp = sta.mcstats[2]+sta.itemstats[slotitemid[inv[8]]][2] +sta.itemstats[slotitemid[inv[9]]][2]
    sta.mc.mana = sta.mcstats[3]+sta.itemstats[slotitemid[inv[8]]][3] +sta.itemstats[slotitemid[inv[9]]][3]
    sta.mc.luck = sta.mcstats[4]+sta.itemstats[slotitemid[inv[8]]][3] +sta.itemstats[slotitemid[inv[9]]][3]
def additem(object):
  if inv[0] == im.Empty:
    inv[0]=(object)
    return True
  elif inv[1] == im.Empty:
    inv[1]=(object)
    return True
  elif inv[2] == im.Empty:
    inv[2]=(object)
    return True
  elif inv[3] == im.Empty:
    inv[3]=(object)
    return True
  elif inv[4] == im.Empty:
    inv[4]=(object)
    return True
  elif inv[5] == im.Empty:
    inv[5]=(object)
    return True
  elif inv[6] == im.Empty:
    inv[6]=(object)
    return True
  elif inv[7] == im.Empty:
    inv[7]=(object)
    return True
  else:
    ds.blit(text6, textRect)
    return False
def colI(obj):
  C = obj.colliderect(sta.mc.rect)
  if C:
      return True