import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from yellowbrick.cluster import KElbowVisualizer
import matplotlib.pyplot as plt
import os
import seaborn as sns; sns.set(style="ticks", color_codes=True)
import statsmodels.api as sm
from statsmodels.formula.api import ols

current_dir = os.getcwd()

# defining the file and dropping genre since pandas doesn't like strings
file_name = "Mall_Customers.csv"
df = pd.read_csv(file_name).rename(columns={'Genre': 'Gender'})

gender_cat = {'Male': 1, 'Female': 2}
# data = df.drop(columns=['Genre'])
df['Gender'] = df['Gender'].map(gender_cat)
data = df

#using a scaler to standardize the data
scaler = StandardScaler()
scaled_features = scaler.fit_transform(data)

#After looking at the elbow_chart function below, determined n_clusters to be 4. Now fitting the scaled_features to
# a 4 seed cluster
kmeans = KMeans(n_clusters=4, init='k-means++', max_iter=300, n_init=15, random_state=0)
kmeans.fit(scaled_features)

#copying the fitted data to a new df, and adding the clusters column
data_with_clusters = data.copy()
data_with_clusters['Clusters'] = kmeans.labels_

# defining clusters into 4 sepearte DFs
c0 = data_with_clusters.loc[data_with_clusters['Clusters'] == 0]
c1 = data_with_clusters.loc[data_with_clusters['Clusters'] == 1]
c2 = data_with_clusters.loc[data_with_clusters['Clusters'] == 2]
c3 = data_with_clusters.loc[data_with_clusters['Clusters'] == 3]

def elbow_chart():
    visualizer = KElbowVisualizer(KMeans(), k=(1,15)).fit(scaled_features)
    plt.legend(bbox_to_anchor= (1,1))
    plt.savefig(current_dir +'/media/elbow_chart.png')
    plt.clf()

def age_chart():
    #building the cluster labels
    c0_min = min(c0['Age'])
    c0_max = max(c0['Age'])
    c0_range = ('{} - {} C0'.format(c0_min, c0_max))

    c1_min = min(c1['Age'])
    c1_max = max(c1['Age'])
    c1_range = ('{} - {} C1'.format(c1_min, c1_max))

    c2_min = min(c2['Age'])
    c2_max = max(c2['Age'])
    c2_range = ('{} - {} C2'.format(c2_min, c2_max))

    c3_min = min(c3['Age'])
    c3_max = max(c3['Age'])
    c3_range = ('{} - {} C3'.format(c3_min, c3_max))

    # the labels would normally have the cluster IDs. i'm replacing them with the cluster ranges above
    data_with_clusters['Cluster Values'] = data_with_clusters['Clusters'].replace([0,1,2,3], [c0_range, c1_range, c2_range, c3_range])
    sns.boxplot(x=data_with_clusters['Cluster Values'], y=data_with_clusters['Age'], palette="Set3", hue=data_with_clusters['Clusters'], dodge=False)
    plt.legend(loc=2, bbox_to_anchor= (1,1), title='Clusters Group ID')
    plt.tight_layout()
    plt.savefig(current_dir +'/media/age_chart.png')
    plt.clf()

def Annual_Income_chart():
    #building the cluster labels
    c0_min = min(c0['Annual Income (k$)'])
    c0_max = max(c0['Annual Income (k$)'])
    c0_range = ('{} - {} C0'.format(c0_min, c0_max))

    c1_min = min(c1['Annual Income (k$)'])
    c1_max = max(c1['Annual Income (k$)'])
    c1_range = ('{} - {} C1'.format(c1_min, c1_max))

    c2_min = min(c2['Annual Income (k$)'])
    c2_max = max(c2['Annual Income (k$)'])
    c2_range = ('{} - {} C2'.format(c2_min, c2_max))

    c3_min = min(c3['Annual Income (k$)'])
    c3_max = max(c3['Annual Income (k$)'])
    c3_range = ('{} - {} C3'.format(c3_min, c3_max))

    # the labels would normally have the cluster IDs. i'm replacing them with the cluster ranges above
    data_with_clusters['Cluster Values'] = data_with_clusters['Clusters'].replace([0,1,2,3], [c0_range, c1_range, c2_range, c3_range])
    sns.boxplot(x=data_with_clusters['Cluster Values'], y=data_with_clusters['Annual Income (k$)'], palette="Set3", hue=data_with_clusters['Clusters'], dodge=False)
    plt.legend(loc=2, bbox_to_anchor= (1,1), title='Clusters Group ID')
    plt.tight_layout()
    plt.savefig(current_dir +'/media/annual_chart.png')
    plt.clf()

def Spending_Score_chart():

    #building the cluster labels
    c0_min = min(c0['Spending Score (1-100)'])
    c0_max = max(c0['Spending Score (1-100)'])
    c0_range = ('{} - {} C0'.format(c0_min, c0_max))

    c1_min = min(c1['Spending Score (1-100)'])
    c1_max = max(c1['Spending Score (1-100)'])
    c1_range = ('{} - {} C1'.format(c1_min, c1_max))

    c2_min = min(c2['Spending Score (1-100)'])
    c2_max = max(c2['Spending Score (1-100)'])
    c2_range = ('{} - {} C2'.format(c2_min, c2_max))

    c3_min = min(c3['Spending Score (1-100)'])
    c3_max = max(c3['Spending Score (1-100)'])
    c3_range = ('{} - {} C3'.format(c3_min, c3_max))

    # the labels would normally have the cluster IDs. i'm replacing them with the cluster ranges above
    data_with_clusters['Cluster Values'] = data_with_clusters['Clusters'].replace([0,1,2,3], [c0_range, c1_range, c2_range, c3_range])
    sns.boxplot(x=data_with_clusters['Cluster Values'], y=data_with_clusters['Spending Score (1-100)'], palette="Set3", hue=data_with_clusters['Clusters'], dodge=False)
    plt.legend(loc=2, bbox_to_anchor= (1,1), title='Clusters Group ID')
    plt.tight_layout()
    plt.savefig(current_dir +'/media/Spending_Score_chart.png')
    plt.clf()

def overall_data():
    #since for the project I selected Cluster 3 for analysis, i'm doing analysis for that cluster.
    #linear regression plots
    df = c3

    sns.lmplot(x='Annual Income (k$)', y='Spending Score (1-100)', data = df)
    sns.lmplot(x='Age', y='Spending Score (1-100)', data = df)
    sns.lmplot(x='Age', y='Annual Income (k$)', data = df)
    print("\n")
    plt.show()

    x = df['Age']
    y = df['Annual Income (k$)']
    z = df['Spending Score (1-100)']

    #performing different correlation analysis
    income_spend_corr = y.corr(z)
    age_spend_corr = x.corr(z)
    age_income_corr = x.corr(y)
    correlation = ' Correlation: \nincome_spending correlation: {} \nage_spending correlation: {}\nage_income correlation: {}\n'.format(income_spend_corr, age_spend_corr, age_income_corr)
    print(correlation)
    print("\n")

    #performing different means of each attribute
    age_mean = df['Age'].mean()
    income_mean = df['Annual Income (k$)'].mean()
    spend_mean = df['Spending Score (1-100)'].mean()
    means = ' means: \nage mean: {} \nspend mean: {} \nincome mean: {}'.format(age_mean, spend_mean, income_mean)
    print(means)
    print("\n")

    #ols can't handle non-alphanumeric characters
    c3.rename(columns={'Spending Score (1-100)': "Spend", 'Annual Income (k$)':'Income'}, inplace=True)
    #performing ANOVA
    lm = ols('Spend ~ Gender + Age + Income', data=c3).fit()
    print('Male & Female stats')
    print(lm.summary())

# 1 is male and 2 is female
def gender_data(gender_id):
    #depending on the gender selected for analysis, the label is updated
    gender_name = ''
    for key, value in gender_cat.items():
        if gender_id == value:
            gender_name = key

    #only selecting values that match the gender ID from the cluster selected
    df = c3.loc[c3['Gender'] == gender_id]
    # print(df)
    #linear regression plots
    sns.lmplot(x='Annual Income (k$)', y='Spending Score (1-100)', data = df)
    sns.lmplot(x='Age', y='Spending Score (1-100)', data = df)
    sns.lmplot(x='Age', y='Annual Income (k$)', data = df)
    plt.show()

    x = df['Age']
    y = df['Annual Income (k$)']
    z = df['Spending Score (1-100)']

    #performing different correlation analysis
    income_spend_corr = y.corr(z)
    age_spend_corr = x.corr(z)
    age_income_corr = x.corr(y)
    correlation = gender_name+' Correlation: \nincome_spending correlation: {} \nage_spending correlation: {}\nage_income correlation: {}\n'.format(income_spend_corr, age_spend_corr, age_income_corr)
    print(correlation)

    #performing different means of each attribute
    age_mean = df['Age'].mean()
    income_mean = df['Annual Income (k$)'].mean()
    spend_mean = df['Spending Score (1-100)'].mean()
    means = gender_name+' means: \nage mean: {} \nspend mean: {} \nincome mean: {}'.format(age_mean, spend_mean, income_mean)
    print(means)

    #ANOVA
    df.rename(columns={'Spending Score (1-100)': "Spend", 'Annual Income (k$)':'Income'}, inplace=True)
    lm = ols('Spend ~ Age + Income', data=df).fit()
    print(lm.summary())

if __name__ == "__main__":
    # age_chart()
    # Annual_Income_chart()
    # Spending_Score_chart()
    # elbow_chart()

    # overall_data()
    # male = gender_data(1)
    female = gender_data(2)


import json

def get_movie_data(files: list) -> list:
    """Parse movie json files into a list of dicts"""
    movies = []
    for file in files:
        with open(file) as f:
            movie_json = json.load(f)
            movies.append(movie_json)
    return movies

def get_single_comedy(movies: list) -> str:
    """return the movie with Comedy in Genres"""
    for movie in movies:
        if "Comedy" in movie["Genre"]:
            return movie["Title"]

def get_movie_most_nominations(movies: list) -> str:
    """Return the movie that had the most nominations"""
    most_nominations = 0
    movie_with_most_nominations = ""
    for movie in movies:
        if "Awards" in movie:
            nominations = int(movie["Awards"].split("&")[1].strip().split()[0])
            if nominations > most_nominations:
                most_nominations = nominations
                movie_with_most_nominations = movie["Title"]
    return movie_with_most_nominations

def get_movie_longest_runtime(movies: list) -> str:
    """Return the movie that has the longest runtime"""
    longest_runtime = 0
    movie_with_longest_runtime = ""
    for movie in movies:
        if "Runtime" in movie:
            runtime = int(movie["Runtime"].split()[0])
            if runtime > longest_runtime:
                longest_runtime = runtime
                movie_with_longest_runtime = movie["Title"]
    return movie_with_longest_runtime


from __future__ import print_function

import os
from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext
from distutils.errors import CompileError, DistutilsError, DistutilsPlatformError, LinkError
import shlex
import subprocess
import sys
import textwrap
import traceback
import re

tensorflow_mpi_lib = Extension('koten.tensorflow.mpi_lib', [])

def is_build_action():
    if len(sys.argv) <= 1:
        return False

    if sys.argv[1].startswith('build'):
        return True

    if sys.argv[1].startswith('bdist'):
        return True

    if sys.argv[1].startswith('install'):
        return True

def get_cpp_flags(build_ext):
    last_err = None
    default_flags = ['-std=c++11', '-fPIC', '-O2']
    if sys.platform == 'darwin':
        flags_to_try = [default_flags + ['-stdlib=libc++'], default_flags]
    else:
        flags_to_try = [default_flags, default_flags + ['-stdlib=libc++']]
    for cpp_flags in flags_to_try:
        try:
            test_compile(build_ext, 'test_cpp_flags', extra_compile_preargs=cpp_flags,
                         code=textwrap.dedent('''\
                    #include <unordered_map>
                    void test() {
                    }
                    '''))

            return cpp_flags
        except (CompileError, LinkError):
            last_err = 'Unable to determine C++ compilation flags (see error above).'
        except Exception:
            last_err = 'Unable to determine C++ compilation flags.  ' \
                       'Last error:\n\n%s' % traceback.format_exc()

    raise DistutilsPlatformError(last_err)

def get_link_flags(build_ext):
    last_err = None
    libtool_flags = ['-Wl,-exported_symbols_list']
    if sys.platform == 'darwin':
        flags_to_try = [libtool_flags]
    else:
        flags_to_try = [libtool_flags]
    for link_flags in flags_to_try:
        try:
            test_compile(build_ext, 'test_link_flags', extra_link_preargs=link_flags,
                         code=textwrap.dedent('''\
                    void test() {
                    }
                    '''))

            return link_flags
        except (CompileError, LinkError):
            last_err = 'Unable to determine C++ link flags (see error above).'
        except Exception:
            last_err = 'Unable to determine C++ link flags.  ' \
                       'Last error:\n\n%s' % traceback.format_exc()

    raise DistutilsPlatformError(last_err)

def get_tf_include_dirs():
    import tensorflow as tf
    tf_inc = tf.sysconfig.get_include()
    return [tf_inc, '%s/external/nsync/public' % tf_inc]

def get_tf_lib_dirs():
    import tensorflow as tf
    tf_lib = tf.sysconfig.get_lib()
    return [tf_lib]

def get_tf_libs(build_ext, lib_dirs, cpp_flags):
    last_err = None
    print(tf_libs)
    for tf_libs in [['tensorflow_framework'], []]:
        try:
            lib_file = test_compile(build_ext, 'test_tensorflow_libs',
                                    library_dirs=lib_dirs, libraries=tf_libs,
                                    extra_compile_preargs=cpp_flags,
                                    code=textwrap.dedent('''\
                    void test() {
                    }
                    '''))

            from tensorflow.python.framework import load_library
            load_library.load_op_library(lib_file)

            return tf_libs
        except (CompileError, LinkError):
            last_err = 'Unable to determine -l link flags to use with TensorFlow (see error above).'
        except Exception:
            last_err = 'Unable to determine -l link flags to use with TensorFlow.  ' \
                       'Last error:\n\n%s' % traceback.format_exc()

    raise DistutilsPlatformError(last_err)

def get_tf_abi(build_ext, include_dirs, lib_dirs, libs, cpp_flags):
    last_err = None
    cxx11_abi_macro = '_GLIBCXX_USE_CXX11_ABI'
    for cxx11_abi in ['0', '1']:
        try:
            lib_file = test_compile(build_ext, 'test_tensorflow_abi',
                                    macros=[(cxx11_abi_macro, cxx11_abi)],
                                    include_dirs=include_dirs, library_dirs=lib_dirs,
                                    libraries=libs, extra_compile_preargs=cpp_flags,
                                    code=textwrap.dedent('''\
                #include <string>
                #include "tensorflow/core/framework/op.h"
                #include "tensorflow/core/framework/op_kernel.h"
                #include "tensorflow/core/framework/shape_inference.h"
                void test() {
                    auto ignore = tensorflow::strings::StrCat("a", "b");
                }
                '''))

            from tensorflow.python.framework import load_library
            load_library.load_op_library(lib_file)

            return cxx11_abi_macro, cxx11_abi
        except (CompileError, LinkError):
            last_err = 'Unable to determine CXX11 ABI to use with TensorFlow (see error above).'
        except Exception:
            last_err = 'Unable to determine CXX11 ABI to use with TensorFlow.  ' \
                       'Last error:\n\n%s' % traceback.format_exc()

    raise DistutilsPlatformError(last_err)

def get_tf_flags(build_ext, cpp_flags):
    import tensorflow as tf
    try:
        return tf.sysconfig.get_compile_flags(), tf.sysconfig.get_link_flags()
    except AttributeError:
        # fallback to the previous logic
        tf_include_dirs = get_tf_include_dirs()
        tf_lib_dirs = get_tf_lib_dirs()
        tf_libs = get_tf_libs(build_ext, tf_lib_dirs, cpp_flags)
        tf_abi = get_tf_abi(build_ext, tf_include_dirs,
                            tf_lib_dirs, tf_libs, cpp_flags)

        compile_flags = []
        for include_dir in tf_include_dirs:
            compile_flags.append('-I%s' % include_dir)
        if tf_abi:
            compile_flags.append('-D%s=%s' % tf_abi)

        link_flags = []
        for lib_dir in tf_lib_dirs:
            link_flags.append('-L%s' % lib_dir)
        for lib in tf_libs:
            link_flags.append('-l%s' % lib)

        return compile_flags, link_flags

def get_mpi_flags():
    show_command = os.environ.get('KOTEN_MPICXX_SHOW', 'mpicxx -show')
    mpi_show_output = subprocess.check_output(
        shlex.split(show_command), universal_newlines=True).strip()
    mpi_show_args = shlex.split(mpi_show_output)
    if not mpi_show_args[0].startswith('-'):
        # Open MPI and MPICH print compiler name as a first word, skip it
        mpi_show_args = mpi_show_args[1:]
        # strip off compiler call portion and always escape each arg
        return ' '.join(['"' + arg.replace('"', '"\'"\'"') + '"'
                         for arg in mpi_show_args])

def test_compile(build_ext, name, code, libraries=None, include_dirs=None, library_dirs=None,
                 macros=None, extra_compile_preargs=None, extra_link_preargs=None):
    test_compile_dir = os.path.join(build_ext.build_temp, 'test_compile')
    if not os.path.exists(test_compile_dir):
        os.makedirs(test_compile_dir)

    source_file = os.path.join(test_compile_dir, '%s.cc' % name)
    with open(source_file, 'w') as f:
        f.write(code)

    compiler = build_ext.compiler
    [object_file] = compiler.object_filenames([source_file])
    shared_object_file = compiler.shared_object_filename(
        name, output_dir=test_compile_dir)

    compiler.compile([source_file], extra_preargs=extra_compile_preargs,
                     include_dirs=include_dirs, macros=macros)
    compiler.link_shared_object(
        [object_file], shared_object_file, libraries=libraries, library_dirs=library_dirs,
        extra_preargs=extra_link_preargs)

    return shared_object_file

def get_cuda_dirs(build_ext, cpp_flags):
    cuda_include_dirs = []
    cuda_lib_dirs = []

    cuda_home = os.environ.get('KOTEN_CUDA_HOME')
    if cuda_home:
        cuda_include_dirs += ['%s/include' % cuda_home]
        cuda_lib_dirs += ['%s/lib' % cuda_home, '%s/lib64' % cuda_home]

    cuda_include = os.environ.get('KOTEN_CUDA_INCLUDE')
    if cuda_include:
        cuda_include_dirs += [cuda_include]

    cuda_lib = os.environ.get('KOTEN_CUDA_LIB')
    if cuda_lib:
        cuda_lib_dirs += [cuda_lib]

    if not cuda_include_dirs and not cuda_lib_dirs:
        # default to /usr/local/cuda
        cuda_include_dirs += ['/usr/local/cuda/include']
        cuda_lib_dirs += ['/usr/local/cuda/lib', '/usr/local/cuda/lib64']

    try:
        test_compile(build_ext, 'test_cuda', libraries=['cudart'], include_dirs=cuda_include_dirs,
                     library_dirs=cuda_lib_dirs, extra_compile_preargs=cpp_flags,
                     code=textwrap.dedent('''\
            #include <cuda_runtime.h>
            void test() {
                cudaSetDevice(0);
            }
            '''))
    except (CompileError, LinkError):
        raise DistutilsPlatformError(
            'CUDA library was not found (see error above).\n'
            'Please specify correct CUDA location with the KOTEN_CUDA_HOME '
            'environment variable or combination of KOTEN_CUDA_INCLUDE and '
            'KOTEN_CUDA_LIB environment variables.\n\n'
            'KOTEN_CUDA_HOME - path where CUDA include and lib directories can be found\n'
            'KOTEN_CUDA_INCLUDE - path to CUDA include directory\n'
            'KOTEN_CUDA_LIB - path to CUDA lib directory')

    return cuda_include_dirs, cuda_lib_dirs

def get_common_options(build_ext):
    cpp_flags = get_cpp_flags(build_ext)
    link_flags = get_link_flags(build_ext)
    mpi_flags = get_mpi_flags()

    have_cuda = True
    cuda_include_dirs, cuda_lib_dirs = get_cuda_dirs(build_ext, cpp_flags)
    MACROS = []
    INCLUDES = []
    SOURCES = ['koten/common/allreduce.cpp',
               'koten/common/operation.cpp']
    COMPILE_FLAGS = cpp_flags + shlex.split(mpi_flags)
    LINK_FLAGS = link_flags + shlex.split(mpi_flags)
    LIBRARY_DIRS = []
    LIBRARIES = []

    INCLUDES += cuda_include_dirs
    LIBRARY_DIRS += cuda_lib_dirs
    LIBRARIES += ['cudart']

    return dict(MACROS=MACROS,
                INCLUDES=INCLUDES,
                SOURCES=SOURCES,
                COMPILE_FLAGS=COMPILE_FLAGS,
                LINK_FLAGS=LINK_FLAGS,
                LIBRARY_DIRS=LIBRARY_DIRS,
                LIBRARIES=LIBRARIES)

def build_tf_extension(build_ext, options):
    tf_compile_flags, tf_link_flags = get_tf_flags(
        build_ext, options['COMPILE_FLAGS'])

    tensorflow_mpi_lib.define_macros = options['MACROS']
    tensorflow_mpi_lib.include_dirs = options['INCLUDES']
    tensorflow_mpi_lib.sources = options['SOURCES'] + \
        ['koten/tensorflow/mpi_ops.cc']
    tensorflow_mpi_lib.extra_compile_args = options['COMPILE_FLAGS'] + \
        tf_compile_flags
    tensorflow_mpi_lib.extra_link_args = options['LINK_FLAGS'] + tf_link_flags
    tensorflow_mpi_lib.library_dirs = options['LIBRARY_DIRS']
    tensorflow_mpi_lib.libraries = options['LIBRARIES']

    build_ext.build_extension(tensorflow_mpi_lib)

# run the customize_compiler
class custom_build_ext(build_ext):
    def build_extensions(self):
        options = get_common_options(self)
        built_plugins = []
        build_tf_extension(self, options)
        built_plugins.append(True)

setup(name='koten',
      version='0.1',
      packages=find_packages(),
      description='Distributed training framework for TensorFlow, Keras, and PyTorch.',
      author='kuroko1t',
      long_description=textwrap.dedent('''\
          koten is multi process excution for Tensorflow in single gpu.'''),
      url='https://github.com/kuroko1t/koten',
      classifiers=[
          'License :: OSI Approved :: Apache Software License'
      ],
      ext_modules=[tensorflow_mpi_lib],
      cmdclass={'build_ext': custom_build_ext},
      zip_safe=False)


from django.shortcuts import render
from django.http import JsonResponse
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.conf import settings

import os
import uuid
import base64
import json
import numpy as np
from PIL import Image
import io

from .networks import CycleGAN
from .models import Images, File, FakeImage
from .preprocessing import make_input, to_PIL

# Create your views here.

def index(request):
    return render(request, 'index.html')

def predict_image(request):
    model = CycleGAN()
    model.log_dir = 'logs'
    model.load('epoch195')

    img_byte = request.body
    img_base64 = img_byte.decode(encoding='utf-8')
    img_data = base64.b64decode(img_base64)
    img_name = str(uuid.uuid4())
    image = ContentFile(img_data, img_name + '.png')

    Images.objects.create(image=image)
    image_path = 'media/real/' + img_name + '.png'

    img_input = make_input(image_path, img_name)
    fake_Y = model.G_X(img_input)

    fake_img_pil = to_PIL(fake_Y)

    fake_img_byte_array = io.BytesIO()
    fake_img_pil.save(fake_img_byte_array, format='JPEG')
    fake_img_data = fake_img_byte_array.getvalue()
    fake_img_name = 'fake_' + img_name + '.jpg'
    fake_image = ContentFile(fake_img_data, fake_img_name)

    FakeImage.objects.create(fake_image=fake_image)
    fake_image_path = 'media/fake/' + fake_img_name

    delete_path = os.path.join(settings.BASE_DIR, img_name + '.jpg')
    print(delete_path)
    os.remove(delete_path)

    context = {'fake_image_path': fake_image_path }
    return JsonResponse(context)

def predict_file(request):
    model = CycleGAN()
    model.log_dir = 'logs'
    model.load('epoch195')

    if request.method == 'POST':
        img_name = str(uuid.uuid4()) + '.jpg'
        path = default_storage.save(img_name, request.FILES['docfile'])
        tmp_file = os.path.join(settings.MEDIA_ROOT, path)

        img_input = make_input('media/' + path, img_name)
        fake_X = model.G_Y(img_input)

        fake_img_pil = to_PIL(fake_X)

        fake_img_byte_array = io.BytesIO()
        fake_img_pil.save(fake_img_byte_array, format='JPEG')
        fake_img_data = fake_img_byte_array.getvalue()
        fake_img_name = 'fake_' + img_name
        fake_image = ContentFile(fake_img_data, fake_img_name)

        FakeImage.objects.create(fake_image=fake_image)
        fake_image_path = 'media/fake/' + fake_img_name

        context = { 'file_path': fake_image_path }
        return JsonResponse(context)
    else:
        context = {'state': '400'}
        return JsonResponse(context)


import os
import numpy as np
import cv2
import torch
from torch.utils import data
from PIL import Image

class Places365(data.Dataset):
  'Characterizes a dataset for PyTorch'
  def __init__(self, path_to_data, path_to_labels, list_IDs=None, resolution=None, transform=None):
        'Initialization'
        self.path_to_data = path_to_data
        self.resolution = resolution
        self.path_to_labels = path_to_labels
        self.transform = transform
        self.d = {} #dictionary to match label to image
        with open(path_to_labels) as f:
            for line in f:
                try:
                    (key, val) = line.split() # switch to "," or ""if you get error
                except ValueError:
                    (key, val) = line.split(",") # switch to "," or ""if you get error

                val = val.replace('\n', "")
                key = key.replace('"', "")
                val = val.replace('"', "")
                self.d[str(key)] = int(val)

        if list_IDs == None:
            self.list_IDs = list(self.d.keys())
        else:
            self.list_IDs = list_IDs

  def __len__(self):
        'Denotes the total number of samples'
        return len(self.list_IDs)

  def __getitem__(self, index):
        'Generates one sample of data'
        # Select sample
        ID = self.list_IDs[index]
        # Load image
        X = Image.open(os.path.join(self.path_to_data, ID))
        # print(X)
        X = X.convert('RGB')
        # print(X)
        if self.resolution!=None:
            X = X.resize(self.resolution)
        # deep learning networks traditionally share many parameters - if you didn't scale your inputs in a way that resulted in similarly-ranged feature values (ie: over the whole dataset by subtracting mean) sharing wouldn't happen very easily because to one part of the image weight w is a lot and to another it's too small.
        # print(X)
        if self.transform:
            X = self.transform(X)
        # print(X)
        # exit()
        # Match to label
        y = torch.tensor(self.d[ID])
        return X, y, ID


import requests
import json
from datetime import datetime

__author__ = 'Steel'

"""Библиотека API для полного контроля и управления серверами на платформе 1Cloud.ru
Документация по API: https://1cloud.ru/api
"""

# Maximum requests per second allowed by 1cloud.ru for each request type
MAX_REQUESTS = {
    'GET': 1.5,
    'POST': 10,
    'PUT': 10,
    'DELETE': 10
}

STATUS_CODES = {
    200: 'request complete',
    401: 'not authorized',
    403: 'request denied',
    404: 'object not found',
    400: 'invalid request parameters',
    500: 'unknown error - contact 1cloud.ru support'
}

URL = 'https://api.1cloud.ru'

class OneCloudApi(object):
    """
    Модуль API 1cloud.ru для полного контроля и управления арендованными серверами

    :param api_key: Ключ API генерируется на сайте 1cloud.ru в настройках профиля. Обязательный параметр.
    :param timeout: Таймаут ожидания ответа от сервера в секундах.
    """
    api_key = None
    timeout = None
    rS = None
    last_request = None

    def __init__(self, api_key, timeout=5):
        self.api_key = api_key
        self.timeout = timeout
        self.rS = requests.Session()
        self.rS.headers.update({
            'Content-Type': 'application/json',
            'Authorization': 'Bearer {}'.format(api_key)
        })

    def req(self, path, method='GET', data=None):
        """
        Автоматически формирует корректный запрос к серверу
        :param path: формируется автоматически вызывающим методом
        :param method: формируется автоматически вызывающим методом
        :param data: формируется автоматически вызывающим методом
        :return: JSON объект с соответствующими запросу данными.
        В случае ошибки: {'ERROR_CODE': код ошибки, 'ERROR_MESSAGE': 'текст ошибки'}
        """
        if self.last_request is None:
            self.last_request = datetime.now()
        elif (self.last_request - datetime.now()).seconds < MAX_REQUESTS[method]:
            return {
                'ERROR_CODE': None,
                'ERROR_MESSAGE': 'Too fast for %s type requests! Try again in %s seconds.' % (
                    method, str(MAX_REQUESTS[method] - (self.last_request - datetime.now()).seconds))
            }

        r = self.rS.request(url=URL + path, method=method, data=data)
        if r.status_code != 200:
            if r.status_code in STATUS_CODES:
                return {
                    'ERROR_CODE': r.status_code,
                    'ERROR_MESSAGE': STATUS_CODES[r.status_code]
                }
            else:
                return {
                    'ERROR_CODE': None,
                    'ERROR_MESSAGE': 'Unknown error. Plain response:' + r.text
                }
        res = json.loads(r.text)
        return res

    def get_balance(self):
        """Запрос состояния счета пользователя

        :return: float значение баланса
        """
        return self.req(path='/customer/balance', method='GET')

    def get_images_list(self):
        """Образы (шаблоны) возможных систем для установки на сервер

        :return: JSON массив с характеристиками каждого образа
        """
        return self.req(path='/image', method='GET')

    def create_image(self, name, tech_name, server_id):
        """Создание шаблона на основе Вашего, уже функионирующего сервера

        :param name: string Пользовательское название шаблона
        :param tech_name: Техническое имя (что-то наподобие id)
        :param server_id: Id сервера, на основе которого будет создан шаблон
        :return: JSON массив с атрибутами шаблона
        """
        data = {
            'Name': name,
            'TechName': tech_name,
            'ServerID': server_id
        }
        return self.req(path='/image', method='POST', data=data)

    def delete_image(self, image_id):
        """Удаление шаблона

        :param image_id: Id удаляемого шаблона
        :return: None
        """
        return self.req(path='/image/{}'.format(image_id), method='DELETE')

    def get_private_networks_list(self):
        """
        Метод возвращает список всех Ваших частных сетей.
        :return: JSON объект, массив объектов, каждый элемент массива описывает атрибуты одной из сетей.
        """
        return self.req(path='/network', method='GET')

    def get_private_network_by_id(self, network_id):
        """
        Подробности об одной из Ваших частных сетей
        :param network_id: Id запрашиваемой сети
        :return: JSON объект, который будет содержать описание значений атрибутов данной частной сети.
        """
        return self.req(path='/network/{}'.format(network_id), method='GET')

    def create_private_network(self, name):
        """
        Создание новой частной сети
        :param name: Имя создаваемой сети
        :return: JSON объект, который будет содержать описание значений атрибутов данной частной сети.
        """
        data = {
            'Name': name
        }
        return self.req(path='/network/', method='POST', data=data)

    def delete_private_network(self, network_id):
        """
        Удаление частной сети
        :param network_id: Id удаляемой сети
        :return:
        """
        return self.req(path='/network/{}'.format(network_id), method='DELETE')

    def get_dc_locations(self):
        """
        Список всех доступных локаций (ЦОД), на которых возможно создать виртуальный серверю
        :return: JSON объект, с массивом объектов - атрибутов одной из доступных локаций.
        """
        return self.req(path='/dcLocation', method='GET')

    def get_servers_list(self):
        """
        Список всех Ваших заказанных серверов
        :return: JSON объект, с массивом объектов - атрибутов заказанных серверов.
        """
        return self.req(path='/server', method='GET')

    def get_server_by_id(self, server_id):
        """
        Подробности о сервере по Id
        :param server_id:
        :return:
        """
        return self.req(path='/server/{}'.format(server_id), method='GET')

    def create_server(self, name, cpu, ram, hdd, image_id, dc_location, hdd_type='SAS', is_high_performance=False):
        """
        Создание сервера
        :param name: Пользовательское название сервера, заданное при создании сервера
        :param cpu: Количество ядер процессора, выделенных на данный сервер [1...8]. Шаг увеличения значений: 1.
        :param ram: Объем ОЗУ в Мб. Linux: [512...16384], Windows: [1024...16384]
                    Шаг увеличения значений: до 1024 шаг 256, после 1024 шаг составляет 1024.
        :param hdd: Количество дискового пространства, выделенного на данный сервер (в Gb).
                    Linux: [10...250], Windows: [40...250] Шаг увеличения значений: 10.
        :param image_id: Уникальный идентификатор образа OS из get_servers_list()
        :param dc_location: Техническое наименование DC, в которой создаем сервер из get_dc_locations()
        :param hdd_type: Тип диска сервера, может быть одним из следующих значений "SAS" или "SSD"
        :param is_high_performance: True - если необходим сервер в высокопроизводительном пуле, false - в базовом пуле.
        :return: JSON объект - атрибуты созданного сервера.
        """
        data = {
            'Name': name,
            'CPU': cpu,
            'RAM': ram,
            'HDD': hdd,
            'ImageID': image_id,
            'HDDType': hdd_type,
            'isHighPerformance': is_high_performance,
            'DCLocation': dc_location
        }
        return self.req(path='/server', method='POST', data=data)

    def update_server_parameters(self, server_id, cpu, ram, hdd, hdd_type, is_high_performance):
        """
        Изменение конфигурации сервера
        :param server_id: Id сервера с изменяемой конфигурацией
        :param cpu: Количество ядер процессора, выделенных на данный сервер [1...8]. Шаг увеличения значений: 1.
        :param ram: Объем ОЗУ в Мб. Linux: [512...16384], Windows: [1024...16384]
                    Шаг увеличения значений: до 1024 шаг 256, после 1024 шаг составляет 1024.
        :param hdd: Количество дискового пространства, выделенного на данный сервер (в Gb).
                    Linux: [10...250], Windows: [40...250] Шаг увеличения значений: 10.
        :param hdd_type: Тип диска сервера, может быть одним из следующих значений "SAS" или "SSD"
        :param is_high_performance: True - если необходим сервер в высокопроизводительном пуле, false - в базовом пуле.
        :return: JSON объект - атрибуты созданного сервера.
        """
        data = {
            'CPU': cpu,
            'RAM': ram,
            'HDD': hdd,
            'HDDType': hdd_type,
            'isHighPerformance': is_high_performance
        }
        return self.req(path='/server/{}'.format(server_id), method='PUT', data=data)

    def delete_server(self, server_id):
        """
        Удаление сервера
        :param server_id:
        :return:
        """
        return self.req(path='/server/{}'.format(server_id), method='DELETE')

    def power_on_server(self, server_id):
        """
        Включить питание сервера
        :param server_id:
        :return:
        """
        data = {
            'Type': 'PowerOn'
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)

    def power_off_server(self, server_id):
        """
        Выключить питание сервера
        :param server_id:
        :return:
        """
        data = {
            'Type': 'PowerOff'
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)

    def shutdown_guest_os(self, server_id):
        """
        Выключение сервера средствами операционной системы. Только для ОС с усановленными VMware комонентами.
        :param server_id:
        :return:
        """
        data = {
            'Type': 'ShutDownGuestOS'
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)

    def reboot_server(self, server_id):
        """
        Перезагрузка сервера
        :param server_id:
        :return:
        """
        data = {
            'Type': 'PowerReboot'
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)

    def connect_server_to_network(self, server_id, network_id):
        """
        Подключить линк созданной ранее частной сети
        :param server_id:
        :param network_id:
        :return:
        """
        data = {
            'Type': 'AddNetwork',
            'NetworkID': network_id
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)

    def disconnect_server_from_network(self, server_id, network_id):
        """
        Отключить от заданной частной сети
        :param server_id:
        :param network_id:
        :return:
        """
        data = {
            'Type': 'RemoveNetwork',
            'NetworkID': network_id
        }
        return self.req(path='/server/{}/action'.format(server_id), method='POST', data=data)


from meetup_dao import find_events_for_clustering, find_groups_for_clustering
from gensim import corpora, models
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk import pos_tag
import nltk
import matplotlib, matplotlib.pyplot as plt
matplotlib.use('Agg')

def get_clusters(city: str, category: str, level_type: str, num_clusters: int, num_words: int, min_word_length: int, use_name: bool, use_desc: bool):

    if level_type == 'group':
        items = find_groups_for_clustering(city, category, use_name, use_desc)
    elif level_type == 'event':
        items = find_events_for_clustering(city, category, use_name, use_desc)
    else:
        print('Error: select correct level type.')
        items = []

    # Get cleaned/tokenized items
    tokenized_items = clean_and_preprocess(items, min_word_length)

    # Setup topic extraction model
    dictionary_LDA = corpora.Dictionary(tokenized_items)
    dictionary_LDA.filter_extremes(no_below=1)
    corpus = [dictionary_LDA.doc2bow(tokenized_event) for tokenized_event in tokenized_items]

    print(corpus)

    # Run topic extraction model
    lda_model = models.LdaModel(corpus, num_topics=num_clusters, \
                                id2word=dictionary_LDA, \
                                passes=10, alpha=[0.01] * num_clusters, \
                                eta=[0.01] * len(dictionary_LDA.keys()))

    clusters = []

    for i, topic in lda_model.show_topics(formatted=True, num_topics=num_clusters, num_words=num_words):
        split_cluster = topic.split("*")

        cluster = [split_cluster[1].strip('\"'), split_cluster[0]]
        if not cluster in clusters:
            clusters.append(cluster)

    return [clusters, lda_model, corpus, tokenized_items, dictionary_LDA] #[['kafka', .65], ['spark', .13]]

def clean_and_preprocess(items, min_word_length):
    tokenized_items = []

    for item in items:
        #     print(event)
        tokenized_event = word_tokenize(item)

        # Clean string:
        # Remove punctuation
        tokenized_event = [word.lower() for word in tokenized_event if word.isalpha()]
        # Remove if word is < X letters
        tokenized_event = [word for word in tokenized_event if len(word) >= min_word_length]
        # Remove stop words
        tokenized_event = [word for word in tokenized_event if word not in stopwords.words('english')]
        # Only keep nouns
        tags = pos_tag(tokenized_event)
        tokenized_event = [tag[0] for index, tag in enumerate(tags) if
                           (tag[1] == 'NN' or tag[1] == 'NNP' or tag[1] == 'NNS' or tag[1] == 'NNPS')]

        tokenized_items.append(tokenized_event)

    return tokenized_items

def run_experiment(num_iterations):
    iterations = num_iterations;
    max_number_words = 10;
    min_word_length = 3;

    city = 'San Francisco'
    category = 'Arts & Culture'

    perplexities_name = []
    coherence_scores_name = []
    perplexities_descr = []
    coherence_scores_descr = []
    perplexities_name_descr = []
    coherence_scores_name_descr = []

    for number_words in range(max_number_words):
        perplexities_name.append(0)
        coherence_scores_name.append(0)
        perplexities_descr.append(0)
        coherence_scores_descr.append(0)
        perplexities_name_descr.append(0)
        coherence_scores_name_descr.append(0)

    for iteration in range(iterations):
        print("Iteration: " + str(iteration));
        for number_words in range(max_number_words):
            print("Num words: "+str(number_words))

            # USE NAME
            # Get model
            [clusters, lda_model, corpus, tokenized_items, dictionary_LDA] = get_clusters(city, category, 'event', 10,
                                                                          number_words+1, min_word_length, True,
                                                                          False)
            # Compute perplexity of model
            perplexity = lda_model.log_perplexity(corpus)

            # Compute coherence score of model
            coherence_model_lda = models.CoherenceModel(model=lda_model, texts=tokenized_items, dictionary=dictionary_LDA, coherence='c_v')
            coherence = coherence_model_lda.get_coherence()

            # Aggregate scores
            perplexities_name[number_words] += perplexity
            coherence_scores_name[number_words] += coherence

            # USE DESCRIPTION
            # Get model
            [clusters, lda_model, corpus, tokenized_items, dictionary_LDA] = get_clusters(city, category, 'event', 10,
                                                                                number_words + 1,
                                                                                min_word_length, False,
                                                                                True)
            # Compute perplexity of model
            perplexity = lda_model.log_perplexity(corpus)

            # Compute coherence score of model
            coherence_model_lda = models.CoherenceModel(model=lda_model, texts=tokenized_items,
                                                        dictionary=dictionary_LDA, coherence='c_v')
            coherence = coherence_model_lda.get_coherence()

            # Aggregate scores
            perplexities_descr[number_words] += perplexity
            coherence_scores_descr[number_words] += coherence

            # USE BOTH
            # Get model
            [clusters, lda_model, corpus, tokenized_items, dictionary_LDA] = get_clusters(city, category, 'event', 10,
                                                                                number_words + 1,
                                                                                min_word_length, True,
                                                                                True)
            # Compute perplexity of model
            perplexity = lda_model.log_perplexity(corpus)

            # Compute coherence score of model
            coherence_model_lda = models.CoherenceModel(model=lda_model, texts=tokenized_items,
                                                        dictionary=dictionary_LDA, coherence='c_v')
            coherence = coherence_model_lda.get_coherence()

            # Aggregate scores
            perplexities_name_descr[number_words] += perplexity
            coherence_scores_name_descr[number_words] += coherence

    # Average the scores
    perplexities_name = [perplexities_name[number_words]/iterations for number_words in range(max_number_words)]
    coherence_scores_name = [coherence_scores_name[number_words]/iterations for number_words in range(max_number_words)]
    perplexities_descr = [perplexities_descr[number_words]/iterations for number_words in range(max_number_words)]
    coherence_scores_descr = [coherence_scores_descr[number_words]/iterations for number_words in range(max_number_words)]
    perplexities_name_descr = [perplexities_name_descr[number_words]/iterations for number_words in range(max_number_words)]
    coherence_scores_name_descr = [coherence_scores_name_descr[number_words]/iterations for number_words in range(max_number_words)]

    num_words = [number_words + 1 for number_words in range(max_number_words)]

    # Print graphs containing results
    plt.xlabel('Number of words')
    plt.ylabel('Perplexity')
    plt.title('Perplexity vs Number of Words in Topic')
    plt.plot(num_words, perplexities_name, label='Use name')
    plt.plot(num_words, perplexities_descr, label='Use description')
    plt.plot(num_words, perplexities_name_descr, label='Use both')
    plt.legend(loc='upper right')
    plt.savefig('perplexity.png')

    plt.clf()

    plt.xlabel('Number of words')
    plt.ylabel('Coherence score')
    plt.title('Coherence score vs Number of Words in Topic')
    plt.plot(num_words, coherence_scores_name, label='Use name')
    plt.plot(num_words, coherence_scores_descr, label='Use description')
    plt.plot(num_words, coherence_scores_name_descr, label='Use both')
    plt.legend(loc='upper right')
    plt.savefig('coherence_score.png')

    # print(perplexities)
    # print(coherence_scores)

    return


import torch

class SGD:
    """Minibatch stochastic gradient descent.
    It is supposed that loss is calculated as a mean over the minibatch.
    params: iterable containing weights and biases, e.g. [W, b]
    lr: learning rate
    """

    def __init__(self, params, lr):
        self.params = params
        self.lr = lr

    def step(self):
        with torch.no_grad():
            for param in self.params:
                param -= self.lr * param.grad
                param.grad.zero_()


from PIL import Image, ImageDraw

class QuadTree:
    def __init__(
        self,
        x: int,
        y: int,
        s: int,
        image: Image.Image,
        name: str,
        color_nodes: list,
        canvas: Image.Image,
        gif: list[Image.Image],
    ) -> None:
        self.x = x
        self.y = y
        self.s = s
        self.bound = (self.x, self.y, self.x + self.s, self.y + self.s)
        self.name = name
        self.image = image
        self.color = self.get_color()
        self.color_nodes = color_nodes
        self.gif = gif
        self.canvas = canvas

        if self.color == None:
            self.divide()
        else:
            self.color_nodes.append(self)
            draw = ImageDraw.Draw(self.canvas)
            # if self.color == 255:
            draw.rectangle(self.bound, fill=self.color)
            self.gif.append(self.canvas.copy())

    def get_color(self) -> int:
        image = self.image.crop(self.bound)
        data = image.getdata()
        return data[0] if all(pixel == data[0] for pixel in data) else None

    def divide(self):
        self.nw = QuadTree(
            self.x,
            self.y,
            self.s // 2,
            self.image,
            self.name + "-" + "nw",
            self.color_nodes,
            self.canvas,
            self.gif,
        )
        self.ne = QuadTree(
            self.x + self.s // 2,
            self.y,
            self.s // 2,
            self.image,
            self.name + "-" + "ne",
            self.color_nodes,
            self.canvas,
            self.gif,
        )
        self.sw = QuadTree(
            self.x,
            self.y + self.s // 2,
            self.s // 2,
            self.image,
            self.name + "-" + "sw",
            self.color_nodes,
            self.canvas,
            self.gif,
        )
        self.se = QuadTree(
            self.x + self.s // 2,
            self.y + self.s // 2,
            self.s // 2,
            self.image,
            self.name + "-" + "se",
            self.color_nodes,
            self.canvas,
            self.gif,
        )


from flask import Flask, request, jsonify
import python_pachyderm
import datetime
from grpc import RpcError

# Pachyderm client to fetch and put file in Pachyderm repos
client = python_pachyderm.PfsClient()
app = Flask(__name__)

def gen_filename(prefix):
    """
    Generate a filename <prefix>-<current time in ISO8601>.txt
    """
    ts = datetime.datetime.now().timestamp()
    readable = datetime.datetime.fromtimestamp(ts).isoformat()
    return prefix+"-"+readable+".txt"

@app.route("/train", methods=["POST"])
def add_training_data():
    """
    Submit training sample(s) to `training` repo, which will trigger
    `train` pipeline.

    POST {
        "data": [sample_1, sample_2, ..., sample_n]
    }
    where sample_i: "<user_id> <movie_id> <rate> <timestamp>"

    Response: {
        "filename": <created_training_data_filename_in_pachyderm>
    }
    """
    json_data = request.get_json()
    filename = gen_filename("train")
    data = "\n".join(json_data['data']).encode()

    with client.commit('training', 'master') as c:
        client.put_file_bytes(c, filename, data)

    return jsonify({"filename": filename})

@app.route("/predict", methods=["POST"])
def predict_data():
    """
    Submit data to predict to `streaming` repo, which will trigger
    `predict` pipeline. Submit is asynchronous and to check the result
    use `POST /result`

    POST {
        "data": [sample_1, sample_2, ..., sample_n]
    }
    where sample_i: "<user_id> <movie_id>"

    Response: {
        "filename": <created_predicting_data_filename_in_pachyderm>
    }
    """
    json_data = request.get_json()
    filename = gen_filename('predict')
    # pachyderm expect a bytes to save, convert str to bytes
    data = "\n".join(json_data['data']).encode()

    with client.commit('streaming', 'master') as c:
        client.put_file_bytes(c, filename, data)

    return jsonify({"filename": filename})

@app.route("/result", methods=["POST"])
def check_result():
    """
    Get the result giving created predicting data filename.

    POST {
        "filename": <filename_returned_by_/predict>
    }

    Response: {
        ready: false
    } or {
        ready: true
        data: <content-of-result-file>
    }
    """
    json_data = request.get_json()
    filename = json_data['filename']
    try:
        data = client.get_file("predict/master", filename)
        return jsonify({
            "ready": True,
            # Data is a iterator of bytes object, convert to str
            "data": list(data)[0].decode()
        })
    except RpcError:
        # If file not exist, either it's a wrong filename or
        # predict-pipeline didn't finish processing
        return jsonify({"ready": False})


import copy
import os
from datetime import datetime
from typing import Any, Tuple

import numpy
import pandas as pd
import tensorflow as tf
import numpy as np
from pandas import DataFrame
from yfinance import Ticker

from env_config import BATCH_SIZE, TRAINING_PATH, EPOCHS, RESULTS
from stonks import process_stock_record

def standardize_data(data: DataFrame) -> DataFrame:
    return data / 100.0

def get_prepped_data(input_filepath: str) -> tuple[int, DataFrame, DataFrame, DataFrame, DataFrame]:
    input_df = pd.read_csv(input_filepath)
    input_df.dropna(inplace=True)
    np.random.shuffle(input_df.values)
    input_df = input_df.iloc[:, 1:]
    # print(input_df)
    target = input_df.pop("84").to_frame()
    standardized_df = standardize_data(input_df)

    # Number Row to split dataframes at, separating training and testing data
    cnt = int(len(standardized_df.index) * 0.75)
    input_size = len(standardized_df.columns)

    train_data = standardized_df.iloc[:cnt, :]
    test_data = standardized_df.iloc[cnt:, :]
    train_target = target.iloc[:cnt, :]
    test_target = target.iloc[cnt:, :]
    return input_size, train_data, train_target, test_data, test_target

def get_prepped_target(input_target: DataFrame) -> Any:
    template_record = {
        "nothing": [0.0],
        "buy": [0.0],
        "put": [0.0]
    }
    df = DataFrame(columns=template_record)
    for idx, record in input_target.iterrows():
        temp_record = copy.deepcopy(template_record)
        temp_record[list(template_record.keys())[int(record.iloc[0])]] = [1.0]
        df = pd.concat([df, DataFrame(temp_record)])
    return tf.convert_to_tensor(numpy.asarray(df.to_numpy()).astype(np.float32))

def get_stock_data(stonk: Ticker, end_date: datetime) -> DataFrame:
    stonk_df = pd.DataFrame(process_stock_record(stonk, end_date=end_date)).T
    input_df = stonk_df.iloc[:, :len(stonk_df.columns)-1]
    return standardize_data(input_df)

def train_model(input_filepath: str) -> None:
    input_size, train_data, train_target, test_data, test_target = get_prepped_data(input_filepath)
    train_target = get_prepped_target(train_target)
    test_target = get_prepped_target(test_target)

    # Create a callback that saves the model's weights
    cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=TRAINING_PATH,
                                                     save_weights_only=True,
                                                     verbose=1)

    model = get_basic_model(input_size)

    model.fit(train_data, train_target, epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[cp_callback])
    model.save(TRAINING_PATH)
    model.save(TRAINING_PATH + '.h5')

    test_loss, test_acc = model.evaluate(test_data, test_target, verbose=2)
    print('\nTest accuracy:', test_acc)

def get_basic_model(input_size: int):
    model = tf.keras.Sequential([
        tf.keras.layers.Flatten(),
        tf.keras.layers.Normalization(axis=-1),
        tf.keras.layers.Dense(input_size, activation='relu'),
        tf.keras.layers.Dense(20, activation='relu'),
        tf.keras.layers.Dense(10, activation='relu'),
        tf.keras.layers.Dense(5, activation='relu'),
        tf.keras.layers.Dense(3)
    ])

    model.compile(optimizer='adam',
                  loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
                  metrics=['accuracy'])
    return model

def get_loaded_model():
    return tf.keras.Sequential([tf.keras.models.load_model(TRAINING_PATH), tf.keras.layers.Softmax()])

def predict_stonk(stonk: Ticker, end_date: datetime) -> Tuple[int, str]:
    prediction_model = get_loaded_model()
    stonk_data = get_stock_data(stonk, end_date)

    result_idx = numpy.argmax(prediction_model.predict(stonk_data))
    print(int(result_idx), RESULTS[result_idx])
    return int(result_idx), RESULTS[result_idx]


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import json
from collections import defaultdict
import pickle
import torch

# Root directory of output files
root_dir = './save/output'
dirs = sorted([f for f in os.listdir(root_dir)])
print(dirs)
# You could overwrite it yourself
# e.g. dirs = ['albert-large-v2-dev', 'roberta-large-dev']

dirs = ['bert-large-dev',
        'bert-large-train',
        'bert-large-batch6-dev',
        'bert-large-batch6-train',
        'bert-large-cls-weight0.5-without-force-pred-eval-dev',
        'bert-large-cls-weight0.5-without-force-pred-eval-train']

def generate(only_dev=True):
    features_train = []
    features_dev = []

    all_results_train = []
    all_results_dev = []

    tokenizers_train = []
    tokenizers_dev = []

    for model_dir in dirs:
        model_name = ''
        tokenizer = None
        all_results = None
        features = None

        with open(os.path.join(root_dir, model_dir, 'config.json')) as f:
            config = json.load(f)
            model_name = config['model_name'].replace('-dev', '').replace('-train', '').replace('-01', '')
        # We only want dev files for vote
        if only_dev and config['type'] == 'train':
            continue
        print(model_name, config['type'])

        with open(os.path.join(root_dir, model_dir, 'features.pkl'), 'rb') as f:
            features = pickle.load(f)

        with open(os.path.join(root_dir, model_dir, 'all_results.pkl'), 'rb') as f:
            all_results = pickle.load(f)

        with open(os.path.join(root_dir, model_dir, 'tokenizer.pkl'), 'rb') as f:
            tokenizer = pickle.load(f)

        if config['type'] == 'train':
            features_train.append(features)
            all_results_train.append(all_results)
            tokenizers_train.append(tokenizer)
        elif config['type'] == 'dev':
            features_dev.append(features)
            all_results_dev.append(all_results)
            tokenizers_dev.append(tokenizer)

    dev_to_save = [features_dev, all_results_dev, tokenizers_dev]
    with open(os.path.join(root_dir, 'saved_data_dev.pkl'), 'wb') as f:
        pickle.dump(dev_to_save, f)

    if not only_dev:
        with open(os.path.join(root_dir, 'saved_data_train.pkl'), 'wb') as f:
            pickle.dump([features_train, all_results_train, tokenizers_train], f)

if __name__ == "__main__":
    generate(only_dev=False)


import copy
import random

from direct.actor.Actor import Actor
from direct.interval.IntervalGlobal import (
    Func,
    LerpAnimInterval,
    LerpScaleInterval,
    Sequence,
)
from direct.particles.ParticleEffect import ParticleEffect
from panda3d.core import CollisionCapsule

from const import MOUSE_MASK, NO_MASK
from gui.character import HealthBar
from utils import address, chance, take_random

from units.shooter import Shooter
from units.unit import Unit
from .character_data import CLASSES, NAMES

# cohesion relation indicator colors
RELATION_COLORS = {
    0: (1, 0.01, 0.24, 1),  # red
    1: (1, 0.45, 0.09, 1),  # orange
    2: (1, 1, 0, 1),  # yellow
    3: (0.67, 0.97, 0.82, 1),  # light green
    4: (0.18, 0.97, 0.57, 1),  # green
    5: (0.18, 0.97, 0.57, 1),  # green
}

class Character(Shooter, Unit):
    """A game character.

    Represents a unit controlled by a player.

    Args:
        id_ (int): This character unique id.
        name (str): This character name.
        class_ (str): This character class.
        sex (str): This character gender.
        team (crew.Crew): The Crew object.
        desc (dict):
            Optional. This character description (used
            for loading character from a saved game).
    """

    def __init__(self, id_, name, class_, sex, team, desc=None):
        Unit.__init__(
            self, "character_" + str(id_), class_, CLASSES[sex + "_" + class_]
        )
        Shooter.__init__(self)

        self._team = team
        self._current_pos = None
        self._current_anim = None
        self._idle_seq = None
        self._cohesion_ball = None
        self._is_stunned = False
        self._is_stimulated = False
        self._health_bar = None

        self.inhale = 15
        self.damage_range = [4, 7]
        self.clear_damage = [4, 7]
        self.effects = {}
        self._yeah_snds = []

        self.name = name
        self.sex = sex
        if sex == "male":
            self.heshe = base.labels.PRONOUNS[0]  # noqa: F821
            self.hisher = base.labels.PRONOUNS[1]  # noqa: F821
            self.himher = base.labels.PRONOUNS[2]  # noqa: F821
        else:
            self.heshe = base.labels.PRONOUNS[3]  # noqa: F821
            self.hisher = base.labels.PRONOUNS[4]  # noqa: F821
            self.himher = base.labels.PRONOUNS[5]  # noqa: F821

        if desc:
            self._energy = desc["energy"]
            self.is_diseased = desc["is_diseased"]
            self.get_well_score = desc["get_well_score"]
            self.traits = desc["traits"]
            self.disabled_traits = desc["disabled_traits"]

            if self.is_diseased:
                taskMgr.doMethodLater(  # noqa: F821
                    60, self.get_well, self.id + "_get_well"
                )
        else:
            self._energy = 100
            self.is_diseased = False
            self.get_well_score = 0

            self.disabled_traits = []
            self.traits = []
            traits = copy.copy(base.labels.TRAITS)  # noqa: F821
            for _ in range(random.randint(0, 2)):
                self.traits.append(random.choice(take_random(traits)))

    @property
    def clear_delay(self):
        """
        Delay between this character death
        and clearing the character object.

        Returns:
            float: Seconds to wait before clearing.
        """
        return 3.5

    @property
    def damage(self):
        """This character one-time calculated damage.

        Returns:
            float: Damage one-time made by this character.
        """
        return random.uniform(*self.damage_range) * self.damage_factor

    @property
    def damage_factor(self):
        """This character damage factor.

        The damage factor depends on cohesion with another
        character on the same part of the Train.

        Returns:
            float: Damage one-time made by this character.
        """
        factor = self._team.calc_cohesion_factor(self.current_part.chars, self)
        if (
            # Loner
            base.labels.TRAITS[4][1] in self.traits  # noqa: F821
            and len(self.current_part.chars) == 1
        ):
            return factor * 1.3

        return factor

    @property
    def description(self):
        """This character state in saveable form.

        Used for saving the character.

        Returns:
            dict: This character description.
        """
        return {
            "id": int(self.id.split("_")[1]),
            "name": self.name,
            "sex": self.sex,
            "class": self.class_,
            "health": self.health,
            "energy": self.energy,
            "place": self.current_part.name,
            "traits": self.traits,
            "disabled_traits": self.disabled_traits,
            "is_diseased": self.is_diseased,
            "get_well_score": self.get_well_score,
        }

    @property
    def energy(self):
        """This character energy.

        Returns:
            int: This character energy points.
        """
        return self._energy

    @energy.setter
    def energy(self, value):
        """This character energy setter.

        Args:
            value (int): New energy value.
        """
        self._energy = min(max(value, 0), 80 if self.is_diseased else 100)

    @property
    def shooting_speed(self):
        """Delay between shots of this unit.

        Returns:
            float: Delay between shots in seconds.
        """
        if base.labels.TRAITS[0][0] in self.traits:  # noqa: F821
            # Fast hands
            return 1.1 + random.uniform(0.1, 0.8)

        if base.labels.TRAITS[0][1] in self.traits:  # noqa: F821
            # Snail
            return 2 + random.uniform(0.1, 1.1)

        return 1.7 + random.uniform(0.1, 0.9)

    @property
    def statuses(self):
        """Return strings describing the current character status.

        Returns:
            list: Status description lines.
        """
        statuses = []
        if base.world.sun.is_dark:  # noqa: F821
            if base.labels.TRAITS[1][0] in self.traits:  # noqa: F821
                statuses.append(base.labels.STATUSES[0])  # noqa: F821
            elif base.train.lights_on:  # noqa: F821
                if "Floodlights" not in base.train.upgrades:  # noqa: F821
                    statuses.append(base.labels.STATUSES[1])  # noqa: F821
            else:
                statuses.append(base.labels.STATUSES[2])  # noqa: F821

        if self.energy <= 95:
            statuses.append(
                base.labels.STATUSES[3].format((100 - self.energy) // 5)  # noqa: F821
            )

        if not base.world.is_in_city and self.current_part is not None:  # noqa: F821
            factor = round(self.damage_factor, 2)
            if factor != 1:
                statuses.append(base.labels.STATUSES[4].format(factor))  # noqa: F821

        if self.health < 50 and base.labels.TRAITS[2][1] in self.traits:  # noqa: F821
            statuses.append(base.labels.STATUSES[5])  # noqa: F821

        if self.is_diseased:
            statuses.append(base.labels.STATUSES[6])  # noqa: F821

        if (
            base.labels.TRAITS[6][1] in self.traits  # noqa: F821
            and base.train.ctrl.current_speed > 0.75  # noqa: F821
        ):
            statuses.append(base.labels.STATUSES[7])  # noqa: F821

        return statuses[:4]

    @property
    def tooltip(self):
        """Tooltip to show on this character mouse pointing.

        Returns:
            str: This character name.
        """
        return self.name

    def attack(self, enemy_unit):
        """Make the given enemy unit this character's target.

        Args:
            enemy_unit (units.enemy.EnemyUnit): Enemy unit to attack.
        """
        if enemy_unit in self.current_part.enemies:
            self._target = enemy_unit

    def prepare(self):
        """Load the character model and positionate it.

        Tweak collision solid and sounds.
        """
        animations = {
            name: address(self.class_ + "-" + name)
            for name in (
                "die",
                "gun_up",
                "incline1",
                "release_gun",
                "stand_and_aim",
                "stand",
                "surrender",
                "tread1",
                "turn_head1",
                "stunned",
                "cough",
                "celebrate",
            )
        }
        self.model = Actor(address(self.sex + "_" + self.class_), animations)
        self.model.enableBlend()
        self.model.setControlEffect("stand", 1)

        self.model.setPlayRate(0.8, "stand_and_aim")
        self.model.setPlayRate(0.6, "stand")
        self.model.loop("stand")
        self._current_anim = "stand"

        self._health_bar = HealthBar(self)

        base.team.init_relations(self)  # noqa: F821

        taskMgr.doMethodLater(  # noqa: F821
            random.randint(40, 60), self._idle_animation, self.id + "_idle_anim"
        )
        self._col_node = self._init_col_node(
            NO_MASK, MOUSE_MASK, CollisionCapsule(0, 0, 0, 0, 0, 0.035, 0.035)
        )
        self.shot_snd = self._set_shoot_snd(self.class_data["shot_snd"])
        self._cough_snd = base.sound_mgr.loadSfx(  # noqa: F821
            "sounds/{sex}_cough.ogg".format(sex=self.sex)
        )
        base.sound_mgr.attachSoundToObject(self._cough_snd, self.model)  # noqa: F821
        self._die_snd = base.sound_mgr.loadSfx(  # noqa: F821
            "sounds/combat/{sex}_death.ogg".format(sex=self.sex)
        )
        base.sound_mgr.attachSoundToObject(self._die_snd, self.model)  # noqa: F821

        for i in range(1, 4):
            yeah_snd1 = base.sound_mgr.loadSfx(  # noqa: F821
                "sounds/{sex}_yes{num}.ogg".format(sex=self.sex, num=str(i))
            )
            base.sound_mgr.attachSoundToObject(yeah_snd1, self.model)  # noqa: F821

            self._yeah_snds.append(yeah_snd1)

        if self.class_ == "soldier":
            z = 0.063 if self.sex == "male" else 0.062
        elif self.class_ == "raider":
            z = 0.047
        elif self.class_ == "anarchist":
            z = 0.06 if self.sex == "male" else 0.057

        self._shoot_anim = self._set_shoot_anim(
            (0.004, 0.045, z), 97, self.class_data["shots_num"]
        )
        taskMgr.doMethodLater(  # noqa: F821
            self.class_data["energy_spend"],
            self._reduce_energy,
            self.id + "_reduce_energy",
        )
        self._prepare_cohesion_particles()
        self._prepare_auras()

    def _prepare_cohesion_particles(self):
        """Prepare cohesion skills particle effects."""
        effect = ParticleEffect()
        effect.loadConfig("effects/recall_the_past.ptf")

        self.effects["recall_the_past"] = {"length": 0.7, "effect": effect}

        effect = ParticleEffect()
        effect.loadConfig("effects/not_leaving_ours.ptf")

        self.effects["not_leaving_ours"] = {"length": 2, "effect": effect}

        effect = ParticleEffect()
        effect.loadConfig("effects/common_rage.ptf")

        self.effects["common_rage"] = {"length": 90, "effect": effect}

    def _prepare_auras(self):
        """Prepare cohesion skills aura effects."""
        aura = loader.loadModel(address("cover_fire_aura"))  # noqa: F821
        aura.hide()
        aura.reparentTo(self.model)
        aura.setY(0.007)

        self.effects["cover_fire"] = {
            "model": aura,
            "seq": Sequence(
                LerpScaleInterval(aura, 1.5, (2, 2, 2), (1, 1, 1)),
                LerpScaleInterval(aura, 1.5, (1, 1, 1), (2, 2, 2)),
            ),
        }

        aura = loader.loadModel(address("common_rage_aura"))  # noqa: F821
        aura.hide()
        aura.reparentTo(self.model)
        aura.setY(0.007)

        self.effects["common_rage_aura"] = {
            "model": aura,
            "seq": Sequence(
                LerpScaleInterval(aura, 1.5, (2, 2, 2), (1, 1, 1)),
                LerpScaleInterval(aura, 1.5, (1, 1, 1), (2, 2, 2)),
            ),
        }

        aura = loader.loadModel(address("hold_together_aura"))  # noqa: F821
        aura.hide()
        aura.reparentTo(self.model)
        aura.setY(0.007)

        self.effects["hold_together"] = {
            "model": aura,
            "seq": Sequence(
                LerpScaleInterval(aura, 1.5, (2, 2, 2), (1, 1, 1)),
                LerpScaleInterval(aura, 1.5, (1, 1, 1), (2, 2, 2)),
            ),
        }

    def play_cohesion_effect(self, name):
        """Play the given cohesion skill effect.

        Args:
            name (str): Name of the skill.
        """
        self.effects[name]["effect"].start(self.model, self.model)
        self.effects[name]["effect"].softStart()

        taskMgr.doMethodLater(  # noqa: F821
            self.effects[name]["length"],
            self.effects[name]["effect"].softStop,
            "stop_cohesion_effect",
            extraArgs=[],
        )

    def play_cohesion_aura(self, name):
        """Play the given aura effect.

        Args:
            name (str): The effect name.
        """
        self.effects[name]["model"].show()
        self.effects[name]["seq"].loop()

    def play_yes(self):
        """Play a voice sound, when the character is chosen."""
        random.choice(self._yeah_snds).play()

    def stop_aura_effect(self, name):
        """Stop the given aura effect.

        Args:
            name (str): The effect name.
        """
        self.effects[name]["model"].hide()
        self.effects[name]["seq"].finish()

    def celebrate(self, task):
        """Play victory celebration animation."""
        if self.current_part.name == "part_rest":
            return task.done

        LerpAnimInterval(self.model, 0.1, self._current_anim, "celebrate").start()
        self._current_anim = "celebrate"
        self.model.play("celebrate")
        return task.done

    def move_to(self, part, to_pos=None):
        """Move this Character to the given Train part.

        If the movement is done with a positions exchange
        between two characters, the method doesn't
        take or release characters cells, just replaces
        one character with another one.

        Args:
            part (train.part.TrainPart):
                Train part to move this Character to.
            to_pos (dict):
                Position to move this Character to.

        Returns:
            bool: True, if the character was moved.
        """
        if to_pos:
            part.chars.append(self)
            pos = to_pos
        else:
            pos = part.give_cell(self)

        if not pos:  # no free cells on the chosen part
            return

        if self.current_part is not None:
            if to_pos:
                self.current_part.chars.remove(self)
            else:
                self.current_part.release_cell(self._current_pos, self)

            if self.current_part.name == "part_rest":
                self.stop_rest()

        if part.name == "part_rest":
            self.rest()

        self.model.wrtReparentTo(part.parent)
        self.model.setPos(pos)
        self.model.setH(part.angle)

        self.current_part = part
        self._current_pos = pos

        if part.name == "part_rest" and base.char_gui.rest_list_shown:  # noqa: F821
            base.char_gui.update_resting_chars(part)  # noqa: F821

        return True

    def do_effects(self, effects):
        """Do outing effects to this character.

        Effects are: adding traits, changing health and energy.

        Args:
            effects (dict): Effects and their values.
        """
        if effects is None:
            return

        effects = copy.deepcopy(effects)
        self.get_damage(-effects.pop("health", 0))

        if "add_trait" in effects and len(self.traits + self.disabled_traits) < 3:
            ind1, ind2 = effects["add_trait"]
            trait = base.labels.TRAITS[ind1][ind2]  # noqa: F821

            if trait and trait not in self.traits + self.disabled_traits:
                self.traits.append(trait)
                base.char_gui.move_status_label(-1)  # noqa: F821
                effects.pop("add_trait")

        for key, value in effects.items():
            if hasattr(self, key):
                setattr(self, key, getattr(self, key) + value)

    def prepare_to_fight(self):
        """Prepare this character for a fight.

        Switch animations and run a task to choose a target.
        """
        self._stop_tasks("_idle_anim")
        if self._idle_seq is not None:
            self._idle_seq.finish()

        self.model.loop("stand_and_aim")
        LerpAnimInterval(self.model, 0.8, self._current_anim, "stand_and_aim").start()
        LerpAnimInterval(self.model, 0.8, "stand", "stand_and_aim").start()
        self._current_anim = "stand_and_aim"

        taskMgr.doMethodLater(  # noqa: F821
            0.5, self._choose_target, self.id + "_choose_target"
        )
        self._health_bar.show_health()

    def rest(self):
        """Make this character rest.

        Stops all the active tasks and starts energy regaining/healing.
        """
        self._stop_tasks(
            "_reduce_energy", "_shoot", "_aim", "_choose_target", "_idle_anim"
        )
        self.model.hide()
        self._col_node.stash()

        taskMgr.doMethodLater(  # noqa: F821
            0.05,
            self._calm_down,
            self.id + "_calm_down",
            extraArgs=[True],
        )
        taskMgr.doMethodLater(  # noqa: F821
            self.class_data["energy_gain"], self._gain_energy, self.id + "_gain_energy"
        )
        if self.health < self.class_data["health"]:
            taskMgr.doMethodLater(  # noqa: F821
                self.class_data["healing"], self._heal, self.id + "_heal"
            )

    def surrender(self):
        """Stop fighting, surrender."""
        self._stop_tasks("_shoot", "_aim", "_choose_target")
        self._col_node.removeNode()
        self._shoot_anim.finish()

        LerpAnimInterval(self.model, 0.5, "stand_and_aim", "surrender").start()
        self.model.play("surrender")

    def stop_rest(self):
        """Stop this character rest."""
        self.model.show()
        self._col_node.unstash()
        base.char_gui.destroy_char_button(self.id)  # noqa: F821

        self._stop_tasks("_gain_energy", "_heal")
        taskMgr.doMethodLater(  # noqa: F821
            self.class_data["energy_spend"],
            self._reduce_energy,
            self.id + "_reduce_energy",
        )
        if base.world.enemy.active_units:  # noqa: F821
            self.prepare_to_fight()

    def _reduce_energy(self, task):
        """
        Reduce the character energy according to
        day part and status: fighting or not.
        """
        if base.world.sun.is_dark:  # noqa: F821
            if not base.train.lights_on:  # noqa: F821
                task.delayTime = 15
            elif "Floodlights" in base.train.upgrades:  # noqa: F821
                task.delayTime = self.class_data["energy_spend"]
            else:
                task.delayTime = 20

            if base.labels.TRAITS[1][1] in self.traits:  # noqa: F821
                # Fear of dark
                task.delayTime /= 2

        elif self._target:
            # Nervousness
            task.delayTime = (
                15 if base.labels.TRAITS[5][1] in self.traits else 20  # noqa: F821
            )
        else:
            task.delayTime = self.class_data["energy_spend"]

        if (
            base.labels.TRAITS[2][1] in self.traits  # noqa: F821
            and self.health < self.class_data["health"] / 2
        ):
            # Hemophobia
            task.delayTime *= 0.75

        self.energy -= 1
        if base.labels.TRAITS[7][0] in self.traits:  # noqa: F821
            # Mechanic
            base.train.get_damage(-3)  # noqa: F821

        return task.again

    def _gain_energy(self, task):
        """Regain this character energy.

        Used as a timed task while the character is resting.
        """
        if (
            base.labels.TRAITS[6][1] in self.traits  # noqa: F821
            and base.train.ctrl.current_speed > 0.75  # noqa: F821
        ):
            # Motion sickness
            return task.again

        self.energy += 3

        for char in self.current_part.chars:
            if char.sex == "female" and self.id != char.id:
                task.delayTime = self.class_data["energy_gain"] - 5
                return task.again

        return task.again

    def _heal(self, task):
        """Regain this character health.

        Used as a timed task while the character is resting.
        """
        if (
            base.labels.TRAITS[6][1] in self.traits  # noqa: F821
            and base.train.ctrl.current_speed > 0.75  # noqa: F821
        ):
            # Motion sickness
            return task.again

        if base.labels.TRAITS[7][1] in self.traits and chance(40):  # noqa: F821
            # Pharmacophobia
            return task.again

        if self.health < self.class_data["health"]:
            self.health += 1
            return task.again

        return task.done

    def _choose_target(self, task):
        """Choose an enemy to shoot.

        Only an enemy from the Train part shooting
        range can be chosen as a target.
        """
        if self.current_part and self.current_part.enemies:
            self._target = random.choice(self.current_part.enemies)
            taskMgr.doMethodLater(0.1, self._aim, self.id + "_aim")  # noqa: F821
            taskMgr.doMethodLater(1, self._shoot, self.id + "_shoot")  # noqa: F821
            return task.done

        # enemies retreated - return to passive state
        if (
            not base.world.enemy.active_units  # noqa: F821
            and base.world.scp_train is None  # noqa: F821
        ):
            taskMgr.doMethodLater(  # noqa: F821
                7,
                self._calm_down,
                self.id + "_calm_down",
                extraArgs=[False],
            )
            return task.done

        return task.again

    def _aim(self, task):
        """Rotate the character to aim on enemy."""
        if (
            self._target
            and self._target.is_dead
            and base.labels.TRAITS[5][0] in self.traits  # noqa: F821
        ):
            # Bloodthirsty
            self.health += 7

        if self.current_part and self._target in self.current_part.enemies:
            if self._target.model:
                self.model.headsUp(self._target.model)
                return task.again

        self._stop_tasks("_shoot")
        self._target = None

        if base.world.enemy.active_units:  # noqa: F821
            taskMgr.doMethodLater(  # noqa: F821
                0.5, self._choose_target, self.id + "_choose_target"
            )
            return task.done

        if base.world.scp_train is not None:  # noqa: F821
            return task.again

        taskMgr.doMethodLater(  # noqa: F821
            7, self._calm_down, self.id + "_calm_down", extraArgs=[False]
        )
        return task.done

    def _calm_down(self, force):
        """Return to passive state on a combat over.

        Args:
            force (bool):
                If True, change to passive animation fast.
                Change animation in usual speed otherwise.
        """
        self._stop_tasks("_shoot", "_calm_down")
        if self.current_part:
            self.model.hprInterval(
                0.1 if force else 2, (self.current_part.angle, 0, 0)
            ).start()

        if force:
            for anim in self.model.get_anim_names():
                self.model.setControlEffect(anim, 0)

            self.model.stop()
            self.model.setControlEffect("stand", 1)
            self.model.loop("stand")
        else:
            LerpAnimInterval(self.model, 2, self._current_anim, "stand").start()

        self._current_anim = "stand"
        taskMgr.doMethodLater(  # noqa: F821
            random.randint(40, 60), self._idle_animation, self.id + "_idle_anim"
        )
        self._health_bar.hide_health()

    def _idle_animation(self, task):
        """Play one of the idle animations.

        Args:
            task (panda3d.core.PythonTask): Task object.
        """
        if self.is_diseased and chance(80):
            self._current_anim = "cough"
            self._cough_snd.play()
        else:
            self._current_anim = random.choice(
                ("incline1", "gun_up", "release_gun", "tread1", "turn_head1")
            )
        LerpAnimInterval(self.model, 0.2, "stand", self._current_anim).start()

        self._idle_seq = Sequence(
            self.model.actorInterval(self._current_anim, playRate=0.75),
            Func(self._stand),
        )
        self._idle_seq.start()

        task.delayTime = random.randint(40, 60)
        return task.again

    def _stand(self):
        """Return to the idle stand animation."""
        LerpAnimInterval(self.model, 0.2, self._current_anim, "stand").start()
        self._current_anim = "stand"

    def _die(self):
        """Character death sequence.

        Stop all the character's tasks, play death animation
        and plan the character object clearing.
        """
        if self._team.hold_together:
            self.health = 1
            return False

        if not Shooter._die(self):
            return False

        Unit._die(self)

        if base.common_ctrl.chosen_char == self:  # noqa: F821
            base.common_ctrl.deselect(clear_resting=False)  # noqa: F821

        self._health_bar.hide_health()
        self._stop_tasks(
            "_reduce_energy", "_get_well", "_infect", "_stop_stimul", "_gain_energy"
        )

        base.char_gui.destroy_char_button(self.id)  # noqa: F821
        self._team.delete_relations(self.id)

        LerpAnimInterval(self.model, 0.3, "stand_and_aim", "die").start()
        LerpAnimInterval(self.model, 0.3, self._current_anim, "die").start()
        self.model.hprInterval(1, (self.current_part.angle, 0, 0)).start()
        self.model.play("die")
        self._die_snd.play()

        taskMgr.doMethodLater(3, self._hide, self.id + "_hide")  # noqa: F821

    def _hide(self, task):
        """Hide the main model."""
        self.model.hide()
        return task.done

    def leave(self):
        """Make this character leave, plan clearing.

        Used only when sending a character away in a city.
        """
        self._stop_tasks("_calm_down", "_gain_energy", "_heal", "_infect", "_get_well")
        taskMgr.doMethodLater(0.05, self.clear, self.id + "_clear")  # noqa: F821

    def clear(self, task):
        """Clear this character.

        Release models and sounds memory, release the part
        cell and delete the character from the crew list.
        """
        self.model.cleanup()
        self._health_bar.removeNode()
        self.model.removeNode()
        base.sound_mgr.detach_sound(self.shot_snd)  # noqa: F821
        base.sound_mgr.detach_sound(self._cough_snd)  # noqa: F821
        base.sound_mgr.detach_sound(self._die_snd)  # noqa: F821
        for snd in self._yeah_snds:
            base.sound_mgr.detach_sound(snd)  # noqa: F821

        self._team.chars.pop(self.id)
        base.res_gui.update_chars()  # noqa: F821
        self.current_part.release_cell(self._current_pos, self)
        self.current_part = None

        return task.done

    def exchange_pos(self, char):
        """Exchange positions of this Character and the given one.

        Args:
            char (units.character.Character):
                Character to exchange the positions with.
        """
        o_part, o_pos = self.current_part, self._current_pos
        self.move_to(char.current_part, char._current_pos)
        char.move_to(o_part, o_pos)

    def hide_relations_ball(self):
        """Hide the relations ball of this character."""
        if self._cohesion_ball is not None:
            self._cohesion_ball.removeNode()
            self._cohesion_ball = None

    def show_relation(self, cohesion):
        """Show a relations ball above this character.

        The color of the ball shows the strength of cohesion
        between the chosen character and this one.

        Args:
            cohesion (float):
                Value of the cohesion between the
                chosen character and this one.
        """
        if self._cohesion_ball is None:
            self._cohesion_ball = loader.loadModel(  # noqa: F821
                address("relation_ball")
            )
            self._cohesion_ball.clearLight()
            base.world.sun.ignore_shadows(self._cohesion_ball)  # noqa: F821

            self._cohesion_ball.reparentTo(self.model)
            self._cohesion_ball.setZ(0.14)

        self._cohesion_ball.setColorScale(*RELATION_COLORS[cohesion // 20])

    def _missed_shot(self):
        """Calculate if character missed the current shot.

        Returns:
            bool: True if character missed, False otherwise.
        """
        miss_chance = 0
        if self.class_ == "soldier":
            if (
                abs(self._target.node.getX()) < 0.56
                and abs(self._target.node.getY()) < 0.95
            ):
                miss_chance += 20
        elif self.class_ == "raider":
            if (
                abs(self._target.node.getX()) > 0.56
                and abs(self._target.node.getY()) > 0.95
            ):
                miss_chance += 20

        if base.world.sun.is_dark:  # noqa: F821
            if base.labels.TRAITS[1][0] in self.traits:  # noqa: F821
                # Cat eyes
                miss_chance -= 5
            elif base.train.lights_on:  # noqa: F821
                if "Floodlights" not in base.train.upgrades:  # noqa: F821
                    miss_chance += 10
            else:
                miss_chance += 20

        miss_chance += (100 - self.energy) // 5
        if self._team.cover_fire:
            miss_chance = min(100, max(0, miss_chance - 25))

        return chance(miss_chance)

    def get_stunned(self):
        """Make this character stunned for some time.

        Stunned unit can't shoot.
        """
        if self._is_stunned or self._is_stimulated:
            return

        self._is_stunned = True
        self._stop_tasks("_aim", "_shoot")

        self._current_anim = "stunned"
        self.model.play("stunned")
        LerpAnimInterval(self.model, 0.05, "stand_and_aim", "stunned").start()

        taskMgr.doMethodLater(  # noqa: F821
            6, self._stop_stunning, self.id + "_stop_stunning"
        )

    def get_sick(self, is_infect=False):
        """Calculations to get this character sick.

        The worse condition the character has the higher is the chance
        for him to get sick. Sick character has lower energy maximum,
        and all his positive traits are disabled until getting well.

        Args:
            is_infect (bool):
                True if the disease came from a
                character on the same Train part.
        """
        if self.is_diseased:
            return

        cond_percent = (self.energy + self.health) / (100 + self.class_data["health"])
        percent = (1 - cond_percent) * 30

        sick_chance = max(
            0,  # not less than zero
            (
                percent
                + (20 if is_infect else 0)
                # Immunity
                - (40 if base.labels.TRAITS[3][0] in self.traits else 0)  # noqa: F821
                # Weak immunity
                + (20 if base.labels.TRAITS[3][1] in self.traits else 0)  # noqa: F821
            ),
        )

        if chance(sick_chance):
            self.is_diseased = True
            self.get_well_score = 0

            for traits_pair in base.labels.TRAITS:  # noqa: F821
                if traits_pair[0] in self.traits:

                    self.traits.remove(traits_pair[0])
                    self.disabled_traits.append(traits_pair[0])

            taskMgr.doMethodLater(  # noqa: F821
                60, self.get_well, self.id + "_get_well"
            )
            taskMgr.doMethodLater(240, self.infect, self.id + "_infect")  # noqa: F821
            self.energy = min(self.energy, 80)

    def get_stimulated(self):
        """Use a stimulator on this character.

        Disables all the negative traits for some time.
        """
        for traits_pair in base.labels.TRAITS:  # noqa: F821
            if traits_pair[1] in self.traits:
                self.traits.remove(traits_pair[1])
                self.disabled_traits.append(traits_pair[1])

        self._is_stimulated = True
        taskMgr.doMethodLater(  # noqa: F821
            300, self._stop_stimul, self.id + "_stop_stimul"
        )

    def _stop_stimul(self, task):
        """Stop stimulation of this character.

        Returns back all the disabled negative traits.
        """
        for trait_pair in base.labels.TRAITS:  # noqa: F821
            if trait_pair[1] in self.disabled_traits:
                self.traits.append(trait_pair[1])
                self.disabled_traits.remove(trait_pair[1])

        self._is_stimulated = False
        return task.done

    def get_well(self, task):
        """Get this character well.

        When the character got well, his energy maximum is restored to
        the default value, and his positive traits are back operational.
        """
        self.get_well_score += 1
        if self.get_well_score < 20:
            return task.again

        self.is_diseased = False
        for trait_pair in base.labels.TRAITS:  # noqa: F821
            if trait_pair[0] in self.disabled_traits:
                self.traits.append(trait_pair[0])
                self.disabled_traits.remove(trait_pair[0])

        self._stop_tasks("_infect")
        return task.done

    def infect(self, task):
        """Infect other characters on the same Train part."""
        for char in self.current_part.chars:
            if char.id != self.id:
                char.get_sick(is_infect=True)

        return task.again

    def _shoot(self, task):
        """Play shooting animation and sound, make damage."""
        if self.current_part and not self.current_part.is_covered:
            return Shooter._shoot(self, task)

        return task.again

    def _stop_stunning(self, task):
        """Stop this character stun and continue fighting."""
        self._is_stunned = False

        LerpAnimInterval(self.model, 0.8, self._current_anim, "stand_and_aim").start()
        self._current_anim = "stand_and_aim"
        self.model.loop("stand_and_aim")

        taskMgr.doMethodLater(0.1, self._aim, self.id + "_aim")  # noqa: F821
        taskMgr.doMethodLater(1, self._shoot, self.id + "_shoot")  # noqa: F821

        return task.done

    def get_damage(self, damage):
        """Getting damage.

        Args:
            damage (int): Damage points to get.
        """
        if base.labels.TRAITS[2][0] in self.traits:  # noqa: F821
            # Masochism
            self.energy += 1

        Unit.get_damage(self, damage)

    def get_scorch_damage(self, task):
        """Get timed damage from the SCP light ray."""
        self.get_damage(0.3)
        return task.again

    def get_stench_damage(self):
        """Get damage from the Stench."""
        if base.labels.TRAITS[6][0] in self.traits and self.inhale > 0:  # noqa: F821
            # Deep breath
            self.inhale -= 1
            return

        if (
            self.current_part
            and self.current_part.name == "part_rest"
            and "Window Frames" in base.train.upgrades  # noqa: F821
        ):
            return

        self.get_damage(1)

def generate_char(id_, class_, sex, team=None):
    """Generate character with the given parameters.

    Args:
        id_ (str): Character id.
        class_ (str): Character class.
        sex (str): Character gender.
        team (crew.Crew): Optional. Crew to add new character into.

    Returns:
        Character: The generated character.
    """
    return Character(id_, random.choice(NAMES[sex]), class_, sex, team)

def load_char(desc, team, parts):
    """Load a character, using the given description.

    Load the model, state and move to the saved position.

    Args:
        desc (dict): Character description.
        team (crew.Crew): The crew object.
        parts (dict): Train parts index.

    Returns:
        crew.character.Character: A ready-to-go character object.
    """
    char = Character(desc["id"], desc["name"], desc["class"], desc["sex"], team, desc)
    char.health = desc["health"]
    char.prepare()
    char.move_to(parts[desc["place"]])
    return char


"""
1) Создать базу данных для хранения информации о книгах в библиотеке.
2) База данных должна содержать две таблицы: "Книги" и "Авторы".
3) В таблице "Книги" должны быть следующие поля: id, название, год издания,
количество экземпляров и id автора.
4) В таблице "Авторы" должны быть следующие поля: id, имя и фамилия.
5) Необходимо создать связь между таблицами "Книги" и "Авторы".
6) Написать функцию-обработчик, которая будет выводить список всех книг с
указанием их авторов.
"""
import random

from flask import Flask, render_template
from task_2_models import db, Books, Authors

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///seminardb.db'
db.init_app(app)

@app.route('/')
def index():
    context = {'title': 'Faculty main page'}
    return render_template('index.html', **context)

@app.cli.command('init-db')
def init_db():
    db.create_all()
    print('OK')

@app.cli.command('fill-auth')
def fill_authors():
    count = 5
    for author in range(1, count + 1):
        new_author = Authors(first_name=f'Name{author}',
                             last_name=f'Surname{author}')
        db.session.add(new_author)
    db.session.commit()
    print('Authors OK')

@app.cli.command('fill-b')
def fill_books():
    count = 5
    for book in range(1, count ** 2):
        author = Authors.query.filter_by(last_name=f'Surname{book % count + 1}').first()
        new_book = Books(name=f'Bookname{book}',
                         year=random.randint(2003, 2023),
                         amount=random.randint(10, 250),
                         author=author)
        db.session.add(new_book)
    db.session.commit()
    print('Books OK')

@app.route('/books/')
def library():
    books = Books.query.all()
    authors = Authors.query.all()
    context = {'title': 'Library', 'books': books, 'authors': authors}
    return render_template('books.html', **context)

if __name__ == '__main__':
    app.run(debug=True)


from typing import Dict, Union
import numpy as np
import pandas as pd
import sklearn
from sklearn.base import BaseEstimator
from scipy.interpolate import interp1d

import metrics

class MetaModel:
    def __init__(self, models_dict: Dict[str, BaseEstimator]) -> None:
        self.models_dict = models_dict
        self.mappings_to_precision: Dict[str, interp1d] = {}

    @staticmethod
    def _get_mapping_to_precision(
        X: Union[pd.DataFrame, np.ndarray],
        y: Union[pd.Series, np.ndarray],
        model: BaseEstimator,
    ) -> interp1d:
        """Compute the threshold corresponding to the ACCEPTABLE RECALL"""
        scores = metrics.get_scores(X, model)
        precisions, _, thresholds = sklearn.metrics.precision_recall_curve(y, scores)
        score_to_precision = interp1d(
            thresholds, precisions[:-1], bounds_error=False, fill_value=(0, 1)
        )
        return score_to_precision

    def fit(
        self,
        X: Union[pd.DataFrame, np.ndarray],
        y: Union[pd.Series, np.ndarray],
    ):
        """Run the fit method on all the submodels, and compute thresholds at 90% recall"""
        for name, model in self.models_dict.items():
            self.models_dict[name] = model.fit(X, y)
            # The threshold is computed on the training data, not on the test data
            # otherwise that would be cheating (we can't compute a threshold on the fly when doing inference
            # it needs to have been computed beforehand)
            self.mappings_to_precision[name] = self._get_mapping_to_precision(
                X, y, model
            )
        return self

    def predict(self, X: Union[pd.DataFrame, np.ndarray]) -> np.ndarray:
        """Predict using the voting rule (if more than half of submodels predict True, than True)"""
        preds = []
        for model in self.models_dict.values():
            preds.append(model.predict(X))
        return (np.mean(preds, axis=0) > 0.5).astype(int)

    def decision_function(self, X: Union[pd.DataFrame, np.ndarray]) -> np.ndarray:
        """Get decision scores by taking the average of the submodels scores rescaled by their precomputed thresholds"""
        all_scores = []
        for name, model in self.models_dict.items():
            scores = metrics.get_scores(X, model)
            # Normalize scores by the threshold at ACCEPTABLE_RECALL
            all_scores.append([self.mappings_to_precision[name](s) for s in scores])
        raw_scores = np.mean(all_scores, axis=0)
        return raw_scores