from collections import OrderedDict
import inspect
from mcdp import logger

from contracts import contract
from contracts.utils import indent, check_isinstance

from .pretty_printing import pretty_print_dict
import copy

class Note(object):

    def __init__(self, msg, locations=None, stacklevel=0, prefix=()):
        self.msg = msg
        if locations is None: locations = OrderedDict()
        self.locations = OrderedDict(locations)
        stack = inspect.stack()
        self.created_function = stack[1+stacklevel][3]
        module = inspect.getmodule(stack[1+stacklevel][0])
        self.created_module = module.__name__
        self.created_file = module.__file__
        self.prefix = prefix

    def __str__(self):
        s = type(self).__name__
        if self.msg:
            s += "\n\n" + indent(self.msg, '  ') + '\n'
        else:
            s += '\n\n   (No messages given) \n'
        if self.locations:
            s += '\nThese are the locations indicated:\n'
            locations = OrderedDict()
            from mcdp_docs.location import LocationUnknown
            for k,v in self.locations.items():
                if isinstance(v, LocationUnknown):
                    locations[k] = '(location unknown)'
                else:
                    locations[k] = v
            s += '\n' + indent(pretty_print_dict(locations), '  ')
        else:
            s += '\n(No locations provided)'
        s += '\n\nCreated by function %s()' % self.created_function
        s += '\n   in module %s' % self.created_module
#         s += '\n   in file %s' % self.created_file
        # TODO: use Location
        if self.prefix:
            p = "/".join(self.prefix)
            s = indent(s, p + '> ')
        return s

class NoteError(Note):
    pass

class NoteWarning(Note):
    pass

class AugmentedResult(object):
    """ Wraps a result with notes and output. """

    def __init__(self):
        """ If result is None => """
        stacklevel = 0
        stack = inspect.stack()
        called_from = stack[1+stacklevel][3]
        self.desc = "Result of %s()" % called_from
        self.result = None
        self.notes = []
        self.output = []

    def get_errors(self):
        return [_ for _ in self.notes if isinstance(_, NoteError)]
    def get_warnings(self):
        return [_ for _ in self.notes if isinstance(_, NoteWarning)]

    def assert_no_error(self):
        errors = self.get_errors()
        if errors:
            msg = 'We have obtained %d errors.' % len(errors)
            d = OrderedDict()
            for i, e in enumerate(errors):
                d[str(i)] = e
            msg += '\n' + indent(pretty_print_dict(d), ' ')
            raise AssertionError(msg)

    def assert_error_contains(self, s):
        """ Asserts that one of the errors contains the string """
        for _ in self.notes:
            if isinstance(_, NoteError):
                ss = str(_)
                if s in ss:
                    return
        msg = 'No error contained the string %r' % s
        raise AssertionError(msg)

    def info(self, s):
        self.log.append('info: %s' % s)

    def set_result(self, x):
        self.result = x

    def get_result(self):
        r = self.result
        if r is None:
            msg = 'Could not get the value of this; the result was None'
            msg += '\n' + indent(self.summary(), '> ')
            raise ValueError(msg) # FIXME
        return r

    def summary(self):
        s = "AugmentedResult (%s)" % self.desc
#         s += '\n' + indent(self.desc, ': ')
        if self.notes:
            d = OrderedDict()
            for i, note in enumerate(self.notes):
                if isinstance(note, NoteWarning):
                    what = 'Warning'
                elif isinstance(note, NoteError):
                    what = 'Error'
                else:
                    assert False, note

                d['%s %d' % (what, i)] = note
            s += "\n" + indent(pretty_print_dict(d), '| ')
        else:
            s += '\n' + '| (no notes found)'
        return s

    def summary_only_errors(self):
        s = "AugmentedResult (%s)" % self.desc
        notes = self.get_errors()

        if notes:
            d = OrderedDict()
            for i, note in enumerate(notes):
                d['error %d' % i] = note
            s += "\n" + indent(pretty_print_dict(d), '| ')
        else:
            s += '\n' + '| (no notes found)'

        return s
    @contract(note=Note)
    def add_note(self, note):
        self.notes.append(note)

    def note_error(self, msg, locations=None):
        logger.error(msg)
        self.add_note(NoteError(msg, locations, stacklevel=1))

    def note_warning(self, msg, locations=None):
        logger.warning(msg)
        self.add_note(NoteWarning(msg, locations, stacklevel=1))

    @contract(prefix='str|tuple')
    def merge(self, other, prefix=()):
        if isinstance(prefix, str):
            prefix = (prefix,)
        check_isinstance(other, AugmentedResult)
        have = set()
        for n in self.notes:
            have.add(n.msg)
        for note in other.notes:
            if not note.msg in have:
                note2 = copy.deepcopy(note)
                note2.prefix = prefix + note2.prefix
                self.notes.append(note2)

        self.output.extend(other.output)


import datetime
fp = open("my_diary.txt", "a")
while True:
    entry = input("What are you doing now? (type quit to exit)")
    now =datetime.datetime.now()
    today = now.strftime("%d/%m/%y %H:%M")
    if "quit" in entry.lower():
        break
    # \n means new line
    fp.write(f"{today} {entry}\n")
fp.close()

#preguntar. Esto no funciona.


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import Imputer

x = pd.read_csv('../Dataset/Heart_v2.csv', encoding='latin-1')

##Columnwise Mode Imputation
imputer = Imputer(missing_values = 'NaN', strategy = 'most_frequent', axis=0) #mean of columns
imputer = imputer.fit(x.iloc[:,[2]])
x.iloc[:,[2]] = imputer.transform(x.iloc[:,[2]])

##Rowwise Mean Imputation
for i in range(4,20,4):
    #impute rows that have all NaN values
    test_df = x.iloc[:,i:i+4]
    nan_count = test_df.apply(lambda x: 4-x.count(), axis=1)
    for j in range(0,len(nan_count)):
       if(nan_count[j] == 4):
          x.iloc[j,i] = test_df.iloc[:,0].mean()
    imputer = Imputer(missing_values = 'NaN', strategy = 'mean', axis=1) #mean of columns
    imputer = imputer.fit(x.iloc[:,i:i+4])
    x.iloc[:,i:i+4] = imputer.transform(x.iloc[:,i:i+4])
x.to_csv('../Dataset/Heart_mean_imputed.csv', encoding='latin-1')


from django import forms

from django.core import validators

class StudentForm(forms.Form):
	name = forms.CharField( max_length=50)
	mail_id = forms.EmailField()
	trainer_name = forms.CharField()
	feedback = forms.CharField(widget=forms.Textarea)

	def clean(self):

		print('complete validation using single clean method')
		total_cleaned_data=super().clean()

		input_name=total_cleaned_data['name']
		if len(input_name)<4:
			raise forms.ValidationError('Name should be greater then 4 characters')

		input_trainer_name=total_cleaned_data['trainer_name']
		if input_trainer_name[0].lower()!='s':
			raise forms.ValidationError('Trainer name should start with s or S letter')


import cv2, os, time
import numpy as np

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Activation, LeakyReLU, Input
from tensorflow.keras import regularizers, optimizers
from tensorflow.keras import initializers
from tensorflow.keras.callbacks import EarlyStopping

from sklearn.model_selection import train_test_split

class Classificator:
    def __init__(self, model, detector):
        self.model = model
        self.detector = detector

    def train(self, images_path_good, images_path_other):
        images_good = DataHandler.from_folder(images_path_good)
        images_other = DataHandler.from_folder(images_path_other)

        X = []
        Y = []

        self.add_class(images_good, 1, X, Y)
        self.add_class(images_other, 0, X, Y)

        X = np.array(X)
        X = X.reshape(X.shape[0], -1)
        Y = np.array(Y)

        if isinstance(self.model, type(Sequential())):
            self.model.compile(optimizer=optimizers.Adam(learning_rate=0.001), loss='mse', metrics=['mse'])
            self.model.fit(X, Y, batch_size=16, epochs=10, verbose=1)
        else:
            self.model.fit(X, Y)

    def predict(self, images_path):
        X = []
        images = DataHandler.from_folder(images_path)
        self.add_class(images, None, X, None)
        X = np.array(X)
        X = X.reshape(X.shape[0], -1)
        X_predict = self.model.predict(X)

        return X_predict

    def add_class(self, images, lable, X, Y=None):
        for i in images:
            _, desc = self.detector.detectAndCompute(i, None)
            try:
                if len(desc) > 30:
                    X.append(desc[:30])
                    if Y != None:
                        Y.append(lable)
                else:
                    print("not enough key points")
            except Exception:
                print("this algorithm failed you")

class DataHandler:
    @staticmethod
    def from_folder(folder_path):
        data_folder= os.path.join(os.curdir, folder_path)
        data_set = [os.path.join(data_folder,file) for file in os.listdir(data_folder) if os.path.isfile(os.path.join(data_folder,file))]
        images = [cv2.imread(x , cv2.IMREAD_GRAYSCALE) for x in data_set]
        return images

    @staticmethod
    def from_video(video_path, output_path):
        DataHandler.video_cut(video_path, output_path)
        return DataHandler.from_folder(output_path)

    @staticmethod
    def video_cut(video_path, output_path):
        vidcap = cv2.VideoCapture(video_path)
        success,image = vidcap.read()
        count = 0
        success = True
        while success and count < 100:
            cv2.imwrite(output_path + "/frame%d.jpg" % count, image)
            success,image = vidcap.read()
            count += 1


"""
Provides base class for static objects. Static objects are those
objects that do not update continuously based on the telemetry
feed.
"""

import abc
from moviepy.video.io.bindings import PIL_to_npimage

class StaticBase(metaclass=abc.ABCMeta):
    """
    Defines base class for static objects, including default object
    return methods.

    To get the representation of the static object, `to_frame` is
    called. Execution chain is `to_frame` -> `_make_material` ->
    `_write_data`.

    To get the mask of the static object, `make_mask` is called.
    Execution chain is `make_mask` -> `_make_material`.
    """
    @abc.abstractmethod
    def to_frame(self):
        """Render the card with data. Override to customize."""
        return PIL_to_npimage(
            self._make_material(False).convert('RGB'))

    @abc.abstractmethod
    def make_mask(self):
        """Default mask creation. Override to customize."""
        return PIL_to_npimage(
            self._make_material(True).split()[-1].convert('RGB'))

    @abc.abstractmethod
    def _write_data(self):
        """Write data to the canvas."""

    @abc.abstractmethod
    def _make_material(self, material_only):
        """Create material used as a canvas."""

    @staticmethod
    def format_time(seconds):
        """
        Converts seconds into seconds, minutes:seconds, or
        hours:minutes.seconds as appropriate.
        """
        minutes, seconds = divmod(float(seconds), 60)
        hours, minutes = divmod(minutes, 60)

        return_value = (int(hours), int(minutes), float(seconds))

        if hours:
            return "{0:d}:{1:0>2d}:{2:0>5.2f}".format(*return_value)
        elif minutes:
            return "{1:d}:{2:0>5.2f}".format(*return_value)
        else:
            return "{2:.2f}".format(*return_value)


# -*- coding: utf-8 -*-

import time
import json
import subprocess
from sys import executable, platform
from pynput.keyboard import GlobalHotKeys
import pyautogui
import pyperclip
from urllib.parse import quote
import requests
from langdetect import detect, DetectorFactory

from apiconf import *

def translator(text, lang_param):
    translate = requests.post(API_URL.format(lang=lang_param,
                                            text=quote(text),
                                            email=EMAIL))

    if translate.status_code == 200:
        response = json.loads(translate.text)
        response_data = response["responseData"]
        response_text = response_data["translatedText"]
        return response_text

def set_lang_param(text):
    lang_param = None
    DetectorFactory.seed = 0
    try:
        detected_language = detect(text)
        print(detected_language)

        if detected_language == 'bg':
            detected_language = 'ru'
        if detected_language == 'ru':
            lang_param = 'ru|en'
        else:
            lang_param = ''.join([detected_language, '|ru'])
    except Exception as e:
        lang_param = 'no language detected'
    return lang_param

def get_xsel_clip():
    text = subprocess.check_output('xsel', universal_newlines=True)
    return text

def get_selection():
    if platform == "linux" or platform == "darwin":
        copied_text = get_xsel_clip();
    elif platform == "win32":
        copied_text = pyperclip.paste()
    return copied_text

def on_activate_t():
    clip = get_selection();
    lang_pair = set_lang_param(clip)
    translated_text = translator(clip, lang_pair)
    subprocess.Popen([executable, 'notify.py', translated_text, lang_pair])

def hotkey_listen():
    with GlobalHotKeys({'<ctrl>+<alt>+t': on_activate_t}) as hot:
        hot.join()

if __name__ == '__main__':
    print(f'running on {platform}')
    hotkey_listen();


from django.shortcuts import render, HttpResponse, redirect
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.contrib.auth.models import User
from . import models
# Create your views here.

def home(request):
    posts = models.Post.objects.all().order_by('-date_posted')
    return render(request, 'blog/blogs_headings.html', {'posts': posts})

def displayBlogContent(request):
    id_val = int(request.POST['id_value'])
    post = models.Post.objects.get(id=id_val)
    return render(request, 'blog/blog_content.html', {'post': post})

@login_required
def createNewBlog(request):
    if(request.method == "POST"):
        title_value = request.POST['title_value']
        content_value = request.POST['content_value']
        user_id = User.objects.get(username=request.user.username).id
        post_obj = models.Post(
            title=title_value, content=content_value, author_id=user_id)  # , user=user_value)
        post_obj.save()
        messages.success(request, 'Blog successfully posted !')
        return redirect('create-blog')
    else:
        return render(request, 'blog/create_blog.html')


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

x_data = [[1, 2], [2, 3], [3, 1], [4, 3], [5,3], [6,2]]
y_data = [[0], [0], [0], [1], [1], [1]]
x_train = torch.FloatTensor(x_data)
y_train = torch.FloatTensor(y_data)

class LogisticClass(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(2, 1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        return self.sigmoid(self.linear(x))

model = LogisticClass()
optimizer = optim.SGD(model.parameters(), lr = 1)

for epoch in range(1000):
    hypothesis = model(x_train)
    loss = F.binary_cross_entropy(hypothesis, y_train)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    if epoch % 10 == 0:
        prediction = hypothesis > torch.FloatTensor([0.5])
        correction_prediction = prediction.float() == y_train
        accuracy = correction_prediction.sum().item() / len(correction_prediction)
        print('epoch:{} loss:{:.4f} accuracy:{:2.2f}%'.format(epoch, loss.item(), accuracy * 100))


#%%
import numpy as np
import os
import matplotlib.pyplot as plt

def plot_data(val_acc_history, loss_acc_history):
    plt.plot(loss_acc_history, label = 'Validation')
    plt.title('Loss per epoch')
    plt.legend()
    plt.show()
    plt.plot(val_acc_history, label = 'Validation')
    plt.title('Accuracy per epoch')
    plt.legend()
    plt.show()
#%%
acc = np.load('GoogLeNet_val_acc_history.npy', allow_pickle=True)
loss = np.load('GoogLeNet_loss_history.npy', allow_pickle=True)

plot_data(acc, loss)

# %%


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/ali/ownCloud/Project/python/django-aparnik-framework-project/testandbuildprojectframework/aparnik/contrib/settings/migrations/0014_auto_20190128_1350.py
# Compiled at: 2020-01-05 09:49:45
# Size of source mod 2**32: 1527 bytes
from django.db import migrations

def add_keys(apps, schema_editor):
    """
    We can't import the Post model directly as it may be a newer
    version than this migration expects. We use the historical version.
    """
    Setting = apps.get_model('settings', 'Setting')
    key = ''
    try:
        key = 'MANAGER_PRODUCT_ID'
        Setting.objects.get(key=key)
    except Exception:
        Product = apps.get_model('products', 'Product')
        ContentType = apps.get_model('contenttypes', 'ContentType')
        product = Product.objects.create(price_fabric=1,
          title='محصول مدیریتی',
          is_show_only_for_super_user=True,
          aparnik_bon_return_value=1,
          maximum_use_aparnik_bon_value=1,
          aparnik_bon_return_expire_value=0)
        new_ct = ContentType.objects.get_for_model(Product)
        Product.objects.filter(polymorphic_ctype__isnull=True).update(polymorphic_ctype=new_ct)
        Setting.objects.create(title='آی دی محصول مدیریتی',
          key=key,
          value=(str(product.id)),
          value_type='i',
          is_show=False,
          is_variable_in_home=False)

class Migration(migrations.Migration):
    dependencies = [
     ('settings', '0013_auto_20190123_1447')]
    operations = [
     migrations.RunPython(add_keys, reverse_code=(migrations.RunPython.noop))]


import pandas as pd
import gspread
from oauth2client.service_account import ServiceAccountCredentials
import numpy as np
from classes import specie, pm_doc
import math
import logging
from theone_con import from_theone
logging.basicConfig(level=logging.INFO) # DEBUG, INFO, WARN

#Storing the links on gdrive for every site
gsheets = {
    'ROSPA0004' : 'https://docs.google.com/spreadsheets/d/1FRh6F8wh9iEcAOI_7gt-h7YDm38YaraEsMl7BCs9c_c/edit#gid=0',
    'ROSPA0012' : 'https://docs.google.com/spreadsheets/d/1oeUT7zVmWmNeIxOuCNX2xNZ8KNxlGA-_Mt7QNYW_Z1o/edit#gid=0',
    'ROSPA0112' : 'https://docs.google.com/spreadsheets/d/1nM5UR7hykwL0mffkkd6YQIvpZzHUPfJBEfrfxf_llHI/edit#gid=0',
    'ROSPA0111' : 'https://docs.google.com/spreadsheets/d/1LrmO_aMmvEyCzuAtsiejMjtSzADsAkQsIGuQPX6hxug/edit#gid=0',
    'ROSPA0109' : 'https://docs.google.com/spreadsheets/d/1H4epQ_iR9jZGugIlo7i9pGrP5Hmf8IocBiuRBUscKLA/edit#gid=0',
    'ROSPA0101' : 'https://docs.google.com/spreadsheets/d/1XNYEqTLr4EuL9Ghb3x8OhWno9VYqRgmtsNpV4icCX5o/edit#gid=0',
    'ROSPA0064' : 'https://docs.google.com/spreadsheets/d/1jxUQQU15eAe6HnLVuNGHDYbi-xFYuc5UbG2BT7eKKdA/edit#gid=0',
    'ROSPA0061' : 'https://docs.google.com/spreadsheets/d/1awwKOi1lY8KTQeRPp3D7BS_mLB3oBXrb98VgyJyvWeY/edit#gid=0',
    'ROSPA0051' : 'https://docs.google.com/spreadsheets/d/1ck3Q4dv55TdI7Uo98qZ4-E0-2-xxaWzq9WBL5X0SSKk/edit#gid=0',
    'ROSPA0042' : 'https://docs.google.com/spreadsheets/d/1A_NPum_EEw5eHNO7AAqm4ZFkezFq07ZRQj32UMQp-YE/edit#gid=0',
    'descrieri' : 'https://docs.google.com/spreadsheets/d/13qN_1-CXxNzcUNV52s8tERspjqxij4Rn0uyHSbPSebg/edit#gid=839134727'}

#Setting the authentification
scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive']
credentials = ServiceAccountCredentials.from_json_keyfile_name('Driveacc-017ff6c04252.json', scope)
gc = gspread.authorize(credentials)

#Getting the site
the_site = 'ROSPA0'+input('What are the last 3 numbers on your site code?  ')

#Getting the table for the site from db
bf = from_theone("select * from poim.sit_table('"+the_site+"')")
bf.fillna('na', inplace=True)

#Getting the master table from gdrive
mst = gc.open_by_url(gsheets[the_site]).worksheet('master')
master = mst.get_all_records()
headers_m = master.pop(0)
df_master = pd.DataFrame(master, columns = headers_m)
df_master = df_master[df_master.id != '']
df_master.fillna('na', inplace=True)

#Getting the impacts table from gdrive
imp = gc.open_by_url(gsheets[the_site]).worksheet('presiuni_sit')
impacts = imp.get_all_records()
df_impacts = pd.DataFrame(impacts)
df_impacts = df_impacts[df_impacts.impact != '']
df_impacts.fillna('na', inplace=True)

#Getting the measures table from gdrive
mas = gc.open_by_url(gsheets[the_site]).worksheet('masuri_sit')
masuri = mas.get_all_records()
df_masuri = pd.DataFrame(masuri)
df_masuri = df_masuri[df_masuri.masura != '']
df_masuri.fillna('na', inplace=True)

#Getting the descriptions table from gdrive
desc = gc.open_by_url(gsheets['descrieri']).worksheet('descrieri_tst')
descrieri = desc.get_all_records()
df_desc = pd.DataFrame(descrieri)
df_desc.dropna(inplace=True)
df_desc.fillna('na', inplace=True)

#Getting the list of the rows from the master table
df_rows = list(df_master.index)

#Creation of the document
doc = pm_doc()

#Open the text file for gis
f=open(the_site + '_gis_final.txt','w+', encoding="utf-8")

                #-------CREATION OF THE CONSERVATION TABLES------

#Itering through rows and creating species objects&tables
for row in df_rows:
    #Initialize sp object
    sp = specie(row, df_master, bf, df_desc, df_impacts, df_masuri)
    f.write(str(sp.cod_sp) +','+ str(sp.lat_sp) + ',' + str(sp.cod_n)+'-'+str(sp.feno) + ',' + str(sp.d3)+'\n')

    #Put the species header
    doc.sp_header(sp)

    #Create the population based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al populației speciei')
    #Add the table title
    doc.chapter_h3('Tabelul A: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al populației')
    #Create the population based conservation tabels (tabels A)
    doc.ftblA_cons()
    doc.ftblA_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 1) Matricea de evaluare a stării de conservare a speciei din punct de vedere al populației speciei')
    doc.ftblM_1()
    doc.ftblM_1_fill(sp)

    #Create the habitat based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al habitatului speciei')
    #Add the table title
    doc.chapter_h3('Tabelul B: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al habitatului speciei')
    #Create the habitat based conservation tabels (tabels B)
    doc.ftblB_cons()
    doc.ftblB_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 2) Matricea pentru evaluarea tendinței globale a habitatului speciei')
    doc.ftblM_2()
    doc.ftblM_2_fill(sp)
    doc.chapter_h3('Matricea 3) Matricea de evaluare a stării de conservare a speciei din punct de vedere al habitatului speciei')
    doc.ftblM_3()
    doc.ftblM_3_fill(sp)

    #Create the perspectives based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al perspectivelor speciei')
    #Add the table title
    doc.chapter_h3('Tabelul C: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al perspectivelor speciei în viitor')
    #Create the perspectives based conservation tabels (tabels C)
    doc.ftblC_cons()
    doc.ftblC_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 4) Matricea pentru evaluarea perspectivelor speciei din punct de vedere al populației speciei')
    doc.ftblM_4()
    doc.ftblM_4_fill(sp)
    doc.chapter_h3('Matricea 5) Perspectivele speciei în viitor, după implementarea planului de management actual')
    doc.ftblM_5()
    doc.ftblM_5_fill(sp)
    doc.chapter_h3('Matricea 6) Matricea evaluării stării de conservare a speciei din punct de vedere al perspectivelor speciei în viitor, după implementarea planului de management actual')
    doc.ftblM_6()
    doc.ftblM_6_fill(sp)

    #Create the global conservation chapter
    doc.chapter_h2('Evaluarea globală a speciei')
    #Add the table title
    doc.chapter_h3('Tabelul D: Parametri pentru evaluarea stării globale de conservare a speciei în cadrul ariei naturale protejate')
    #Create the global conservation tabels (tabels D)
    doc.ftblD_cons()
    doc.ftblD_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 7) Evaluarea stării globale de conservare a speciei')
    doc.ftblM_7()
    doc.ftblM_7_fill(sp)

f.close()
                #-------CREATION OF THE MEASURES TABLES------

doc.chapter_h1('Măsurile de conservare pentru menținerea sau îmbunătățirea stării de conservare a speciilor de păsări')

doc.measures_ch(df_masuri, df_impacts)

#Salvation ^^ of the document
doc.save(the_site+'_export_final'+'.docx')


import matplotlib.pyplot as plt

lines = plt.plot([0, 1, 2, 3], [1, 2, 3, 5], 'b-', linewidth=2.0)
# plot返回一个线对象(line_2d)列表
# 填入的是x,y轴坐标，对应的x值为[0, 1, 2, 3]
# 如果只有一个参数，则为y轴坐标
# 第三个参数为点格式
# linewidth 控制线的宽度

lines[0].set_antialiased(False)
# set_antialiased方法实现开启/去除平滑

plt.setp(lines, color='r', linewidth=2.0)
plt.setp(lines, 'color', 'r', 'linewidth', 2.0)  # matlab风格
# setp()可以作用于一个列表对象或者是一个单一的对象。
# 你可以使用python风格的关键字参数或者是MATLAB风格的string/value对为参数
# line_2d对象具有的属性和取值,见OneNote:人工智能-数据挖掘

plt.ylabel("First numbers")
# 添加y轴标签

plt.axis([0, 6, 0, 10])
# axis规定 x 轴显示范围为 0-6  规定y轴显示范围为 0-10

plt.show()


from django import forms
from sales.utils import PRODRECWIDGETS, MINIPRODWIDGETS
from survey.forms import ProdRecCreateForm
from trapTransmitter.models import Traptransmitterordermodel, SalesSalesinputtraptransmitter

class TrapTransmitterUpdateForm(forms.ModelForm):
    class Meta:
        model = Traptransmitterordermodel
        fields = ('number_of_collars',
                  'vhf_beacon_frequency', 'notification_mail', 'notification_sms',
                  'vhf_beacon_schedule', 'id_tag', 'globalstar', 'iridium', 'contact_name_airtime_fee',
                  'contact_mail_airtime_fee', 'world_location', 'interval', 'comment', 'delivery_time',
                  'payment_option', 'operation_number')

class TrapTransmitterProdRecForm(ProdRecCreateForm):
    class Meta:
        model = SalesSalesinputtraptransmitter
        fields = ('co_worker',
                  'further_instructions_programming',
                  'path_customer_folder',
                  'order')

        widgets = PRODRECWIDGETS

class SalesInputTrapTransCreateForm(forms.ModelForm):
    class Meta:
        model = SalesSalesinputtraptransmitter
        # exclude = ('order',)
        fields = ('further_instructions_programming',)

        widgets = MINIPRODWIDGETS

        labels = {
            'gl_no_of_attempts': 'GLOBALSTAR no of attempts',
            'gl_fixes_per_message': 'GLOBALSTAR fixes per message',
        }


# Default Imports
import pandas as pd
import matplotlib.pyplot as plt
from greyatomlib.descriptive_stats.q01_calculate_statistics.build import calculate_statistics

dataframe = pd.read_csv('data/house_prices_multivariate.csv')
sale_price = dataframe.loc[:, 'SalePrice']

def plot():
    mean = sale_price.mean()
    median = sale_price.median()
    mode = np.int64(sale_price.mode())

    plt.axvline(x=mean)
    plt.axvline(x=median)
    plt.axvline(x=mode)

   # plt.hist(median, bins=10)

plt.show()


# -*- coding: utf-8 -*-
# Copyright 2010 Thamini S.à.R.L    This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

import time
import xlwt
from ad_account_optimization.report.report_engine_xls import report_xls
from ad_report_sales.report.sale_order_item import ReportStatus
import cStringIO
from xlwt import Workbook, Formula
from tools.translate import _

class report_sales_order_xls(report_xls):
    no_ind = 0

    def get_no_index(self):
        self.set_no_index()
        return self.no_ind
    def set_no_index(self):
        self.no_ind += 1
        return True

    def create_source_xls(self, cr, uid, ids, data, report_xml, context=None):
        if not context:
            context = {}
        context = context.copy()
        rml_parser = self.parser(cr, uid, self.name2, context=context)
        objs = self.getObjects(cr, uid, ids, context=context)
        rml_parser.set_context(objs, data, ids, 'xls')

        n = cStringIO.StringIO()
        wb = xlwt.Workbook(encoding='utf-8')
        self.generate_xls_report(rml_parser, data, rml_parser.localcontext['objects'], wb)
        wb.save(n)
        n.seek(0)
        return (n.read(), 'xls')

    def generate_xls_report(self, parser, data, obj, wb):
        c = parser.localcontext['company']
        ws = wb.add_sheet(('Sale Order Report'))#- %s - %s' % (c.partner_id.ref, c.currency_id.name))[:31])
        ws.panes_frozen = True
        ws.remove_splits = True
        ws.portrait = 0 # Landscape
        ws.fit_width_to_pages = 1
        ws.col(0).width = len("ABCD")*1024
        ws.col(1).width = len("ABCDEFG")*1024
        ws.col(2).width = len("ABCDEFGHI")*1024
        ws.col(3).width = len("ABC")*1024
        ws.col(4).width = len("AB")*1024
        ws.col(5).width = len("ABCD")*1024
        ws.col(6).width = len("ABCD")*1024
        ws.col(7).width = len("ABCDC")*1024
        ws.col(8).width = len("ABCD")*1024

        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        #         Spesifikasi Kolom                                                         #
        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        cols_specs = [
                # Headers data
                ('Title',  13, 0, 'text',
                    lambda x, d, p: 'Sale Order Report'),
                ('Kosong', 13, 0, 'text',
                    lambda x, d, p: ""),
                ('Space', 1, 0, 'text',
                    lambda x, d, p: ""),
                ('Space2', 9, 0, 'text',
                    lambda x, d, p: ""),

                # Balance column
                #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
                #        Kolom Text                                                                 #
                #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

                ('Date', 1, 200, 'text',
                    lambda x, d, p: 'Date'),
                ('Reference', 1, 200, 'text',
                    lambda x, d, p: 'Reference'),
                ('Order No.', 1, 200, 'text',
                    lambda x, d, p: 'Order No.'),
                ('Customer', 1, 200, 'text',
                    lambda x, d, p: 'Customer'),
                ('Curr', 1, 200, 'text',
                    lambda x, d, p: 'Curr'),
                ('Sub Total', 1, 200, 'text',
                    lambda x, d, p: 'Sub Total'),
                ('Discount', 1, 200, 'text',
                    lambda x, d, p: 'Discount'),
                ('Tax', 1, 200, 'text',
                    lambda x, d, p: 'Tax'),
                ('Sales Total', 1, 200, 'text',
                    lambda x, d, p: 'Sales Total'),
                ('Payment', 1, 200, 'text',
                    lambda x, d, p: 'Payment'),
                ('Balance', 1, 200, 'text',
                    lambda x, d, p: 'Balance'),
                #'Date','Reference','Order No.','Customer','Curr','Sub Total','Discount','Tax','Sales Total','Payment','Balance'
                ('Date1', 1, 200, 'text',
                    lambda x, d, p: x['date_order']),
                ('Reference1', 1, 200, 'text',
                    lambda x, d, p: x['client_order_ref']),
                ('Order No.1', 1, 200, 'text',
                    lambda x, d, p: x['name']),
                ('Customer1', 1, 200, 'text',
                    lambda x, d, p: x['partner_id']['name']),
                ('Curr1', 1, 200, 'text',
                    lambda x, d, p: x['pricelist_id']['currency_id']['name']),
                ('Sub Total1', 1, 200, 'number',
                    lambda x, d, p: x['amount_undiscount']),
                ('Discount1', 1, 200, 'number',
                    lambda x, d, p: x['add_disc_amt']),
                ('Tax1', 1, 200, 'number',
                    lambda x, d, p: x['amount_tax']),
                ('Sales Total1', 1, 200, 'number',
                    lambda x, d, p: x['amount_total']),
                ('Payment1', 1, 200, 'number',
                    lambda x, d, p: 0),
                ('Balance1', 1, 200, 'number',
                    lambda x, d, p: 0),
                #item
                ('CodeItem', 1, 200, 'text',
                    lambda x, d, p: x['product_id']['default_code']),
                ('NameItem', 1, 200, 'text',
                    lambda x, d, p: x['product_id']['name']),
                ('Quantity', 1, 200, 'number',
                    lambda x, d, p: x['product_uom_qty']),
                ('PriceItem', 1, 200, 'number',
                    lambda x, d, p: x['price_unit']),
                ('DiscItem', 1, 200, 'number',
                    lambda x, d, p: x['discount']),
                ('SubtotalItem', 1, 200, 'number',
                    lambda x, d, p: x['price_subtotal']),

        ]
        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        #         ISI KOLOM                                                                 #
        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        row_hdr0 = self.xls_row_template(cols_specs, ['Space','Title'])
        row_hdr1 = self.xls_row_template(cols_specs, ['Date','Order No.','Customer',
                                                      'Curr','Sub Total','Discount','Tax','Sales Total',
                                                      #'Payment','Balance'
                                                      ])
        row_hdr2 = self.xls_row_template(cols_specs, ['Date1','Order No.1','Customer1',
                                                      'Curr1','Sub Total1','Discount1','Tax1','Sales Total1',
                                                      #'Payment1','Balance1'
                                                      ])
        row_hdr3 = self.xls_row_template(cols_specs, ['Space','CodeItem','NameItem',
                                                      'Quantity','PriceItem','DiscItem','SubtotalItem',
                                                      #'Space'
                                                      ])

        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        #         ####ISI KOLOM####                                                         #
        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

        ## Style variable Begin
        hdr_style = xlwt.easyxf('pattern: pattern solid, fore_color sky_blue;')
        row_normal_style=  xlwt.easyxf(num_format_str='#,##0.00;(#,##0.00)')
        row_bold_style = xlwt.easyxf('font: bold on;pattern: pattern solid, fore_color ice_blue;',num_format_str='#,##0.00;(#,##0.00);')
        row_bold_style_line = xlwt.easyxf('font: bold on;pattern: pattern solid, fore_color ice_blue;',num_format_str='#,##0.00;(#,##0.00);')
        row_bold_bottom_style = xlwt.easyxf('font: bold on;borders: bottom thin;',num_format_str='#,##0.00;(#,##0.00)')

        tittle_style = xlwt.easyxf('font: height 240, name Arial Black, colour_index black, bold on; align: wrap on, vert centre, horiz left; pattern: pattern solid, fore_color white;')
        subtittle_left_style = xlwt.easyxf('font: name Arial, colour_index brown, bold on, italic on; align: wrap on, vert centre, horiz right; pattern: pattern solid, fore_color white;')
        subtittle_right_style = xlwt.easyxf('font: name Arial, colour_index brown, bold on, italic on; align: wrap on, vert centre, horiz left; pattern: pattern solid, fore_color white;')
        subtittle_top_and_bottom_style = xlwt.easyxf('font: height 240, name Arial, colour_index black, bold off, italic on; align: wrap on, vert centre, horiz left; pattern: pattern solid, fore_color white;')
        blank_style = xlwt.easyxf('font: height 650, name Arial, colour_index brown, bold off; align: wrap on, vert centre, horiz left; pattern: pattern solid, fore_color white;')
        normal_style = xlwt.easyxf('font: height 240, name Arial, colour_index black, bold off; align: wrap on, vert centre, horiz left;')
        total_style = xlwt.easyxf('font: height 200, name Arial, colour_index black, bold on, italic on; align: wrap on, vert centre;pattern: pattern solid, fore_color gray25;', num_format_str='#,##0.00;(#,##0.00)')
        ## Style variable End

        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        #         Write To Excel                                                            #
        #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
        self.xls_write_row(ws, None, data, parser, 1, row_hdr0, tittle_style)
        self.xls_write_row(ws, None, data, parser, 2, row_hdr1, row_bold_style)
        row_count=3
        for inv in parser._get_basedon(data):
            self.xls_write_row(ws, inv, data, parser, row_count, row_hdr2, row_normal_style)
            row_count+=1
            if data['form']['with_item']:
                for line in inv.order_line:
                    self.xls_write_row(ws, line, data, parser, row_count, row_hdr3, row_normal_style)
                    row_count+=1
            row_count+=1
        pass

report_sales_order_xls(
        'report.sale.order.form.xls',
        'sale.line.report',
        'addons/ad_report_sales/report/report_sales_webkit.mako',
        parser=ReportStatus,       header=False)


import os
import sys
import logging
import datetime

class mylog(object):
    '''
    # ERROR
    # WARNING
    # INFO
    # DEBUG
    '''

    def __init__(self):
        path = os.path.dirname(os.path.dirname(__file__))
        version = datetime.datetime.now().strftime("%Y-%m-%dH%H")
        self.logname = 'all_%s.log' % version  # all.log
        self.log_path = "temp"
        self.logger = logging.getLogger("logger")
        self.tem_path = os.path.join(path, self.log_path)
        self.log_path = os.path.join(self.tem_path, self.logname)

    def log_name(self):
        return self.log_path

    def log(self):
        formatter = logging.Formatter(
            "%(asctime)s %(levelname)-8s: %(message)s")
        #tem_path  = os.path.join(os.path.dirname(os.getcwd()),self.log_path)
        #log_path = os.path.join(self.tem_path,self.logname)
        if not os.path.isdir(self.tem_path) or not os.path.isfile(self.log_path):
            try:
                os.mkdir(self.tem_path)
            except FileExistsError as e:
                pass
            with open(self.log_path, 'w', encoding='utf-8') as f:
                pass

        if not self.logger.handlers:
            # 文件日志
            file_handler = logging.FileHandler(self.log_path)
            file_handler.setLevel(logging.INFO)  # 指定日志的最低输出级别，默认为WARN级别
            file_handler.setFormatter(formatter)  # 可以通过setFormatter指定输出格式

            # 控制台日志
            console_handler = logging.StreamHandler(sys.stdout)
            console_handler.setLevel(logging.DEBUG)  # 指定日志的最低输出级别，默认为WARN级别
            console_handler.formatter = formatter  # 也可以直接给formatter赋值

            # 为logger添加的日志处理器
            self.logger.addHandler(file_handler)
            self.logger.addHandler(console_handler)

            # 指定日志的最低输出级别，默认为WARN级别
            self.logger.setLevel(logging.INFO)

        return self.logger

if __name__ == '__main__':
    # l = log()
    # l.info('1234566789')
    m = mylog()
    m.log().info('8888888888888')
    # for i in range(0,500):
    #     m.info(i)
    print(m.log_name())


import pytest

from ..common import PluginTestCase as PTC, run_plugin_test_case

@pytest.mark.parametrize(
    "test_case",
    [
        PTC(
            "manifest_keys_unknown_keys",
            expected=[
                {
                    "slug": "unknown_manifest_key",
                    "description": 'Unknown manifest key "foo"',
                    "categories": ["correctness"],
                    "locations": [(["__manifest__.py"], 1, 47)],
                }
            ],
        ),
        PTC(
            "manifest_keys_unknown_keys_complex",
            expected=[
                {
                    "slug": "unknown_manifest_key",
                    "description": 'Unknown manifest key "foo"',
                    "categories": ["correctness"],
                    "locations": [(["__manifest__.py"], 13, 5)],
                }
            ],
        ),
        PTC(
            "manifest_keys_active",
            expected=[
                {
                    "slug": "deprecated_manifest_key",
                    "description": (
                        '"active" manifest key was renamed to "auto_install"'
                    ),
                    "categories": ["correctness", "deprecated"],
                    "locations": [(["__manifest__.py"], 1, 41)],
                }
            ],
        ),
        PTC(
            "manifest_keys_deprecated_xml",
            expected=[
                {
                    "slug": "deprecated_manifest_key",
                    "description": (
                        '"demo_xml" manifest key was deprecated in favor of "demo"'
                    ),
                    "categories": ["correctness", "deprecated"],
                    "locations": [(["__manifest__.py"], 1, 49)],
                }
            ],
        ),
        PTC(
            "manifest_keys_deprecated_xml_init_xml",
            expected=[
                {
                    "slug": "deprecated_manifest_key",
                    "description": (
                        '"init_xml" manifest key was deprecated in favor of "data"'
                    ),
                    "categories": ["correctness", "deprecated"],
                    "locations": [(["__manifest__.py"], 1, 58)],
                }
            ],
        ),
        PTC("manifest_keys_deprecated_xml_init_xml_csv"),
        PTC(
            "duplicate_value",
            expected=[
                {
                    "slug": "duplicate_manifest_value",
                    "description": (
                        '"depends" manifest key has a duplicated value: foo'
                    ),
                    "categories": ["correctness"],
                    "locations": [(["__manifest__.py"], 2, 5)],
                }
            ],
        ),
    ],
    ids=PTC.test_case_id,
)
def test_manifest_keys(test_data_dir, test_case):
    run_plugin_test_case(
        test_data_dir,
        "manifest_keys",
        test_case,
        extra_plugins=("addon_path_emitter", "python_emitter"),
    )


from Day2 import getPuzzleinput
import numpy as np
import itertools
import collections

def proneInput(Inp):
    return [(x.split(' ')[1], x.split(' ')[-3])for x in Inp]
#
#
def presort(Inp):
    d=collections.defaultdict(list)
    for k,v in Inp:
        d[k].append(v)
        d[k].sort()

    return d

def work(Inp):
    manual=[]
#------------------------start poin and queue------------------------------------
    start=[[i,ord(i)-64+61] for i in Inp.keys() if not any(i in x for x in Inp.values())]
    start.sort()
    # startval=start[0]
    # queue=start[1:]
#---------------------------working----------------------------------------------
    Q = []
    P = []
    M = []
    Order = Inp.copy()
    t=0

    while Order or Q or P:

        if t ==0:
            P.extend(start)

        if len(P)<6 and Q:
            for i in Q:
                if not any(i in x for x in Order.values()):
                    P.append([i,ord(i)+60-64])
                    print('Begin ',i,t)
                    Q=[k for k in Q if k != i]
                    if len(P)>=5:
                        break

        for k in P:
            k[1]-=1
            if k[1]==0:
                M.append(k[0])
                print('Finished ',k[0],t)
                P=[x for x in P if x != k]
                Q.extend(Order[k[0]])
                Q=[t for t in Q if t not in M]
                Q=set(Q)
                Q=sorted(list(Q))
                del Order[k[0]]

        #t += 1

        t+=1

    return t

if __name__=="__main__":

    Inp=getPuzzleinput(7)
    pronedInp=proneInput(Inp)
    res=presort(pronedInp)
    duration=work(res)

#-------------------------shorter Internet Solution Part 1------------------------------
# from collections import defaultdict, deque
# from Day2 import getPuzzleinput
#  # Edges
# E = defaultdict(list)
#  # In-degree
# D = defaultdict(int)
# Inp=getPuzzleinput(7)
# for line in Inp:
#     words = line.split()
#     x = words[1]
#     y = words[7]
#     E[x].append(y)
#     D[y] += 1
#
# Q=[]
# for k in E:
#     if D[k] == 0:
#         Q.append(k)
#
#
# ans = ""
# while Q:
#     x = sorted(Q)[0]
#     Q = [y for y in Q if y!=x]
#     ans += x
#     for y in E[x]:
#         D[y] -= 1
#         if D[y] == 0:
#             Q.append(y)
# print (ans)


"""Simple Subscriber"""

import asyncio
from typing import Optional, List
from uuid import UUID

from jetblack_messagebus import Client, DataPacket

class SimpleSubscriber(Client):
    """A simple subscriber"""

    async def on_authorization(
            self,
            client_id: UUID,
            host: str,
            user: str,
            feed: str,
            topic: str
    ) -> None:
        print(f'authorization: client={client_id}, host="{host}", user="{user}", feed="{feed}"",topic="{topic}"')

    async def on_data(
            self,
            user: str,
            host: str,
            feed: str,
            topic: str,
            data_packets: Optional[List[DataPacket]],
            content_type: str
    ) -> None:
        print(f'data: user="{user}",host="{host}",feed="{feed}",topic="{topic}"')
        if not data_packets:
            print("no data - closing")
            self.stop()
        else:
            for packet in data_packets:
                message = packet.data.decode('utf8') if packet.data else None
                print(f'received: entitlements={packet.entitlements},message={message}')

    async def on_forwarded_subscription_request(
            self,
            client_id: UUID,
            user: str,
            host: str,
            feed: str,
            topic: str,
            is_add: bool
    ) -> None:
        print(f'notification: client_id={client_id},user={user},host={host}, feed={feed},topic={topic},is_add={is_add}')

    async def on_closed(self, is_faulted):
        print(f'closed: is_faulted={is_faulted}')

async def main():
    client = await SimpleSubscriber.create('localhost', 9001)
    await client.add_subscription('TEST', 'FOO')
    await client.start()

if __name__ == '__main__':
    asyncio.run(main())


# TODO: Wrap it as a class and connect it to GUI
# A script to transform anomaly data to d3m format
import pandas as pd
import numpy as np
import os
import json

##############################
# Some information for the dataset to be transformed
# Designed for time series data
name = 'yahoo_sub_5'
src_path = './raw_data/yahoo_sub_5.csv'
label_name = 'anomaly'
timestamp_name = 'timestamp'
value_names = ['value_{}'.format(i) for i in range(5)]
ratio = 0.9 # Ratio of training data, the rest is for testing

###############################

dst_root = './' + name
dirs = ['./', 'SCORE', 'TEST', 'TRAIN']
maps = {'./': None, 'SCORE': 'TEST', 'TEST': 'TEST', 'TRAIN': 'TRAIN'}

# Create the corresponding directories
for d in dirs:
    if maps[d] is not None:
        dataset_name = 'dataset_' + maps[d]
        problem_name = 'problem_' + maps[d]
    else:
        dataset_name = name + '_dataset'
        problem_name = name + '_problem'
    tables_dir = os.path.join(dst_root, d, dataset_name, 'tables')
    if not os.path.exists(tables_dir):
        os.makedirs(tables_dir)
    problem_dir = os.path.join(dst_root, d, problem_name)
    if not os.path.exists(problem_dir):
        os.makedirs(problem_dir)

# Process data
_df = pd.DataFrame()
df = pd.read_csv(src_path)
_df['d3mIndex'] = df.index
_df['timestamp'] = df[timestamp_name]
for value_name in value_names:
    _df[value_name] = df[value_name]
_df['ground_truth'] = df[label_name]
df = _df
cols = df.columns.tolist()

# Save all the data
df.to_csv(os.path.join(dst_root, name+'_dataset', 'tables', 'learningData.csv'), index=False)

# Save training and testing data
train_df, test_df = df[:int(df.shape[0]*ratio)], df[int(df.shape[0]*ratio):]

train_df.to_csv(os.path.join(dst_root, 'TRAIN', 'dataset_TRAIN', 'tables', 'learningData.csv'), index=False)
test_df.to_csv(os.path.join(dst_root, 'TEST', 'dataset_TEST', 'tables', 'learningData.csv'), index=False)
test_df.to_csv(os.path.join(dst_root, 'SCORE', 'dataset_TEST', 'tables', 'learningData.csv'), index=False)

# Data splits
row_0 = train_df.shape[0]
row_1 = train_df.shape[0]
row = row_0 + row_1
df = pd.DataFrame(np.array([[i for i in range(row)], ['TRAIN' for _ in range(row_0)] + ['TEST' for _ in range(row_1)], [0 for _ in range(row)], [0 for _ in range(row)]]).transpose(), columns = ['d3mIndex', 'type', 'repeat', 'fold'])

# Save data splits for all data
train_df.to_csv(os.path.join(dst_root, name+'_problem', 'dataSplits.csv'), index=False)

# Save training and testing splits
train_df, test_df = df[:row_0], df[row_0:]
train_df.to_csv(os.path.join(dst_root, 'TRAIN', 'problem_TRAIN', 'dataSplits.csv'), index=False)
test_df.to_csv(os.path.join(dst_root, 'TEST', 'problem_TEST', 'dataSplits.csv'), index=False)
test_df.to_csv(os.path.join(dst_root, 'SCORE', 'problem_TEST', 'dataSplits.csv'), index=False)

# Dataset JSON files
# Load template
with open('template/datasetDoc.json') as json_file:
    data = json.load(json_file)
columns = []
for i in range(len(cols)):
    c = {}
    c['colIndex'] = i
    c['colName'] = cols[i]
    if i == 0:
        c['colType'] = 'integer'
        c['role'] = ['index']
    elif i == 1:
        c['colType'] = 'integer'
        c['role'] = ['attribute']
    elif i == len(cols)-1:
        c['colType'] = 'integer'
        c['role'] = ['suggestedTarget']
    else:
        c['colType'] = 'real'
        c['role'] = ['attribute']
    columns.append(c)
data['dataResources'][0]['columns'] = columns
data['dataResources'][0]['columnsCount'] = len(cols)

data['about']['datasetID'] = name + '_dataset'
data['about']['datasetName'] = name
with open(os.path.join(dst_root, name+'_dataset', 'datasetDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

data['about']['datasetID'] = name +'_dataset_TRAIN'
data['about']['datasetName'] = "NULL"
with open(os.path.join(dst_root, 'TRAIN', 'dataset_TRAIN', 'datasetDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

data['about']['datasetID'] = name + '_dataset_TEST'
data['about']['datasetName'] = 'NULL'
with open(os.path.join(dst_root, 'TEST', 'dataset_TEST', 'datasetDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

data['about']['datasetID'] = name + '_dataset_TEST'
data['about']['datasetName'] = 'NULL'
with open(os.path.join(dst_root, 'SCORE', 'dataset_TEST', 'datasetDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

# Problem JSON files
# Load template
with open('template/problemDoc.json') as json_file:
    data = json.load(json_file)

data['about']['problemID'] = name+'_problem'
data['about']['problemName'] = name+'_problem'
data['about']['problemDescription'] = 'Anomaly detection'
data['about']['taskKeywords'] = ['classification', 'binary', 'tabular']
data['inputs']['data'][0]['datasetID'] = name + '_dataset'
data['inputs']['data'][0]['targets'][0]['colIndex'] = len(cols)-1
data['inputs']['data'][0]['targets'][0]['colName'] = cols[-1]
data['inputs']['dataSplits']['datasetViewMaps']['train'][0]['from'] = name+'_dataset'
data['inputs']['dataSplits']['datasetViewMaps']['test'][0]['from'] = name+'_dataset'
data['inputs']['dataSplits']['datasetViewMaps']['score'][0]['from'] = name+'_dataset'
data['inputs']['dataSplits']['datasetViewMaps']['train'][0]['to'] = name+'_dataset_TRAIN'
data['inputs']['dataSplits']['datasetViewMaps']['test'][0]['to'] = name+'_dataset_TEST'
data['inputs']['dataSplits']['datasetViewMaps']['score'][0]['to'] = name+'_dataset_SCORE'

with open(os.path.join(dst_root, name+'_problem', 'problemDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

with open(os.path.join(dst_root, 'TRAIN', 'problem_TRAIN', 'problemDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

with open(os.path.join(dst_root, 'TEST', 'problem_TEST', 'problemDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

with open(os.path.join(dst_root, 'SCORE', 'problem_TEST', 'problemDoc.json'), 'w') as outfile:
    json.dump(data, outfile, indent=4)

# Make an empty targets.csv
with open(os.path.join(dst_root, 'SCORE', 'targets.csv'), 'w') as outfile:
    outfile.write('')


from category_encoders import *
import pandas as pd
import numpy as np
from scipy.special import comb

import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.utils import shuffle
from torch.autograd import Variable

class FeatureProcessor:
    def __init__(self,date_features, categorical_features,continous_features,label_column,max_cardinality=4,verbose=0):
        date_features.sort()
        categorical_features.sort()
        continous_features.sort()

        self.date_feats  = date_features
        self.cat_feats   = categorical_features
        self.conti_feats = continous_features
        self.label_column = label_column
        self.verbose      = verbose
        self.max_cardinality = max_cardinality

    def categorical_processor(self,train_X,train_y,cat_feats,label_column,test_X=None):

        """
        input : feature df and label column separtly
        output : do ohe for low cardinality features and catboost for the remaining.

        """
        if self.verbose>0:
            print('Me is here in categorical_processor')
        #ohe low cardinality
        low_cardinality_feats = [col for col in cat_feats if train_X[col].nunique() <= self.max_cardinality]

        if len(low_cardinality_feats)>0:
            print('doing ohe..............')
            if self.verbose>0:
                print(low_cardinality_feats)
            ohe_lb = OneHotEncoder(cols=low_cardinality_feats)
            train_X = ohe_lb.fit_transform(train_X)
            if test_X:
                test_X  = ohe_lb.transform(test_X)

            cat_feats = list(set(cat_feats)-set(low_cardinality_feats))
            if len(cat_feats)<1:
                if test_X:

                    return train_X,test_X
                else:
                    return train_X

        #catboost feats remaining
        print('cat boost cols')
        print(cat_feats)
        catboost_lb = CatBoostEncoder(cols=cat_feats)
        train_X = catboost_lb.fit_transform(train_X,train_y)
        if test_X:
            if self.verbose>0:
                print('both train and test datasets are passed')
            X_test = catboost_lb.transform(X_test)
            return train_X,test_X
        else:
            if self.verbose>0:
                print('only train dataset is passed')
            return train_X

    def date_parser(self,df,date_features):

        if self.verbose>0:
            print('Me is here in date_parser')
        initial_no_columns = df.shape[1]
        combination_tracker = []
        for col in date_features:
            df[col] = pd.to_datetime(df[col])

        for col in date_features:
            df['day_{}'.format(col)] = df[col].dt.day
            df['month_{}'.format(col)] = df[col].dt.month
            df['year_{}'.format(col)] = df[col].dt.month
            for second_col in set(date_features)-set([col]):
                if set([second_col,col]) in combination_tracker:
                    continue
                else:
                    combination_tracker.append(set([second_col,col]))

                df['diff_{a}_{b}'.format(a=col,b=second_col)] = (df[col] - df[second_col]).dt.days

        #sanity check
        no_date_cols = len(date_features)
        new_cols_added_by_diff = comb(no_date_cols,2)
        day_month_type_features = 3*no_date_cols
        total_new_cols = new_cols_added_by_diff + day_month_type_features

        if df.shape[1]-initial_no_columns != total_new_cols:
            exit('somthing went wrong in date parsing')
        else:
            if self.verbose>0:
                print('date parsing completed sucessfully')
        if self.verbose>0:
            print('no of new columns added for {a} date cols {b}'.format(a=len(date_features),b=total_new_cols))
        return df

    def continuous_categorical_maskup(self,train_X,cat_feats,conti_feats,test_X=None):
        """
        """
        if self.verbose>0:
            print('Me is here in cocontinuous_categorical_maskup')

        initial_columns = train_X.columns.tolist()
        for i,col in enumerate(cat_feats):
            if self.verbose>0:
                print('checking out {a}th ,ie {b} out of {c}'.format(a=i,b=col,c=len(cat_feats)))
            temp = train_X.groupby(by=col)[conti_feats].median(skipna=True).reset_index()
            temp.columns = [col+'_median_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

            temp = train_X.groupby(by=col)[conti_feats].max(skipna=True)
            temp.columns = [col+'_max_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

            temp = train_X.groupby(by=col)[conti_feats].min(skipna=True)
            temp.columns = [col+'_max_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

        #sanity check
        count_cols_added = 3*len(cat_feats)*len(conti_feats)
        new_cols_added = list(set(train_X.columns.tolist())-set(initial_columns))
        if new_cols_added !=count_cols_added:
            exit('Booyakasha! Check Dis! Column mismatch ')
        if self.verbose > 1:
            print('{a} columns added in continuous_categorical_maskup. They are {b}'.format(a=len(new_cols_added),b=new_cols_added))

        return train_X,test_X

    def arithmetic_mashup(self,train_X,train_y,conti_feats,test_X=None):
        """
        """
        for col in train_X.columns:
            med_val = train_X[col].median(skpna=True)
            train_X[col] = train_X[col].fillna(med_val)

            if test_X:
                test_X[col] = test_X[col].fillna(med_val)

    def deep_feat_generator(self,train_X,train_y,conti_feats,test_X=None):

        class Net(nn.Module):

            def __init__(self,continous_features):
                super(Net, self).__init__()
                n1 = len(continous_features)
                print(n1)
                self.fc1 = nn.Linear(n1,800)
                self.fc2 = nn.Linear(800,800)

            def forward(self, x):
                x = self.fc1(x)
                x = F.relu(x)
                x = self.fc2(x)
                x = F.dropout(x, p=0.1)
                x = F.relu(x)
                x = self.fc2(x)

                return x

        x_train,y_train,x_test = train_X,train_y,test_X

        #preprocessing for neural net

        for col in x_train.columns:
            med_val = train_X[col].median(skpna=True)
            x_train[col] = x_train[col].fillna(med_val)
            x_train[col] = x_train[col].fillna(med_val)

            train_min = x_train[col].min()
            train_max = x_train[col].max()
            x_train[col] -= train_min
            x_train[col] /= train_max

            if x_test:
                x_test[col] = x_test[col].fillna(med_val)
                x_test[col] -= train_min
                x_test[col] /= train_max

        #network training
        net = Net(continous_features=conti_feats)

        batch_size = 50
        num_epochs = 5
        learning_rate = 0.001
        batch_no = len(x_train) // batch_size

        criterion = nn.CrossEntropyLoss()
        optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

        for epoch in range(num_epochs):

            # x_train, y_train = shuffle(x_train, y_train)
            # Mini batch learning
            for i in range(batch_no):
                start = i * batch_size
                end = start + batch_size

                x_var = Variable(torch.FloatTensor(x_train[start:end].values))
                y_var = Variable(torch.LongTensor(y_train[start:end].values))
                # Forward + Backward + Optimize
                optimizer.zero_grad()
                ypred_var = net(x_var)
                loss =criterion(ypred_var, y_var)
                loss.backward()
                optimizer.step()
            # if epoch % 100 == 0:
            print('Epoch {}'.format(epoch+1))
            print(loss.detach().numpy())

        #getting features
        train_var = Variable(torch.FloatTensor(x_train.values), requires_grad=True)
        with torch.no_grad():
            result_train = net(train_var)

        out_size = 800
        train = pd.DataFrame(data=result_train.numpy(),columns=['deep_feat_{}'.format(i) for i in range(1,out_size+1)])
        # train['target'] = y_train
        print(train.head(2))

        if test_X:
            test_var = Variable(torch.FloatTensor(x_test.values), requires_grad=True)
            with torch.no_grad():
                result_test = net(test_var)

            test = pd.DataFrame(data=result_test.numpy(),columns=['deep_feat_{}'.format(i) for i in range(1,out_size+1)])
            # test['target'] = y_val
            print(test.head(2))
            exit('Not implemented')
        else:
            return train

    def fit_transform(self,train_X,train_y,test_X=None):
        # #all arithmetic operations
        # if test_X:
        #     pass
        # else:
        #     pass
        # #deep feat
        # if test_X:
        #     pass

        # else:
        #     print('only train')
        #     train_X_conti = self.deep_feat_generator(train_X=train_X[self.conti_feats],train_y=train_y,conti_feats=self.conti_feats,test_X=None)
        #     train_X = pd.concat([train_X,train_X_conti],axis=1)

        if test_X:
            train_X_cat, test_X_cat = self.categorical_processor(train_X=train_X[self.cat_feats],train_y=train_y,cat_feats=self.cat_feats,label_column=self.label_column,test_X=test_X[self.cat_feats])
            train_X = train_X.drop(self.cat_feats,axis=1)
            train_X = pd.concat([train_X,train_X_cat],axis=1)

            test_X = test_X.drop(self.cat_feats,axis=1)
            test_X = pd.concat([test_X,test_X_cat],axis=1)
        else:
            train_X_cat = self.categorical_processor(train_X=train_X[self.cat_feats],train_y=train_y,cat_feats=self.cat_feats,label_column=self.label_column,test_X=None)
            train_X = train_X.drop(self.cat_feats,axis=1)
            train_X = pd.concat([train_X,train_X_cat],axis=1)
        # print(train_X[self.cat_feats].head())
        # if test_X:
        #     train_X, test_X = self.continuous_categorical_maskup(train_X,self.cat_feats,self.conti_feats,test_X=test_X)
        # else:
        #     train_X = self.continuous_categorical_maskup(train_X,self.cat_feats,self.conti_feats)

        if test_X:
            df = pd.concat([train_X,test_X])
        else:
            train_X[self.label_column[0]]=train_y

        df = self.date_parser(df=train_X,date_features=self.date_feats)

        return df


import numpy as np
import matplotlib.pyplot as plt

#Annotating text

ax = plt.subplot(111)

t = np.arange(0.0, 5.0, 0.01)
s = np.cos(2*np.pi*t)
line, = plt.plot(t, s, lw=2)

plt.annotate('local max', xy=(2, 1), xytext=(1.5, 1.5), arrowprops=dict(facecolor='brown', shrink=0.05),)
plt.annotate('local min', xy=(2.5,-1), xytext =(3,-1.5), arrowprops=dict(facecolor='grey', shrink=0.05), )
x = np.linspace(0,5 ,6)
y= 0 + 0*x
plt.plot(x,y, lw=0.5, color='red')
plt.ylim(-2,2)
plt.show()


from setuptools import setup

requires = ["requests"]

packages = ["ReversingLabs",
            "ReversingLabs.SDK"]

with open("README.md", "r") as readme:
    long_description = readme.read()

setup(
    name="reversinglabs-sdk-py3",
    version="1.1.2",
    description="Python SDK for using ReversingLabs services - Python 3 version.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    author="ReversingLabs",
    author_email="support@reversinglabs.com",
    url="https://github.com/reversinglabs/reversinglabs-sdk-py3",
    packages=packages,
    python_requires=">=3.6",
    install_requires=requires,
    license="MIT",
    zip_safe=False,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Natural Language :: English",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: Implementation :: CPython",
        "Topic :: Security",
        "Topic :: Software Development :: Libraries :: Python Modules"
    ],
    project_urls={
        "Documentation": "https://github.com/reversinglabs/reversinglabs-sdk-py3/blob/main/README.md",
        "Source": "https://github.com/reversinglabs/reversinglabs-sdk-py3"
    },
)


import h5py
import pytest

from hexrd.material import Material, load_materials_hdf5

# Tolerance for comparing floats
FLOAT_TOL = 1.e-8

# Use consistent units to simplify testing
DEFAULT_LENGTH_UNIT = 'angstrom'
DEFAULT_ANGLE_UNIT = 'degrees'

@pytest.fixture
def default_material():
    return Material()

@pytest.fixture
def test_materials_file(example_repo_path):
    return example_repo_path / 'NIST_ruby/single_GE/include/materials.h5'

def normalize_unit(v):
    if hasattr(v, 'unit'):
        # Assume it's a val unit
        if v.unit in ('radians', 'degrees'):
            # Assume it's an angle
            return v.getVal(DEFAULT_ANGLE_UNIT)
        else:
            # Assume it's a length
            return v.getVal(DEFAULT_LENGTH_UNIT)

    return v

def are_close(v1, v2, tol=FLOAT_TOL):
    return abs(normalize_unit(v1) - normalize_unit(v2)) < tol

def lparms_are_close(lparms, indices, tol=FLOAT_TOL):
    first = lparms[indices[0]]
    return all(are_close(first, lparms[i], tol) for i in indices[1:])

def test_sgnum_setter(default_material):
    # Just in case we change the default...
    default_material.sgnum = 225

    # Should have a == b == c and alpha == beta == gamma == 90
    lparms = default_material.latticeParameters
    assert lparms_are_close(lparms, [0, 1, 2])
    assert lparms_are_close(lparms, [3, 4, 5])
    assert are_close(lparms[5], 90)

    default_material.sgnum = 165
    lparms = default_material.latticeParameters

    # Gamma should be 120, the other angles should be 90
    assert not lparms_are_close(lparms, [3, 4, 5])
    assert lparms_are_close(lparms, [3, 4])
    assert are_close(lparms[3], 90)
    assert are_close(lparms[5], 120)

def test_load_materials(test_materials_file):
    materials = load_materials_hdf5(test_materials_file)

    with h5py.File(test_materials_file, 'r') as f:
        # Check that it loaded all of the materials
        mat_names = list(f.keys())
        assert all(x in materials for x in mat_names)

        # Check that the values for ruby match
        ruby = materials['ruby']
        params = f['ruby']['LatticeParameters'][()]

        # Convert to angstroms...
        for i in range(3):
            params[i] *= 10

        for i in range(6):
            assert are_close(params[i], ruby.latticeParameters[i])


import numpy as np
import math
import matplotlib.pyplot as plt

L = 10
delta_u = 0.1
E = 110e13
p = 4300000000
Nx = 50
h = L / Nx
tau = h / np.sqrt(E/p)
T = 1
a = 0
b = a + L
ta = 0
tb = ta + T
Ua = Ub = 0
Nt = int(T/tau)

def started_u(x):
   #return -0.4 * (x ** 2) + 0.04 * x
   return (-0.004*x + 0.04) * x

def phi(x):
    return 0

def m(x):
    return ((tau**2) * E) / (p * (h ** 2))

def second_row(x):
    return tau * phi(x) + x

def get_matrix(Nt, Nx):
    matrix = np.zeros((Nt, Nx))
    x = np.linspace(a, b, Nx)
    t = np.linspace(ta, tb, Nt)
    matrix[0] = started_u(x)
    matrix[1] = list(map(second_row, matrix[0]))
    matrix[:, 0] = Ua
    matrix[:, -1] = Ub
    i = 2
    j = 1
    while i < Nt:
        _t = t[i]
        while j < Nx - 1:
            _x = x[j]
            matrix[i][j] = -matrix[i - 2][j] + m(_x) * matrix[i - 1][j - 1] + 2 * matrix[i - 1][j] * (-m(_x) + 1) + m(_x) * matrix[i - 1][j + 1]
            j += 1
        i += 1
        j = 1
    return matrix

def graph_builder(matrix):
    x_list = np.linspace(a, b, Nx)
    y = np.arange(0, Nt, 40)
    for i in y:
        plt.plot(x_list, matrix[int(i)])
    plt.show()
    plt.show()

print(tau)
matrix = get_matrix(Nt, Nx)
graph_builder(matrix)


"""Third step.
This step takes the templates (B matrix) and the activation matrix (A matrix) and the annotation
of data to elaborate a Hidden Markov Model to predict the bursting or non-bursting state."""

import sys
import numpy as np

from scipy.signal import lfilter, butter
from scipy.stats import gennorm

import warnings
warnings.simplefilter(action='ignore', category=DeprecationWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)
from hmmlearn import hmm
from sklearn.externals import joblib

expName = sys.argv[1]
runN = sys.argv[2]
chan = sys.argv[3]

K = 1
L = 25

lmbda = 100.0 # sparsity coeff
alpha = 0.25 # sparsity exponent

def butter_highpass(cutoff, fs, order=5):
    nyq = 0.5 * fs
    normal_cutoff = cutoff / nyq
    b, a = butter(order, normal_cutoff, btype='highpass', analog=False)
    return b, a

hb, ha = butter_highpass(300, 20000.0, 5)
settleTime = 1000

F = 5
decay_coeffs = np.array([1.0 - i*(1.0/F) for i in range(F)])
decay_coeffs = decay_coeffs / np.sum(decay_coeffs)
decay_coeffs[0] = 1.0

time = None

template_data = []
try:
    annotations = open("data/annotations/annotation_"+expName+"_"+str(runN)+"_"+chan+".txt").readlines()
    annStartPoint = int(annotations.pop(0))
    for line in annotations:
        if line == "":
            continue
        line = line.split(" ")
        s, e = annStartPoint-settleTime+int(float(line[0])*20000), annStartPoint-settleTime+int(float(line[1])*20000)
        template_data.append((s, e))

    data = np.load("E:\\bryan\\recordings\\4Jun2018\\"+expName+"\\"+runN+"\\chan_" + chan + ".npy")
    burst = lfilter(hb, ha, data)[settleTime:]
    X = burst.T.astype(np.float32)

    T = len(burst)
    A = np.zeros((T, K))
    A_templates = np.zeros((T, K))
    for ann in template_data:
        A_templates[ann[0]-L:ann[1],0] = 1

    for k in range(K):
        A[:,k] = np.load("data/activations/spikeActivation_"+expName+"_"+str(runN)+"_"+chan+"_"+str(k)+".npy")[settleTime:]
        A[:,k] = np.convolve(A[:,k], decay_coeffs)[len(decay_coeffs)-1:]

    (beta, loc, scale) = gennorm.fit(X, fbeta=1.3)

    Avalues = []
    Aantivalues = []
    levels = 5

    for t in range(annStartPoint, annStartPoint+15000):
        value = A[t,0]//scale
        if value > A[t,0]//levels:
            value = levels-1
        value += 0.5

        if A_templates[t,0] != 0:
            Avalues.append(value)
        else:
            Aantivalues.append(value)

    bins = range(levels+1)

    eBursting, _ = np.histogram(Avalues, bins=bins, density=True)
    eNotBursting, _ = np.histogram(Aantivalues, bins=bins, density=True)

    p = 0.74 # P(stay off)
    q = p # P (stay on)

    transition_p = np.array([
        [p, 1-p],
        [1-q, q]
    ])

    emission_p = np.array([
        eNotBursting,
        eBursting
    ])

    starting_p = np.array([
        1.0,
        0.0
    ])

    multinomial_A = (A[:,0] // scale).astype(np.int)
    multinomial_A = np.minimum(multinomial_A, levels-2)
    multinomial_A.shape = (T,1)

    for t in range(levels-1):
        multinomial_A[t] = t

    hmmmodel = hmm.MultinomialHMM(n_components=2, params="", init_params="")
    hmmmodel.transmat_ = transition_p
    hmmmodel.emissionprob_ = emission_p
    hmmmodel.startprob_ = starting_p
    hmmmodel.fit(multinomial_A)

    np.save("data/markov/distribution_"+expName+"_"+str(runN)+"_"+str(chan), np.array((beta, loc, scale)))
    joblib.dump(hmmmodel, "data/markov/model_"+expName+"_"+str(runN)+"_"+str(chan)+".mod")

except Exception as e:
    print("Could not load: ", e)


from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _
import gitlab

class Question(models.Model):
    """Main element to drive the flow of the website. Questions are what the user can ask and which will trigger the
    rest of the actions on the website
    """
    UNDECIDED = 0
    APPROVED = 1
    ANSWERED = 2
    REJECTED = 3

    STATUS_CHOICES = (
        (UNDECIDED, _('Undecided')),
        (APPROVED, _('Approved')),
        (ANSWERED, _('Answered')),
        (REJECTED, _('Rejected'))
        )

    question = models.TextField(verbose_name=_('Your Question'), blank=False, null=False)
    original_question = models.TextField(verbose_name=_('Backup information'), blank=True, null=True)
    relevant_timespan = models.TextField(verbose_name=_('Relevant timespan'), blank=True, null=True)
    relevant_location = models.TextField(verbose_name=_('Relevant location'), blank=True, null=True)
    extra_info = models.TextField(verbose_name=_('Extra information'), blank=True, null=True)
    categories = models.TextField(verbose_name=_('Categories'), blank=True, null=True)

    user_email = models.EmailField(verbose_name=_('User Email'), blank=True, null=True)
    asked_by_ip = models.GenericIPAddressField(null=True, blank=True)

    date_asked = models.DateTimeField(auto_now_add=True)
    status = models.IntegerField(choices=STATUS_CHOICES, default=UNDECIDED)

    def get_card_data(self):
        return {
            'title': self.question,
            }

    def get_as_home_row_card(self):
        return render_to_string('core/includes/question_list_block.html',
                                context=self.get_card_data())

    def save(self, **kwargs):
        super().save(**kwargs)
        try:
            self.issue
        except ObjectDoesNotExist:
            if self.status == self.APPROVED:
                GitlabIssues.objects.create(question=self)

class GitlabIssues(models.Model):
    question = models.OneToOneField(Question, on_delete=models.SET_NULL, null=True, related_name='issue')
    issue_id = models.IntegerField(null=False)
    creation_date = models.DateTimeField(auto_now_add=True)

    def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
        if self.issue_id is None:
            if not settings.DEBUG:
                gl = gitlab.Gitlab('https://gitlab.com', private_token=settings.GITLAB_PERSONAL_TOKEN)
                project = gl.projects.get(settings.GITLAB_PROJECT_ID)
                template_issue = project.files.get(
                    file_path='Templates/template_question_issue.md', ref='master').decode().decode('utf-8')

                issue_body = f"Question: {self.question.question}\n" \
                             f"Original Question: {self.question.original_question}\n\n" \
                             f"- Categories: {self.question.categories}\n" \
                             f"- Timespan: {self.question.relevant_timespan}\n" \
                             f"- Location: {self.question.relevant_location}\n" \
                             f"- Extra information: {self.question.extra_info}\n" \
                             f"- Asked by: {self.question.user_email}\n" \
                             f"\n\n{template_issue}"
                issue_title = f"Question: {self.question.question}"
                issue = project.issues.create({
                    'title': issue_title[:254],
                    'description': issue_body,
                    })
                issue.labels = ['Editor needed', ]
                issue.save()
                self.issue_id = issue.get_id()
            else:
                self.issue_id = 1234
        super().save(force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields)


# -*- coding: utf-8 -*-
"""
Created on Mon Apr  1 13:37:25 2019

@author: Win7
"""
import kutuphane
from sklearn.feature_selection import SelectKBest, f_classif
#1- Verileri yükle
giris, cikis, kisi_bilgisi =  kutuphane.dosya_oku('data/odev.csv')
olcekli_giris = kutuphane.olceklendir(giris)
dogruluk,f1skor = kutuphane.basari_hesapla(olcekli_giris, cikis, kisi_bilgisi)

#Chi-square
for k in range(1,30,1):
    ozellikler  = kutuphane.chi2_ozellik_cikar(giris,cikis,k)
    azaltilmis_olcekli_giris = olcekli_giris[:,ozellikler]
    dogruluk,f1skor = kutuphane.basari_hesaplaCV(azaltilmis_olcekli_giris, cikis, kisi_bilgisi,10)
    print("k="+str(k) + " acc="+str(dogruluk))

for k in range(1,30,1):
    azaltilmis_olcekli_giris = SelectKBest(f_classif, k=k).fit_transform(X=olcekli_giris,y=cikis)
    dogruluk,f1skor = kutuphane.basari_hesaplaCV(azaltilmis_olcekli_giris, cikis, kisi_bilgisi)
    print("k="+str(k) + " acc="+str(dogruluk))
ozellikler  = kutuphane.chi2_ozellik_cikar(giris,cikis,140)


import os
import numpy as np
import cv2
import csv
from .custom import CustomDataset
from pycocotools.coco import COCO
import pyclipper
from .registry import DATASETS

"""
{
	"segmentation": [[125.12,539.69,140.94,522.43...]],
	"num_keypoints": 10,
	"area": 47803.27955,
	"iscrowd": 0,
	"keypoints": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,142,309,1,177,320,2,191,398...],
	"image_id": 425226,"bbox": [73.35,206.02,300.58,372.5],"category_id": 1,
	"id": 183126
},

"""

@DATASETS.register_module
class TextDataSet(CustomDataset):
    CLASSES = ('text')
    list_file = []

    def load_annotations(self, ann_file):
        ext = ['jpg', 'png', 'jpeg', 'JPG']
        img_infos = []
        self.coco = COCO()
        self.coco.dataset = {}
        annotations = []
        images = []
        categories = []
        category = {"supercategory": "text",
                    "id": 1,
                    "name": "text"}
        categories.append(category)
        anno_id = 0
        for f in os.listdir(ann_file):
            if f.rpartition('.')[2] in ext:
                info = {}

                image = {}

                img_path = os.path.join(ann_file, f)
                im = cv2.imread(img_path)
                h, w, _ = im.shape
                txt_fn = os.path.join(ann_file, 'gt_%s.%s' % (os.path.basename(img_path).rpartition('.')[0], 'txt'))
                if not os.path.exists(txt_fn):
                    print('text file {} does not exists'.format(txt_fn))
                # if not self.test_mode and "img" in img_path:
                #    continue
                text_boxes, text_polys, text_tags, text_labels = self.load_txt_annotation(txt_fn)
                img_id = int(os.path.basename(img_path).rpartition('.')[0].split("_")[-1])
                info["bboxes"] = text_boxes
                info["polys"] = text_polys
                info["labels"] = text_labels
                info["filename"] = img_path
                info["file_name"] = img_path
                info["width"] = w
                info["height"] = h

                image["file_name"] = img_path
                image["height"] = h
                image["width"] = w
                image["id"] = img_id

                images.append(image)

                for box, poly in zip(text_boxes, text_polys):
                    anno_id += 1
                    # poly = self.shrink_poly(poly)
                    coco_box = [box[0], box[1], box[2] - box[0], box[3] - box[1]]
                    segmentation = [[poly[0][0], poly[0][1], poly[1][0], poly[1][1], poly[2][0], poly[2][1], poly[3][0],
                                     poly[3][1]]]
                    area = polygon_area(poly)
                    annotation = dict(id=anno_id, image_id=img_id, category_id=1, segmentation=segmentation, poly=poly,
                                      area=area, bbox=coco_box, iscrowd=0)
                    annotations.append(annotation)
                img_infos.append(info)
        self.coco.dataset = dict(info={}, licenses=[], images=images, annotations=annotations, categories=categories)
        self.coco.createIndex()
        self.cat_ids = self.coco.getCatIds()
        self.cat2label = {cat_id: i + 1 for i, cat_id in enumerate(self.cat_ids)}
        self.img_ids = self.coco.getImgIds()
        return img_infos

    def shrink_poly(self, poly, r=0.96):
        '''
        fit a poly inside the origin poly, maybe bugs here...
        used for generate the score map
        :param poly: the text poly
        :param r: r in the paper
        :return: the shrinked poly
        '''
        try:
            area_poly = abs(pyclipper.Area(poly))
            perimeter_poly = perimeter(poly)
            poly_s = []
            pco = pyclipper.PyclipperOffset()
            if perimeter_poly:
                d = area_poly * (1 - r * r) / perimeter_poly
                pco.AddPath(poly, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
                poly_s = pco.Execute(-d)
            shrinked_poly = poly_s[0]
            if len(shrinked_poly) < 4:
                return poly
            return shrinked_poly

        except Exception as e:
            return poly

    def load_txt_annotation(self, path):
        text_polys = []
        labels = []
        text_boxes = []
        text_tags = []
        if not os.path.exists(path):
            return text_boxes, text_polys, text_tags, labels

        with open(path, 'r') as f:
            reader = csv.reader(f)
            for line in reader:
                label = line[-1]
                line = [i.strip('\ufeff').strip('\xef\xbb\xbf') for i in line]

                x1, y1, x2, y2, x3, y3, x4, y4 = list(map(int, line[:8]))
                if '#' in label and "img" in path:
                    continue
                box = [min(x1, x2, x3, x4), min(y1, y2, y3, y4), max(x1, x2, x3, x4), max(y1, y2, y3, y4)]
                text_boxes.append(box)
                labels.append(1)
                text_polys.append(np.array([[x1, y1], [x2, y2], [x3, y3], [x4, y4]], dtype=np.int32))
                text_tags.append(label)
        return text_boxes, text_polys, text_tags, labels

    def get_ann_info(self, idx):
        info = self.img_infos[idx]
        gt_bboxes = np.array(info["bboxes"], dtype=np.float32)
        gt_labels = np.array(info["labels"], dtype=np.int64)
        gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32)
        labels_ignore = np.zeros((0, 1), dtype=np.int64)
        gt_masks = []
        segm_mask = np.zeros((info["height"], info["width"]), np.uint8)
        # gt_mask = np.zeros((info["height"], info["width"]), np.uint8)
        for poly in info["polys"]:
            poly = self.shrink_poly(poly)
            poly = np.array(poly, np.int32)
            gt_mask = np.zeros((info["height"], info["width"]), np.uint8)
            cv2.fillPoly(gt_mask, [poly], 1)
            gt_masks.append(gt_mask)
            segm_mask = cv2.bitwise_and(gt_mask, segm_mask)
        # cv2.fillPoly(gt_mask, info["polys"], 1)
        # if gt_masks:
        # labels_ignore    gt_masks=np.array(gt_masks,dtype=np.int64)
        ann = dict(bboxes=gt_bboxes, labels=gt_labels, labels_ignore=labels_ignore, segm=segm_mask, polys=info["polys"],
                   bboxes_ignore=gt_bboxes_ignore,
                   masks=gt_masks)
        return ann

def perimeter(poly):
    try:
        p = 0
        nums = poly.shape[0]
        for i in range(nums):
            p += abs(np.linalg.norm(poly[i % nums] - poly[(i + 1) % nums]))
        # logger.debug('perimeter:{}'.format(p))
        return p
    except Exception as e:
        traceback.print_exc()
        raise e

def polygon_area(poly):
    '''
    compute area of a polygon
    :param poly:
    :return:
    '''
    edge = [
        (poly[1][0] - poly[0][0]) * (poly[1][1] + poly[0][1]),
        (poly[2][0] - poly[1][0]) * (poly[2][1] + poly[1][1]),
        (poly[3][0] - poly[2][0]) * (poly[3][1] + poly[2][1]),
        (poly[0][0] - poly[3][0]) * (poly[0][1] + poly[3][1])
    ]
    return -np.sum(edge) / 2.


import re
import collections
import os
import sys
from io import StringIO
from lxml import etree
import lxml.html
import yaml
from xmldiff import main as xd
import pprint

from utils import file_utils, yaml_utils

def get_html_file_list(html_dir):
    file_list = file_utils.get_file_list(root_path_str=html_dir, file_stem_str='*', file_ext_str='*', recursive_bool=False, rtn_abs_path_bool=True, rtn_uri=False)
    if 0 < len(file_list) < 2:
        print('Only one file found, exiting.')
        return None
        # xml = file_list[0].read_text()
    elif 1 < len(file_list) :
        # print(file_list[0])
        return file_list
    else:
        print('File not found, exiting.')
        return None

def make_tree(fn):
    try:
        print('Reading file: ' + str(fn))
        html_text = fn.read_text()
    except:
        print('Could not read: ' + str(fn))
    html = etree.parse(StringIO(html_text), parser=html_parser)
    return html

def compare_trees():
    if 'dirlist' in config:
        for html_dir in config['dirlist']:
            html_file_list = get_html_file_list(html_dir=html_dir)
            base_html_tree = make_tree(fn=html_file_list.pop(0))
            for html_filename in html_file_list:
                html_tree = make_tree(fn=html_filename)
                # diff_formatter = xd.formatting.XMLFormatter(pretty_print=True)
                # diff_formatter = xd.formatting.XmlDiffFormatter(normalize='WS_TAGS', pretty_print=True)
                diff_formatter = xd.formatting.DiffFormatter(pretty_print=True)
                diff_o = {"F": 0.5}
                html_diff = xd.diff_trees(left=base_html_tree,
                                          right=html_tree,
                                          diff_options=diff_o,
                                          formatter=diff_formatter)
                with open(str(html_filename).replace('html', 'diff'), 'w') as diff_out:
                    diff_out.write(html_diff)

def setup():
    global html_parser, config
    html_parser = etree.HTMLParser()
    config = yaml_utils.load_config()
    return True

if __name__ == "__main__":
    success = setup()
    if not success:
        print('Failed to complete setup, exiting.')
        sys.exit()

    compare_trees()

# raw_tree = etree.ElementTree(html)
# nice_tree = collections.OrderedDict()

# print(html.body)
# tag_body_flag = False
# for tag in html.iter():
#     #print(tag)
#     path = re.sub('\[[0-9]+\]', '', html.getpath(tag))
#     if path not in nice_tree:
#         nice_tree[path] = []
#     if len(tag.keys()) > 0:
#         nice_tree[path].extend(attrib for attrib in tag.keys() if attrib not in nice_tree[path])

# for path, attribs in nice_tree.items():
#     indent = int(path.count('/') - 1)
#     print('{0}{1}: {2} [{3}]'.format('    ' * indent, indent, path.split('/')[-1], ', '.join(attribs) if len(attribs) > 0 else '-'))