import pandas as pd
import numpy as np
from gensim.models import Word2Vec
import jieba
import time
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.ensemble import RandomForestClassifier

# 读取动作片评论数据集
data = pd.read_csv(r'./整理后数据/merged_action.csv')

# 加载Word2Vec模型
model = Word2Vec.load(r'./segmentation_wordcloud/word2vec模型/word2vec_action.model').wv

# 定义获取特征矩阵的函数
def getVector_v1(cutWords, word2vec_model):
    count = 0
    article_vector = np.zeros(word2vec_model.vector_size)
    for cutWord in cutWords:
        if cutWord in word2vec_model.key_to_index:
            article_vector += word2vec_model.get_vector(cutWord)
            count += 1
    if count != 0:
        return article_vector / count
    else:
        return np.zeros(word2vec_model.vector_size)

# 分词处理
list_all = []
data['评论内容'] = data['评论内容'].astype(str)
for sentence in data['评论内容']:
    words = list(jieba.cut(sentence))
    list_all.append(words)

# 生成特征矩阵
startTime = time.time()
vector_list = []
i = 0
for cutWords in list_all:
    i += 1
    if i % 1000 == 0:
        print('前%d 篇评论形成词向量花费%.2f 秒' % (i, time.time() - startTime))
    vector = getVector_v1(cutWords, model)
    vector_list.append(vector)
X = np.array(vector_list)
print('Total Time You Need To Get X: %.2f 秒' % (time.time() - startTime))
X.dump('sat_dissat_vector.txt')

# 标签编码
labelEncoder = LabelEncoder()
y = labelEncoder.fit_transform(data['情感'])

# 划分数据集
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=0)

# 构建逻辑回归模型
logistic_model = LogisticRegression()
logistic_model.fit(train_X, train_y)
logistic_score = logistic_model.score(test_X, test_y)

# 输出分类报告
report = classification_report(test_y, logistic_model.predict(test_X), labels=[0, 1], target_names=['正向情感', '负向情感'],zero_division=1)
print(report)

# 计算AUC值
auc_score = roc_auc_score(test_y, logistic_model.predict_proba(test_X)[:, 1], multi_class='ovr')
print('AUC Score:', auc_score)

# 构建随机森林模型
rf = RandomForestClassifier(
    criterion='entropy',
    n_estimators=10000,
    min_samples_split=100,
    min_weight_fraction_leaf=0.05
)
rf.fit(train_X, train_y)
y_pred = rf.predict(test_X)
print('分类预测值：', y_pred)


import time

import pandas as pd
from pandas import DataFrame, Series
import numpy as np
from itertools import permutations

pd.options.mode.chained_assignment = None  # default='warn'

def build_empty_assignment_detail(users):
    """build the empty assignment detail obj from a list of users"""
    result = {user: {'total_time_used': 0, 'own_tasks': []} for user in users}
    return result

def assign_sub_matrix(sub_performance_matrix, assignment_detail):
    """Calculate the VDC users with sub-tasks"""
    users = sub_performance_matrix.columns.tolist()
    task_count = len(sub_performance_matrix.index)  # How many tasks in this sub-matrix, maybe less then users

    # Add offset to each users' baseline
    for idx, user in enumerate(users):
        sub_performance_matrix[user] += assignment_detail[user]['total_time_used']

    user_permutations = permutations(users, task_count)  # The permutations of all users on tasks = A(users, task_count)

    minimum_permutation_time_used = -1
    minimum_permutation = None

    # Loop each permutation to find the smallest
    for idx_p, permutation in enumerate(user_permutations):
        # print(permutation)

        maximum_time_used_in_permutation = -1  # the time_used of smallest permutation
        for idx_u, user in enumerate(permutation):
            time_used_for_user_on_task = sub_performance_matrix.iloc[idx_u][user]

            # the max time used in one permutation
            maximum_time_used_in_permutation = time_used_for_user_on_task if (
                time_used_for_user_on_task > maximum_time_used_in_permutation) else maximum_time_used_in_permutation

        # Find the smallest max time used permutation
        if (maximum_time_used_in_permutation < minimum_permutation_time_used) or (minimum_permutation_time_used == -1):
            minimum_permutation_time_used = maximum_time_used_in_permutation
            minimum_permutation = permutation

    print('Smallest permutation: ', minimum_permutation)

    for idx, user in enumerate(minimum_permutation):
        time_used_for_user_on_task = sub_performance_matrix.iloc[idx][user]
        task_name = sub_performance_matrix.index[idx]
        assignment_detail[user]['total_time_used'] = time_used_for_user_on_task
        assignment_detail[user]['own_tasks'].append(task_name)

    print(sub_performance_matrix)
    return

def assign_tasks(performance_matrix, assignment_detail):
    users = performance_matrix.columns.tolist()  # the columns of the DataFrame is userIds
    user_count = len(users)
    task_count = len(performance_matrix.index)  # The number of tasks
    # offsets = [0] * user_count  # Offsets means the init value of each user when starting a round of calculation

    start_row = 0  # the start index of task in a round of calculation
    stop_row = 0
    remain_task_count = task_count  # How many tasks left to be calculated

    # Loop of each round of calculation
    while remain_task_count > 0:
        # Split the sub matrix from performance_matrix
        stop_row += remain_task_count if (remain_task_count < user_count) else user_count
        sub_performance_matrix = performance_matrix.iloc[start_row:stop_row]

        # Do calculation!
        assign_sub_matrix(sub_performance_matrix, assignment_detail)

        # Update the remain count & start index after one round
        remain_task_count -= (stop_row - start_row)
        start_row = stop_row

        print(assignment_detail)

    return

def main():
    # Sample data
    users = ['P1', 'P2', 'P3']
    performance_2darray = np.array([
        [2, 5, 3],
        [2, 1, 4],
        [4, 3, 1],
        [3, 2, 1],
        [4, 5, 8],
        [7, 4, 4],

        [6, 3, 3],
        [3, 5, 2]
    ])
    performance_matrix = DataFrame(performance_2darray,
                                   columns=users,
                                   index=['T1', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8'])

    assignment_detail = build_empty_assignment_detail(users)

    assign_tasks(performance_matrix, assignment_detail)
    return

if __name__ == '__main__':
    start_time = time.time()
    main()
    stop_time = time.time()
    print('用时: ', (stop_time - start_time), ' (s)')


import os
import re
import pprint
from xml.dom import minidom

DEBUG = False

class Config(object):
	def __init__(self, filename):
		with open(filename, 'r') as file_:
			lines = file_.readlines()

		lines = self._remove_line_feed(lines)
		lines = self._remove_comment(lines)
		lines = self._remove_blank_lines(lines)

		basic, beats, visions = self._split_parts(lines)

		self._read_basics(basic)
		self._read_beats(beats)
		self._read_visions(visions)
		self._validate()

	def _remove_line_feed(self, lines):
		return [re.sub(r'\n', '', line) for line in lines]

	def _remove_comment(self, lines):
		return [re.sub(r'#.*', '', line) for line in lines]

	def _remove_blank_lines(self, lines):
		return [line for line in lines if line]

	def _split_parts(self, lines):
		basic, beats, visions = [], [], []

		target = basic
		for line in lines:
			if line.startswith('='):
				if target is basic:
					target = beats
				elif target is beats:
					target = visions
				continue

			target.append(line)

		return basic, beats, visions

	def _read_basics(self, basic):
		for line in basic:
			key, value = self._parse_key_value(line)
			setattr(self, key, value)

			if DEBUG:
				print('{}: {}'.format(key, value))

	def _read_beats(self, beats):
		self.beats = []

		position = 'left'
		for line in beats:
			singer, lyric, beats = self._parse_lyrics_beats(line)
			beat = {}
			beat['singer'] = singer
			beat['lyric'] = lyric
			beat['beats'] = beats
			if lyric is None:
				position = 'left'
			else:
				beat['position'] = position
				if position == 'left':
					position = 'right'
				elif position == 'right':
					position = 'left'
				else:
					assert False

			self.beats.append(beat)

			if DEBUG:
				pprint.pprint(beat)

	def _read_visions(self, visions):
		self.visions = []
		self._read_visions_basic(visions)
		self._read_visions_components(visions)

		if DEBUG:
			pprint.pprint(self.visions)

	def _read_visions_basic(self, visions):
		curr_beat = 0
		for line in visions:
			if line == 'clear':
				for vision in self.visions:
					if vision['end'] == -1:
						vision['end'] = curr_beat
			else:
				name, beat = self._parse_visions_beats(line)
				if name is not None:
					self.visions.append({
						'name': name,
						'start': curr_beat,
						'end': -1,
						'components': [],
					})
				curr_beat += beat

	def _read_visions_components(self, visions):
		for vision in self.visions:
			basename = '{}.svg'.format(vision['name'])
			src_filename = os.path.join(self.visions_dir_name, basename)
			document = minidom.parse(src_filename)
			groups = document.getElementsByTagName('g')
			for group in groups:
				layer_name = group.getAttribute('inkscape:label')
				if not layer_name: # not a layer
					continue
				rects = group.getElementsByTagName('rect')
				if not rects:
					continue
				rect = rects[0]
				style = rect.getAttribute('style')
				color = re.search('fill:#([0-9a-f]+)', style).group(1)
				vision['components'].append({
					'id': layer_name,
					'color': color,
				})

	def _validate(self):
		for key, type_ in self._get_settings_types().items():
			value = getattr(self, key)
			if type(value) is not type_:
				raise Exception('Invalid config type: {} - {}'.format(key, value))

	def _parse_key_value(self, line):
		match = re.match(r'^\s*(\S+)\s*:\s*(\S+)\s*$', line)
		if not match:
			raise Exception('Invalid config basic setting format: {}'.format(line))

		key = match.group(1)
		key = re.sub('-', '_', key)

		value = match.group(2)
		value = self._get_settings_types()[key](value)

		return key, value

	def _parse_lyrics_beats(self, line):
		match = re.match(r'^\s*((\S)\s+)*(\S+)\s*(.*?)\s*$', line)
		if not match:
			raise Exception('Invalid config beats format: {}'.format(line))

		singer = match.group(2)
		if singer:
			singer = self._get_singer_name(singer)

		lyric = match.group(3)
		if self._is_meta_label(lyric):
			lyric = None

		beats = match.group(4)
		beats = beats.split(',')
		beats = [beat.strip() for beat in beats]
		beats = [float(beat) for beat in beats]

		return singer, lyric, beats

	def _parse_visions_beats(self, line):
		match = re.match(r'^\s*(\S+)\s*(\S*)\s*$', line)
		if not match:
			raise Exception('Invalid config beats format: {}'.format(line))

		name = match.group(1)
		if self._is_meta_label(name):
			name = None

		beat = match.group(2)
		beat = float(beat)

		return name, beat

	def _is_meta_label(self, label):
		return label.startswith('[') and label.endswith(']')

	def _get_singer_name(self, singer_symbol):
		mapping = {
			'+': 'boy',
			'-': 'girl',
			'*': 'both',
		}

		if singer_symbol not in mapping:
			raise Exception('No such singer symbol: {}'.format(singer_symbol))

		singer_name = mapping[singer_symbol]
		return singer_name

	def _get_settings_types(self):
		return {
			'song_name': str,
			'lyric_writer': str,
			'melody_writer': str,
			'audio_filename': str,
			'output_dir_name': str,
			'visions_dir_name': str,
			'beat_per_minute': float,
			'begin_time': float,
			'beats': list,
		}


from divisible import __version__
from divisible.main import is_divisible, is_even, is_odd

import pytest

def test_version():
    assert __version__ == "0.1.0"

@pytest.mark.parametrize(
    "numerator,denominator,expected",
    [(10, 2, True), (10, 3, False), (9, 7, False), (9, 10, False), (10, 0, False)],
)
def test_is_divisible(numerator, denominator, expected):
    assert is_divisible(numerator, denominator) == expected

@pytest.mark.parametrize(
    "number,expected",
    [(10, True), (3, False), (-2, True), (0, True)]
)
def test_is_even(number: int, expected: bool):
    assert is_even(number) == expected

@pytest.mark.parametrize(
    "number,expected",
    [(10, False), (3, True), (-5, True), (0, False)]
)
def test_is_odd(number: int, expected: bool):
    assert is_odd(number) == expected


import cv2
import numpy as np
import math
from get_reference_frame import get_ref_frame
import os
import csv

def pixel_distance(img1, img2):
    '''
    Get the histogram of an image. For an 8-bit, grayscale image, the
    histogram will be a 256 unit vector in which the nth value indicates
    the percent of the pixels in the image with the given darkness level.
    The histogram's values sum to 1.
    '''
    img1 = img1.astype(np.int16)
    img2 = img2.astype(np.int16)

    return np.sum(np.absolute(np.subtract(img1.flatten(),
        img2.flatten())))

# function for taking frames from video and combining into video to save # images sent to OCR
def combine_images(images, num_images: int, num_horizontal: int):
    '''
    Return a new image with the images glued together
    Arguments:
        images: N x H x W ndarray of images
        num_horizontal: determines the number of images to put next to each other
        on the same row
    '''
    combined_image = None
    for i in range (math.ceil(num_images / num_horizontal)):
        row_image = None
        for j in range (num_horizontal):
            if row_image is None:
                row_image = images[i*num_horizontal + j]
            elif (i*num_horizontal + j >= num_images):
                row_image = cv2.hconcat([row_image, np.zeros((images.shape[1],images.shape[2],3), np.uint8)])
            else:
                row_image = cv2.hconcat([row_image, images[i*num_horizontal + j]])

        if combined_image is None:
            combined_image = row_image
        else:
            combined_image = cv2.vconcat([combined_image, row_image])

    return combined_image

def process_avi_pixels(file_location, time_of_ref=200, window_len=10,
    images_per_combine:int=240, images_per_horizontal:int=12, start_time=None, end_time=None,
    skip=20, threshold=60, fps=25, name=None, out_folder=None, ref_frame_loc=None):
    '''
    Function for taking avi and grabbing cropped frames (fixed) from them
    '''
    ## Getting the reference frame to compare with
    #### !!! This was required for automatically getting ref frame !!!
    # x, y, w, h = get_ref_frame(file_location, time_of_ref, fps, window_len, ref_frame_loc)
    vc = cv2.VideoCapture(file_location)
    ret, frame = vc.read()
    print(frame.shape)
    vc.set(1, 63900)

    ### For cricket matches
    x = 69
    y = 321
    w = 87
    h = 21
    ret, frame = vc.read()
    # print(frame.shape)
    cv2.imwrite(ref_frame_loc + name + '_ref.jpg', frame[y:y+h, x:x+w])

    vc = cv2.VideoCapture(file_location)
    compare_frame = None # this frame is what we compare for skipping
    if vc.isOpened():
        rval, frame = vc.read()
    else:
        rval = False

    timestamps = [] # dictionary mapping from crop to frame
    list_timestamps = [] # list of composite dictionary mappings
    framenum_li = []
    all_framenum_li = []
    frames = None
    time_window = None
    curr_time_window_cnt = 0

    if start_time is not None:
        start = start_time * fps
    else:
        start = 0

    end = end_time * fps

    # frame num, num frames processed, num images generated
    framenum, cnt, cnt_composite = 0, 0, 0

    # loop to start of time window of interest
    while (framenum < start):
        rval, frame = vc.read()
        framenum += 1
    # compare_frame = frame[y:y+h, x:x+w].copy()
    compare_frame = cv2.imread(ref_frame_loc + name + '_ref.jpg')

    while framenum < end:
        # process every 'skip'th frame
        for i in range (skip):
            rval, frame = vc.read()
            framenum += 1

        if frame is None:
            return list_timestamps

        # skip frames disimilar to what we want
        # calibrate treshold for pixel similarity
        while ((pixel_distance(compare_frame, frame[y:y+h, x:x+w]) / (w*h*3)) > threshold):
            rval, frame = vc.read()
            framenum += 1
            if frame is None:
                return list_timestamps

        if frame is None:
            return list_timestamps
        # while (pixel_distance(compare_frame, frame[y:y+h, x:x+w]) > 10):
        #     rval, frame = vc.read()
        #     framenum += 1

        # cv2.imwrite('test.jpg', frame)
        # print(frame.shape)
        # if cnt==21:
        #     cv2.imwrite('test1.jpg', frame[y:y+h, x:x+w])
        #     print(framenum)
        # if cnt==22:
        #     cv2.imwrite('test2.jpg', frame[y:y+h, x:x+w])
        #     print(framenum)
        cropped_frame = frame[y:y+h, x:x+w].copy()
        bordersize = 8
        padded_frame = cv2.copyMakeBorder(
            cropped_frame,
            top=bordersize,
            bottom=bordersize,
            left=bordersize,
            right=bordersize,
            borderType=cv2.BORDER_CONSTANT,
            value=[255, 255, 255]
        )

        cropped_frame = np.expand_dims(padded_frame, axis=0)
        cnt += 1
        timestamps.append(framenum / fps)
        framenum_li.append(framenum)
        # timestamps[cnt % images_per_combine] = framenum # map index to current timestamp

        if frames is None:
            frames = cropped_frame.copy()
        else:
            frames = np.concatenate((frames, cropped_frame), axis=0)

        if (cnt % images_per_combine == 0):
            new_image = combine_images(frames, images_per_combine, images_per_horizontal)
            list_timestamps.append(timestamps)
            all_framenum_li.append(framenum_li)
            timestamps = []
            framenum_li = []

            # send to ocr here
            if not os.path.exists(out_folder):
                os.makedirs(out_folder)

            path = os.path.join(out_folder, name + "_" + str(cnt_composite) + ".jpg")
            cv2.imwrite(path, new_image)
            cnt_composite += 1

            print('Image: ', path, ' SAVED')

            frames = None # reset frames

        # for debugging purposes
        # if (cnt > images_per_combine):
        #     return list_timestamps

    remaining_frames = images_per_combine - frames.shape[0]
    white_frames = np.full((remaining_frames, frames.shape[1], frames.shape[2], frames.shape[3]), 255)

    frames = np.concatenate((frames, white_frames), axis=0)
    new_image = combine_images(frames, images_per_combine, images_per_horizontal)
    list_timestamps.append(timestamps)
    all_framenum_li.append(framenum_li)
    # print(image.shape[0])
    # print(image.shape[1])
    timestamps = {}

    # send to ocr here

    path = os.path.join(out_folder, name + "_" + str(cnt_composite) + ".jpg")
    cv2.imwrite(path, new_image)
    cnt_composite += 1

    frames = None # reset frames
    vc.release()
    return list_timestamps, all_framenum_li, new_image.shape[0], new_image.shape[1]

def save_csv(list_of_timestamps, list_framenum, img_per_comp, name):
    '''
    Save a csv
    Arguments:
        list_of_timestamps: list of dictionaries, with each dictionary mapping
        an image in the crop (row-major order) to a timestamp
        img_per_crop: number of images in each crop
        name: path/name to save the csv to/as
    '''
    if not os.path.exists(name):
        os.makedirs(name)
    num_comps = len(list_of_timestamps)
    with open(name + 'run1.csv', mode='w') as csv_file:
        fieldnames = ['cmpid', 'imgid', 'time', 'frame']
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        writer.writeheader()
        for i in range (num_comps):
            num_imgs = len(list_of_timestamps[i])
            for j in range(num_imgs):
                # print(str(i) + ' , ' + str(j))
                writer.writerow({'cmpid': i, 'imgid': j, 'time': list_of_timestamps[i][j], 'frame': list_framenum[i][j]})

# save timestamp information as JSON
def save_json (list_of_timestamps):
    pass


import json

file = open("fr.json")
france_cities = json.load(file)
idf_major_cities = []

PARIS_AREA_NAME = "Île-de-France"
LARGE_POPULATION_NUMBER = 50000

for index, city in enumerate(france_cities):
    if (
        city["admin_name"] == PARIS_AREA_NAME
        and int(city["population"]) >= LARGE_POPULATION_NUMBER
    ):
        major_idf_city = {
            "City_index": index,
            "Latitude": city["lat"],
            "Longitude": city["lng"],
            "City": city["city"],
        }
        idf_major_cities.append(major_idf_city)

print(f"len of major cities in idf: {len(idf_major_cities)}")

with open("idf_major_cities.json", "w") as f:
    json.dump(idf_major_cities, f, ensure_ascii=False)


import os
import moviepy.video.io.ImageSequenceClip as mv

ro = '/home/na/1_Face_morphing/2_data/3_ganformer_v3/5_latentcode_exp/'
src_path = ro + '1_use_fixed_image/'
fps = 1

folder = os.listdir(src_path)
for fold in folder:
    if os.path.isdir(src_path + fold) is False:
        continue
    dst_path = src_path + fold + '.mp4'

    img_list = os.listdir(src_path + fold)
    img_list.sort()

    image_files = [os.path.join(src_path + fold, img)
                   for img in img_list
                   if img.endswith(".png")]
    clip = mv.ImageSequenceClip(image_files, fps=fps)

    clip.write_videofile(dst_path)


import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(128, 256, 1)
        self.conv2 = nn.Conv2d(128, 256, 3, padding=1)
        self.conv3 = nn.Conv2d(128, 256, 5, padding=2)
        self.max_pooling = nn.MaxPool2d(3, stride=1, padding=1)

    def forward(self, x):
        out1 = F.relu(self.conv1(x))  # 1,256, 14, 14
        out2 = F.relu(self.conv2(x)) # 1, 256, 14, 14
        out3 = F.relu(self.conv3(x))  # 1, 256, 14, 14
        out_max = self.max_pooling(x)  # 1,128, 14, 14

        output = torch.cat((out1, out2, out3, out_max), dim=1)
        return output

def function(x, y):
    x, y = torch.tensor(x, requires_grad=True), torch.tensor(y, requires_grad=True)
    loss = 2 * x * y + x ** 2
    loss.backward()
    return x.grad

print (function(3, 1))


import asyncio
import datetime

async def gerar_dados(quantidade: int, dados: asyncio.Queue) -> None:
    print(f'Aguarde a geração de {quantidade} dados...')
    for idx in range(1, quantidade + 1):
        item: int = idx * idx
        await dados.put((item, datetime.datetime.now()))
        await asyncio.sleep(0.001)
    print(f'{quantidade} dados gerados com sucesso!')

async def processar_dados(quantidade: int, dados: asyncio.Queue) -> None:
    print(f'Aguarde o processamento de {quantidade} dados...')
    processados = 0
    while processados < quantidade:
        await dados.get()
        processados += 1
        await asyncio.sleep(0.001)
    print(f'Foram processados {processados} itens')

if __name__ == '__main__':
    total: int = 5_000
    dados = asyncio.Queue()
    el = asyncio.new_event_loop()
    asyncio.set_event_loop(el)

    print(f'Computando {total * 2:.2f} dados.')

    el.run_until_complete(gerar_dados(total, dados))
    el.run_until_complete(gerar_dados(total, dados))
    el.run_until_complete(processar_dados(total * 2, dados))

    el.close()


# -*- coding:utf-8 -*-
"""
Description: A python 2.7 implementation of gcForest proposed in [1]. A demo implementation of gcForest library as well as some demo client scripts to demostrate how to use the code. The implementation is flexible enough for modifying the model or
fit your own datasets.
Reference: [1] Z.-H. Zhou and J. Feng. Deep Forest: Towards an Alternative to Deep Neural Networks. In IJCAI-2017.  (https://arxiv.org/abs/1702.08835v2 )
Requirements: This package is developed with Python 2.7, please make sure all the demendencies are installed, which is specified in requirements.txt
ATTN: This package is free for academic usage. You can run it at your own risk. For other purposes, please contact Prof. Zhi-Hua Zhou(zhouzh@lamda.nju.edu.cn)
ATTN2: This package was developed by Mr.Ji Feng(fengj@lamda.nju.edu.cn). The readme file and demo roughly explains how to use the codes. For any problem concerning the codes, please feel free to contact Mr.Feng.
"""
import numpy as np
from sklearn.model_selection import StratifiedKFold

from .base_layer import BaseLayer
from ..estimators import get_estimator_kfold
from ..utils.metrics import accuracy_pb, accuracy_win_vote, accuracy_win_avg
from ..utils.win_utils import get_windows
from ..utils.debug_utils import repr_blobs_shape
from ..utils.log_utils import get_logger

LOGGER = get_logger("gcforest.layers.fg_win_layer")

#CV_POLICYS = ["data", "win"]
#CV_POLICYS = ["data"]

class FGWinLayer(BaseLayer):
    def __init__(self, layer_config, data_cache):
        """
        est_config (dict):
            estimator的config
        win_x, win_y, stride_x, stride_y, pad_x, pad_y (int):
            configs for windows
        n_folds(int): default=1
             1 means do not use k-fold
        n_classes (int):

        """
        super(FGWinLayer, self).__init__(layer_config, data_cache)
        # estimator
        self.est_configs = self.get_value("estimators", None, list, required=True)
        self.win_x = self.get_value("win_x", None, int, required=True)
        self.win_y = self.get_value("win_y", None, int, required=True)
        self.stride_x = self.get_value("stride_x", 1, int)
        self.stride_y = self.get_value("stride_y", 1, int)
        self.pad_x = self.get_value("pad_x", 0, int)
        self.pad_y = self.get_value("pad_y", 0, int)
        self.n_classes = self.get_value("n_classes", None, int, required=True)
        #self.cv_policy = layer_config.get("cv_policy", "data")
        #assert(self.cv_policy in CV_POLICYS)
        assert len(self.bottom_names) >= 2
        assert len(self.est_configs) == len(self.top_names), "Each estimator shoud produce one unique top"
        # self.eval_metrics = [("predict", accuracy_pb), ("vote", accuracy_win_vote), ("avg", accuracy_win_avg)]
        self.eval_metrics = [("predict", accuracy_pb), ("avg", accuracy_win_avg)]
        self.estimator1d = [None for ei in range(len(self.est_configs))]

    def _init_estimators(self, ei, random_state):
        """
        ei (int): estimator index
        """
        top_name = self.top_names[ei]
        est_args = self.est_configs[ei].copy()
        est_name ="{}/{}_folds".format(top_name, est_args["n_folds"])
        # n_folds
        n_folds = int(est_args["n_folds"])
        est_args.pop("n_folds")
        # est_type
        est_type = est_args["type"]
        est_args.pop("type")
        # random_state
        random_state = (random_state + hash("[estimator] {}".format(est_name))) % 1000000007
        return get_estimator_kfold(est_name, n_folds, est_type, est_args, random_state=random_state)

    def fit_transform(self, train_config):
        LOGGER.info("[data][{}], bottoms={}, tops={}".format(self.name, self.bottom_names, self.top_names))
        phases = train_config.phases
        X_train_win, y_train_win = None, None
        test_sets = None

        for ti, top_name in enumerate(self.top_names):
            LOGGER.info("[progress][{}] ti={}/{}, top_name={}".format(self.name, ti, len(self.top_names), top_name))
            # check top cache
            if np.all(self.check_top_cache(phases, ti)):
                LOGGER.info("[data][{}] all top cache exists. skip progress".format(self.name))
                continue

            # init X, y, n_classes
            if X_train_win is None:
                for pi, phase in enumerate(phases):
                    bottoms = self.data_cache.gets(phase, self.bottom_names)
                    LOGGER.info('[data][{},{}] bottoms.shape={}'.format(self.name, phase, repr_blobs_shape(bottoms)))
                    X, y = np.concatenate(bottoms[:-1], axis=1), bottoms[-1]
                    # n x n_windows x channel
                    X_win = get_windows(X, self.win_x, self.win_y, self.stride_x, self.stride_y, self.pad_x, self.pad_y)
                    _, nh, nw, _ = X_win.shape
                    X_win = X_win.reshape((X_win.shape[0], -1, X_win.shape[-1]))
                    y_win = y[:,np.newaxis].repeat(X_win.shape[1], axis=1)
                    if pi == 0:
                        assert self.n_classes == len(np.unique(y)), \
                                "n_classes={}, len(unique(y))={}".format(self.n_classes, len(np.unique(y)))
                        X_train_win, y_train_win = X_win, y_win
                    else:
                        test_sets = [("test", X_win, y_win), ]

            # fit
            est = self._init_estimators(ti, train_config.random_state)
            y_probas = est.fit_transform(X_train_win, y_train_win, y_train_win[:,0], cache_dir=train_config.model_cache_dir,
                    test_sets = test_sets, eval_metrics=self.eval_metrics,
                    keep_model_in_mem=train_config.keep_model_in_mem)

            for pi, phase in enumerate(phases):
                y_proba = y_probas[pi].reshape((-1, nh, nw, self.n_classes)).transpose((0, 3, 1, 2))
                LOGGER.info('[data][{},{}] tops[{}].shape={}'.format(self.name, phase, ti, y_proba.shape))
                self.data_cache.update(phase, self.top_names[ti], y_proba)
            if train_config.keep_model_in_mem:
                self.estimator1d[ti] = est

    def transform(self):
        phase = "test"
        for ti, top_name in enumerate(self.top_names):
            LOGGER.info("[progress][{}] ti={}/{}, top_name={}".format(self.name, ti, len(self.top_names), top_name))

            bottoms = self.data_cache.gets(phase, self.bottom_names[:-1])
            LOGGER.info('[data][{},{}] bottoms.shape={}'.format(self.name, phase, repr_blobs_shape(bottoms)))
            X = np.concatenate(bottoms, axis=1)
            # n x n_windows x channel
            X_win = get_windows(X, self.win_x, self.win_y, self.stride_x, self.stride_y, self.pad_x, self.pad_y)
            _, nh, nw, _ = X_win.shape
            X_win = X_win.reshape((X_win.shape[0], -1, X_win.shape[-1]))

            est = self.estimator1d[ti]
            y_proba = est.predict_proba(X_win)
            y_proba = y_proba.reshape((-1, nh, nw, self.n_classes)).transpose((0, 3, 1, 2))
            LOGGER.info('[data][{},{}] tops[{}].shape={}'.format(self.name, phase, ti, y_proba.shape))
            self.data_cache.update(phase, self.top_names[ti], y_proba)

    def score(self):
        eval_metrics = [("predict", accuracy_pb), ("avg", accuracy_win_avg)]
        for ti, top_name in enumerate(self.top_names):
            for phase in ["train", "test"]:
                y = self.data_cache.get(phase, self.bottom_names[-1])
                y_proba = self.data_cache.get(phase, top_name)
                y_proba = y_proba.transpose((0,2,3,1))
                y_proba = y_proba.reshape((y_proba.shape[0], -1, y_proba.shape[3]))
                y = y[:,np.newaxis].repeat(y_proba.shape[1], axis=1)
                for eval_name, eval_metric in eval_metrics:
                    acc = eval_metric(y, y_proba)
                    LOGGER.info("Accuracy({}.{}.{})={:.2f}%".format(top_name, phase, eval_name, acc*100))


import argparse
import changelog_parser
import changelog_formatter

argparser = argparse.ArgumentParser()
argparser.add_argument("file")
args = argparser.parse_args()

file = open(args.file, "r")
changelog = changelog_parser.parse(file)
if changelog:
    formatted_changelog = changelog_formatter.format(changelog)

    # Format message to print newline character and not actual newlines
    message = "{}".format(formatted_changelog.replace("\n", r"\n"))
    print(message)


#Imagen y prepocesamiento
#Librerías principales
import sys
import os
from os import scandir, getcwd
# Librerias preprocesamiento
import cv2 as cv
import numpy as np
from skimage import io
from skimage import filters

from math import copysign, log10

# Pre-procesamiento de imagen
def preprocesamiento(nombre):
    #Adquisición de imagen

    imagen = cv.imread("Images/KBImages/" + nombre)

    # Eliminación de ruido - Preprocesamiento (2)
    im_ruido = cv.pyrMeanShiftFiltering(imagen,sp=30,sr=50)

    # Escala de grises - Preprocesamiento (3)
    im_gris = cv.cvtColor(im_ruido, cv.COLOR_BGR2GRAY)

    # Binarización | Binarización de Otsu - Segmentación (4)
    ret, im_binaria = cv.threshold(im_gris, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)

    # Detección de bordes - Segmentación (5)
    img_filtro1 = filters.sobel(im_binaria)

    # Operaciones morfologicas - Segmentación (6)
    kernel = cv.getStructuringElement(cv.MORPH_RECT, (3, 3))
    apertura = cv.morphologyEx(im_binaria, cv.MORPH_OPEN, kernel=kernel, iterations=2)
    dilatacion = cv.dilate(apertura, kernel, iterations=3)

    # Umbralización - Segmentación (7)
    th, im_umbral = cv.threshold(apertura, 80, 150, cv.THRESH_BINARY_INV)

    # Copia de la imagen umbralizada - Segmentación (8)
    im_umbral_copia = im_umbral.copy()

    # Mascara para rellenar con 2 pixeles mayores a la imagen - Segmentación (9)  [Relleno]
    a, l = im_umbral.shape[:2]
    mascara = np.zeros((a+2, l+2), np.uint8)

    # Rellenando desde el punto (0, 0) al 255
    cv.floodFill(im_umbral_copia, mascara, (0,0), 255);

    # Invertido del relleno de la imagen - Segmentación (10) [Relleno invertido]
    im_seg_inv = cv.bitwise_not(im_umbral_copia)

    # Combinación de imagenes para obtener imagen rellenada - Segmentación (11)
    im_combinada = im_umbral | im_seg_inv

    # Binarización de otsu - Segmentación (12)
    ret, im_binaria2 = cv.threshold(im_combinada, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)
    #--- se usará para momentos ---
    #tamanio_imagen('Imagen binaria rellena')

    #momentos(im_binaria2)
    #distance(im_binaria2)

    #Crear una carpeta para las nuevas imágenes si no existe aún
    try:
        # Se crea la carpeta
        if not os.path.exists('Images/NewImages'):
            os.makedirs('Images/NewImages')

    # Si no se puede crear, muestra un error
    except OSError:
        print ('Error: No se ha podido crear la carpeta nueva :(')

    Nueva = cv.imwrite("Images/PreImages/" + nombre,im_binaria2)

def ls(ruta='Images/KBImages/'):
    return [arch.name for arch in scandir(ruta) if arch.is_file()]

for original in ls():
        x = original.split(".")
        nombre = x[0]
        formato = x[1]
        if(nombre.endswith("_3")):
            imagen = nombre + "." + formato
            preprocesamiento(imagen)


from flask import Flask, render_template, request, jsonify, send_from_directory
import logging
import time
import sys
import os
from llama_index import (GPTVectorStoreIndex, LLMPredictor,
                         SimpleDirectoryReader, ServiceContext,
                         StorageContext, load_index_from_storage)
from llama_index import QuestionAnswerPrompt, RefinePrompt
from llama_index.query_engine.transform_query_engine import TransformQueryEngine
from langchain.chat_models import ChatOpenAI
import backoff
import openai
import socket
from llama_index.indices.postprocessor import MetadataReplacementPostProcessor

# Constants
MODEL = "gpt-4"

# Get API key from environment variable or prompt user
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
    api_key = input("Enter OpenAI API key: ")

# Get folder path from command line argument or prompt user
folder_path = sys.argv[1] if len(sys.argv) > 1 else input("Enter folder path: ")

# Set OpenAI API key
os.environ["OPENAI_API_KEY"] = api_key
openai.api_key = api_key

# Setup logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

# Initialize LLM Predictor and ServiceContext
llm_predictor = LLMPredictor(llm=ChatOpenAI(temperature=0.4, model_name=MODEL))
service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)

# Load the index from storage
storage_context = StorageContext.from_defaults(persist_dir=folder_path)
index = load_index_from_storage(storage_context)

# Define your ENGLISCH_QA_PROMPT and ENGLISCH_REFINE_PROMPT templates here
# ...

# Time function
def timeit(func):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        elapsed_time = end_time - start_time
        return result, elapsed_time
    return wrapper

# Function to handle retries with exponential backoff
@backoff.on_exception(backoff.expo,
                      Exception,
                      max_tries=10,
                      giveup=lambda e: False,  # Don't give up on any exception
                      base=10)
@timeit
def query_index(query_engine, input_text):
    return query_engine.query(input_text)

# Create a QueryEngine
query_engine = index.as_query_engine(
    service_context=service_context,
    similarity_top_k=15,
    response_mode="compact",
    text_qa_template=ENGLISCH_QA_PROMPT,
    refine_template=ENGLISCH_REFINE_PROMPT,
    node_postprocessors=[
        MetadataReplacementPostProcessor(target_metadata_key="window")
    ],
)

def handle_user_query(casedesc):
    input_text = casedesc
    response, response_time_query = query_index(query_engine, input_text)
    output_accGPT = response.response

    filenames_pages = dict()
    for doc_id, metadata in response.metadata.items():
        filename_label = metadata.get("file_name")
        page_label = metadata.get("page_label")
        if filename_label is not None:
            filename_label = os.path.basename(filename_label)
            if filename_label not in filenames_pages:
                filenames_pages[filename_label] = set()
            if page_label is not None:
                filenames_pages[filename_label].add(f"{page_label}")
    summary = []
    for filename, pages in filenames_pages.items():
        sorted_pages = sorted(pages, key=lambda x: (not x.isdigit(), int(x) if x.isdigit() else x))

        summary.append(f"{filename} Page: ({', '.join(sorted_pages)})" if pages else filename)
    return output_accGPT, ", ".join(summary), response_time_query

app = Flask(__name__)

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/query', methods=['POST'])
def query():
    try:
        casedesc = request.form['casedesc']
        result, filenames, response_time_query = handle_user_query(casedesc)
        return jsonify({
            'result': result,
            'filenames': filenames,
            'time': str(response_time_query)
        })
    except Exception as e:
        logging.error("Error while processing the query: ", exc_info=True)
        return jsonify({'error': str(e)}), 500

@app.route('/pdfs/<filename>')
def serve_pdf(filename):
    response = send_from_directory(folder_path, filename, as_attachment=False, mimetype='application/pdf')
    response.headers['Content-Disposition'] = f'inline; filename={filename}'
    return response

def find_unused_port(start_port=8000):
    port = start_port
    while True:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            try:
                s.bind(('', port))
                s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                return port
            except OSError:
                port += 1

if __name__ == "__main__":
    port = find_unused_port()
    app.run(host='0.0.0.0', port=port, use_reloader=False)


# -*- coding: utf-8 -*-
from odoo.tests.common import TransactionCase
from odoo.addons.report_docx.report.report_docx import DataModelProxy
from odoo.tools import misc
import tempfile
import shutil

class test_ReportDocx(TransactionCase):

    def setUp(self):
        '''准备数据'''
        super(test_ReportDocx, self).setUp()
        self.ir_actions = self.env.ref('sell.report_sell_order_1')
        self.sell_order = self.env.ref('sell.sell_order_1')
        self.report_docx_sell = self.ir_actions._lookup_report('sell.order')

        self.ir_actions_pdf = self.env.ref('sell.report_sell_order_2')
        self.sell_order = self.env.ref('sell.sell_order_1')
        self.report_pdf_sell = self.ir_actions_pdf._lookup_report('sell.order')

    def test_lookup_report(self):
        '''测试docx报表模板'''
        self.report_docx_sell.create(self.cr, self.uid, self.sell_order.id, self.ir_actions, self.env.context)

    def test_lookup_report_pdf(self):
        '''测试docx报表模，输出类型为pdf'''
        self.ir_actions.write({'output_type':'pdf'})
        self.report_docx_sell.create(self.cr, self.uid,self.sell_order.id, self.ir_actions, self.env.context)

    def test_get_docx_data(self):
        model = self.report_docx_sell.get_docx_data(self.cr, self.uid, self.ir_actions.id, self.ir_actions, self.env.context)

    def test_save_file(self):
        doxc_file = self.report_docx_sell.create(self.cr, self.uid, self.sell_order.id, self.ir_actions, self.env.context)
        tempname = tempfile.mkdtemp()
        shutil.copy(misc.file_open('sell/template/sell.order.docx').name, tempname)
        self.report_docx_sell._save_file(tempname+"/sell.order.docx",doxc_file)

    def test_render_to_pdf(self):
        doxc_file = self.report_pdf_sell.create(self.cr, self.uid, self.sell_order.id, self.ir_actions_pdf,
                                                 self.env.context)

    def test_datamodelproxy(self):
        data=DataModelProxy([{"type":'selection'}])
        data.__getitem__(0)
        data = DataModelProxy([])
        data.__getattr__(0)

    def test_by_selection(self):
        data = DataModelProxy(self.env['home.page'])
        home = self.env['home.page']
        data._compute_by_selection(home._fields.get('menu_type'),"all_business")
        home = self.env['sell.order']
        data = DataModelProxy(home)
        data._compute_by_selection(home._fields.get('state'),"done")


# -*- coding: utf-8 -*-
"""
Created on Sun Apr 28 17:32:38 2019

@author: Anne
"""

import numpy as np
import cv2
from matplotlib import pyplot as plt

def variance_of_laplacian(image):
	# compute the Laplacian of the image and then return the focus
	# measure, which is simply the variance of the Laplacian
	return cv2.Laplacian(image, cv2.CV_64F).var()

path="C:/Image_Aesthetic/3blue.png"

img = cv2.imread(path)

#cv2.imshow('image',img
plt.imshow(img)

#cv2.waitKey(0)

mask = np.zeros(img.shape[:2],np.uint8)

bgdModel = np.zeros((1,65),np.float64)

fgdModel = np.zeros((1,65),np.float64)
#from left, from top, from right, from bottom
rect = (100,75,200,150)

cv2.grabCut(img,mask,rect,bgdModel,fgdModel,5,cv2.GC_INIT_WITH_RECT)

mask2 = np.where((mask==2)|(mask==0),0,1).astype('uint8')

mask3 = np.where(~((mask==2)|(mask==0)),0,1).astype('uint8')

img1 = img*mask3[:,:,np.newaxis]

plt.imshow(img1)

gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
fm = variance_of_laplacian(gray)
text = "Not Blurry"

	# if the focus measure is less than the supplied threshold,
	# then the image should be considered "blurry"

print(fm)
if fm < 650:                            #THRESHOLD
	text = "Blurry\nAppealing"
else:
   text="Not Blurry\nNot Appealing"
print(text)

	# show the image
#cv2.putText(img1, "{}: {:.2f}".format(text, fm), (5, 30),
#	cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 1)
#cv2.imshow("Image", img1)
key = cv2.waitKey(0)


from django.db import models

from accounts.models import User

class Customer(models.Model):
    user = models.OneToOneField(User, related_name="customer", on_delete=models.CASCADE)
    profile_picture = models.URLField(blank=True)
    created = models.TimeField(auto_now_add=True)
    updated = models.TimeField(auto_now=True)

    class Meta:
        ordering = ('-created',)

    def __str__(self):
        return f'{self.user.full_name} for user {self.user.username}'


from datetime import datetime
from calendar import monthrange
from ntplib import NTPClient

def get_timestamp(ntp_client):
    ntp_response = ntp_client.request('ntp.shoa.cl')
    return datetime.fromtimestamp(ntp_response.tx_time)

def main():
    ntp_client = NTPClient()
    date_obj = get_timestamp(ntp_client)

    first_day = date_obj.replace(day=1).day
    days_in_month = monthrange(date_obj.year, date_obj.month)[1]

    month = [day for day in range(first_day, days_in_month)]
    print(month)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from pandas import DataFrame
from pandas import concat
from itertools import chain
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn import metrics
import matplotlib.pyplot as plt
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.utils import to_categorical
from data_operate import getDataSet
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import cohen_kappa_score
from sklearn.model_selection  import train_test_split

def getdata(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    data_set = df.iloc[:, 15:].values
    data_set = data_set.astype('float64')
    #归一化
    scaler = StandardScaler()
    # scaler = MinMaxScaler()

    train_data_set = np.array(data_set)
    #得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y,True,list).values)
    reframed_train_data_set = reframed_train_data_set[:,:16*chunk_size_x+1]
    # 数据集划分,选取前60%天的数据作为训练集,中间20%天作为验证集,其余的作为测试集
    train_days = int(len(reframed_train_data_set) * 0.6)
    valid_days = int(len(reframed_train_data_set) * 0.2)

    data = reframed_train_data_set[:,:-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)

    train_x = data[:train_days, :]
    valid_x = data[train_days:train_days + valid_days, :]
    test_x = data[train_days + valid_days:, :]

    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    label = to_categorical(label, num_classes=4)
    train_y = label[:train_days]
    valid_y = label[train_days:train_days + valid_days]
    test_y = label[train_days + valid_days:]

    # 将数据集重构为符合LSTM要求的数据格式,即 [样本，时间步，特征维度：16]
    train_x = train_x.reshape((train_x.shape[0], chunk_size_x, 16))
    valid_x = valid_x.reshape((valid_x.shape[0], chunk_size_x, 16))
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))

    return train_x, train_y, valid_x, valid_y, test_x, test_y , scaler

def gettrain_rand(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    trainsize = int(df.shape[0]*0.9)
    data_set = df.iloc[:trainsize, 15:].values
    data_set = data_set.astype('float64')
    # 归一化
    scaler = StandardScaler()
    train_data_set = np.array(data_set)
    # 得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y, True, list).values)
    reframed_train_data_set = reframed_train_data_set[:, :16*chunk_size_x + 1]
    data = reframed_train_data_set[:, :-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)
    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    label = to_categorical(label, num_classes=4)
    train_x,rest_x,train_y,rest_y = train_test_split(data,label,test_size = 0.2,random_state= 0)
    valid_x,test_x,valid_y,test_y = train_test_split(rest_x,rest_y,test_size = 0.5,random_state= 0)

    train_x = train_x.reshape((train_x.shape[0], chunk_size_x, 16))
    valid_x = valid_x.reshape((valid_x.shape[0], chunk_size_x, 16))
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))

    return train_x,train_y,valid_x,valid_y,test_x,test_y

def lstm_model(train_X,chunk_size_y):
    model = Sequential()
    #第一层
    model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(train_X.shape[1], train_X.shape[2])))
    # 第二层
    model.add(LSTM(64, return_sequences=False))
    model.add(Dropout(0.5))
    # 第三层 因为是回归问题所以使用linear
    model.add(Dense(4 * chunk_size_y, activation='softmax'))
    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')
    return model

def getdata_test(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    data_set = df.iloc[:, 15:].values
    data_set = data_set.astype('float64')
    #归一化
    scaler = StandardScaler()

    train_data_set = np.array(data_set)
    #得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y,True,list).values)
    reframed_train_data_set = reframed_train_data_set[:,:16*chunk_size_x+1]

    data = reframed_train_data_set[:,:-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)
    test_x = data

    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    # label = to_categorical(label, num_classes=4)
    test_y = label
    # 将数据集重构为符合LSTM要求的数据格式,即 [样本，时间步，特征维度：16]
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))
    return test_x, test_y , scaler

def train_vote(path):
    df = pd.read_csv(path + '.csv')
    chunk_size_x = 6
    chunk_size_y = 1
    input_epochs = 20
    input_batch_size = 100

    train_x_, label_y_, valid_x, valid_y, test_x, test_y = \
        gettrain_rand(df, chunk_size_x, chunk_size_y)
    for i in range(10):
        train_x, _,label_y,_ =  train_test_split(train_x_, label_y_,test_size = 0.4,random_state= None)
        model = lstm_model(train_x, chunk_size_y)
        model.fit(train_x, label_y, epochs=input_epochs, batch_size=input_batch_size,
                  validation_data=(valid_x, valid_y), verbose=2, shuffle=False)
        model.save(path + 'lstm' + str(i) + '.h5')
        # test_predict = model.predict(test_x)
        # pred_cls = np.argmax(test_predict, axis=1)
        # test_y_cl = np.argmax(test_y, axis=1)
        # kappa = cohen_kappa_score(test_y_cl, pred_cls)
        # print(kappa)
        # acc = accuracy_score(test_y_cl, pred_cls)
        # acc0 = accuracy_score(test_y_cl, np.zeros_like(test_y_cl))
        # print(acc, acc0)
        # recall = metrics.recall_score(test_y_cl, pred_cls, average='macro')
        # print('recall:%f' % recall)

def vote(path,pathtest):
    models = []
    for i in range(10):
        model = load_model(path + 'lstm' + str(i) + '.h5')
        models.append(model)

    df = pd.read_csv('./testset/' + pathtest + '.csv')
    chunk_size_x = 6
    chunk_size_y = 1
    test_x, test_y,scaler =  getdata_test(df, chunk_size_x, chunk_size_y)

    pred_cls = np.zeros((test_x.shape[0],10))
    for i in range(10):
        model = models[i]
        test_predict = model.predict(test_x)
        pred_cls[:,i] = np.argmax(test_predict, axis=1)
    pred_cl = np.zeros((test_x.shape[0],1))
    for i in range(test_x.shape[0]):
        counts = np.bincount(pred_cls[i,:].astype(np.int32))
        pred_cl[i] = np.argmax(counts)

    kappa = cohen_kappa_score(test_y, pred_cl)
    print('kappa:%f'%kappa)
    acc = accuracy_score(test_y, pred_cl)
    print('accuracy:%f'%acc)
    recall = metrics.recall_score(test_y, pred_cl, average='macro')
    print('recall:%f' % recall)

if __name__ == '__main__':
    trainSets = ['station178_filled','station303_filled','station304_filled','station305_filled','station306_filled']
    testSets = ['station1','station2','station3','station4','station5']
    for i in range(5):
        print(trainSets[i]+' start training')
        train_vote(trainSets[i])
        print('Bagging trained')

    # for i in range(5):
    #     print('\n' + trainSets[i]+' start testing')
    #     vote(trainSets[i],testSets[i])
    #     print('Bagging result:')


from enum import Enum
from typing import TypeVar, Generic, List
from unittest import TestCase

from pydictable import GenericDictAble, ListField, EnumField, DictAble, ObjectField, DataValidationError

class TestGeneric(TestCase):
    def test_generic(self):
        class Gender(Enum):
            male = 'male'
            female = 'female'

        class City(Enum):
            Bangalore = 'Bangalore'
            Chennai = 'Chennai'

        T = TypeVar('T')

        class SelectField(GenericDictAble, Generic[T]):
            options: List[T] = None

            @staticmethod
            def inject(item: T):
                return {'options': ListField(EnumField(item))}

        class Profile(DictAble):
            gender: SelectField[Gender] = ObjectField(SelectField.make(Gender))

        class Lead(DictAble):
            city: SelectField[City] = ObjectField(SelectField.make(City))

        p = Profile(dict={'gender': {'options': ['male']}})
        l = Lead(dict={'city': {'options': ['Bangalore']}})
        self.assertEqual(p.gender.options[0], Gender.male)
        self.assertNotEqual(l.city.options[0], Gender.male)
        self.assertEqual(l.city.options[0], City.Bangalore)

        self.assertRaises(DataValidationError, lambda: Profile(dict={'gender': {'options': ['invalid']}}))


"""
This experiment shows an example of how the loader and preprocess modules can be used to load a dataset and
do some simple preprocessing
"""
from coburn.data import preprocess, loader
from showit import tile
from matplotlib import pyplot as plt

def main():
    # these samples will be automatically downloaded if they are not found locally
    dataset = loader.load(samples=['4bad52d5ef5f68e87523ba40aa870494a63c318da7ec7609e486e62f7f7a25e8',
                                   'a7e37600a431fa6d6023514df87cfc8bb5ec028fb6346a10c2ececc563cc5423',
                                   '70a6300a00dbac92be9238252ee2a75c86faf4729f3ef267688ab859eed1cc60'])

    # this transform will cause the dataset to find the mean image of a movie
    # anytime an item is requested from the dataset
    transform = preprocess.Mean()
    dataset.set_transform(transform)

    mean_images = list()
    for i in range(len(dataset)):
        sample = dataset[i]  # mean transform has already been applied!
        mean_images.append(sample)

    tile(mean_images)
    plt.show()


from enum import Enum

import pyxel

class UiState(Enum):
    IN_PLAY = 1,
    WIN = 2,
    LOSE = 3

class UI:

    def __init__(self) -> None:
        self.state = UiState.IN_PLAY

    def draw(self):
        pyxel.text(157, 54, "Press 'R' to restart.", pyxel.COLOR_YELLOW)

        if self.state == UiState.IN_PLAY:
            self._draw_in_play()
        elif self.state == UiState.WIN:
            self._draw_win()
        elif self.state == UiState.LOSE:
            self._draw_lose()

    def set_state(self, state: UiState):
        self.state = state

    def _draw_in_play(self):
        pyxel.text(80, 10, "Legend", pyxel.COLOR_YELLOW)

        pyxel.rect(80, 20, 4, 4, pyxel.COLOR_LIME)
        pyxel.text(88, 20, "Fuel", pyxel.COLOR_LIME)

        pyxel.rect(80, 30, 4, 4, pyxel.COLOR_YELLOW)
        pyxel.text(88, 30, "Treasure", pyxel.COLOR_YELLOW)

        pyxel.rect(80, 40, 4, 4, pyxel.COLOR_RED)
        pyxel.text(88, 40, "Bombs", pyxel.COLOR_RED)

        pyxel.rect(80, 50, 4, 4, pyxel.COLOR_LIGHT_BLUE)
        pyxel.text(88, 50, "Granite", pyxel.COLOR_LIGHT_BLUE)

    def _draw_win(self):
        pyxel.text(100, 30, "YOU\nWIN!", 10)

    def _draw_lose(self):
        pyxel.text(100, 30, "GAME\nOVER!", 10)


import matplotlib
matplotlib.use('Agg')

import argparse
import glob
import os
import shutil
import sys
import json
from tqdm import tqdm
import tensorflow as tf

import numpy as np
from gym import spaces

from deepagent.agents.common import AbstractAgentTrainer
from deepagent.agents.random_agent import DebugAgentTester, RacerDijsktraTester
from deepagent.common.auto_dir import get_numbered_directories, get_next_numbered_directory
from deepagent.envs.environment_wrappers import TrainingEnvironment
from deepagent.envs.factory import EnvironmentFactory
from deepagent.envs.spaces import DeepAgentSpace
from deepagent.envs.env_constants import EnvType
from deepagent.experiments.params import params
from deepagent.experiments.params.params import ExperimentType, ImitationLearning
from deepagent.experiments.params.utils import load_params
from deepagent.recording.recording import record_atari, MultiPlayerDemoReader
from deepagent.agents.memory import BCMemory

import logging
logging.basicConfig(level=logging.INFO)
from deepagent.experiments.params import utils as param_utils

def setup_environment(env_file_str, env_type, params_name, random_seed=0, arena_nbr=0):
    seed = 123 + random_seed
    np.random.seed(seed)

    env1 = EnvironmentFactory.get_env(env_file_str, env_type, seed, params_name, random_seed, arena_nbr)
    env_wrapper = TrainingEnvironment(env1)

    return env_wrapper

def setup_multi_agent(env_wrapper):
    agent_factory = params.AgentParams.agent_factory
    multi_agent_training_class = agent_factory.multi_agent_training_class()
    return multi_agent_training_class(environment_wrapper=env_wrapper, agent_factory=agent_factory)

import os
from typing import List, Tuple
import numpy as np

INITIAL_POS = 33
SUPPORTED_DEMONSTRATION_VERSIONS = frozenset([0, 1])

def get_demo_files(path: str) -> List[str]:
    """
    Retrieves the demonstration file(s) from a path.
    :param path: Path of demonstration file or directory.
    :return: List of demonstration files

    Raises errors if |path| is invalid.
    """
    if os.path.isfile(path):
        if not path.endswith(".demo"):
            raise ValueError("The path provided is not a '.demo' file.")
        return [path]
    elif os.path.isdir(path):
        paths = [
            os.path.join(path, name)
            for name in os.listdir(path)
            if name.endswith(".demo")
        ]
        if not paths:
            raise ValueError("There are no '.demo' files in the provided directory.")
        return paths
    else:
        raise FileNotFoundError(
            f"The demonstration file or directory {path} does not exist."
        )

def get_state_shape(file_path: str) -> Tuple[int]:
    state = None
    with open(file_path) as f:
        for line_idx, line in enumerate(tqdm(f)):
            if 'Prev Obs: [ ' not in line:
                continue
            items = line.strip().split(' | ')
            state = items[2][12:-1]
            state = np.fromstring(state, dtype=np.float32, sep=' ')
            break
    if state is None:
        raise Exception('Could not find state shape from recording file')
    return state.shape

def create_pretraining_generator(recording_files: List[str], action_shape: Tuple[int]):
    def load_data_generator():
        num_episodes = 0
        for recording_file in recording_files:
            with open(recording_file) as f:
                episode_idx = 0
                for line_idx, line in enumerate(f):
                    if 'Prev Obs: [ ' not in line:
                        continue
                    items = line.strip().split(' | ')
                    state = items[2][12:-1]
                    state = np.fromstring(state, dtype=np.float32, sep=' ')
                    action = items[1].split()[-1]
                    action = np.eye(action_shape[0])[int(action)]
                    episode_idx = int(line.strip().split('UnitId: (0, ')[1].split(')')[0])
                    yield state, action
                num_episodes += episode_idx
        print(f'Loaded {num_episodes} episodes from recording files.')
    return load_data_generator

def get_pretraining_length(recording_files):
    steps = 0
    for recording_file in recording_files:
        with open(recording_file) as f:
            for line_idx, line in enumerate(tqdm(f)):
                if 'Prev Obs: [ ' in line:
                    steps += 1
    return steps

def pre_train():
    for brain_name, pre_train_args in params.ImitationLearning.pre_training_args.items():
        # init policies
        if pre_train_args.recording_files is not None:
            # Loads pretraining data from recording files
            if pre_train_args.action_shape_override is None:
                raise Exception(
                    'Action shape override was found to be None. Action shape override must be supplied in '
                    'ImitationLearning.PreTrainArgs if recording files are used.'
                )
            action_shape = pre_train_args.action_shape_override
            state_shape = get_state_shape(pre_train_args.recording_files[0])

            # TF dataset is used so training can be done even if the full data doesn't fit in memory
            print(f'Loading pretraining data from recording files: {", ".join(pre_train_args.recording_files)}')
            dataset = tf.data.Dataset.from_generator(create_pretraining_generator(pre_train_args.recording_files, action_shape),
                                                     output_signature=(
                                                      tf.TensorSpec(shape=state_shape, dtype=tf.float32),
                                                      tf.TensorSpec(shape=action_shape, dtype=tf.float32,))
                                                     )
            dataset_len = get_pretraining_length(pre_train_args.recording_files)
            print(f'{dataset_len} examples found in pretraining data')
            dataset = dataset.apply(tf.data.experimental.assert_cardinality(dataset_len))
            vec_space = spaces.Box(-1.0, 1.0, state_shape)
            state_space = DeepAgentSpace(vector_space=vec_space, agent_name=brain_name)
            vector_space = spaces.Box(0.0, 1.0, shape=action_shape, dtype=np.float32)
            action_space = DeepAgentSpace(vector_space=vector_space, agent_name=brain_name)
        else:
            reader = MultiPlayerDemoReader(pre_train_args.player_runs)
            meta = reader.load_meta()
            state_shape = meta.state_shape
            action_shape = meta.action_shape
            brain_name = meta.student_brain

            print(state_shape)
            print(action_shape)

            if state_shape is None:
                state_shape = pre_train_args.state_shape_override

            if action_shape is None:
                action_shape = pre_train_args.action_shape_override

            if params.EnvironmentParams.env_type.is_atari():
                frame_stack = 4
            elif params.EnvironmentParams.env_type.is_unity_ml():
                frame_stack = params.UnityParams.unity_frame_stack
            else:
                raise ValueError(f"Don't know how to pre_train for env_type: {params.EnvironmentParams.env_type}")

            num_actions = action_shape[0]

            state_shape = (state_shape[0], state_shape[1], state_shape[2] * 4)

            # TODO: I don't think the low and high actually matter here, but space management needs to be cleaned up and made consistent in general
            state_space = DeepAgentSpace(image_spaces=spaces.Box(low=-1.0, high=1.0, shape=state_shape))

            if params.EnvironmentParams.env_type.is_atari():
                action_space = DeepAgentSpace(vector_space=spaces.Discrete(n=action_shape[0]))
            elif params.EnvironmentParams.env_type.is_unity_ml():
                action_space = DeepAgentSpace(vector_space=spaces.Box(low=-1.0, high=1.0, shape=action_shape))
            else:
                raise ValueError(f"Don't know how to pre_train for env_type: {params.EnvironmentParams.env_type}")

            state_shape = [x.shape for x in state_space.image_spaces]

        agent_factory = params.AgentParams.agent_factory

        policy = agent_factory.create_agent(agent_type=brain_name, state_space=state_space, action_space=action_space)

        policy.print_summary()
        # import tensorflow as tf
        #policy.set_tensorboard_writer(tf.summary.create_file_writer(params.ModuleParams.tensorboard_dir))
        from deepagent.loggers.logging import get_tensorboard_writer
        policy.set_tensorboard_writer(get_tensorboard_writer())
        print('Loading pre-training data ...')
        if pre_train_args.recording_files is not None:
            memory = BCMemory()
            memory.dataset = dataset
        else:
            memory = reader.bc_memory(frame_stack=frame_stack, num_actions=num_actions,
                                      use_terminals_as_start_state=params.EnvironmentParams.env_type.is_atari(),
                                      tile_first=params.EnvironmentParams.env_type.is_unity_ml())

            action_counts, action_distribution = memory.action_distribution()
            print(f'\nAction Counts: {action_counts}, Action Distributions: {action_distribution}')

        # np.set_printoptions(threshold=500, precision=4, linewidth=150)
        # mean, std = memory.state_mean_and_std()
        # print(f'State Index Mean:\n{mean}')
        # print(f'\nState Index Standard Deviations:\n{std}')
        # np.set_printoptions(threshold=1000, precision=8, linewidth=75) #defaults
        # print(mean.shape)
        # print(std.shape)

        policy.pre_train(memory)

def train(multi_agent: AbstractAgentTrainer):
    multi_agent.train(steps=params.TrainingParams.steps, gamma=params.TrainingParams.gamma,
                      lam=params.TrainingParams.lam)
    #multi_agent.write_report_summary()

def test(multi_agent: AbstractAgentTrainer, test_suffix: str = ''):
    multi_agent.test(num_episodes=params.TestingParams.num_episodes, gamma=params.TestingParams.gamma,
                            testing_suffix=test_suffix)
    #multi_agent.write_report_summary()

def population_test(multi_agent: AbstractAgentTrainer):
    return multi_agent.population_test()

def copy_weights(from_dir, to_dir):
    print('Copying previous weights from {} to {}'.format(from_dir, to_dir))
    files = glob.iglob(os.path.join(from_dir, '*.h5'))
    for file in files:
        shutil.copy2(file, to_dir)

def copy_reward_masks(from_dir, to_dir):
    print('Copying previous reward masks from {} to {}'.format(from_dir, to_dir))
    files = glob.iglob(os.path.join(from_dir, '*.pkl'))
    for file in files:
        shutil.copy2(file, to_dir)

def set_weights_dir(experiment_type, prev_weights, weights, weights_num):
    if experiment_type == ExperimentType.train or experiment_type == ExperimentType.pre_train:
        params.ModuleParams.weights_dir = get_next_numbered_directory(params.ModuleParams.params_dir)
        if prev_weights is not None:
            prev_weights = os.path.join(params.ModuleParams.params_dir, prev_weights)
            copy_weights(from_dir=prev_weights, to_dir=params.ModuleParams.weights_dir)
            copy_reward_masks(from_dir=prev_weights, to_dir=params.ModuleParams.weights_dir)
    elif experiment_type == ExperimentType.test or experiment_type == ExperimentType.record or experiment_type == ExperimentType.population_test:
        if params.AgentParams.agent_factory.multi_agent_training_class() == DebugAgentTester or params.AgentParams.agent_factory.multi_agent_training_class() == RacerDijsktraTester:
            weights = params.ModuleParams.params_dir
        elif weights == "latest":
            weights = str(sorted(get_numbered_directories(params.ModuleParams.params_dir))[-1])
        elif weights is None or weights == '' or weights not in os.listdir(params.ModuleParams.params_dir):
            raise ValueError(
                'Weights number must be specified for testing or recording if not using RandomAgentTester for the AgentTrainer.'
                '\n\tTraining directories are created in ascending order and found in subdirectories of the parameters folders.'
                '\n\tAvailable training folders for {} are {}'.format(params.ModuleParams.name, sorted(
                    get_numbered_directories(params.ModuleParams.params_dir))))

        params.ModuleParams.weights_dir = os.path.join(params.ModuleParams.params_dir, weights)
        params.ModuleParams.weights_num = weights_num

def set_tensorboard_dir():
    tensorboard_top_dir = os.path.join(params.ModuleParams.params_dir, 'tensorboard')
    if not os.path.exists(tensorboard_top_dir):
        os.makedirs(tensorboard_top_dir)
    params.ModuleParams.tensorboard_dir = get_next_numbered_directory(tensorboard_top_dir)

parser = argparse.ArgumentParser()
parser.add_argument('--params', '-p', help="The params module to use: ex: 'params1', 'params2', etc.")
parser.add_argument('--experiment_type', '-et', type=ExperimentType, choices=list(ExperimentType),
                    help="The type of experiment to run:\n\tperform supervised imitation learning\n\tperform "
                         "reinforcement learning training\n\ttest the trained network", required=True)
parser.add_argument('--weights', '-w', type=str,
                    help='The training weights folder to test with. Training directories are numbered in ascending '
                         'order and found in subdirectories of the parameters folders.')
parser.add_argument('--weights_num', '-wn', type=str,
                    help='# The step number at which the weights were saved (or None to use the final weights)'
                         'For example: 400000')
parser.add_argument('--prev_weights', '-pw', type=str,
                    help='A weights folder to use at the start of training. Training will still occur in the next '
                         'highest numbered subdirectory of the parameters folders, but the weights can be loaded '
                         'from any of the previous numbered directories.')
parser.add_argument('--test_suffix', '-ts', type=str, default='',
                    help='# This string is added to the test log filename (default="")')
parser.add_argument('-u', action='store_true', help='This argument is just here because pycharm is dumb at '
                                                    'executing modules on remote interpreters. The workaround means '
                                                    'that it adds a "-u" field in the wrong place.')
parser.add_argument('--env_args', '-ea', nargs='*')
parser.add_argument('--param_overrides', '--po', '-po', type=str,
                    help='Parameters to override in the params module in JSON format. '
                         'Ex: --param_overrides="{\'UnityParams.unity_port\': 5006, \'TrainingParams.batch_size\': 8}"')

def jsonstr_to_kwarg(jsonStr):
    param_overrides = jsonStr.replace("\'", "\"")
    param_overrides = json.loads(param_overrides)
    param_kwargs = {}

    # parse params into a dict (class name) of kwargs (param_name:param_val)
    for param_key, param_value in param_overrides.items():
        param_class, param_var = param_key.split('.')
        if param_class not in param_kwargs.keys():
            param_kwargs[param_class] = {}  # first time seeing override for this class, create default/empty kwargs
        param_kwargs[param_class][param_var] = param_value

    return param_kwargs

def main(argv=sys.argv[1:]):
    print('CLEARED FOR RELEASE')
    print('''

                                ▓▀^^"""  """""▀▒
                                ╬              ╬
                               ╒▓              ▒∩
                              ,▓▄              ╓╬⌐
                                ╬              ╬⌐
                                ╬              ╬
                                ▀▒ÆÆÆÆÆ  ╦╦╦╦Æ▒▀

   #▒▀▀▀▒       ,,                       ╚▀▀▀▓▓▀▀▀              ,
  ▐╬        ,▒▀^ `╙▒╦    @▀▒⌐    ▒▌^^^╙▒▄    ╬Γ    ║╬^^^^^` ╓▒▀^ ^▀▒ j╬     ▒Γ
   "▀▒▒Æ╗   ╬⌐      ▒∩  @▀  ▒µ   ▒▌    ╫▀    ╬Γ    ║╬,,,,, ]╬        j╬,,,,,▒Γ
         ╬∩ ╬∩      ▓∩ @╬╗╗╗╣╬⌐  ▒▌╙╙▀▓      ╬Γ    ║╬      └╬        j╬     ▒Γ
   ▒Æ╗╖╖▒▀   ▀▒╖╓╓#▓` @▀      ▒µ ▒▌   ╙▓╕    ╬Γ    ║▓╖╖╖╖╖⌐ ^▀▄╖╓╓#▀ j╬     ▒Γ

 _____     ______     ______     ______   ______     ______     ______     __   __     ______
/\  __-.  /\  ___\   /\  ___\   /\  == \ /\  __ \   /\  ___\   /\  ___\   /\ "-.\ \   /\__  _\
\ \ \/\ \ \ \  __\   \ \  __\   \ \  _-/ \ \  __ \  \ \ \__ \  \ \  __\   \ \ \-.  \  \/_/\ \/
 \ \____-  \ \_____\  \ \_____\  \ \_\    \ \_\ \_\  \ \_____\  \ \_____\  \ \_\\\\"\_\    \ \_\
  \/____/   \/_____/   \/_____/   \/_/     \/_/\/_/   \/_____/   \/_____/   \/_/ \/_/     \/_/

''')
    args = parser.parse_args(argv)
    print('argv=',argv)
    params_name = args.params
    experiment_type = args.experiment_type
    params.RuntimeParams.experiment_type = experiment_type
    prev_weights = args.prev_weights
    weights = args.weights
    weights_num = args.weights_num
    test_suffix = args.test_suffix
    load_params(name=params_name)
    if args.env_args:
        params.AgentParams.agent_kwargs = dict([pair.split('=') for pair in args.env_args])
    else:
        params.AgentParams.agent_kwargs = {}

    # Override params if they are in the command line args
    if args.param_overrides:
        param_kwargs = jsonstr_to_kwarg(args.param_overrides)
        params.param_override(param_kwargs)

    invalid_keys = ['']
    for k in invalid_keys:
        if k in params.AgentParams.agent_kwargs:
            del params.AgentParams.agent_kwargs[k]
    if len(params.AgentParams.agent_kwargs) == 0:
        params.AgentParams.agent_kwargs = None

    run_experiment(experiment_type, prev_weights, weights, weights_num, params_name, test_suffix, args=args)

def run_experiment(experiment_type, prev_weights, weights, weights_num, params_name, test_suffix='', tensorboard_logging=True, args=None):
    print(f'setting weights->{weights}')
    set_weights_dir(experiment_type=experiment_type, prev_weights=prev_weights, weights=weights, weights_num=weights_num)

    param_utils.write_commandline_params(args.__str__(), "commandline_full")
    if args.param_overrides:
        param_kwargs: dict = jsonstr_to_kwarg(args.param_overrides)
        param_utils.write_commandline_params(param_kwargs.__str__(), "commandline_overrides")
    param_utils.write_params_state()

    if tensorboard_logging:
        set_tensorboard_dir()
    else:
        params.ModuleParams.tensorboard_dir = None

    print('Experiment type:{}'.format(experiment_type))
    print("Loaded params:{}".format(params.ModuleParams.params_dir))
    print('Weights dir set to {}'.format(params.ModuleParams.weights_dir))
    if prev_weights is not None:
        params.ModuleParams.prev_weights = prev_weights
        print('Using prev weights: {}'.format(prev_weights))

    env_type = params.EnvironmentParams.env_type
    runtime = params.RuntimeParams

    # Pre-Train the network from previously recorded data
    if runtime.is_pre_training():
        pre_train()
        return

    # Record an atari environment
    if runtime.is_recording() and env_type.is_atari():
        # This is always the recording setup to use for atari, setting the teacher_brain and student_brain
        # just affects the data in the meta data files for the atari recordings.
        record_atari(player_id=params.ImitationLearning.atari_player_id)
        return

    # Setup environments for training, testing, or unity recording.
    # Population trainers setup and tear-down environments by themselves
    agent_factory = params.AgentParams.agent_factory
    multi_agent_training_class = agent_factory.multi_agent_training_class()
    if multi_agent_training_class.instantiates_envs():
        multi_agent = multi_agent_training_class()
    else:
        env_wrapper = setup_environment(params.EnvironmentParams.environment, params.EnvironmentParams.env_type, params_name)
        multi_agent = setup_multi_agent(env_wrapper)

    # Train atari or unity
    if runtime.is_training():
        train(multi_agent)
    # Test atari or unity
    elif runtime.is_testing():
        test(multi_agent, test_suffix=test_suffix)

if __name__ == '__main__':
    main()


# This file is part of sner4 project governed by MIT license, see the LICENSE.txt file.
"""
sner agent testssl
"""

import json
import re
import subprocess
import sys
from collections import defaultdict
from pprint import pprint
from zipfile import ZipFile

from sner.lib import file_from_zip
from sner.server.parser import ParsedItemsDb, ParserBase

class ParserModule(ParserBase):  # pylint: disable=too-few-public-methods
    """testssl parser"""

    ARCHIVE_PATHS = r'output\-[0-9]+\.json'
    FINDINGS_IGNORE = ['OK', 'INFO']

    @classmethod
    def parse_path(cls, path):
        """parse data from path"""

        pidb = ParsedItemsDb()

        with ZipFile(path) as fzip:
            for fname in filter(lambda x: re.match(cls.ARCHIVE_PATHS, x), fzip.namelist()):
                pidb = cls._parse_data(file_from_zip(path, fname).decode('utf-8'), pidb)

        return pidb

    @classmethod
    def _parse_data(cls, data, pidb):
        """parse raw string data"""

        json_data = json.loads(data)
        if not isinstance(json_data['scanTime'], int):  # pragma: no cover
            return pidb

        result = json_data['scanResult'][0]
        host_address = result['targetHost']
        service_port = int(result['port'])
        service_proto = 'tcp'

        note_data = {
            'output': data,
            'data': {},
            'findings': defaultdict(list)
        }

        for section_name, section_data in result.items():
            if isinstance(section_data, list):
                # pop findings for section
                for finding in section_data:
                    if finding['id'] == 'cert':
                        # rewrap certificate data
                        tmp = finding['finding'].split(' ')
                        tmp = ' '.join(tmp[:2]) + '\n' + '\n'.join(tmp[2:-2]) + '\n' + ' '.join(tmp[-2:])
                        # parse as auror tool
                        try:
                            note_data['cert_txt'] = subprocess.run(
                                ['openssl', 'x509', '-text', '-noout'],
                                input=tmp, check=True, capture_output=True, text=True
                            ).stdout
                        except subprocess.CalledProcessError as exc:  # pragma: no cover
                            note_data['cert_txt'] = str(exc)

                    if finding['severity'] not in cls.FINDINGS_IGNORE:
                        note_data['findings'][section_name].append(finding)
            else:
                # pop scalar data
                note_data['data'][section_name] = section_data

        pidb.upsert_note(host_address, 'testssl', service_proto, service_port, data=json.dumps(note_data))

        return pidb

if __name__ == '__main__':  # pragma: no cover
    pprint(ParserModule.parse_path(sys.argv[1]))


import vcf
from Bio import SeqIO
# builtin libs
from typing import Any, List, Iterator, Tuple, NoReturn, Dict, Callable, Union, Sequence
from itertools import groupby, starmap, chain
import re
import argparse
import copy
import csv
import os
import sys
# for types
from Bio.SeqRecord import SeqRecord

def raise_(e: Exception) -> NoReturn:
   raise e

def sanitize(fn: str) -> str: return re.sub('[^\w\-_\. ]', '_', fn)
debug_log = sys.stderr.write

def write_subfiles(vcf_reader: vcf.parser.Reader, seqio_refs: dict, refid: str, vcf_grp: Iterator[Any]) -> Tuple[str, str, str, str]:
   refseq: Dict[str, SeqRecord] =  seqio_refs.get(refid) or raise_(ValueError(f"{refid} not found inreference fasta file"))
   basename = f"{sanitize(vcf_reader.filename)}.{sanitize(refid)}"
   vcf_name = f"{basename}.vcf"
   ref_name = f"{basename}.fasta"
   vartable_fn = f"{basename}.tsv"
   with open(vcf_name, 'w') as vcf_out_file:
       vcf_writer = vcf.Writer(vcf_out_file, vcf_reader)
       for r in vcf_grp: vcf_writer.write_record(r)
   with open(ref_name, 'w') as ref_out_file:
       SeqIO.write([refseq], ref_out_file, 'fasta')
   return (refid, ref_name, vcf_name, vartable_fn)

def make_args(in_args: argparse.Namespace, chrom: str, ref_fn: str, vcf_fn: str, vartable_out_fn: str) -> argparse.Namespace:
    ns = copy.deepcopy(in_args)
    ns.vcf_path = vcf_fn
    ns.ref = ref_fn
    ns.out = vartable_out_fn
    return ns
    #subprocess.check_call('vartable', '--bam', args.bam, '--vcf', vcf_fn, '--ref'
def chrom_(r: vcf.model._Record) -> str:
   return r.CHROM
# def read_tsv
DEBUG = False
COMPILE_SUB_RESULTS = True
def validate_file_overwrites(arg_sets: Sequence[argparse.Namespace]) -> Union[None,NoReturn]:
    for arg_set in arg_sets:
       problems=[]
       if (not arg_set.allow_overwrite) and os.path.exists(arg_set.out):
           problems.append( (f"--allow-overwrite option is off, but file {arg_set.out} exists! Stopping.") )
    if any(problems):
       raise ValueError("\n".join(problems))
    return None

def dispatch(args: argparse.Namespace, vartable_dispatch: Callable[[argparse.Namespace], int], writer: Callable) -> int:
    vcf_fn, ref_fn = args.vcf_path, args.ref
    with open(ref_fn) as ref_file, open(vcf_fn) as vcf_file:
        refs = { x.id : x for x in SeqIO.parse(ref_file, 'fasta') }
        vcf_reader = vcf_reader = vcf.Reader(vcf_file)
        vcf_groups = groupby(vcf_reader, key=chrom_) # lambda x: x.CHROM)
        chrom_files =  [write_subfiles(vcf_reader, refs, chrom, grp) for chrom, grp in vcf_groups]
    subrun_args = [make_args(args, *info) for info in chrom_files]
    validate_file_overwrites(subrun_args)
    for arg_set in subrun_args:
        debug_log(f"Running: {arg_set}")
        if not DEBUG:
            headers = vartable_dispatch(arg_set)
            # debug_log(result)
    if COMPILE_SUB_RESULTS:
        readers = [csv.DictReader(open(args.out), delimiter='\t') for args in subrun_args ]
        headers.sort()
        sub_headers = [ sorted(r.fieldnames) for r in readers ]
        assert all( (x == headers for x in sub_headers)  ), f"Header missmatch of {headers} of output csv files, see:\n {subrun_args}\n{sub_headers}"
        all_rows = chain.from_iterable( readers )
        writer(args.out, all_rows, headers)
    return 0


from deta import Deta
import numpy as np
import pandas as pd
import os
from dotenv import load_dotenv
import datetime
import streamlit as st

class datamodel:
	'''
	Dataclass model for getting and processing data for visualization
	'''
	def __init__(self, ):
		# load enviroment variables and extract database key
		# for localhost version
		#load_dotenv(".env")
		#os.getenv('key')
		# for streamlit app in live mode
		key = st.secrets['key']
		# create database object
		deta = Deta(key)
		# create databse connection
		self.db = deta.Base("YgoEloBase")
		# set default miesteschaft dictonary for saving and updating
		self.df_default_meisterschaft = pd.DataFrame(index=['Wanderpokal', 'Local', 'Fun Pokal'], columns=['Teilnahme', 'Top', 'Win']).fillna(0).to_dict()
		# filter parameter for deck building page
		amount_dict = {'Starter':0, 'Handtrap':0, 'Boardbreaker':0, 'Extender':0, 'Brick':0,
                       'Floodgate':0, 'Searcher':0, 'Monster':0, 'Zauber':0, 'Falle':0,
                       'Engine 1':0, 'Engine 2':0, 'Engine 3':0, 'Engine 4':0,'Draw':0,
					   'Feuer':0, 'Wasser':0, 'Licht':0, 'Finsternis':0, 'Erde':0, 'Wind':0,
					   'Divine':0}
		cardtyp_default = ['Starter', 'Extender', 'Handtrap', 'Boardbreaker', 'Brick']
		# load data to session state of get data from session state
		if "reload_flag" not in st.session_state:
			# get data from data base
			self.df, self.cols, self.hist_cols = self.__create_data_model()
			# setup session state
			st.session_state = {'owner':self.df['Owner'].unique(),
								'types_select':self.df['Type'].unique(),
								'tier':self.df['Tier'].unique(),
								'sorting':['Elo-Rating'],
								'tournament':'Alle',
								'deck_i':'',
								'deck':[],
								'reload_flag':False,
								'dataframe':self.df,
								'hist_cols':self.hist_cols,
								'cols':self.cols,
								'login':None,
								'card_amounts':amount_dict,
								'cardtyp_defaults':cardtyp_default
								}
		# if page get a forced reload after data base insertion reload data
		elif st.session_state['reload_flag']:
			# get data from data base
			self.df, self.cols, self.hist_cols = self.__create_data_model()
			# setup session state
			st.session_state = {'owner':self.df['Owner'].unique(),
								'types_select':self.df['Type'].unique(),
								'tier':self.df['Tier'].unique(),
								'sorting':['Elo-Rating'],
								'tournament':'Alle',
								'deck_i':'',
								'deck':[],
								'reload_flag':False,
								'dataframe':self.df,
								'hist_cols':self.hist_cols,
								'cols':self.cols,
								'login':None,
								'card_amounts':amount_dict,
								'cardtyp_defaults':cardtyp_default
								}
		# use already loaded data
		else:
			self.df = st.session_state['dataframe']
			self.hist_cols = st.session_state['hist_cols']
			self.cols = st.session_state['cols']

	def get(self):
		'''
		Method for returng the processed data from the databse to
		the dashbord.
		'''
		return self.df, self.cols, self.hist_cols

	def __create_data_model(self):
		'''
		Method for getting and preprocessing the yugioh data from the
		database
		:return df: dataframe with data
		:return cols: list with sorted columns of the dataframe
		'''
		# get data from data base
		df, hist_cols, save_cols = self.__get_all_data()
		hist_cols = self.__sort_hist_cols(hist_cols)
		df = self.__clean_data(df, hist_cols)
		return df, save_cols, hist_cols

	def __get_all_data(self, ):
		'''
		Method to get all data from the Database
		:use db: as detabase keys
		:return df_all: Dtaframe with all data like deck name, player and elo
		:return hist_cols: list of columns with historic elo data
		:return save_cols:list with all columns with further infos
		'''
		# get all data
		res = self.db.fetch()
		# create a dataframe
		df = pd.DataFrame(res.items)
		#df.to_csv('db.csv')
		# build a dataframe from the history
		tmp = df.loc[:, 'History'].to_list()
		hist_df = pd.DataFrame(tmp).fillna(0).astype(int)
		hist_cols = list(np.flip(hist_df.columns.to_list()))
		del df['History']
		save_cols = df.columns.to_numpy()
		save_cols = np.delete(save_cols, np.where(save_cols == 'key'))
		save_cols = np.delete(save_cols, np.where(save_cols == 'Meisterschaft'))
		# merge history dataframe and rest
		df_all = pd.merge(df, hist_df, right_index=True, left_index=True)

		return df_all, hist_cols, save_cols

	def update_elo_ratings(self, deck1, deck2, erg1, erg2, df_elo, hist_cols, save_cols):
		'''
		Method for updating Elo Rating and result stats after a new match
		'''
		# get index
		idx1 = int(df_elo[df_elo['Deck']==deck1].index.to_numpy())
		idx2 = int(df_elo[df_elo['Deck']==deck2].index.to_numpy())
		# update siege/niederlagen
		if not erg1 == erg2:
				df_elo.at[idx1, 'Siege'] += erg1
				df_elo.at[idx1, 'Niederlage'] += erg2

				df_elo.at[idx2, 'Siege'] += erg2
				df_elo.at[idx2, 'Niederlage'] += erg1
		else:
				df_elo.at[idx1, 'Remis'] +=erg1
				df_elo.at[idx2, 'Remis'] +=erg2
		# update matches
		df_elo.at[idx1, 'Matches'] += 1
		df_elo.at[idx2, 'Matches'] += 1
		# update dgp
		df_elo.at[idx1, 'dgp'] = (df_elo.at[idx1, 'dgp'] + 2*df_elo.at[idx2, 'Elo'])//3
		df_elo.at[idx2, 'dgp'] = (df_elo.at[idx2, 'dgp'] + 2*df_elo.at[idx1, 'Elo'])//3
		# update elo
		norm = erg1 + erg2
		score1 = erg1/norm
		score2 = erg2/norm

		# calculate winning probabilities
		# Formula from Chess ELO System
		alpha = 10**((df_elo.at[idx1, 'Elo']-df_elo.at[idx2, 'Elo'])/400)
		p1 = alpha/(alpha+1)
		p2 = 1-p1

		# Update ELO (Chess Formula)
		df_elo.at[idx1, 'Elo'] += int(32*(score1-p1))
		df_elo.at[idx2, 'Elo'] += int(32*(score2-p2))

		df_elo = self.__update_tiers(df_elo)

		key1 = df_elo.at[idx1, 'key']
		key2 = df_elo.at[idx2, 'key']

		self.__update_element(key1, df_elo.loc[idx1, :], save_cols, hist_cols)
		self.__update_element(key2, df_elo.loc[idx2, :], save_cols, hist_cols)

	def __update_tiers(self, df):
		'''
		Method for updating tier standing of a deck after a match
		'''
		Label = ['Kartenstapel','Fun', 'Good', 'Tier 2', 'Tier 1', 'Tier 0']
		NCl = len(Label)

		mean = int(df['Elo'].mean())
		std = float(df['Elo'].std())
		bins = np.array([0, mean-2*std, mean-std, mean, mean+std, mean+2*std, 10e6])
		liste =[]
		for jj in range(len(df)):
			for kk in range(NCl):
				if(df.at[jj, 'Elo']>=bins[kk] and df.at[jj, 'Elo']<=bins[kk+1]):
					liste.append(Label[kk])
					break

		df['Tier'] = liste
		return df

	def __update_element(self, unique_key, ds, save_cols, hist_cols):
		'''
		Method for updating an element in the database
		:param unique_key: key of the element to updated
		:param ds: data series with the data with sould be updated
		:param save_cols: list of columns with element infos
		:param hist_cols: list with column names to build elo history
		:param db: key to database
		:return: nothing
		'''
		# empty update dictionaries
		dict_update = {}
		dict_hist = {}
		# build historic elo dictionary
		for col in save_cols:
			dict_update[col] = str(ds[col])
		# build ifon dictionary
		for col in hist_cols:
			dict_hist[col] = str(ds[col])
		# include historic elo dictonary to info dictionary
		dict_update['History'] = dict_hist
		dict_update['Meisterschaft'] = ds['Meisterschaft']
		# update database
		self.db.put(dict_update, unique_key)

	def update_tournament(self, deck, tour, result, df, save_cols, hist_cols):
		'''
		Method for updating the tournament column of a spezific deck
		:param deck: name des decks to update
		:param tour: name of the tournament to update
		:param df: dataframe with all daa
		:param save_cols:list with columns with deck infos
		:param hist_cols: list with all columns with historic elo data
		:param db: key to database
		:return: nothing
		'''
		# find index of the deck to update
		idx = int(df[df['Deck']==deck].index.to_numpy())
		# increase choosen tournament
		df.at[idx, 'Meisterschaft']['Teilnahme'][tour] += 1
		if result == 'Win':
			df.at[idx, 'Meisterschaft']['Win'][tour] += 1
		if result == 'Top':
			df.at[idx, 'Meisterschaft']['Top'][tour] += 1
		# get key of the deck
		unique_key = df.at[idx, 'key']
		# update element in database
		self.__update_element(unique_key, df.loc[idx, :], save_cols, hist_cols)

	def update_stats(self, deck_choose, in_stats, modif_in, in_stats_type, new_type, df_elo, save_cols, hist_cols):
		'''
		Method for updating deck characteristica
		:param deck_choosen: name of the deck to update
		:param in_stats: info column to update
		:parm modif_in: value of the midification
		:param in_stats_type: update type of tthe choosen deck
		:param new_type: new_type to update
		:param df_elo: dataframe with all data
		:param save_cols: list with info columns
		:param hist_cols: columns with historic elo data
		:param db: key to database
		:return: nothing
		'''
		# find index of choosen deck to update
		idx = int(df_elo[df_elo['Deck']==deck_choose].index.to_numpy())
		# update characteristics if choosen
		if len(in_stats) > 0:
			df_elo.at[idx, in_stats] = modif_in
		# update deck type if choosen
		if len(in_stats_type)>0:
			df_elo.at[idx, in_stats_type] = new_type
		# get deck key and update the database
		unique_key = df_elo.at[idx, 'key']
		df_elo['Meisterschaft'] = df_elo['Meisterschaft'].astype(object)
		self.__update_element(unique_key, df_elo.loc[idx, :], save_cols, hist_cols)

	def update_history(self, df, hist_cols, save_cols):
		'''
		Method for increase the historic elo columns with the actual elo
		:param df: dataframe with all data
		:param hist_cols: list with historic elo columns
		:param save_cols: list with deck info columns
		:param db: key to database
		:return: nothing
		'''
		# get actual date and modify layout
		date = datetime.date.today()
		name = date.strftime("%m/%Y")
		# if actual date is not in historic columns include new column
		if not name in hist_cols:
			# include new column
			df[name] = df['Elo']
		# add new column to hist columns
		hist_cols += [name]
		# update all decks in database with new historic deck column
		for i in range(len(df)):
				unique_key = df.at[i, 'key']
				self.__update_element(unique_key, df.loc[i, :], save_cols, hist_cols)

	def insert_new_deck(self, deck, name, atk, contr, rec, cons, combo, resi, typ):
		'''
		Function for inserting a new deck to the database
		:param deck: name of the deck
		:param name: name of the player
		:param atk: attack rating
		:param contr: control rating
		:param rec: recovery rating
		:param cons: consistency rating
		:param combo: combo rating
		:param resi: resilience rating
		:param typ; type of the deck
		:param db: key to database
		:return: nothing
		'''
		# build info dictionary
		input_dict = {
		'Deck':deck,
		'Owner':name,
		'Tier':'Tier 2',
		'Attack':atk,
		'Control':contr,
		'Recovery':rec,
		'Consistensy':cons,
		'Combo':combo,
		'Resilience':resi,
		'Type':typ,
		'Wanderpokal':0,
		'Meisterschaft':self.df_default_meisterschaft,
		'Liga Pokal':0,
		'Fun Pokal':0,
		'Matches':0,
		'Siege':0,
		'Remis':0,
		'Niederlage':0,
		'dgp':0,
		'Elo':1500,
		'History':{}}
		# inserting the new deck
		self.db.put(input_dict)

	def __clean_data(self, df, hist_cols):
		'''
		Method for cleaning and calculation of new data with data from the database
		:param df: dataframe with data from the database
		:param hist_cols: list with all historic elo columns
		:return df: dataframe with data in right format and new calculated values
		'''
		# columns with integer values
		cols_to_int = ['Elo', 'dgp', 'Wanderpokal', 'Liga Pokal', 'Fun Pokal', 'Attack', 'Combo', 'Resilience', 'Recovery',
					'Consistensy', 'Control','Matches', 'Siege', 'Remis', 'Niederlage']+hist_cols
		# fill nulls and transform float data to interes
		for k in cols_to_int:
			df[k] = df[k].fillna(0).astype(float)
			df[k] = df[k].astype(int)
		# calculate last 3 month difference
		df['Letzte 3 Monate'] = 0
		idx = df[df[hist_cols[-1]]>0].index
		df.loc[idx, 'Letzte 3 Monate'] = df.loc[idx, 'Elo']-df.loc[idx, hist_cols[-1]]
		df['Letzte 3 Monate'] = df['Letzte 3 Monate'].astype(int)
		# calculate last 6 month difference
		df['Letzte 6 Monate'] = 0
		idx = df[df[hist_cols[-2]]>0].index
		df.loc[idx, 'Letzte 6 Monate'] = df.loc[idx, 'Elo']-df.loc[idx, hist_cols[-2]]
		df['Letzte 6 Monate'] = df['Letzte 6 Monate'].astype(int)
		# calculate last 12 month difference
		df['Letzte 12 Monate'] = 0
		idx = df[df[hist_cols[-4]]>0].index
		df.loc[idx, 'Letzte 12 Monate'] = df.loc[idx, 'Elo']-df.loc[idx, hist_cols[-4]]
		df['Letzte 12 Monate'] = df['Letzte 12 Monate'].astype(int)
		# calculate mean and std of the last year
		df['Mean 1 Year'] = np.nanmean(df[hist_cols[-5:-1]+['Elo']],1)
		df['Mean 1 Year'] = df['Mean 1 Year'].astype(int)
		df['Std. 1 Year'] = np.nanstd(df[hist_cols[-5:-1]+['Elo']],1)
		df['Std. 1 Year'] = df['Std. 1 Year'].astype(int)
		# calculate win rate
		df['Gewinnrate'] = 100*np.round(df['Siege']/(df['Siege']+df['Remis']+df['Niederlage']), 2)
		return df.fillna(0)

	def __sort_hist_cols(self, hist_cols):
		'''
		Method for sorting the list with historic elo in ascending order
		:param hist_cols: list with columns with historic elo
		:return out: sorted list with historic elo
		'''
		# empty list for transformed (to date) column names
		t = []
		# loop over historic columns
		for c in hist_cols:
			t.append(datetime.date(year = int(c[-4:]), month = int(c[:2]), day=1))
		# sort dates
		t = pd.Series(t).sort_values().reset_index(drop=True)
		# list for sorted cback transformed (to string) column names
		out = []
		# loop over all dates in ascending order
		for i in range(len(t)):
			out.append(t[i].strftime('%m/%Y'))
		return out


import keras.backend as K
import math
import numpy as np
import h5py
import matplotlib.pyplot as plt

def mean_predctn(y_true, y_pred):
    return K.mean(y_pred)

def load_dataset():
    dataset = h5py.File('datasets/train.h5', "r")
    train_set_x = np.array(dataset["train_set_x"][:]) # your train set features
    train_set_y = np.array(dataset["train_set_y"][:]) # your train set labels

    test_dataset = h5py.File('datasets/test.h5', "r")
    test_set_x = np.array(test_dataset["test_set_x"][:]) # your test set features
    test_set_y = np.array(test_dataset["test_set_y"][:]) # your test set labels

    classes = np.array(test_dataset["list_classes"][:]) # the list of classes

    train_set_y = train_set_y.reshape((1, train_set_y.shape[0]))
    test_set_y = test_set_y.reshape((1, test_set_y.shape[0]))

    return train_set_x, train_set_y, test_set_x, test_set_y, classes