import os
import shutil
import subprocess
import sys
import time

import cv2
import numpy as np
import torch
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import transforms

from utils.database.Pixelmap import PixelMap_fold_STmap

raw_video = "./VideoCapture/raw_video.MOV"
time_stamps_txt = "./VideoCapture/time_stamps.txt"
camera_dir = "./Camera/"
landmark_exe = "points81.exe"
face_landmarks_dir = "./VideoCapture/rt_face_landmarks/"
st_maps_dir = "./VideoCapture/STmaps/"
matlab_dir = "./Matlab for STmap/"
matlab_exe = "map_generation.exe"
out_video_dir = "./Test Record/"

# Clear Cache
face_landmarks_cache = os.listdir(face_landmarks_dir)
for f in face_landmarks_cache:
    os.remove(face_landmarks_dir + f)

st_maps_cache = os.listdir(st_maps_dir)
for f in st_maps_cache:
    shutil.rmtree(st_maps_dir + f)

if os.path.exists(raw_video):
    os.remove(raw_video)

if os.path.exists(time_stamps_txt):
    os.remove(time_stamps_txt)

# Show Welcome Prompt
welcome_window = "Welcome to Heart Rate Estimation Based on Face Perception"
frame_welcome_raw = np.zeros((540, 960, 3), np.uint8)
frame_welcome_raw[0:, 0:, 0], frame_welcome_raw[0:, 0:, 1], frame_welcome_raw[0:, 0:, 2] \
    = 239, 173, 0  # BGR
cv2.namedWindow(welcome_window, cv2.WINDOW_NORMAL)
cv2.resizeWindow(welcome_window, 960, 540)
cv2.setWindowProperty(welcome_window, cv2.WND_PROP_TOPMOST, 1)
welcome_end_time = time.time() + 5
while time.time() < welcome_end_time:
    frame_welcome = frame_welcome_raw.copy()
    cv2.putText(
        frame_welcome, "Welcome!", (50, 80),
        cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 5
    )
    cv2.putText(
        frame_welcome, "Heart Rate Estimation Based on Face Perception", (50, 140),
        cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 3
    )
    cv2.putText(
        frame_welcome,
        "The camera will be opened in " + str(round(welcome_end_time - time.time())) + "s.",
        (50, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
    )
    cv2.putText(
        frame_welcome,
        "After being opened, it may take another few seconds for the camera to be ready.",
        (50, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2
    )
    cv2.putText(
        frame_welcome, "Press Esc before the camera is opened to exit the program.", (50, 300),
        cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
    )
    cv2.imshow(welcome_window, frame_welcome)
    if cv2.waitKey(10) == 27:
        print("Keyboard Interrupt: Program ended by Esc")
        cv2.destroyAllWindows()
        os.system("pause")
        sys.exit(0)

# Record Raw Video
# Open Camera
camera_id = 0
capture = cv2.VideoCapture(camera_id)
capture.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)

# Close Welcome Window
cv2.destroyWindow(welcome_window)

# Video Writer
vw = cv2.VideoWriter()
codec = vw.fourcc('a', 'v', 'c', '1')
vw.open(raw_video, codec, 30.0, (1280, 720))
if not vw.isOpened():
    print("Error: Failed to instantiate video writer")
    os.system("pause")
    sys.exit(1)

# Process Each Frame
frame_count = 0
while True:
    cap_suc, frame = capture.read()
    if not cap_suc:
        print("Error: Failed to open camera")
        os.system("pause")
        sys.exit(1)
    frame_count += 1

    # Write Time
    wf = open(time_stamps_txt, mode='a')
    wf.write(str(round(time.time() * 1000)) + "\n")
    wf.close()

    # Write Video
    vw.write(frame)

    cv2.putText(
        frame, time.ctime() + "    Frame: " + str(frame_count), (10, 30),
        cv2.FONT_HERSHEY_SIMPLEX, 1, (215, 120, 0), 2
    )
    cv2.putText(
        frame, "You can record for as long as you want, but at least 360 frames.", (10, 80),
        cv2.FONT_HERSHEY_SIMPLEX, 0.8, (64, 64, 255), 2
    )
    cv2.putText(
        frame, "Press Esc to Stop Recording After 360 Frames", (10, 120),
        cv2.FONT_HERSHEY_SIMPLEX, 1, (64, 64, 255), 2
    )

    camera_window = "Heart Rate Estimation Based on Face Perception"
    cv2.namedWindow(camera_window, cv2.WINDOW_NORMAL)
    cv2.resizeWindow(camera_window, 960, 540)
    cv2.setWindowProperty(camera_window, cv2.WND_PROP_TOPMOST, 1)
    cv2.imshow(camera_window, frame)

    if cv2.waitKey(1) == 27 and frame_count > 360:
        break

capture.release()
vw.release()
cv2.destroyWindow(camera_window)

# Face Landmarking
os.chdir(camera_dir)
try:
    subprocess.check_output(landmark_exe, creationflags=subprocess.CREATE_NEW_CONSOLE)
except subprocess.CalledProcessError:
    print("Error: No face detected in the current frame. Please keep steady and try again.")
    os.system("pause")
    sys.exit(1)

# Show prompt
stmap_window = "Generating STmaps..."
frame_stmap = np.zeros((540, 960, 3), np.uint8)
frame_stmap[0:, 0:, 0], frame_stmap[0:, 0:, 1], frame_stmap[0:, 0:, 2] \
    = 239, 173, 0  # BGR
cv2.namedWindow(stmap_window, cv2.WINDOW_NORMAL)
cv2.resizeWindow(stmap_window, 960, 540)
cv2.setWindowProperty(stmap_window, cv2.WND_PROP_TOPMOST, 1)
cv2.putText(
    frame_stmap, "Heart Rate Estimation Based on Face Perception",
    (50, 80), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 3
)
cv2.putText(
    frame_stmap, "Generating Spatial-Temporal Maps (STmaps) of the video...",
    (50, 150), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
)
cv2.putText(
    frame_stmap, "A console application implemented by MATLAB should be running.",
    (50, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
)
cv2.putText(
    frame_stmap, "This may take up to several minutes. Please wait...",
    (50, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
)
cv2.putText(
    frame_stmap, "This window will close automatically after completion.",
    (50, 300), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 255, 255), 2
)
cv2.imshow(stmap_window, frame_stmap)
cv2.waitKey(10)

# STmap Generation
os.chdir("../")
os.chdir(matlab_dir)
try:
    subprocess.check_output(matlab_exe, creationflags=subprocess.CREATE_NEW_CONSOLE)
except subprocess.CalledProcessError:
    print("Error: An unexpected error occurred while generating STmaps.")
    cv2.destroyAllWindows()
    os.system("pause")
    sys.exit(1)

# Close Prompt
cv2.destroyWindow(stmap_window)

# Get Heart Rate from Network
os.chdir("../")

video_length = 300

toTensor = transforms.ToTensor()
resize = transforms.Resize(size=(320, 320))

test_dataset = PixelMap_fold_STmap(root_dir='./VideoCapture/STmaps/',
                                   Training=False,
                                   transform=transforms.Compose([resize, toTensor]),
                                   VerticalFlip=False,
                                   video_length=video_length)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=0)

################################################################################
net_id = "1662170218"

net_file = "./model.pt"
if os.path.exists("./HR_models/"):
    net_file = './HR_models/hr_model_cuda_' + net_id + ".pt"

net = torch.load(net_file, map_location=torch.device('cpu'))

net.to(torch.device("cpu"))
net.eval()

################################################################################
output_list = []
for (data, fps, idx) in test_loader:
    data = Variable(data)

    data = data.to(torch.device("cpu"))

    feat_hr, feat_n, output, img_out, \
    feat_hrf1, feat_nf1, hrf1, idx1, \
    feat_hrf2, feat_nf2, hrf2, idx2 = net(data)

    output_list.append(output.item() / (video_length / fps.item()) * 60)

################################################################################
# Show Output
print("Estimation completed: ", end="")
print(output_list)

capture_o = cv2.VideoCapture(raw_video)

vw_o = cv2.VideoWriter()
out_codec = vw_o.fourcc('a', 'v', 'c', '1')
out_video_file = out_video_dir + "Test_" + str(round(time.time())) + ".MOV"
vw_o.open(out_video_file, out_codec, 30.0, (1280, 720))

# Read All Timestamps
rf = open(time_stamps_txt, 'r')
time_list = []
line = rf.readline()
while line:
    time_list.append(eval(line[:-1]))
    line = rf.readline()
rf.close()

# Calculate Average FPS
frame_num = len(time_list)
avr_fps = frame_num / (time_list[-1] - time_list[0]) * 1000.0
frame_interval = round(avr_fps / 2)
clip_num = int((frame_num - 300) / avr_fps * 2)

# Update Output Value Every 0.5s
frm_o_count = 0
clip_idx = 0
while frm_o_count < frame_num:
    out_cap_suc, frame_o = capture_o.read()
    if not out_cap_suc:
        print("Error: Failed to read video file " + raw_video)
        cv2.destroyAllWindows()
        os.system("pause")
        sys.exit(1)
    frm_o_count += 1

    # Update Output
    if frm_o_count % frame_interval == 0 and clip_idx < clip_num - 1:
        clip_idx += 1

    # Out of Bound
    if clip_idx >= len(output_list):
        print("Warning: No more output values @ Frame " + str(frm_o_count) + " of " + str(frame_num))
        clip_idx = len(output_list) - 1

    # Put Text
    cv2.putText(
        frame_o,
        time.ctime(time_list[frm_o_count - 1] / 1000.0) + "    Frame: " + str(frm_o_count),
        (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (215, 120, 0), 2
    )
    cv2.putText(
        frame_o, "Estimated Heart Rate: ",
        (10, 80), cv2.FONT_HERSHEY_SIMPLEX, 1, (215, 120, 0), 2
    )
    cv2.putText(
        frame_o, str(round(output_list[clip_idx], 1)),
        (50, 160), cv2.FONT_HERSHEY_SIMPLEX, 2, (215, 120, 0), 5
    )

    # Write Output Video
    vw_o.write(frame_o)

    # Last Frame
    if frm_o_count == frame_num - 1:
        cv2.putText(
            frame_o, "Successfully finished! You can close this window and view the video at ",
            (10, 210), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (64, 64, 255), 2
        )
        cv2.putText(
            frame_o, out_video_file,
            (10, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (64, 64, 255), 2
        )

    output_window = "Estimated Heart Rates"
    cv2.namedWindow(output_window, cv2.WINDOW_NORMAL)
    cv2.resizeWindow(output_window, 960, 540)
    cv2.setWindowProperty(output_window, cv2.WND_PROP_TOPMOST, 1)
    cv2.imshow(output_window, frame_o)

    if frm_o_count == frame_num - 1:
        cv2.waitKey(0)
    else:
        cv2.waitKey(round(1000 / avr_fps))

capture_o.release()
vw_o.release()


#coding=utf-8

import tornado.web
import tornado.ioloop
class IndexHandle(tornado.web.RequestHandler):

	def get(self):
	    """定义请求函数"""
	    self.write('hello wukang')

if __name__ == '__main__':
    app=tornado.web.Application([
            (r'/',IndexHandle),
                              ])
    app.listen(8000)
    tornado.ioloop.IOLoop.current().start()


import streamlit as st
import pandas as pd
import requests
import cv2

captions_df = pd.read_csv(".\data\captions.csv")
captions_df = captions_df.drop_duplicates(subset=['image'], keep='first').reset_index(drop=True)
filenames = captions_df['image'].tolist()
PATH_IMAGES = ".\data\Images\\"

def search_images():
    matches = requests.get(
        url="http://localhost:3020/query_to_image/",
        params={
            "query": query
        }
    ).json()
    matched_indices = matches['matches']
    scores = matches['scores']
    for score, inx in zip(scores, matched_indices):
        image = cv2.imread(PATH_IMAGES+filenames[inx])
        st.image(image, channels="BGR")
        st.caption(str(score))

st.markdown("# Search Images based on the query")
st.sidebar.markdown("# Search Images")
query = st.text_input("Describe an image you are looking for!", value="", max_chars=256)
if st.button("Search images"):
    matches = requests.get(
        url="http://localhost:3020/query_to_image/",
        params={
            "query": query
        }
    ).json()
    matched_indices = matches['matches']
    scores = matches['scores']
    for score, inx in zip(scores, matched_indices):
        image = cv2.imread(PATH_IMAGES+filenames[inx])
        st.image(image, channels="BGR")
        st.caption(str(score))


# determiner.py
# determine choices with complicated conditions

import re
import reader # to read color stds to determine std color

def determine_matching_field(desired_field_name, current_field_name):
    print("desired_field_name: " + desired_field_name)
    print("current_field_name: " + current_field_name)

def determine_standard_key(raw_key):
    #print("\n===Determine Standard Key for " + raw_key + "===\n")

    standard_key = ''

    all_field_keywords = {
        'sku':['sku','item#'],
        'collection':['collection'],
        'features':['features','acme.description'],
        'type':['product type','description'],
        'intro':['intro','short description'],
        'color':['color'],
        'material':['material'],
        'finish':['finish'],
        'width':['width'],
        'depth':['depth','length'],
        'height':['height'],
        'weight':['weight'],
        'cost':['cost','price'],
        'images':['image'],
        'barcode':['barcode']
    }

    for field_key, field_keywords in all_field_keywords.items():
        #print("field_key, field_keywords: " + field_key + ", " + str(field_keywords))
        for keyword in field_keywords:
            raw_key_no_space = re.sub('(\\s+|_)','',raw_key.lower()) # unpredictable typos OR format in headers given by vendor such as 'D E S C R I P T I O N'
            keyword_no_space = re.sub('\\s', '', keyword)
            if re.search(keyword_no_space, raw_key_no_space):
                #print("keyword " + keyword + " in raw_key " + raw_key)
                standard_key = field_key
                break
        if standard_key != '':
            break

    #print("standard_key: " + standard_key)
    return standard_key

def determine_field_name(field, sheet_df):
    print("\n===Determine Field Name: " + field + "===\n")
    sheet_headers = sheet_df.columns.values

    field_keywords = {
        'sku':['sku','item#'],
        'collection':['collection'],
        'features':['features','acme.description'],
        'type':['product type','description'],
        'intro':['intro','short description'],
        'color':['color'],
        'material':['material'],
        'finish':['finish'],
        'width':['width'],
        'depth':['depth','length'],
        'height':['height'],
        'weight':['weight'],
        'cost':['cost','price'],
        'images':['image'],
        'barcode':['barcode']
    }
    keywords = field_keywords[field]
    matching_field = False
    field_name = ''
    for keyword in keywords:
        for header in sheet_headers:
            header_no_space = re.sub('(\\s+|_)','',header.lower()) # unpredictable typos OR format in headers given by vendor such as 'D E S C R I P T I O N'
            keyword_no_space = re.sub('\\s', '', keyword)
            if re.search(keyword_no_space, header_no_space):
                field_name = header
                #print("field_name: " + field_name)
                matching_field = True
                break
        if matching_field:
            break
    return field_name

def determine_measurement_type(measurement, handle):

	#print("=== Determine Measurement Type=== ")

	meas_type = 'rectangular' # default to rectangular WxD b/c most common. alt option: round

	measurement = measurement.lower() # could contain words such as "Round" or "n/a"
	#print("Measurement: " + measurement)

	blank_meas = True
	if measurement != 'n/a' and measurement != '':
		blank_meas = False

	if not blank_meas:
		# eg "9\' x 2\'"
		if re.search('\'|\"',measurement): # given in units of feet and/or inches

			#print("===Measurement Given in Custom Format===")

			meas_ft_value = meas_in_value = 0.0

			#if re.search("round",measurement):
				#print("Measurement is of Round Object, so Diam or Rad?")

			meas_vars = []
			meas_value = measurement # from here on use value

			if re.search('\'\\d+\"\\s*\\d+(\'|\")',meas_value):
				meas_type = 'combined_rect' #already set by default
				print("Warning for " + handle + ": Width and Depth given in same field, while determining measurement type: \"" + meas_value + "\"!")
			elif re.search('(\")\\s*.+(\")',meas_value) or re.search('(\')\\s*.+(\')',meas_value):
				meas_type = 'invalid'
				print("Warning for " + handle + ": 2 values with the same unit given while determining measurement type: \"" + meas_value + "\"!")
			# assuming meas value followed by meas type
			elif re.search('\\s+',meas_value):
				#print("Measurement contains a space character.")
				meas_vars = re.split('\\s+',meas_value)
				#print("Meas vars: " + str(meas_vars))
				meas_value = meas_vars[0]
				meas_type = meas_vars[1]

	#print("Measurement value: " + meas_value)
	#print("Measurement type: \"" + meas_type + "\"\n")

	return meas_type

def determine_given_colors(product_details):
	given_colors = True

	for variant_details in product_details:
		colors = ''

		if len(variant_details) > 4:
			colors = variant_details[4].strip()

			if colors == '' or colors.lower() == 'n/a':
				# no colors given
				given_colors = False

		else:
			given_colors = False

	return given_colors

def determine_given_materials(product_details):
	given_materials = True

	for variant_details in product_details:
		materials = ''

		if len(variant_details) > 5:
			materials = variant_details[5].strip()

			if materials == '' or materials.lower() == 'n/a':
				# no colors given
				given_materials = False

		else:
			given_materials = False

	return given_materials

def determine_given_finishes(product_details):
	given_finishes = True

	for variant_details in product_details:
		finishes = ''

		if len(variant_details) > 6:
			finishes = variant_details[6].strip()

			if finishes == '' or finishes.lower() == 'n/a':
				# no finishes given
				given_finishes = False

		else:
			given_finishes = False

	return given_finishes

def determine_given_dimensions(product_details):
	given_dims = True

	for variant_details in product_details:
		width = depth = height = ''

		if len(variant_details) > 7:
			width = variant_details[7].strip()

			if len(variant_details) > 8:
				depth = variant_details[8].strip()

				if len(variant_details) > 9:
					height = variant_details[9].strip()
		else:
			given_dims = False

		if width == '' or width.lower() == 'n/a':
			# no width given but maybe other dims given
			if depth == '' or depth.lower() == 'n/a':
				# no width or depth given but maybe height given
				if height == '' or height.lower() == 'n/a':
					given_dims = False

	return given_dims

def determine_inv_location_key(location, item_inv):
	loc_key = ''
	for key in item_inv.keys():
		if re.search(location.lower(),key.lower()):
			#print("key: " + key)

			if re.search('qty', key.lower()):
				loc_key = key
				break

	return loc_key

def determine_stocked(sheet1_sku, all_inv, locations=[]):
	#print("\n===Determine Stocked===\n")
	#print("sheet1_sku: " + sheet1_sku)
	#print("all_inv: " + str(all_inv))
	stocked = False
	given_info = False # if we are given an item but no stock info we still want to display it
	if len(locations) == 0:
		locations = ['ny', 'nj', 'la', 'sf']

	for item_inv in all_inv:
		if item_inv['sku'] == sheet1_sku:
			#print("item_inv: " + str(item_inv))

			for loc in locations:
				#print("loc: " + loc)
				for key, val in item_inv.items():
					if re.search("qty",key) and re.search(loc,key):
						if round(float(val)) > 0: # problem dataframe reading decimal
							#print(sheet1_sku + " is stocked in " + loc)
							stocked = True
							break
					elif re.search('eta',key): # if there is an eta then it will be stocked
						if val != '' and val.lower() != 'none':
							location = re.sub('locations\\.|_warehouse_eta','',key)
							#print(sheet1_sku + " will be stocked in " + location)
							stocked = True
							break

				if stocked:
					break

			# if not stocked:
			# 	print(sheet1_sku + " Not Stocked")

			break

	return stocked

def determine_eta_header(item_inv):
	eta_header = ''
	for key in item_inv.keys(): # later may need to check both eta and loc to see if valid location but currently only eta given for la which is valid
		if re.search('eta',key):
			eta_header = key
			break
	return eta_header

def determine_inv_tracking(sku, all_inv):
	inv_tracking = False
	for item_inv in all_inv:
		item_sku = item_inv['sku']
		if sku == item_sku:
			inv_tracking = True
			break

	return inv_tracking

# solo product is from final item info split by ; delimiter
# could use catalog instead if we need to access product intro specifically but body html should be good enough
def determine_product_bundle(solo_product):
	print("\n===Determine Product Bundle===\n")
	#print("solo_product: " + str(solo_product))
	bundle = False

	# determine if loft bed bc treated differently
	# loft bed determined by handle bc type still bed, and handle same for all vrnts in fp
	product_handle_idx = 0
	product_handle = solo_product[0][product_handle_idx] # same for all vrnts so use first vrnt
	if re.search('loft-bed', product_handle):
		print("Product is loft bed, so check if optional items to bundle.")
		# how do we know if there is an optional bundle in the loft bed product? description says optional queen bed, but what if that is referring to the size of the loft bed? can we assume loft bed is only twin? no bc that is restrictive.
		# in this case the descrip says optional bed underneath but not reliable. usually we can tell by img so descrip may need to be required here.
		# in this case we know the queen is not loft bc loft is not in the raw type but in the raw description it says loft bed
		# could check if twin and queen, and then add bundle of the 2
		# most reliable to say if no loft in raw type then not loft but that requires adding parameter
		# although less reliable, for simplicity check descrip for optional bed underneath to confirm not referring to size of loft bed
		# if only colors or other options, no bundle

		# most reliable to draw from raw type bc we can bundle the two together whether or not referring to underbed
		# raw types are loft bed and queen bed
		# component option vals would be loft bed, queen bed, loft+queen

		#intro_idx = 3
		body_html_idx = 2
		for vrnt in solo_product:
			#vrnt_intro = vrnt[intro_idx]
			vrnt_body_html = vrnt[body_html_idx]
			print("vrnt_body_html: " + vrnt_body_html)
			if re.search('optional.*bed.*under', vrnt_body_html.lower()):
				bundle = True
				print("Bundle Product")
				break

	# for loft bed, if no descrip stating optional bed, check if options size twin and queen

	return bundle

# if we do not find a standard color, do we still want to use the custom color as a tag?
# yes bc need to see color else no color no filter. plus it will show which custom colors could be grouped into a standard color by new keyword
# problem is we need to minimize filter list to only standard colors so we cannot allow custom colors to make tags.
# instead pass warning so we can correct it by adding new keyword to colors.json
def determine_standard_color(color):
	#print("\n===Determine Standard Color===\n")
	standard_color = ''
	standard_colors = reader.read_standards('colors')
	#print("standard_colors: " + str(standard_colors))
	for std_color, keywords in standard_colors.items():
		#print("std_color: " + std_color)
		for keyword in keywords:
			if re.search(keyword, color.lower()):
				standard_color = std_color
				break # found keyword so no need to see more
		if standard_color != '': # standard color would only be set if we determined color so no need to see more
			break

	if standard_color == '':
		print("Warning: could not find standard color for color " + color + "! ")

	return standard_color

# need value type to read standards file by name
def determine_standard(init_value, value_type):
	#print("\n===Determine Standard for value: " + init_value + "===\n")
	standard = ''
	standards = reader.read_standards(value_type)
	#print("standards: " + str(standards))
	for std, keywords in standards.items():
		#print("std: " + std)
		for keyword in keywords:
			if re.search(keyword, init_value.lower()):
				standard = std
				break # found keyword so no need to see more
		if standard != '': # standard would only be set if we determined value so no need to see more
			break

	if standard == '':
		print("Warning: could not find standard for value " + init_value + "! ")

	return standard

# if we see duplicate opts bt 2 vrnts in same product this will cause error so show warning and handle
def determine_duplicate_opts(product):
	print("\n===Determine Duplicate Options===\n")
	# product = [[1,2,3...],[1,2,3...],...]
	#print("product: " + str(product))

	duplicate_opts = []

	vrnt_opt1_name_idx = 8
	vrnt_opt1_val_idx = 9
	vrnt_opt2_name_idx = 10
	vrnt_opt2_val_idx = 11
	vrnt_opt3_name_idx = 12
	vrnt_opt3_val_idx = 13

	option_strings = []
	option_data = []
	for vrnt in product:

		vrnt_opt_string = vrnt[vrnt_opt1_name_idx] + vrnt[vrnt_opt1_val_idx] + vrnt[vrnt_opt2_name_idx] + vrnt[vrnt_opt2_val_idx] + vrnt[vrnt_opt3_name_idx] + vrnt[vrnt_opt3_val_idx]
		option_strings.append(vrnt_opt_string)

		opt_names = [vrnt[vrnt_opt1_name_idx], vrnt[vrnt_opt2_name_idx], vrnt[vrnt_opt3_name_idx]]
		opt_vals = [vrnt[vrnt_opt1_val_idx], vrnt[vrnt_opt2_val_idx], vrnt[vrnt_opt3_val_idx]]
		option_data.append([opt_names,opt_vals])

	# if we find matching options bt 2 vrnts in same product then it will be considered invalid unless we use other info to create more options bc they are 2 vrnts so must have different options but the info in the description is limited
	for option_string in option_strings:
		count = option_strings.count(option_string)
		if count > 1:
			# found 2 vrnts with same options so not enough info in single vrnt so we must compare with another variant to see what options to add
			print("Duplicate Opts")
			duplicate_opts = option_data
			break

	print("duplicate_opts: " + str(duplicate_opts))
	return duplicate_opts

def determine_duplicate_product_opts(product_opt_data): # [[[names],[values]]]
	print("\n===Determine Duplicate Product Options===\n")
	# product_opt_data = [[names],[vals]]
	#print("product_opt_data: " + str(product_opt_data))

	duplicate_opts = []

	option_strings = []
	option_data = []
	for vrnt_opt_data in product_opt_data:

		opt_names = vrnt_opt_data[0] #[vrnt[vrnt_opt1_name_idx], vrnt[vrnt_opt2_name_idx], vrnt[vrnt_opt3_name_idx]]
		opt_vals = vrnt_opt_data[1] #[vrnt[vrnt_opt1_val_idx], vrnt[vrnt_opt2_val_idx], vrnt[vrnt_opt3_val_idx]]
		vrnt_opt_string = ''
		for opt_idx in range(len(opt_names)):
			vrnt_opt_string += opt_names[opt_idx] + opt_vals[opt_idx]
		option_strings.append(vrnt_opt_string)

		option_data.append([opt_names,opt_vals])

	# if we find matching options bt 2 vrnts in same product then it will be considered invalid unless we use other info to create more options bc they are 2 vrnts so must have different options but the info in the description is limited
	print("All Option Strings")
	for option_string in option_strings:
		print(option_string)
	print("Search for Duplicate Option Strings")
	for option_string in option_strings:
		print(option_string)
		count = option_strings.count(option_string)
		if count > 1:
			# found 2 vrnts with same options so not enough info in single vrnt so we must compare with another variant to see what options to add
			print("Duplicate Opts for " + option_string)
			duplicate_opts = option_data
			break

	print("duplicate_opts: " + str(duplicate_opts))
	return duplicate_opts

# determine no. sizes by given different dims
def determine_single_size(product):
	single_size = True

	width_idx = 7
	depth_idx = 8
	height_idx = 9
	widths = []
	depths = []
	heights = []
	for vrnt in product:
		width = vrnt[width_idx]
		depth = vrnt[depth_idx]
		height = vrnt[height_idx]

		widths.append(width)
		depths.append(depth)
		heights.append(height)

	# if any dims are different then we have multiple sizes so not single size
	dims = [widths, depths, heights]
	for dim in dims:
		# if all same check next dims for differ
		# if all dims same then single size = true
		for val in dim:
			if dim.count(val) != len(dim): # if any dims different then single size = false
				single_size = False
				break

		if single_size == False:
			break

	return single_size

def determine_dimensionful_opts(product_options):
	# if no valid opt vals but different dims, then show only largest dim
	#print("\n===Determine Dimensionful Options===\n")
	opt_string = ''
	dimensionful_opts = True # false
	# for vrnt_opts in product_options:
	# 	print("vrnt_opts: " + str(vrnt_opts))
		#opt_string += opt_val
		# check the first vrnt to see if dimensionful opts
		# print("vrnt1: " + str(vrnts[0]))
		# vrnt1_opts = vrnts[0][1]
		# print("vrnt1_opts: " + str(vrnt1_opts))

	return dimensionful_opts


from __future__ import print_function

import logging

import click
import tqdm

from .. import core
from .. import fs_to_csv
from .. import tools

from . import shared

log = logging.getLogger(__name__)

@shared.git_log_args
@click.option('--output', '-o', type=click.File(mode='w'), default='-',
              help="Output destination (default is stdout)")
@click.option('--force', '-f', is_flag=True, help="Force refresh of cache")
@click.option('--filter', '-F', 'filters', multiple=True,
              help=('Report only for changes to files matching this artifact. '
                    'Default is to show just stats changes for entire tree'))
@click.pass_context
def command(ctx, rev_range=None, path=None, **options):
    stats_fs = ctx.obj['fs_class'](ctx.obj['cache_dir'], init=options['force'])

    total, source_commits = tools.tee_count(
        core.generate_shas_for_source_range(ctx.obj['repo'], rev_range))

    fs_commits = tqdm.tqdm(
        core.generate_stats_fs_commits_for_source_commits(
            stats_fs, ctx.obj['repo'], source_commits,
            paths=[path] if path else None, **ctx.obj['cache_kwargs']),
        unit='commit', total=total, unit_scale=True)

    filters = options['filters'] or ['^SUBTREE_TOTAL']
    log.info("Generating csv for range '{}'.".format(rev_range) or 'all')

    fs_to_csv.fs_commits_to_csv(
        stats_fs, fs_commits, options['output'],
        test_patterns=ctx.obj['test_patterns'], filters=filters)


from PIL import Image
import sys
import os

pic_name = sys.argv[1]
json_name = pic_name.split('.')[0]

img_path = os.getcwd() + '/pic/' + pic_name

output_file = open(os.getcwd() + '/pic-json/' + json_name + '.json',"w")

img = Image.open(img_path)
img_rgb = img.convert('RGB')

tuple_value = (0,0,0)
hex_color = ""
r_hex = ""
g_hex = ""
b_hex = ""

output_file.write("[")
for i in range(img_rgb.size[0]):
    for j in range(img_rgb.size[1]):
        tuple_value = img_rgb.getpixel((i,j))
        r_hex = hex(tuple_value[0])[2:]
        if len(r_hex) == 1 :
            r_hex = "0" + r_hex
        g_hex = hex(tuple_value[1])[2:]
        if len(g_hex) == 1 :
            g_hex = "0" + g_hex
        b_hex = hex(tuple_value[2])[2:]
        if len(b_hex) == 1 :
            b_hex = "0" + b_hex
        hex_color = r_hex + g_hex + b_hex
        if((i == img_rgb.size[0] - 1) and (j == img_rgb.size[1] - 1)):
            output_file.write('{"x":' + str(i) + ',"y":' + str(j) + ',"hex":"' + hex_color + '"}')
        else:
            output_file.write('{"x":' + str(i) + ',"y":' + str(j) + ',"hex":"' + hex_color + '"},')
output_file.write("]")

output_file.close()


from confluent_kafka import Producer
from confluent_kafka.serialization import StringSerializer, SerializationContext, MessageField
from confluent_kafka.schema_registry import SchemaRegistryClient
from confluent_kafka.schema_registry.avro import AvroSerializer
from Config import config, srConfig, rootDirectory
from Module.ClassStock import Stock, stockToDict
from Module.ClassCompany import Company, companyToDict
from datetime import datetime
import os, json, random, time

def deliveryReport(err, event):
    if err is not None:
        print(f"Error ID: {event.key().decode('utf8')}: {err}")
    else:
        print(f"Stored ID: {event.key().decode('utf8')} to Topic: {event.topic()}")

def sendProducer(topic, object, producer, avroSerializer):
    producer.produce(topic=topic,
                     key=StringSerializer('utf_8')(str(object.id)),
                     value=avroSerializer(object, SerializationContext(topic, MessageField.VALUE)),
                     on_delivery=deliveryReport)

if __name__ == '__main__':
    topic1 = 'streaming.goapi.idx.stock.json'
    topic2 = 'streaming.goapi.idx.companies.json'
    schema1 = 'IDX-Stock'
    schema2 = 'IDX-Company'
    jsonFile = 'kafka.ksql-join-stock-company 2023-07-28.json'

    string_serializer = StringSerializer('utf_8')
    schema_registry_client = SchemaRegistryClient(srConfig)

    stockSchema = schema_registry_client.get_latest_version('IDX-Stock')
    stockAvroSerializer = AvroSerializer(schema_registry_client,
                                         stockSchema.schema.schema_str,
                                         stockToDict)
    companySchema = schema_registry_client.get_latest_version('IDX-Company')
    companyAvroSerializer = AvroSerializer(schema_registry_client,
                                           companySchema.schema.schema_str,
                                           companyToDict)
    # Read Dummy Input
    jsonPath = os.path.join(rootDirectory, 'Resource', jsonFile)

    # Opening JSON file
    file = open(jsonPath)

    # returns JSON object as a dictionary
    jsonData = json.load(file)

    producer = Producer(config)
    while True:
        try:
            for row in jsonData:
                close = random.uniform(row['low'], row['high'])  # For testing purpose
                stock = Stock(row['id'], row['ticker'], row['date'], row['open'], row['high'], row['low'], close,
                              row['volume'])
                company = Company(row['ticker'], row['ticker'], row['name'], row['logo'],
                                  str(datetime.today().strftime('%Y-%m-%d %H:%M:%S')))
                print(f"id:{row['id']} high:{row['high']} open:{row['open']} low:{row['low']} close:{close} "
                      f"name:{row['name']} logo:{row['logo']}")
                sendProducer(topic1, stock, producer, stockAvroSerializer)
                sendProducer(topic2, company, producer, companyAvroSerializer)
            time.sleep(5)
            # Save and flush
            producer.flush()
        except KeyboardInterrupt:
            print("Shutdown Starting...")
            break


class Student:
    def setValue(self,name,roll):
        self.name=name
        self.roll=roll
    def printVal(self):
        print(self.name)
        print(self.roll)
    def __str__(self):  #to display object
        return self.name
obj=Student()
obj.setValue("vjay",105)
ob2=Student()
ob2.setValue("ajay",105)
ob2.printVal()
print(obj)

'''o/p
(w/o def__str)
ajay
105
<__main__.Student object at 0x7fb6802788d0>

(with def__str__)
ajay
105
vjay'''

object oriented programmimg/instance variable.py


dados = dict()
pessoas = []
mulheres = []
acima = []
cont = somaid = 0
while True:
    dados['nome'] = str(input('Nome: '))
    while True:
        dados['sexo'] = str(input('Sexo [F/M]: '))
        if dados['sexo'] in 'FfmM':
            break
        else:
            print('Ops. Dados inválidos!!', end=' ')
    dados['idade'] = int(input('Idade: '))
    somaid += dados['idade']
    pessoas.append(dados)
    dados = dict()
    cont += 1
    while True:
        a = str(input('Quer continuar ?[S/N] '))
        if a not in 'sSnN':
            print('Ops. Dados inválidos!!', end=' ')
        else:
            break
    if a in 'nN':
        break
print(pessoas)
print(f'Foram adicionadas {cont} pessoas.')
print(f'A média de idade das pessoas adicionadas é de {somaid/cont} anos.')
for p in pessoas:
    if p['sexo'] in 'fF':
        mulheres.append(p['nome'])
    if p['idade'] > somaid/cont:
        acima.append(p['nome'])
print(f'As mulheres cadastradas foram {mulheres}.')
print(f'As pessoas acima da média são {acima}')


import math, numpy
import cluster, movie_lens_data, my_util

class UserProfile(my_util.Model):
    """Model for a single user.
    ::
        User profile:
            tag_profile - {tag_id: score}
            genre_profile - {genre_id: score}
            x_factors0 - 3 variable vector for movies with tags
            x_factors1 - 2 variable vector for movies without tags

        Movie data:
            _movie_tags - {movie_id: {tag id: ln(count) + 1} }
            _movie_genres - {movie id : set of genre ids}
            _movie_medians - {movie id : movie median rating}
    """

    def __init__(self, movie_genres, movie_ratings, movie_tags,
                 tag_counts, genre_counts, movie_medians):
        """
        :param movie_genres: {movie id : set of genre ids}
        :param movie_ratings: [(movie id, rating)]
        :param movie_tags: {movie_id: {tag_id: ln(count) + 1} }
        :param tag_counts: {tag_id: count}
        :param genre_counts: {genre id : genre count}
        :param movie_medians: {movie id : movie median rating}
        """
        self._movie_tags = movie_tags
        self._movie_genres = movie_genres
        self._movie_medians = movie_medians

        self.tag_profile = self._compute_tag_profile(movie_ratings, movie_tags,
                                                     tag_counts)
        self.genre_profile = self._compute_genre_profile(movie_ratings,
                                                         movie_genres, genre_counts)
        self.x_factors0, self.x_factors1 = \
            self._compute_x_factors(movie_ratings, movie_tags,
                                    movie_genres, movie_medians)

    def _compute_tag_profile(self, movie_ratings, movie_tags, tag_counts):
        """Returns a tag_profile object.

        :param movie_ratings: [(movie id, rating)]
        :param movie_tags: {movie_id: {tag_id: ln(count) + 1}}
        :param tag_counts: {tag_id: count}
        :return: {tag_id: score}
        """
        tag_profile = {} # {tag_id: score}

        # for each movie review, add to "tag_profile"
        for movie_id, rating in movie_ratings:
            offset = rating - 3

            # it's possible for a movie to be untagged
            if (offset != 0) and (movie_id in movie_tags):
                for tag_id in movie_tags[movie_id]:
                    if tag_id not in tag_profile:
                        tag_profile[tag_id] = 0

                    tag_profile[tag_id] += offset * movie_tags[movie_id][tag_id]

        # normalize the values in "tag_profile"
        for tag_id in tag_profile:
            tag_profile[tag_id] /= tag_counts[tag_id]

        return tag_profile

    def _compute_genre_profile(self, movie_ratings, movie_genres, genre_counts):
        """Returns a genre_profile object.

        :param movie_ratings: [(movie id, rating)]
        :param movie_genres: {movie_id: set of genre ids}
        :param genre_counts: {genre_id: genre count}
        :return: {genre_id: score}
        """
        genre_profile = {} # {genre_id: score}

        # for each movie review, add to "genre_profile"
        for movie_id, rating in movie_ratings:
            offset = rating - 3

            # it's possible for a movie to have no genre classified
            if (offset != 0) and (movie_id in movie_genres):
                for genre_id in movie_genres[movie_id]:
                    if genre_id not in genre_profile:
                        genre_profile[genre_id] = 0

                    genre_profile[genre_id] += offset

        # normalize the values in "genre_profile"
        for genre_id in genre_profile.keys():
            genre_profile[genre_id] /= genre_counts[genre_id]

        return genre_profile

    def _dot_product_with_movie_genre_ids(self, movie_genre_ids):
        """
        :param movie_genre_ids: a set of genre ids
        """
        dot_product = 0
        genre_profile = self.genre_profile

        for genre_id in movie_genre_ids:
            if genre_id in genre_profile:
                dot_product += genre_profile[genre_id]

        return dot_product

    def _dot_product_with_movie_tag_scores(self, movie_tag_scores):
        """
        :param movie_tag_scores: {tag_id: score}
        """
        dot_product = 0
        tag_profile = self.tag_profile

        for tag_id in movie_tag_scores:
            if tag_id in tag_profile:
                dot_product += tag_profile[tag_id] * movie_tag_scores[tag_id]

        return dot_product

    def _compute_x_factors(self, movie_ratings, movie_tags,
                           movie_genres, movie_medians):
        """ Fit this user to the Ax = b model using least squares.

        :param movie_ratings: [(movie id, rating)]
        :param movie_tags: {movie_id: {tag_id: ln(count) + 1} }
        :param movie_genres: {movie_id: set of genre ids}
        :param movie_medians: {movie_id : movie median rating}
        :return: x_factor0, x_factor1 - x_factor0 has 3
            variables, x_factor1 has two variables
        """
        # model 0: A0 * x = b0 - when tags are present
        # model 1: A1 * x = b1 - when tags are not present

        # build up "A" and "b" in "Ax" = "b"
        A0 = []
        b0 = []
        A1 = []
        b1 = []

        for movie_id, rating in movie_ratings:
            # each movie rating is a single row
            # the b value is always the same
            b = rating - movie_medians[movie_id]

            # tag and genre dot products
            tag_dot_product = 0
            genre_dot_product = 0

            if movie_id in movie_tags.keys():
                tag_dot_product = self._dot_product_with_movie_tag_scores(movie_tags[movie_id])

            if movie_id in movie_genres.keys():
                genre_dot_product = self._dot_product_with_movie_genre_ids(movie_genres[movie_id])

            if math.fabs(tag_dot_product > 0.1):
                # tags are present
                A0.append([tag_dot_product, genre_dot_product, 1.0])
                b0.append([b])

            # Always contribute data to the alternative model, where
            # "tag_dot_product" is not needed
            A1.append([genre_dot_product, 1.0])
            b1.append([b])

            # An alternative is to not always contribute data to A1. The
            # problem here is that if the user reviews very few movies,
            # then the lack of A1 means no "x_factors1", which then
            # means there is no fall back model for movies that have
            # no tag information.

        # least square solutions
        x_factors0 = None
        if len(A0) >= 3:
            result = numpy.linalg.lstsq(A0, b0, rcond=None)
            x_factors0 = result[0]

        x_factors1 = None
        if len(A1) >= 2:
            result = numpy.linalg.lstsq(A1, b1, rcond=None)
            x_factors1 = result[0]

        return x_factors0, x_factors1

    def predict(self, movie_id):
        """ Predicts a rating for the given movie_id. Returns None if
        no prediction can be made.
        """
        # movie median is a required value
        if movie_id not in self._movie_medians:
            return None

        movie_median = self._movie_medians[movie_id]

        # compute "tag_product" and "genre_product"
        movie_tag_scores = None # {tag_id: movie tag score}
        movie_genre_ids = None # a set of genre ids

        if movie_id in self._movie_tags:
            movie_tag_scores = self._movie_tags[movie_id]

        if movie_id in self._movie_genres:
            movie_genre_ids = self._movie_genres[movie_id]

        tag_product = 0
        genre_product = 0

        if movie_tag_scores is not None:
            tag_product = self._dot_product_with_movie_tag_scores(movie_tag_scores)

        if movie_genre_ids is not None:
            genre_product = self._dot_product_with_movie_genre_ids(movie_genre_ids)

        # preferred situation: use model 0
        x = self.x_factors0
        A = [tag_product, genre_product, 1.0]

        # fall back situation: no tag, or if x_factors0 is None
        if (tag_product < 1e-6) or x is None:
            x = self.x_factors1
            A = [genre_product, 1.0]

        # it's possible that x is still None
        if x is None:
            return movie_median # last fall back

        # standard model: Ax + movie_median
        A = numpy.array([A])
        return A.dot(x)[0][0] + movie_median

def main():
    length = movie_lens_data.get_input_obj("user_ratings_test_length")

    # Sends a "tag_count_eval" command to cluster nodes.
    if cluster.cluster_info is None:
        print("Cannot connect to cluster.")
        return

    cluster.send_command({
        "op": "distribute",
        "worker_op": "tag_count_eval",
        "length": length
    })

    cluster.wait_for_completion()
    cluster.print_status()
    print()

    file_name = "tag_count_eval_results.bin"
    results = cluster.merge_list_results(file_name)
    user_ids, agreements = zip(*results)

    my_util.print_rank_agreement_results(agreements, "tag counting")
    print('\a')

if __name__ == "__main__":
    main()


def insertion_sort(my_list):
    """
    • Always start with the second item in the list.
	• Compare it to the previous item.
	• If it's less, then swap.
    • Move to the next item (and compare it to the previous ones (all the way back)
        until its' not less than the previous - hence it's in the right spot.
    """
    for i in range(1, len(my_list)):
        temp = my_list[i]
        j = i - 1
        while temp < my_list[j] and j > -1:
            # Slide it backward
            my_list[j+1] = my_list[j]
            my_list[j] = temp
            j -= 1
    return my_list

if __name__ == '__main__':
    my_list = [4,2,6,5,1,3]
    print(insertion_sort(my_list))


import os

class TrafficControlError(Exception):
    pass

class TrafficControl(object):
    def __init__(self, interface):
        """
        Class constructor.

        :param interface: Name of traffic controller interface
        """

        self.interface = interface

    def tc(self, command, ignore_fails=False):
        """
        Executes a traffic control command.
        """

        command = 'tc %s' % (command)
        if ignore_fails:
            command += ' 2>/dev/null'
        if os.system(command) != 0 and not ignore_fails:
            raise TrafficControlError

    def reset(self):
        """
        Clears all existing traffic control rules.
        """

        self.tc('qdisc del dev %s root handle 1: htb default 0' % self.interface, ignore_fails=True)
        self.tc('qdisc add dev %s root handle 1: htb default 1' % self.interface)

    def set_fixed_bandwidth(self, bandwidth):
        """
        Configures a fixed bandwidth class for this interface.

        :param bandwidth: Bandwidth limit in kbps
        """

        self.tc('class add dev %s parent 1: classid 1:1 htb rate %dkbit ceil %dkbit' % (
            self.interface, bandwidth, bandwidth
        ))
        self.tc('qdisc add dev %s parent 1:1 fq_codel' % (
            self.interface
        ), True)


## Exercícios propostos no site: https://www.ime.usp.br/~macmulti/exercicios/inteiros/index.html
## Dado um sequencia de números inteiros não-nulos, seguida por 0, impirmir seus quadrados.

def main():
    print("Calculo dos quadrados de uma sequência de numeros!")
    print("Para finalizar digite 0 (zero).\n")

    num = int(input("Digite um número: "))
    while num != 0:
        quadrado = num * num
        print("O número ", num, "ao quadrado é", quadrado)
        num = int(input("Digite um numero: "))
main()


# 唯一性，不可用下标访问
List = list((1, 2, 3, 4, 4, 2, 0))
Set = set(List)
print(Set)                #自动排序
Set.add(6)
print(Set)
Set.remove(4)
print(Set)

Set = frozenset([1, 2, 3])  #不可变集合


import sys

DAY = '2'
PUZ = 'b'
infile_name = "./dat/day" + DAY + "-" + "a" + "-in.dat"
outfile_name = "./res/day" + DAY + "-" + PUZ + "-out.dat"

infile = open(infile_name, 'r')
#BEGIN:

result = 0
for line in infile:
	nums = line.split()
	numerator = 0
	denominator = 0  # Don't try this at home
	for x in range(len(nums)):
		for y in range(len(nums)):
			if x == y:
				continue
			if int(nums[x]) % int(nums[y]) == 0:
				numerator = nums[x]
				denominator = nums[y]

	result += (int(numerator) / int(denominator))

outfile = open(outfile_name, 'w')
outfile.write(str(int(result)))
outfile.close()


import pytorch_lightning as pl
import torch
from xmuda.models.modules import Net2DFeat, Net3DFeat, FuseNet
from xmuda.models.LMSCNet import LMSCNet
from xmuda.common.utils.metrics import Metrics
import pickle
import numpy as np
import time
import os.path as osp

class RecNetLMSC(pl.LightningModule):
    def __init__(self, preprocess_dir):
        super().__init__()
        self.class_frequencies = np.array([5.41773033e+09, 1.57835390e+07, 1.25136000e+05, 1.18809000e+05,
                                           6.46799000e+05, 8.21951000e+05, 2.62978000e+05, 2.83696000e+05,
                                           2.04750000e+05, 6.16887030e+07, 4.50296100e+06, 4.48836500e+07,
                                           2.26992300e+06, 5.68402180e+07, 1.57196520e+07, 1.58442623e+08,
                                           2.06162300e+06, 3.69705220e+07, 1.15198800e+06, 3.34146000e+05])
        self.class_num = 20
        self.lmscnet = LMSCNet(
            class_num=self.class_num,
            class_frequencies=self.class_frequencies)

        with open(osp.join(preprocess_dir, "visible_voxels.pkl"), 'rb') as f:
            self.invisible_voxels = pickle.load(f)

        self.train_metrics = Metrics(self.class_num)
        self.val_metrics = Metrics(self.class_num)
        self.train_metrics_visible = Metrics(self.class_num)
        self.val_metrics_visible = Metrics(self.class_num)
        # tensorboard = self.logger.experiment

    def forward(self, batch):
        occupancy = batch['voxel_occupancy'].cuda()
        # n_points_3d = batch['n_points_3d']
#        img = batch['img']
#        bs = img.shape[0]
#        coords_3d = batch['coords_3d']
#        occupancy = torch.zeros(bs, 256, 256, 32, device=self.device)
#        # print(coords_3d.shape)
#        # prev = 0
#        for i in range(bs):
#            idx = coords_3d[:, 3] == i
#            b_coords = coords_3d[idx]
#            occupancy[i, b_coords[:, 0], b_coords[:, 1], b_coords[:, 2]] = 1.0
#            # prev = n_point
#        occupancy = occupancy.transpose(2, 3)

        out = self.lmscnet(occupancy)
        return out

    def training_step(self, batch, batch_idx):

        pred = self(batch)

        target = batch['ssc_label_1_4'].cuda()
        loss = self.lmscnet.compute_loss(pred, target)

        self.train_metrics.add_batch(prediction=pred, target=target)
        self.train_metrics_visible.add_batch(prediction=pred,
                                             target=target,
                                             scenes=batch['scene'],
                                             invisible_data_dict=self.invisible_voxels)

        self.log('train/loss', loss.item())
        self.log('train/loss_ssc', loss.item())
        for metrics, suffix in [(self.train_metrics, ""), (self.train_metrics_visible, "_visible")]:

            self.log("train/mIoU" + suffix,
                     metrics.get_semantics_mIoU().item())
            self.log("train/IoU" + suffix, metrics.get_occupancy_IoU().item())
            self.log("train/Precision" + suffix,
                     metrics.get_occupancy_Precision().item())
            self.log("train/Recall" + suffix,
                     metrics.get_occupancy_Recall().item())
            self.log("train/F1" + suffix, metrics.get_occupancy_F1().item())
            metrics.reset_evaluator()

        return loss

    def validation_step(self, batch, batch_idx):
        pred = self(batch)

        target = batch['ssc_label_1_4'].cuda()
        loss = self.lmscnet.compute_loss(pred, target)

        # loss = self.bce_logits_loss(logits, occ_labels)
        self.log('val/loss', loss.item())
        self.log('val/loss_ssc', loss.item())

        self.val_metrics.add_batch(prediction=pred, target=target)
        self.val_metrics_visible.add_batch(prediction=pred,
                                           target=target,
                                           scenes=batch['scene'],
                                           invisible_data_dict=self.invisible_voxels)

        # pred_occ_labels = (torch.sigmoid(logits) > 0.5).float()
        # acc = (pred_occ_labels == occ_labels).float().mean()
        # self.log('train/acc', acc.item())

    def validation_epoch_end(self, outputs):
        for metrics, suffix in [(self.val_metrics, ""), (self.val_metrics_visible, "_visible")]:
            self.log("val/mIoU" + suffix,
                     metrics.get_semantics_mIoU().item())
            self.log("val/IoU" + suffix, metrics.get_occupancy_IoU().item())
            self.log("val/Precision" + suffix,
                     metrics.get_occupancy_Precision().item())
            self.log("val/Recall" + suffix,
                     metrics.get_occupancy_Recall().item())
            self.log("val/F1" + suffix, metrics.get_occupancy_F1().item())
            metrics.reset_evaluator()

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=1e-4)
        return optimizer


import numpy as np
from .correlated import correlated,correlated_ts
from .mc import mc
from .AddPaths import delta,x_hat,Sfunc
import time
import pandas as pd
from numba import njit,jit_module
from numba import prange,objmode
import sys
from datetime import datetime
import matplotlib.pyplot as plt
from scipy.stats import kurtosis

# @njit(nogil=True)
def select_levels(t0,T,M_t,eps,r,F,strategy=1,N=100,boundary=None):
    if strategy==1:
        dt_0 = np.minimum((eps)**2/r(np.array([0])),T-t0)
        levels = dt_0/M_t**np.arange(0,2)
        N_out=np.zeros(2,dtype=np.int64)
        N_diff=np.ones(2,dtype=np.int64)*N
        SS_out = np.zeros(2);C_out = np.zeros(2); E_out=np.zeros(2)
    else:
        dt_1 = np.minimum(eps**2/r(np.array([0])),T-t0)
        levels = np.append(np.array([T-t0]),dt_1/M_t**np.arange(0,2))
        N_out=np.zeros(3,dtype=np.int64)
        N_diff=np.ones(3,dtype=np.int64)*N
        SS_out = np.zeros(3);C_out = np.zeros(3); E_out=np.zeros(3)
    return levels,N_out,N_diff,E_out,SS_out,C_out

def rnd1(x, decimals, out):
    return np.round_(x, decimals, out)#.astype(np.int64)

# @njit(nogil=True,parallel=True)
def update_paths(I,E,SS,C,N,N_diff,levels,t0,T,M_t,eps,Q,M,r,F,boundary,strategy=1,rev=False,diff=False,v_ms=1,std=False,update=True):
    cores = 1 if update else 8
    n = np.maximum(2,rnd1(N_diff/cores,0,np.empty_like(N_diff/cores)).astype(np.int64))
    for j in range(len(I)):
        i=I[j]
        dt_f = levels[i]
        for k in prange(cores):
            if i!=0:
                if strategy==1 or i!=1:
                    with objmode(start1 = 'f8'):
                        start1 = time.perf_counter()
                    if rev:
                        x_f,x_c = correlated_ts(dt_f,M_t,t0,T,eps,n[i],Q,M,r,boundary=boundary,strategy=strategy,diff=diff,std=std)
                    else:
                        x_f,x_c = correlated(dt_f,M_t,t0,T,eps,n[i],Q,M,r,boundary=boundary,strategy=strategy,diff=diff)
                    with objmode(end1 = 'f8'):
                        end1 = time.perf_counter()
                else:
                    with objmode(start1 = 'f8'):
                        start1 = time.perf_counter()
                    x_f,x_c = correlated(dt_f,round(levels[0]/dt_f),t0,T,eps,n[i],Q,M,r,boundary=boundary,strategy=strategy,diff=diff,v_ms=v_ms)
                    with objmode(end1 = 'f8'):
                        end1 = time.perf_counter()
                est_f = F(x_f);est_c=F(x_c)
                C_temp = (end1-start1)/n[i]
                E_temp = np.mean(est_f-est_c)
                SS_temp = np.sum((est_f-est_c-E_temp)**2)
            else:
                with objmode(start2 = 'f8'):
                    start2 = time.perf_counter()
                x = mc(dt_f,t0,T,n[i],eps,Q,M,r,boundary=boundary,rev=rev,diff=diff,v_ms=v_ms)
                with objmode(end2 = 'f8'):
                    end2 = time.perf_counter()
                est = F(x)
                C_temp = (end2-start2)/n[i]
                E_temp = np.mean(est)
                SS_temp = np.sum((est-E_temp)**2)
            Delta = delta(E[i],E_temp,N[i],n[i])
            E[i] = x_hat(N[i],n[i],E[i],E_temp,Delta)
            if update:
                SS[i] = Sfunc(N[i],n[i],SS[i],SS_temp,Delta)
                '''Update cost like updating an average'''
                C[i] = x_hat(N[i],n[i],C[i],C_temp,delta(C[i],C_temp,N[i],n[i]))
            N[i] = N[i] + n[i]
    return E,SS,N,C

# @njit(nogil=True)
def ml(e2,Q,t0,T,M_t,eps,M,r,F,N_warm=40,boundary=None,strategy=1,alpha=None,beta=None,gamma=None,rev=False,diff=False,v_ms=1,std=False):
    '''
    e2: bound on mean square error
    Q: initial distribution
    t0: starting time
    T: end time
    M_t: dt_c/dt_f, where dt_c is coarse step size and dt_f is fine step size
    eps: mean free path, i.e. epsilon in model
    M: equilibrium distribution for velocity
    r: collision rate
    F: function used to find quantity of interest, E(F(X,V)).
    '''
    levels,N,N_diff,E,SS,C = select_levels(t0,T,M_t,eps,r,F,N=N_warm,strategy=strategy)
    '''
    levels: a list of step sizes for each level
    N: list of number of paths used
    N_diff: list of number of paths needed at every level
    E: estimation of quantity of interest at each level
    SS: sum of squares at each level
    C: Cost at each level
    '''
    L = len(levels)-1
    '''While loop to continue until RMSE fits with e2'''
    while True:
        '''Update paths based on N_diff'''
        while np.max(N_diff)>0:
            I = np.where(N_diff > 0)[0] #Index for Levels that need more paths
            N_diff = np.minimum(N_diff,np.ones(len(N_diff),dtype=np.int64)*400_000)
            # print(f'index where more paths are needed: {I}, N_diff: {N_diff}')
            first = np.sum(N)==0 or (beta is None and gamma is None)
            E,SS,N,C = update_paths(I,E,SS,C,N,N_diff,levels,t0,T,M_t,eps,Q,M,r,F,boundary,strategy,rev=rev,diff=diff,v_ms=v_ms,std=std,update=first)
            if first: V = SS/(N-1) #Update variance
            '''Determine number of paths needed with new information'''
            N_diff = np.ceil(2/e2*np.sqrt(V/C)*np.sum(np.sqrt(V*C))).astype(np.int64) - N
        '''Test bias is below e2/2 - to do so it is necessary to know exponent
        in weak error'''
        if (E.size>=4 and strategy==1) or (strategy==3 and E.size>4):
            if alpha is None:
                L1 = max(1,np.where(levels<=eps**2/r(1))[0][0])
                pa = lin_fit(np.arange(L1,L),np.log2(np.abs(E[L1:L+1]))); alpha = -pa[0]
            test = np.max(np.abs(E[-3:]))/(M_t**alpha-1) < np.sqrt(e2/2)
        else:
            test=False
        if test:
            break
        L += 1;
        if L==17:
            print('WARNING: maximum capacity reached!')
            break
        print('Level added')
        if beta is None and gamma is None:
            N_diff = np.append(N_diff,N_warm).astype(np.int64)
            N = np.append(N,0).astype(np.int64)
            E = np.append(E,0.0); V = np.append(V,0.0); C = np.append(C,0.0); SS = np.append(SS,0.0)
        else:
            #Calculate constant from Theorem
            c2 = V[-1]*2**((L-1)*beta)
            V = np.append(V,c2*2**(-L*beta))
            c3 = C[-1]*2**(-(L-1)*gamma)
            C = np.append(C,c3*2**(L*gamma))
            E = np.append(E,0.0);
            N = np.append(N,0).astype(np.int64)
            N_diff = np.ceil(2/e2*np.sqrt(V/C)*np.sum(np.sqrt(V*C))).astype(np.int64) - N
        if strategy==1:
            levels = np.append(levels,levels[0]/(M_t**(L)))
        else:
            levels = np.append(levels,levels[1]/(M_t**(L)))
    return E,V,C,N,levels

def lin_fit(x,y):
    p=1; n = x.size
    X = np.stack((np.ones(n),x),axis=1)
    normal_matrix = (X.T).dot(X)
    moment_matrix = (X.T).dot(y)
    return np.linalg.inv(normal_matrix).dot(moment_matrix)

@njit(nogil=True,parallel=True)
def convergence_tests(N,dt_list,Q,t0,T,M_t,eps,M,r,F,boundary,strategy,rev=False,diff=False,v_ms=1,std=False):
    '''Calculates values for consistency test for each level given by dt_list'''
    cores = 8 #Controls parrelisation

    L = dt_list.size
    b = np.zeros(L) #mean(F(x^f)-F(X^c))
    b2 = np.zeros(L) #mean((F(x^f)-F(X^c))^2)
    b3 = np.zeros(L) #mean((F(x^f)-F(X^c))^3)
    b4 = np.zeros(L) #mean((F(x^f)-F(X^c))^4)
    v = np.zeros(L) #mean(F(X))
    v2 = np.zeros(L) #mean(F(X)^2)
    var1 = np.zeros(L) #var(F(X))
    var2 = np.zeros(L) #var(F(X^f)-F(X^c))
    # kur1 = np.empty(L-1) #kurtosis calculated for biases
    cons = np.zeros(L) #consistency calculated for all biases
    cost1 = np.zeros(L) #cost for each level
    cost2 = np.zeros(L) #cost for each level bias

    if N%cores!=0:
        print('WARNING -  Number of samples is not divisble by 8!\n Please change N for optimal functionality')
    n = round(N/cores)
    # all_diff = np.zeros((N,L))

    for l in range(L):
        print(l)
        diff_vec = np.empty((cores,n))
        val = np.empty((cores,n))
        for j in prange(cores):
            if l<L-1:
                # print(f'l={l}')
                with objmode(start1 = 'f8'):
                    start1 = time.perf_counter()
                if rev:
                    x_f,x_c = correlated_ts(dt_list[l+1],M_t,t0,T,eps,n,Q,M,r,boundary=boundary,diff=diff,std=std)
                else:
                    x_f,x_c = correlated(dt_list[l+1],M_t,t0,T,eps,n,Q,M,r,boundary=boundary,diff=diff,v_ms=v_ms)
                with objmode(end1 = 'f8'):
                    end1 = time.perf_counter()
                cost2[l+1] += (end1-start1)
                diff_vec[j,:] = F(x_f)-F(x_c)
            with objmode(start2 = 'f8'):
                start2 = time.perf_counter()
            x = mc(dt_list[l],t0,T,n,eps,Q,M,r,boundary=boundary,rev=rev,diff=diff,v_ms=v_ms)
            with objmode(end2 = 'f8'):
                end2 = time.perf_counter()
            cost1[l] += (end2-start2)
            val[j,:] = F(x)
        cost1[l] = cost1[l]/N
        v[l] = np.mean(val)
        v2[l] = np.mean(val**2)
        var1[l]  = v2[l] - v[l]**2
        if l<L-1:
            # all_diff[:,l+1] = diff.flatten()
            b[l+1] = np.mean(diff_vec)
            b2[l+1] = np.mean(diff_vec**2)
            b3[l+1] = np.mean(diff_vec**3)
            b4[l+1] = np.mean(diff_vec**4)
            var2[l+1] = b2[l+1]-b[l+1]**2
            cost2[l+1] = cost2[l+1]/N
            # cons[l+1] = np.abs(b[l+1]+v[l]-v[l+1])/(3*(np.sqrt(var2[l+1])+ np.sqrt(var1[l])+np.sqrt(var1[l+1]))/np.sqrt(N))
    cons[1:] = np.abs(b[1:]+v[:-1]-v[1:])/(3*(np.sqrt(var2[1:])+np.sqrt(var1[:-1])+np.sqrt(var1[1:]))/np.sqrt(N))
    kur1 = np.append(0,(b4[1:]-4*b3[1:]*b[1:]+6*b2[1:]*b[1:]**2-3*b[1:]**4)/(b2[1:]-b[1:]**2)**2)
    return b,b2,b3,b4,v,v2,var1,var2,kur1,cons,cost1,cost2#,all_diff

jit_module(nopython=True,nogil=True)

def ml_test(N,N0,dt_list,E2,Q,t0,T,M_t,eps,M,r,F,logfile,boundary=None,strategy=1,convergence=True,complexity=True,rev=False,diff=False,v_ms=1,std=False):
    ''''
    filename for logfile should always begin with 'logfile_APS'

    '''
    save_file= logfile is not None
    L = dt_list.size
    if save_file:
        if logfile.name[0:11]!='logfile_APS':
            sys.exit('ERROR: name of logfile should start with "logfile_APS"')
        now = datetime.now().strftime("%d-%B-%Y %H:%M:%S")
        logfile.write("\n")
        logfile.write("*********************************************************\n")
        logfile.write(f"***Python ml_test for APS method on {now}         ***\n")
        if rev:
            logfile.write(f"***    Reverse splitting is used!!      ***\n")
        if diff:
            logfile.write(f"***    Altered diffusive coefficient is used!!      ***\n")
        logfile.write("\n")
        logfile.write("*********************************************************\n")
        logfile.write("*** Experiemnt setup  ***\n")
        logfile.write("*** S(x,v) = 1/sqrt(2*pi)e^{-v^2/2}(1+cos(2pi*(x+1/2))) ***\n")
        logfile.write(f"*** r(x) = (-a(x-0.5)+b)(x<=0.5) + (a(x-0.5)+b)(x>0.5), with a={(r(np.array([1]))-r(np.array([0])))}, b={r(np.array([0]))} and eps = {eps}  ***\n")
        if 2!=F(2):
            logfile.write(f"*** OBS!! replace this text with specification of quantity of interest F(3) = {F(np.array([3]))} ***\n")
        else:
            logfile.write("*** Quantity of interest is F(X) = X  ***\n")
        if boundary is None:
            logfile.write("*** No boundary conditions  ***\n")
        else:
            logfile.write("*** OBS!! replace this text with specification of boundary conditions  ***\n")
        logfile.write("*********************************************************\n")
        logfile.write("Convergence tests, kurtosis, telescoping sum check \n")
        logfile.write(f"*** using {N} samples and {L} levels ***\n")
        logfile.write(" l dt^f mean(F(X^f)-F(X^c)) mean(F(X^c))  var(F(X^f)-F(X^c)) var(F(X^c))")
        logfile.write(" cost(F(X^f)-F(X^c)) cost(F(X)) kurtosis consistency \n")
        logfile.write("---------------------------------------------------------\n")

    if convergence:
        print('Convergence test')
        b,b2,b3,b4,v,v2,var1,var2,kur1,cons,cost1,cost2 = convergence_tests(N,dt_list,Q,t0,T,M_t,eps,M,r,F,boundary=boundary,strategy=strategy,rev=rev,diff=diff,v_ms=v_ms,std=std)
        print('Convergence test DONE')
        print(kur1)
        if save_file:
            for i in range(dt_list.size):
                logfile.write(f'{i} {dt_list[i]} {b[i]} {v[i]} {var2[i]} {var1[i]} {cost2[i]} {cost1[i]} {kur1[i]} {cons[i]}\n')

        # Linear regression to estimate alpha, beta and gamma. Only test for dt << eps^2
        i = 0
        old = 0
        for va in var2[1:]:
            if va < old:
                break
            i+=1
            old=va
        alpha = -np.polyfit(range(i,var2.size),np.log2(np.abs(b[i:])),1)[0]
        beta = -np.polyfit(range(i,var2.size),np.log2(np.abs(var2[i:])),1)[0]
        gamma = np.polyfit(range(i,var2.size),np.log2(np.abs(cost2[i:])),1)[0]
        print(f'alpha= {alpha}, beta = {beta}, gamma= {gamma}')
        if save_file:
            np.savetxt('resultfile'+logfile.name[7:],(dt_list,v,b,var1,var2,cost1,cost2,kur1,cons))
            logfile.write('\n*********************************************************\n')
            logfile.write('\n*** Linear regression estimates of MLMC paramters ***\n')
            logfile.write(f'\n*** regression is done for levels with dt << eps^2 = {eps**2} ***\n')
            logfile.write('*********************************************************\n')
            logfile.write(f'alpha = {alpha} (exponent for weak convergence) \n')
            logfile.write(f'beta = {beta} (exponent for variance of bias estimate) \n')
            logfile.write(f'gamma = {gamma} (exponent for cost of bias estimate) \n')

    if complexity:
        if save_file:
            logfile.write("\n*********************************************************\n")
            logfile.write("*** MLMC complexity test ***\n")
            logfile.write("*********************************************************\n")
            logfile.write(" e2 value mlmc_cost N_l dt \n")
            logfile.write("---------------------------------------------------------\n")
        data = {}
        pd.set_option('max_columns',None)
        # for e2 in E2:
        #     print(f'MSE= {e2}')
        #     E,V,C,N,levels = ml(e2,Q,t0,T,M_t,eps,M,r,F,N0,boundary=boundary,strategy=strategy,alpha=1,rev=rev,diff=diff)
        #     data[e2] = {'dt':levels,'N_l':N,'E':E,'V_l':V,'V[E]':V/N,'C_l':C,'N_l C_l':N*C}
        #     df = pd.DataFrame(data[e2])
        #     df = df.append(pd.DataFrame({'dt':' ','N_l':' ','E':[np.sum([e for e in data[e2]['E']])],'V_l':' ','V[E]':[np.sum([v for v in data[e2]['V[E]']])],'C_l':[np.sum([c for c in data[e2]['C_l']])],'N_l C_l':[np.sum(n*c for n,c in zip(data[e2]['N_l'],data[e2]['C_l']))]}))
        #     print(df)
        #     if save_file:
        #         name = f'resultfile_complexity_{e2}'+logfile.name[7:]
        #         df.to_csv(name,index=False)
        #         logfile.write(f" {e2} {np.sum(E)} {np.dot(C,N)} {N} {levels} \n")

        if convergence:
            '''Find index where they transition'''
            i = 0
            old = 0
            for va in var2[1:]:
                if va < old:
                    break
                i+=1
                old=va
            # print(f'slope of variance before:{np.polyfit(range(1,i-2),np.log2(np.abs(var2[1:i-2])),1)}')
            # print(f'slope of variance after:{np.polyfit(range(i+2,var2.size),np.log2(np.abs(var2[i+2:])),1)}')
            # print(f'slope of bias before:{np.polyfit(range(1,i-2),np.log2(np.abs(bias[1:i-2])),1)}')
            alpha = -np.polyfit(range(i,var2.size),np.log2(np.abs(b[i:])),1)[0]
            # print(f'slope of bias after:{alpha}')
            # print(f'slope of cost:{np.polyfit(range(i,cost2.size),np.log2(np.abs(cost2[i:])),1)}')
            '''Complexity test based on estimations in convergence test'''
            dfs = {}
            pd.set_option('display.float_format', '{:.2E}'.format)
            for e2 in E2:
                L=4 + i
                E = np.append(v[i],b[i:L])
                V = np.append(var1[i],var2[i:L])
                C = np.append(cost1[i],cost2[i:L])
                N = np.ceil(2/e2*np.sqrt(V/C)*np.sum(np.sqrt(V*C)))
                test = (np.max(np.abs(E[-3:]))/(2**alpha-1) < np.sqrt(e2/2))
                out=False
                while not test:
                    L+=1
                    # print(L)
                    if L>=v.size:
                        out = True
                        print(f'Warning!. Need more results to adhere to bias requirement for e2= {e2}')
                        break
                    N = np.append(N,0)
                    E = np.append(E,b[L])
                    V = np.append(V,var2[L])
                    C = np.append(C,cost2[L])
                    N += np.maximum(0,np.ceil(2/e2*np.sqrt(V/C)*np.sum(np.sqrt(V*C))).astype(np.int64))
                    test = (np.max(np.abs(E[-3:]))/(2**alpha-1) < np.sqrt(e2/2))
                if not out:
                    df = pd.DataFrame({'E':E,'V':V,'V[Y]':V/N,'C':C,'N':N,'N C':C*N,'dt':dt_list[i-1:min(L,v.size-1)]})
                    df = df.append(pd.DataFrame({'E':np.sum(E),'V':' ','V[Y]':[np.sum(V/N)],'C':[np.sum(C)],'N':' ','N C':[np.dot(C,N)],'dt':' '}),ignore_index=True)
                    # print(df)
                    dfs[e2] = df
                    if save_file:
                        name = f'resultfile_complexity_{e2}'+logfile.name[7:]
                        df.to_csv(name,index=False)
                        logfile.write(f" {e2} {np.sum(E)} {np.dot(C,N)} {N} {dt_list[i-1:min(L,v.size-1)]} \n")
                else:
                    break
            '''Plotting number of paths and computational costs as functions of MSE'''
            fig, (ax1, ax2) = plt.subplots(2, 1)
            N = []
            Cost = []
            end = len(dfs)
            start = end-min(end,6)
            for e2 in E2[start:end]:
                N += [dfs[e2]['N'][:-1]]
                Cost += [dfs[e2]['N C'][N[-1].size]]
                # print(Cost)
                ax1.plot(range(N[-1].size),N[-1],label=r'$E^2 = ${:.2E}'.format(e2))
            ax2.loglog(E2[start:end],Cost)
            c4 = np.mean(Cost*E2[start:end]/np.log(E2[start:end]))
            c4 *= 2
            ax2.plot(E2[start:end],c4*np.log(E2[start:end])/E2[start:end],label= r'$\mathcal{O}(\log{E}^2E^{-2})$')
            ax1.set_yscale('log')
            ax2.set_xlabel(r'$E^2$')
            ax2.set_ylabel(r'Total costs')
            ax1.set_xlabel(r'Levels')
            ax1.set_ylabel(r'Paths')
            ax2.legend()
            ax1.legend()
            if save_file:
                filename = f'complexity_results'+logfile.name[7:-3]
                fig.savefig(filename)

        else:
            # Projected cost based on Multilevel Theorem
            cost_e=0; l = 1
            e2 = 1/2**l
            # Wall clock time
            WC = []
            print('Compile')
            # ml(e2,Q,t0,T,M_t,eps,M,r,F,N0,boundary=boundary,strategy=strategy,alpha=1.0,beta=1.0,gamma=1.0,rev=rev,diff=diff)
            print('Compilation done')
            while cost_e<3000:
                print(f'MSE= {e2}')
                start = time.perf_counter()
                E,V,C,N,levels = ml(e2,Q,t0,T,M_t,eps,M,r,F,N0,boundary=boundary,strategy=strategy,alpha=1.0,rev=rev,diff=diff)
                WC += [time.perf_counter()-start]
                data[e2] = {'dt':levels,'N_l':N,'E':E,'V_l':V,'V[E]':V/N,'C_l':C,'N_l C_l':N*C}
                df = pd.DataFrame(data[e2])
                df = df.append(pd.DataFrame({'dt':' ','N_l':' ','E':[np.sum([e for e in data[e2]['E']])],'V_l':' ','V[E]':[np.sum([v for v in data[e2]['V[E]']])],'C_l':[np.sum([c for c in data[e2]['C_l']])],'N_l C_l':[np.sum(n*c for n,c in zip(data[e2]['N_l'],data[e2]['C_l']))]}))
                print(df)
                if save_file:
                    name = f'resultfile_complexity_{e2}'+logfile.name[7:]
                    df.to_csv(name,index=False)
                    logfile.write(f" {e2} {np.sum(E)} {np.dot(C,N)} {N} {levels} \n")
                #Actual simulation cost
                cost_s = WC[-1]#np.dot(C,N)
                #Constant from Theorem
                c4 = c4 = cost_s*e2/(np.log(np.sqrt(e2))**2)
                l+=1
                e2 = 1/2**l
                cost_e = c4*np.log(np.sqrt(e2))**2/e2
            if save_file:
                name = f'wall_clock_time_{e2}'+logfile.name[7:]
                np.savetxt(name,WC)

    if save_file:
        logfile.write('\n')
        logfile.close()

    if convergence:
        plt.figure()
        plt.plot(dt_list[1:],var2[1:],':',label='var(F(x^f)-F(X^c))')
        plt.plot(dt_list,var1,'--',color = plt.gca().lines[-1].get_color(),label='var(F(X))')
        plt.plot(dt_list[1:],np.abs(b[1:]),':',label='mean(|F(x^f)-F(X^c)|)')
        plt.plot(dt_list,v,'--',color = plt.gca().lines[-1].get_color(),label='mean(F(X))')
        # plt.title(f'Plot of variance and bias')
        plt.xscale('log')
        plt.yscale('log')
        plt.xlim(max(dt_list),min(dt_list))
        plt.legend()
        if save_file:plt.savefig(f'var_bias'+logfile.name[7:-3])
        plt.figure()
        plt.plot(range(1,kur1.size),kur1[1:],':')
        plt.title(f'Plot of kurtosis')
        if save_file:plt.savefig(f'kurtosis'+logfile.name[7:-3])
        plt.figure()
        plt.plot(range(1,kur1.size),cons[1:],':')
        plt.title(f'Plot check of consistency')
        if save_file:plt.savefig(f'consistency_check'+logfile.name[7:-3])
        if not save_file:plt.show()


#PAG PRACTICA
import tkinter as tk
import Entrenamiento as e

ventanaP = tk.Tk()

ventanaP.withdraw()
ventanaP.geometry("600x600")

entradaIndex=None

def ocultarBotones():
    papa.place_forget()
    uva.place_forget()
    limon.place_forget()
    guardar.place_forget()
    frutaMaestro.place_forget()

    if(entradaIndex==0):
        maduro.place(x=230, y=100)
        inmaduro.place(x=230, y=300)
        guardarEstado.place(x=500,y=500)
    elif(entradaIndex==1):
        caso()

def caso():
        ventanaP.withdraw()
        e.llamarLimon()
        e.entradaMaduraOInmadura(estadoYfruta())
        e.mostrarTitulo(variable.get())
def mostrarBotones():
   papa.place(x=230, y=100)
   uva.place(x=230, y=200)
   limon.place(x=230, y=300)
   guardar.place(x=500,y=500)
   frutaMaestro.place(x=230,y=400)

   maduro.place_forget()
   inmaduro.place_forget()
   guardarEstado.place_forget()

# Variable de control para el botón
variable = tk.IntVar()
variableEstado =tk.IntVar()

maduro = tk.Button(ventanaP, text="FRUTA MADURA", width=20, height=5,command=lambda: variableEstado.set(0))#valor 0 o sea que es maduro
inmaduro = tk.Button(ventanaP, text="FRUTA NO MADURA", width=20, height=5,command=lambda :variableEstado.set(1))

guardarEstado=tk.Button(ventanaP,text="GUARDAR ESTADO",width=10,height=5,command=caso)

papa = tk.Button(ventanaP, text="PAPA", width=20, height=5,command=lambda : variable.set(0))
uva = tk.Button(ventanaP, text="UVA", width=20, height=5,command=lambda : variable.set(1))
limon = tk.Button(ventanaP, text="LIMON", width=20, height=5,command=lambda : variable.set(2))
frutaMaestro=tk.Button(ventanaP,text="FRUTA NUEVA",width=20,height=5,command=lambda:variable.set(3))

guardar = tk.Button(ventanaP,text="GUARDAR FRUTA",width=10,height=3,command=ocultarBotones)

def estadoYfruta():
    #variableEstado=MaduroInmaduro,varibale=PapaUvaLimonFrutaProfe entradaIndex=EntrenarClasificar
    arreglo=[variableEstado.get(),variable.get(),entradaIndex]
    return arreglo

def sacarClase(entradaI):
    global entradaIndex
    entradaIndex=entradaI

guardar.place(x=500,y=500)
papa.place(x=230, y=100)
uva.place(x=230, y=200)
limon.place(x=230, y=300)
frutaMaestro.place(x=230,y=400)

def retrocederInicio(event):
    from Index import mostrarVentana
    mostrarVentana()

def ocultarVentana():
    ventanaP.withdraw()

ventanaP.bind("<Escape>",retrocederInicio)


from turtle import *
speed(0)

def draw(size, levels, angle):
    if levels == 0:
        color("green")
        dot(size)
        color("black")
        return

    forward(size)
    right(angle)

    draw(size * 0.67, levels - 1, angle)

    left(angle * 2)

    draw(size * 0.67, levels - 1, angle)
    right(angle)
    back(size)

left(90)
draw(100, 7, 45)

mainloop()


from itertools import combinations
from typing import Callable, Optional, Union

import numpy as np
import numpy.typing as npt
from scipy.spatial import Delaunay

from .reference_elements import Cell, referenceInterval, referenceTriangle

class MaskedList():
    def __init__(self,
                 arr: npt.NDArray,
                 mask: Optional[npt.NDArray[np.bool_]] = None):
        if mask is None:
            self._mask = np.array([False]*arr.shape[0], dtype=np.bool_)
        else:
            self._mask = mask

        self._arr = arr

        if self.arr.shape[0] != self.mask.shape[0]:
            raise ValueError("Mask length must match arr length")

    @property
    def masked_view(self):
        return self._arr[~self.mask].view()

    @property
    def arr(self):
        return self._arr.view()

    @property
    def mask(self):
        return self._mask.view()

    @mask.setter
    def mask(self, mask):
        if self.arr.shape[0] != mask.shape[0]:
            raise ValueError("Mask length must match arr length")
        self._mask = mask

    def halfdeepcopy(self, mask: Optional[npt.NDArray[np.bool_]]):
        if mask is None:
            return MaskedList(self._arr.view(), self._mask.copy())

        return MaskedList(self._arr.view(), mask)

class SimplexMesh:
    '''
    n dimensional simplex mesh. Mesh is a k-dim submanifold of an n>=2
    dimensional space. Obviously k<=n.
    '''

    @classmethod
    def Create_2d_unit_square_unstructured(cls, n):
        # include all corner points
        corners = np.array([[0,0], [1,0], [1,1], [0,1]])

        # include sufficient points on all edges
        pts = np.random.uniform(low=[0], high=[1], size=(4, n-2))
        e_0 = np.array([pts[0], np.zeros(n-2)]).T
        e_1 = np.array([pts[1], np.zeros(n-2)]).T[:,[1, 0]]
        e_2 = np.array([pts[2], np.ones(n-2)]).T
        e_3 = np.array([pts[3], np.ones(n-2)]).T[:,[1, 0]]

        inner = np.random.uniform(low=[0,0], high=[1,1], size=(n**2-4*(n-2)-4,2))
        all_pts = np.vstack([corners, e_0, e_1, e_2, e_3, inner])

        mesh = Delaunay((all_pts))

        return cls(mesh.points, mesh.simplices, referenceTriangle)

    @classmethod
    def Create_2d_refined(cls, n):
        n = n if n > 2 else 2

        def square(n):
            return np.array([[i/n, j/n]
                             for i in range(n+1)
                             for j in range(n+1)])

        vs = np.unique(np.concatenate([
            square(int(n/4)) * np.array([0.3, 1]),
            square(int(n/4)) * np.array([0.3, 1]) + np.array([0.7, 0]),
            square(int(n/2)) * np.array([0.4, 1]) + np.array([0.3, 0])
        ]), axis=0)

        mesh = Delaunay((vs*2)-1)

        return cls(mesh.points, mesh.simplices, referenceTriangle)

    @classmethod
    def Create_2d_unit_square_structured(cls, n, scaled=False):
        if n < 2:
            n = 2

        h = 1 / (n-1)
        k = n*n            # Number of vertices
        l = 2*(n-1)*(n-1)  # Number of faces
        vertices = np.empty((k,2), dtype=np.double)
        faces    = np.empty((l,3), dtype=np.uint)

        # Build vertex list
        # bottom left to top right, row wise
        for i in range(n):
            for j in range(n):
                vertices[i*n+j] = [j * h, i * h]

        for i in range(n-1):
            for j in range(n-1):
                ind = 2 * (i*(n-1) + j)
                faces[ind]   = [i*n + j  , i*n + j + 1  , (i+1)*n + j]
                faces[ind+1] = [i*n + j+1, (i+1)*n + j+1, (i+1)*n + j]

        if scaled:
            return cls((vertices*2)-1, faces, referenceTriangle)
        return cls((vertices), faces, referenceTriangle)

    @classmethod
    def Create_2d_manifold(cls, n):
        if n < 2:
            n = 2

        h = 1 / (n-1)
        k = n*n            # Number of vertices
        l = 2*(n-1)*(n-1)  # Number of faces
        vertices = np.empty((k,3), dtype=np.double)
        faces    = np.empty((l,3), dtype=np.uint)

        # Build vertex list
        # bottom left to top right, row wise
        f = lambda x,y: np.sin(-(x-0.5)*(y-.5))
        for i in range(n):
            for j in range(n):
                vertices[i*n+j] = [j * h, i * h, f(i*h,j*h)]

        for i in range(n-1):
            for j in range(n-1):
                ind = 2 * (i*(n-1) + j)
                faces[ind]   = [i*n + j  , i*n + j + 1  , (i+1)*n + j]
                faces[ind+1] = [i*n + j+1, (i+1)*n + j+1, (i+1)*n + j]

        return cls(vertices, faces, referenceTriangle)

    @classmethod
    def Create_1d_unit_interval_structured(cls, n):
        if n < 2:
            n = 2

        vertices = np.expand_dims(np.linspace(0,1,n), 1)
        faces    = np.array([[i,i+1] for i in range(n-1)])

        return cls(vertices, faces, referenceInterval)

    def __init__(self,
                 _v: Union[MaskedList, npt.ArrayLike],
                 _c: Union[MaskedList, npt.ArrayLike],
                 element : Cell):
        '''
        :param vertices: list of vertices in n dimensional space.
                         Must be a numpy ndarray of shape (k,n). Where k is the
                         number of vertices an n is the dimension of the space.

        :param cells: list of elements in form of a vertex list.
                         Must be a numpy ndarray of shape (l, d). Where l is
                         the number of elements and d is the number of points
                         describing each simplex.
                         Thus, d-1 is the dimensionality of the submanifold the
                         mesh describes.
                         I.e.,
                         d = 2 => mesh of lines.
                         d = 3 => mesh of triangles.

        Caution! Only works for orientable surfaces. Gives garbage results for
        möbius strips!

        Create numberings for all topological entities within their dimension.
        I.e., number all vertices from 0 to n, all edges from 0 to m, all faces
        from 0 to l, and so on.

        Also create lookup tables for adjacent, lower dimensional entites in
        all dimensions.
        I.e., Which vertices are the edges made of.
              Which edges are the faces made of,
              Which vertices are the faces made of

        For a 2D simplex Mesh, a global numbering for all vertices, edges and
        faces is needed.
        vertices are numbered implicitly by the order in :param vertices:
        faces are numbered implicitly by the order in :param elements:
        An edge numbering must be created

        For a 1D simplex mesh, the entire numbering is implicitly given.
        Only 0D and 1D numbering is needed. This is contained in :param
        vertices: and :param elements:
        '''
        vertices = _v if type(_v) is MaskedList else MaskedList(np.array(_v))
        cells    = _c if type(_c) is MaskedList else MaskedList(np.array(_c))

        # dimension of vertex coordinate vectores
        self.dim             = vertices.arr.shape[1]

        # dimension of the mesh
        self.dim_submanifold = element.dim

        # if self.dim != 2 and self.dim_submanifold != 2:
        #     raise NotImplementedError()

        if self.dim_submanifold > self.dim:
            raise ValueError(
                'Cannot embed a {} dimensional manifold into a {} dimensional'
                + ' space'.format(self.dim_submanifold, self.dim)
            );

        if element.dim != self.dim_submanifold:
            raise ValueError("Mesh Element does not match manifold dimension")

        self._element = element

        # self.nfaces doubles as the global numbering for all entities within
        # their dimension.
        # And vertex list for dimension n to dimension 0, for n > 0
        # And actual spatial coordinates for n = 0
        # Everything in self.nfaces has a globally implied direction.
        #   n = 0: (Implied to be of [x_1, x_2, ..., x_n] form
        #   n = 1: Lower vertex id to higher vertex id
        #   n = 2: Counterclockwise
        self.nfaces = {
            0: vertices
        }

        for n in range(1, self.dim_submanifold):
            # create global edge numbering, where the global direction is always
            # low to high
            _edges = list(set(tuple(sorted(e))
                              for t in cells.masked_view
                              for e in combinations(t, 2)))
            self.nfaces[n] = MaskedList(np.array(_edges))

        self.nfaces[self.dim_submanifold] = cells

        self._entities_per_dimension = np.array(
            [self.nfaces[n].masked_view.shape[0] for n in sorted(self.nfaces)]
        )

        if self.dim_submanifold == 2:
            # self.klookup[i][j] is a mapping from j-entities to i-entites
            # where j > i, and i > 0. For i = 0 refer to self.nfaces
            # self.klookup[i][j] -> [[({-1,1}, a)]*b]*c
            #   where {-1,1} is the local direction relative to the global one
            #         a is the id of the respective global entity ID
            #         b is how many i entities each j-entity consists of
            #         c is how many j-entities the mesh is made of
            self._klookup = {
                1: {2: None}
            }

            # create inverse function of self.nfaces[1]
            # _edge_lookup: Edge -> (Direction, GlobalEdgeID)
            # Where Edge \in (VertexID, VertexID)
            _edge_lookup = {tuple(e): (d, i)
                            for i, e_ in enumerate(self.nfaces[1].arr)
                            for d, e in ((1, e_), (-1, reversed(e_)))}

            self._klookup[1][2] = [[_edge_lookup[(e[i], e[(i+1)%3])]
                                   for i in range(3)]
                                  for e in self.nfaces[2].masked_view]

        # The boundary is the list of all (self.dim_submanifold - 1)-entities
        # that are adjacent to exactly one (self.dim_submanifold)-entity
        _adjacency_count = np.repeat(
            2,
            self.nfaces[self.dim_submanifold-1].arr.shape[0]
        )

        d_sub = self.dim_submanifold
        for es in self.adjacency(d_sub,d_sub-1):
            for e in es:
                if type(e) is tuple:
                    _, e = e

                _adjacency_count[e] -= 1

        _adjacency_count[self.nfaces[d_sub-1].mask.nonzero()[0]] = 2

        self._interior_facets = np.where(_adjacency_count == 0)[0]
        self._boundary_mesh = None

        n_facets = self.nfaces[d_sub-1].masked_view.shape[0]
        n_interior_facets = self._interior_facets.shape[0]

        if (n_facets - n_interior_facets) > 0 and d_sub > 1:
            masked_cells = MaskedList(self.nfaces[d_sub-1].arr.view())
            masked_cells.mask[self._interior_facets] = True

            self._boundary_mesh = SubSimplexMesh(outer=self,
                                                 cells=masked_cells)

    @property
    def element(self):
        return self._element

    @property
    def boundary_mesh(self):
        return self._boundary_mesh

    @property
    def entities_per_dimension(self):
        return self._entities_per_dimension

    @property
    def global_entities_per_dimension(self):
        return self._entities_per_dimension

    def adjacency(self, d1, d2):
        '''
        Get d2-entities adjacent to d1-entities

        Only implemented properly for d2 < d1
        For d2==d1: Each element is only ajacent to itself

        For d2=1, wrapper for self.nfaces
        Otherwise wrapper for self.klookup
        '''
        if d1 < 0 or d2 < 0:
            raise ValueError("dimensions must be positive")

        if d1 > self.dim_submanifold:
            raise ValueError("d1 must be less or equal to self.dim_submanifold")

        if d2 > d1:
            raise NotImplementedError()

        if d1 == d2:
            l = len(self.nfaces[d1].arr)
            mask = self.nfaces[d1].mask
            return np.arange(l)[~mask].reshape(-1,1)
            # return np.arange(l).reshape(-1,1)

        # from here d2 < d1, both positive and d1 meaningful

        if d2 == 0:
            return self.nfaces[d1].masked_view

        return self._klookup[d2][d1]

    def split(self, predicate: Callable[[npt.NDArray],bool]):
        d_sub = self.dim_submanifold

        cond = np.array(
            [all(map(predicate, self.nfaces[0].arr[cells]))
             for cells in self.nfaces[d_sub].arr]
        )
        ma_1 = np.logical_or(self.nfaces[d_sub].mask, ~cond)
        ma_2 = np.logical_or(self.nfaces[d_sub].mask, cond)

        m_1 = SubSimplexMesh(outer=self,
                             cells=self.nfaces[d_sub].halfdeepcopy(ma_1))
        m_2 = SubSimplexMesh(outer=self,
                             cells=self.nfaces[d_sub].halfdeepcopy(ma_2))
        return m_1, m_2

    # TODO: figure out how to implement boundary operator. This should return a
    # list of all connected boundaaies of the submanifold.

class SubSimplexMesh(SimplexMesh):
    def __init__(self, outer: SimplexMesh, cells: MaskedList):
        # all vertices initially masked
        vertices = MaskedList(outer.nfaces[0].arr,
                              np.array([True]*outer.nfaces[0].arr.shape[0]))

        # unmask all needed vertices
        for c in cells.masked_view:
            vertices.mask[c] = False

        d_sub = cells.arr[0].shape[0] - 1
        if outer.dim_submanifold == d_sub:
            element = outer.element
        elif outer.dim_submanifold == d_sub +1:
            element = outer.element.lower_element
        else:
            raise NotImplementedError()

        SimplexMesh.__init__(self,
                             _v=vertices,
                             _c=cells,
                             element=element)
        self._outer = outer

        # TODO: correct edge numbering
        # if d_sub == 2:
        #     self.nfaces[1] = outer.nfaces[1]
        #     self.nfaces[1].mask = np.array([True]*len(self.nfaces[1].mask))
        #     # create inverse function of self.nfaces[1]
        #     # _edge_lookup: Edge -> (Direction, GlobalEdgeID)
        #     # Where Edge \in (VertexID, VertexID)
        #     _edge_lookup = {tuple(e): (d, i)
        #                     for i, e_ in enumerate(self.nfaces[1].arr)
        #                     for d, e in ((1, e_), (-1, reversed(e_)))}

        #     for e in self.nfaces[2].masked_view:
        #         for i in range(3):
        #             _, n = _edge_lookup[(e[i], e[(i+1)%3])]
        #             self.nfaces[1].mask[n] = False

        #     self._klookup[1][2] = [[_edge_lookup[(e[i], e[(i+1)%3])]
        #                            for i in range(3)]
        #                           for e in self.nfaces[2].masked_view]
        #     print(self.nfaces[1].masked_view)

    @property
    def global_entities_per_dimension(self):
        return self._outer.entities_per_dimension

    def split(self, predicate: Callable[[npt.NDArray],bool]):
        d_sub = self.dim_submanifold

        cond = np.array(
            [all(map(predicate, self.nfaces[0].arr[cells]))
             for cells in self.nfaces[d_sub].arr]
        )
        ma_1 = np.logical_or(self.nfaces[d_sub].mask, ~cond)
        ma_2 = np.logical_or(self.nfaces[d_sub].mask, cond)

        m_1 = SubSimplexMesh(outer=self._outer,
                             cells=self.nfaces[d_sub].halfdeepcopy(ma_1))
        m_2 = SubSimplexMesh(outer=self._outer,
                             cells=self.nfaces[d_sub].halfdeepcopy(ma_2))
        return m_1, m_2

def import_gmsh(file: str):
    tags = [dict(), dict(), dict(), dict()]

    f = open(file, 'r')
    it = iter(f.readlines())

    while(next(it).strip() != '$MeshFormat'): pass
    v = next(it).strip().split(' ')

    # only v4.1 supported
    assert float(v[0]) == 4.1

    while(next(it).strip() != '$Entities'): pass
    t = next(it).strip().split(' ')
    numTags = [int(t[0]), int(t[1]), int(t[2]), int(t[3])]

    # all point tags
    for i in range(numTags[0]):
        l = next(it).strip().split(' ')
        if(int(l[3]) != 0):
            tags[0][int(l[0])] = int(v[4])

    # all multi dimensional tags
    for i in [1,2,3]:
        for j in range(numTags[i]):
            l = next(it).strip().split(' ')
            if(int(l[7]) != 0):
               tags[i][int(l[0])] = int(l[8])

    # skip to nodes
    while(next(it).strip() != '$Nodes'): pass

    l = next(it).strip().split(' ')
    blocks = int(l[0])
    nnodes = int(l[1])

    nodes = []
    nodeNumbering = dict()

    n = 0

    for i in range(blocks):
        l = next(it).strip().split(' ')
        nodesInBlock = int(l[3])

        # The node numbers
        for j in range(nodesInBlock):
            l = next(it).strip().split(' ')
            nodeNumbering[int(l[0])] = n
            n += 1

        # The actual coordinates
        for j in range(nodesInBlock):
            l = next(it).strip().split(' ')
            nodes.append((float(l[0]), float(l[1])))

    # skip to elements
    while(next(it).strip() != '$Elements'): pass

    l = next(it).strip().split(' ')
    blocks = int(l[0])

    edges = set()
    triangles = set()

    for i in range(blocks):
        l = next(it).strip().split(' ')
        elemDim = int(l[0])
        elemEntity = int(l[1])
        elemType = int(l[2])
        elemsInBlock = int(l[3])

        for j in range(elemsInBlock):
            l = next(it).strip().split(' ')

            if elemType == 1:
                edges.add( (0, int(l[1]), int(l[2])) )
                # edges.add( (tags[elemDim][elemEntity], int(l[1]), int(l[2])) )
            elif elemType == 2:
                triangles.add((0, int(l[1]),
                               int(l[2]), int(l[3])))
                # triangles.add((tags[elemDim][elemEntity], int(l[1]),
                #                int(l[2]), int(l[3])))
            else:
                print("Unsupported Element Type: {}".format(elemType))
                continue

    renumbering = dict()
    vertices = []
    faces = []
    d_boundaries = []
    n_boundaries = []

    for t in triangles:
        _t = (nodeNumbering[t[1]], nodeNumbering[t[2]],nodeNumbering[t[3]])

        for k in _t:
            if k not in renumbering:
                vertices.append(nodes[k])
                renumbering[k] = len(vertices)-1

        faces.append((renumbering[_t[0]], renumbering[_t[1]], renumbering[_t[2]]))

    for e in edges:
        d_boundaries.append(renumbering[nodeNumbering[e[1]]])
        d_boundaries.append(renumbering[nodeNumbering[e[2]]])

    return SimplexMesh(vertices, faces, referenceTriangle)