import json
import random
import re

import pubchempy as pcp

from csv_to_json import formulate_code

# -------- \begin constants ----------------------
atoms_list = ["	Hydrogen	",
              "	Helium	",
              "	Lithium	",
              "	Beryllium	",
              "	Boron	",
              "	Carbon	",
              "	Nitrogen	",
              "	Oxygen	",
              "	Fluorine	",
              "	Neon	",
              "	Sodium	",
              "	Magnesium	",
              "	Aluminium	",
              "	Silicon	",
              "	Phosphorus	",
              "	Sulfur	",
              "	Chlorine	",
              "	Argon	",
              "	Potassium	",
              "	Calcium	",
              "	Scandium	",
              "	Titanium	",
              "	Vanadium	",
              "	Chromium	",
              "	Manganese	",
              "	Iron	",
              "	Cobalt	",
              "	Nickel	",
              "	Copper	",
              "	Zinc	",
              "	Gallium	",
              "	Germanium	",
              "	Arsenic	",
              "	Selenium	",
              "	Bromine	",
              "	Krypton	",
              "	Rubidium	",
              "	Strontium	",
              "	Yttrium	",
              "	Zirconium	",
              "	Niobium	",
              "	Molybdenum	",
              "	Technetium	",
              "	Ruthenium	",
              "	Rhodium	",
              "	Palladium	",
              "	Silver	",
              "	Cadmium	",
              "	Indium	",
              "	Tin	",
              "	Antimony	",
              "	Tellurium	",
              "	Iodine	",
              "	Xenon	",
              "	Cesium	",
              "	Barium	",
              "	Lanthanum	",
              "	Cerium	",
              "	Praseodymium	",
              "	Neodymium	",
              "	Promethium	",
              "	Samarium	",
              "	Europium	",
              "	Gadolinium	",
              "	Terbium	",
              "	Dysprosium	",
              "	Holmium	",
              "	Erbium	",
              "	Thulium	",
              "	Ytterbium	",
              "	Lutetium	",
              "	Hafnium	",
              "	Tantalum	",
              "	Tungsten	",
              "	Rhenium	",
              "	Osmium	",
              "	Iridium	",
              "	Platinum	",
              "	Gold	",
              "	Mercury	",
              "	Thallium	",
              "	Lead	",
              "	Bismuth	",
              "	Polonium	",
              "	Astatine	",
              "	Radon	",
              "	Francium	",
              "	Radium	",
              "	Actinium	",
              "	Thorium	",
              "	Protactinium	",
              "	Uranium	",
              "	Neptunium	",
              "	Plutonium	",
              "	Americium	",
              "	Curium	",
              "	Berkelium	",
              "	Californium	",
              "	Einsteinium	",
              "	Fermium	",
              "	Mendelevium	",
              "	Nobelium	",
              "	Lawrencium	",
              "	Rutherfordium	",
              "	Dubnium	",
              "	Seaborgium	",
              "	Bohrium	",
              "	Hassium	",
              "	Meitnerium	",
              "	Darmstadtium	",
              "	Roentgenium	",
              "	Copernicium	",
              "	Nihonium	",
              "	Flerovium	",
              "	Moscovium	",
              "	Livermorium	",
              "	Tennessine	",
              "	Oganesson	",
              ]
not_a_compound_list = ["the", "was", "and", "get", "of", "in", "as", "an"]

# -------- \end constants ----------------------

# -------- \begin question types ----------------------
def if_the_temprature_passes(row):
    if re.search("If the temperature passes", row["contexts"]) is None:
        return None, None, None, None
    else:
        additional_info = re.search(
            "If the temperature passes \d+ degrees when heating .* for more than \d+ seconds .*", row["contexts"])
        temp_threshold_str = re.findall("\d+ degrees", additional_info.group())
        heat_duration_str = re.findall("\d+ seconds", additional_info.group())
        loss_str = re.findall("loss of (\D*\d+\.?\d* [grams|milligrams|\%]+)", additional_info.group())
        name = re.findall(" when heating (.*) for more", additional_info.group())

        return temp_threshold_str, loss_str, heat_duration_str, name

def We_discovered_that_if_the_amount_of(row):
    if re.search("We discovered that if the amount of", row["contexts"]) is None:
        return None, None
    else:
        additional_info = re.search(
            "We discovered that if the amount of .*", row["contexts"])
        quantity_threshold = re.findall("above (\D*\d+\.?\d* [grams|milligrams|\%]+)", additional_info.group())
        threshold_comp_name = re.findall("the amount of (.*) in", additional_info.group())
        temp_threshold_str = re.findall("the temperature is less than (\D*\d+\.?\d* degrees)", additional_info.group())
        loss_str = re.findall("the product of the process decreases by (\d+\%)", additional_info.group())
        return temp_threshold_str, loss_str, quantity_threshold, threshold_comp_name

def overheat_the(row):
    if re.search("Overheat the .*", row["contexts"]) is None:
        return None, None, None
    else:
        additional_info = re.search(
            "Overheat the .*", row["contexts"])
        temp_threshold_str = re.findall("the temperature is above (\D*\d+\.?\d* degrees)", additional_info.group())
        decrease_ratio_str = re.findall("for each (\d+\.\d+ [second|hour]+)", additional_info.group())
        loss_str = re.findall("a loss of (\D*\d+\.?\d* [grams|milligrams|ml|ML\%]+)", additional_info.group())
        product_name = re.findall("Overheat the (.*) will result", additional_info.group())

        return temp_threshold_str, loss_str, decrease_ratio_str, product_name

def if_we_heat(row):
    if re.search("If we heat .*", row["contexts"]) is None:
        return None, None
    else:
        additional_info = re.search(
            "If we heat .*", row["contexts"])
        temp_threshold_str = re.findall("to temperature higher than (\D*\d+\.?\d* degrees)", additional_info.group())
        loss_str = re.findall("at a rate of (\D*\d+\.?\d* [grams|milligrams|milliliters|\%]+) per minute.",
                              additional_info.group())
        name = re.findall("If we heat (.*) to tem", additional_info.group())
        return temp_threshold_str, loss_str, name

def stirring_the_mixture_longer(row):
    if re.search("stirring the mixture longer.*", row["contexts"]) is None:
        return None, None
    else:
        additional_info = re.search(
            "stirring the mixture longer.*", row["contexts"])
        loss_str = re.findall("will cause a loss of (\D*\d+\.?\d* [grams|milligrams|milliliters|\%]+)",
                              additional_info.group())
        decrease_ratio_str = re.findall("for each (minute|hour) above the original time",
                                        additional_info.group())
        name = ["the mixture"]
        return loss_str, decrease_ratio_str, name

def if_the_temperature_exceed(row):
    if re.search(" If the temperature exceed .*", row["contexts"]) is None:
        return None, None
    else:
        additional_info = re.search(
            "If the temperature exceed .*", row["contexts"])
        temp_threshold_str = re.findall("If the temperature exceed (\D*\d+\.?\d* degrees)",
                                        additional_info.group())
        decrease_ratio_str = re.findall("it will result in (\d+\% decrease) in the final products",
                                        additional_info.group())
        name = re.findall("when heating (.*) it will result", additional_info.group())
        return temp_threshold_str, decrease_ratio_str, name

def if_we_cool_the_mixture(row):
    if re.search("If we cool the mixture.*", row["contexts"]) is None:
        return None, None
    else:
        additional_info = re.search(
            "If we cool the mixture.*", row["contexts"])
        temp_threshold_str = re.findall("below (\D*\d+\.?\d* degrees)",
                                        additional_info.group())
        decrease_ratio_str = re.findall("the product of the process decreases by (\d+\%)",
                                        additional_info.group())
        name = ["the mixture"]
        return temp_threshold_str, decrease_ratio_str, name

# -------- \end question types ----------------------

# --------- \begin function for generating the question -------
def randomize_product_amount(vars_and_vals):
    weight_units = ['gr', 'mg', 'kg']
    volume_units = ['mL', 'L']

    product_amount = vars_and_vals[1]
    amount_unit_split = product_amount.split()
    amount_unit_split[0] = float(amount_unit_split[0]) * (random.uniform(0, 2))
    amount_unit_split[1] = random.choice(weight_units) if amount_unit_split[1] in weight_units else \
        amount_unit_split[1]
    amount_unit_split[1] = random.choice(weight_units) if amount_unit_split[1] in volume_units else \
        amount_unit_split[1]
    return "{:.2f}".format(amount_unit_split[0]) + " " + amount_unit_split[1]

def generate_reactors(components):
    available_reactors = ""
    reactors_list = []
    for comp in components:
        if comp[0].startswith("("):
            comp[0] = comp[0][1:]
        if random.random() < 0.2:  # w.p. 20% change the units
            weight_units = ['gr', 'mg', 'kg']
            volume_units = ['mL', 'L']
            amount_unit_split = comp[1].split()
            if amount_unit_split[1] == 'g':
                amount_unit_split[1] = 'gr'
            amount_unit_split[1] = random.choice(weight_units) if amount_unit_split[1] in weight_units else \
                amount_unit_split[1]
            amount_unit_split[1] = random.choice(weight_units) if amount_unit_split[1] in volume_units else \
                amount_unit_split[1]
            amount_unit_split[0] = str(float(amount_unit_split[0]) * (random.uniform(0, 2)))
            available_reactors += amount_unit_split[0] + " " + amount_unit_split[1] + " of " + comp[0] + ", "
            reactors_list.append([amount_unit_split[0], amount_unit_split[1]])
        else:
            available_reactors += comp[1] + " of " + comp[0] + ", "
            reactors_list.append([comp[1], comp[0]])
    return available_reactors, reactors_list

def get_vars_from_question(q_vars):
    variables = q_vars.split(", ")
    code_vars = "["
    for var in variables[:-1]:
        amount_name = var.split(" of ")
        amount_name[1] = amount_name[1].replace("'", "")
        amount_name[0] = amount_name[0].replace("'", "")
        code_vars += f"( ' {amount_name[1]} ' , ' {amount_name[0]} ' ) ,"
    return code_vars + "]"

def get_reactors_and_output(row):
    context = row["contexts"]
    check2switch = "\(\d+\.?\d* mmol, \d+\.?\d* mg\)|\(\d+\.?\d* mmol, \d+\.?\d* g\)\(\d+\.?\d* mmol, \d+\.?\d* mL\)\(\d+\.?\d* mmol, \d+\.?\d* gr\)\(\d+\.?\d* mmol, \d+\.?\d* ml\)"
    while re.search(check2switch, context) is not None:
        cc = re.search(check2switch, context)
        split_to_reorder = re.split("\(|,|\)", cc.group())
        context = context.replace(cc.group(), "( " + split_to_reorder[2] + ", " + split_to_reorder[1] + " )")
    split_by_sentence = context.split(". ")
    if len(split_by_sentence[-1]) == 0:
        dropped_additional = context.replace(split_by_sentence[-2], '')
        # dropped_additional = dropped_additional.replace(split_by_sentence[-3], '')
        start_search = -3
    else:
        dropped_additional = context.replace(split_by_sentence[-1], '')
        # dropped_additional = dropped_additional.replace(split_by_sentence[-2], '')
        start_search = -2

    succeed = False
    while not succeed:
        try:
            product_amount = re.search("(\d+\.?\d* g)|(\d+\.?\d* mg)|(\d+\.?\d* mL)|(\d+\.?\d* ml)",
                                       split_by_sentence[start_search]).group()
        except:
            if start_search < -20:
                return None, None
            start_search -= 1
            continue
        succeed = True

    vars_and_vals_list = re.split("(\d+\.?\d* g)|(\d+\.?\d* mg)|(\d+\.?\d* mL)|(\d+\.?\d* ml)", dropped_additional)
    vars_and_vals_list = [i for i in vars_and_vals_list if i is not None and i is not '']
    for item in vars_and_vals_list:
        if re.search("[a-z]+", item) is None:
            vars_and_vals_list.remove(item)
    vars_and_vals = []
    for i in range(len(vars_and_vals_list) // 2):
        if re.search("(\d+\.?\d* g)|(\d+\.?\d* mg)|(\d+\.?\d* mL)|(\d+\.?\d* ml)",
                     vars_and_vals_list[2 * i]) is not None:
            prev_sentence = vars_and_vals_list[2 * i + 1].split()
            vars_and_vals.append(
                [prev_sentence[0] if len(prev_sentence) == 1 else (
                    prev_sentence[1][1:] if prev_sentence[1].startswith("(") else prev_sentence[1]),
                 vars_and_vals_list[2 * i]])
        else:
            idx = -1
            comp_name = ""
            prev_parts = vars_and_vals_list[2 * i].split()
            while re.search("[a-z|0-9]+", comp_name) is None:
                comp_name += prev_parts[idx]
                idx -= 1
            vars_and_vals.append(
                [comp_name[1:] if comp_name.startswith("(") else comp_name, vars_and_vals_list[2 * i + 1]])
    return vars_and_vals, product_amount

def get_time_from_question(question):
    return re.findall(", for (.*),", question)[0]

def generate_duration(row, desired_output):
    # function_lists = [if_the_temprature_passes, We_discovered_that_if_the_amount_of, overheat_the, if_we_heat,
    #                   stirring_the_mixture_longer, if_the_temperature_exceed, if_we_cool_the_mixture]
    generated_temp, temp_threshold_str, loss_str, generated_duration, heat_duration_str, name = None, None, None, None, None, None
    unit = None

    if None not in if_the_temprature_passes(row):
        temp_threshold_str, loss_str, heat_duration_str, name = if_the_temprature_passes(row)
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-100, 100)
        generated_duration = float(re.findall("(\d+)", heat_duration_str[0])[0]) + random.randint(0, 10)
        unit = random.choice(["minutes", "hours"])
        question = f"if we heat to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in We_discovered_that_if_the_amount_of(row):
        temp_threshold_str, loss_str, quantity_threshold, threshold_comp_name = We_discovered_that_if_the_amount_of(row)
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-100, 100)
        generated_duration = random.randint(0, 10)
        generate_quantity = float(quantity_threshold[0].split()[0]) + random.uniform(
            float(quantity_threshold[0].split()[0]), 10)
        unit = random.choice(["minutes", "hours"])
        question = f"if the {threshold_comp_name[0]} was over {generate_quantity}, we cool the mixture to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in overheat_the(row):
        temp_threshold_str, loss_str, decrease_ratio_str, product_name = overheat_the(row)
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-100, 100)
        generated_duration = random.randint(0, 10)
        unit = random.choice(["minutes", "hours"])
        question = f"if we heat the {product_name[0]} to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in if_we_heat(row):
        temp_threshold_str, loss_str, name = if_we_heat(row)
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-100, 100)
        generated_duration = random.randint(0, 10)
        unit = random.choice(["minutes", "hours"])
        question = f"if we heat the {name[0]} to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in stirring_the_mixture_longer(row):
        loss_str, _, name = stirring_the_mixture_longer(row)
        name = name[0]
        generated_temp = random.randint(-100, 100)
        unit = random.choice(["minutes", "hours"])
        generated_duration = random.randint(1, 10) if unit == "hours" else random.choice([30 * i for i in range(20)])
        question = f"if we heat the {name} to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in if_the_temperature_exceed(row):
        temp_threshold_str, loss_str, name = if_the_temperature_exceed(row)
        unit = random.choice(["minutes", "hours"])
        generated_duration = random.randint(1, 10) if unit == "hours" else random.choice([30 * i for i in range(20)])
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-100, 100)
        question = f"if we heat the {name} to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    if None not in if_we_cool_the_mixture(row):
        temp_threshold_str, loss_str, name = if_we_cool_the_mixture(row)
        unit = random.choice(["minutes", "hours"])
        generated_duration = random.randint(1, 10) if unit == "hours" else random.choice([30 * i for i in range(20)])
        generated_temp = float(temp_threshold_str[0].split()[0]) + random.randint(-50, 50)
        question = f"if we cool the {name} to {generated_temp} degrees, for {generated_duration} {unit}, how much of the initial reactors to get {desired_output} of the final product?"

    return generated_temp, temp_threshold_str, loss_str, str(
        generated_duration) + " " + unit, heat_duration_str, name, question

# ---------  \end function for generating the question -------

def generate_question_type5(row):
    vars_and_vals = get_reactors_and_output(row)
    question = "how many moles of the product does the process yield ?"
    try:
        if vars_and_vals[0][-1][0] in not_a_compound_list:
            raise NameError
        validity_check = pcp.get_compounds(vars_and_vals[0][-1][0], 'name')[0].exact_mass
    except:
        return "", ""
    code = f"molar_mass = pcp.get_compounds( \"{vars_and_vals[0][-1][0]}\", 'name')[0].exact_mass [EOL]" \
           f"molar_mass = float ( molar_mass ) [EOL]" \
           f"yielded_grams = to_gr(\" {vars_and_vals[0][-1][1]} \") [EOL]" \
           f"return  yielded_grams  /  molar_mass  [EOL]"
    return question, code

def generate_question_type6(row):  # debugged
    vars_and_vals = get_reactors_and_output(row)
    product_quantity = vars_and_vals[0][-1]
    desired_output = randomize_product_amount(vars_and_vals)
    try:
        reactor_chosen = random.choice(vars_and_vals[0][:-1])
    except:
        return "", ""
    reactor_name = reactor_chosen[0]
    reactor_weight = reactor_chosen[1]
    try:
        if reactor_name in not_a_compound_list:
            raise NameError
        validity_check = pcp.get_compounds({reactor_name}, 'name')[0].exact_mass
    except:
        return "", ""
    question = f"how many moles of {reactor_name} do we need to get {desired_output} of the product ?"
    code = f"desired_product = to_gr( ' {desired_output} ' ) [EOL]" \
           f"product_described = to_gr( ' {product_quantity[1]} ' ) [EOL]" \
           f"described_component = to_gr( ' {reactor_weight} ') [EOL]" \
           f"needed_reactor = desired_product / product_described * described_component [EOL]" \
           f"reactor_molar_weight = pcp.get_compounds( \"{reactor_name}\" , 'name')[0].exact_mass [EOL]" \
           f"return ( needed_reactor / float( reactor_molar_weight ) ) [EOL]"
    return question, code

def generate_question_type7(row):  # debugged
    vars_and_vals = get_reactors_and_output(row)
    chosen_atom = random.choice(atoms_list).strip()
    compound_name = vars_and_vals[0][-1][0].replace('.', '')
    try:
        if compound_name in not_a_compound_list:
            raise NameError
        validity_check = pcp.get_compounds(compound_name, 'name')[0].elements
    except:
        return "", ""
    print("detected compound : ", compound_name)
    question = f"Is {chosen_atom} present in the product ?"
    code = f"chosen_atom = pcp.get_compounds( \" {chosen_atom} \" , 'name')[0].molecular_formula [EOL]" \
           f"product_elements = pcp.get_compounds( \"{compound_name}\" , 'name')[0].elements [EOL]" \
           f"return chosen_atom in product_elements [EOL]"
    return question, code

def generate_question_type1(row):  # debugged
    vars_and_vals = get_reactors_and_output(row)
    desired_output = randomize_product_amount(vars_and_vals)
    question_1 = f"how much do we need from each of the reactors to get {desired_output} of the final product ?"  # TODO V2 : add an environmental condtion
    code_1 = f"desired_product = to_gr( \" {desired_output} \" )[EOL]" \
             f"components = {vars_and_vals[0][:-1]} [EOL]" \
             f"product_described = to_gr( \" {vars_and_vals[1]} \" )[EOL]" \
             f"portions_needed = ( desired_product ) /100 [EOL]" \
             f"needed_reactors = [[reactor [ 0 ] , to_gr( reactor [ 1 ] ) * portions_needed] for reactor in components] [EOL]" \
             f"return needed_reactors [EOL]"
    code_1 = formulate_code(code_1)
    return question_1, code_1

def generate_question_type2(row):  # debugged
    vars_and_vals = get_reactors_and_output(row)
    q_vars, q_list = generate_reactors(vars_and_vals[0][:-1])
    q_list = [[q[1], q[0]] for q in q_list]
    if len(vars_and_vals[0][:-1]) < 1:
        return "", ""
    question2 = f"we have {q_vars}, how can we optimize the process?"  # TODO V2 : add an environmental conditions
    code_2 = f"components = {vars_and_vals[0][:-1]} [EOL]" \
             f"have_components = {q_list} [EOL]" \
             f"min_portion = float( 'inf' ) [EOL]" \
             f"for component, needed in zip ( components , have_components ) : [EOL]" \
             f"[TAB]portions = to_gr( component [ 1 ] ) / to_gr( needed [ 1 ] ) [EOL]" \
             "[TAB]if portions < min_portion : [EOL]" \
             "[TAB][TAB]min_portion = portions [EOL]" \
             "optimized = [] [EOL]" \
             "for need, have in zip ( components , have_components ) : [EOL]" \
             "[TAB]optimized.append( [ have[0] , to_gr( have [1] ) - to_gr ( need [1] ) * min_portion ] ) [EOL]" \
             "return optimized [EOL]"
    code_2 = formulate_code(code_2)
    return question2, code_2

def generate_question_type3(row):  # debugged
    vars_and_vals = get_reactors_and_output(row)
    q_vars, q_list = generate_reactors(vars_and_vals[0][:-1])
    question_3 = f"we have {q_vars} how much can we create of the final product?"  # TODO V2 : add an environmental conditions
    code_3 = f"available_reactors = {get_vars_from_question(q_vars)} [EOL]" \
             f"components =  {vars_and_vals[0][:-1]} [EOL]" \
             f"product_described = to_gr( \" {vars_and_vals[1]} \" ) [EOL]" \
             f"minimal_product_portion = float( 'inf' ) [EOL]" \
             f"for needed, have in zip ( components , available_reactors ): [EOL]" \
             f"[TAB]tmp_min_portion = to_gr( have [ 1 ] ) / to_gr( needed[1] ) [EOL]" \
             f"[TAB]if tmp_min_portion < minimal_product_portion : [EOL]" \
             f"[TAB][TAB]minimal_product_portion = tmp_min_portion [EOL]" \
             f"return minimal_product_portion * product_described [EOL]"
    code_3 = formulate_code(code_3)
    return question_3, code_3

def generate_question_type4(row):  # CONTINUE HEREEEE
    vars_and_vals = get_reactors_and_output(row)
    desired_output = randomize_product_amount(vars_and_vals)
    generated_temp, temp_threshold_str, loss_str, generated_duration, heat_duration_str, name, question_4 = generate_duration(
        row, desired_output)
    if heat_duration_str is None:
        return None, None
    code_4 = f"time = to_minute( \" {get_time_from_question(question_4)} \" ) [EOL]" \
             f"loss = \'{loss_str[0]}\' [EOL]" \
             f"components = {vars_and_vals[0][:-1]} [EOL]" \
             f"described_product_amount = to_gr( \" {desired_output} \" ) [EOL]" \
             f"threshold_duration = to_minute( \" {heat_duration_str} \" ) [EOL]" \
             f"temprature = {generated_temp} [EOL]" \
             f"threshold_temp = {temp_threshold_str} [EOL]" \
             f"final_product_amount = described_product_amount [EOL]" \
             f"for t in range( time ): [EOL]" \
             f"[TAB]if t > threshold_duration and temprature > threshold_temp: [EOL]" \
             f"[TAB][TAB]final_product_amount = compensate_for_loss( loss= loss[0], current_value= final_product_amount) [EOL]" \
             f"portions = final_product_amount / described_product_amount [EOL]" \
             f"return [[component[0], to_gr(component[1]) * portions] for component in components] [EOL]"
    code_4 = formulate_code(code_4)
    answe_4 = execute
    return question_4, code_4

data = json.load(open("gpt4-parsed-uspto.json", "r", encoding="utf-8"))

for idx, entry in enumerate(data):
    q1, c1 = generate_question_type1(row=entry)
    q2, c2 = generate_question_type2(row=entry)
    q3, c3 = generate_question_type3(row=entry)
    q4, c4 = generate_question_type4(row=entry)

    print(c4)


import urllib.request, urllib.parse
import bs4 as BeautifulSoup

# 建立与用户以及网络的会话
base = input("Enter the URL: ")
try:
    page = urllib.request.urlopen(base)
except:
    print("Cannot open %s" % base)
    quit()

# 准备soup
soup = BeautifulSoup.BeautifulSoup(page)

# 提取链接，并用（名称，网址）的元组表示
links = [(link.string, link['href']) for link in soup.find_all("a") if link.has_attr("href")]

# 尝试打开每个链接
broken = False

for name, url in links:
    # 将base和链接目标组合在一起，因为页面内的链接都是相对地址，所以这里要与base结合在一起
    # 例如： base=http://hanwen.me     url=/about
    # dest = http://hanwen.me/about
    dest = urllib.parse.urljoin(base, url)
    print(dest)
    try:
        page = urllib.request.urlopen(dest)
        page.close()
    except:
        print("Link \"%s\" to \"%s\" is probably broken." %(name, dest))
        broken = True
# 显示好消息
if not broken:
    print("Page %s does not seem to have broken links. " %base)


import matplotlib.pyplot as plt
import numpy as np
from numpy import *
from mpl_toolkits import mplot3d
import random

# Presets
ax = plt.axes(projection='3d')

def randomcolor():
    colorArr = ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F']
    color = ""
    for i in range(6):
        color += colorArr[random.randint(0, 14)]
    return "#" + color

class Group(object):
    def __init__(self):
        self.vectors = []

    def register(self, *v: ndarray):
        for i in v:
            self.vectors.append(i)

    def display(self):
        temp = []
        color_p = randomcolor()
        for i in self.vectors:
            for j in range(len(i)):
                temp.append(i[j])
            ax.quiver(0, 0, 0, temp[0], temp[1], temp[2], arrow_length_ratio=0.1, color=color_p)
            ax.scatter3D(temp[0], temp[1], temp[2], color=color_p, s=2500)
            temp = []

    def rotate(self, theta: 2 * pi):
        rotation_matrix = array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]])
        for i in range(len(self.vectors)):
            self.vectors[i] = dot(rotation_matrix, self.vectors[i])

def initial():
    x = np.arange(-6, 6, 0.1)
    y = np.zeros(120)

    ax.plot(x, y, y, color='#000000')
    ax.plot(y, x, y, color='#000000')
    ax.plot(y, y, x, color='#000000')

    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)
    ax.set_zlim(-2, 2)
    plt.gca().set_box_aspect((1, 1, 1))

    plt.show()

def main():
    a1 = array([[1], [1], [1]])
    a2 = array([[-1], [-1], [1]])
    a3 = array([[1], [-1], [-1]])
    a4 = array([[-1], [1], [-1]])

    G1 = Group()
    G1.register(a1, a2, a3, a4)

    G1.display()
    G1.rotate(pi / 6)
    G1.display()

    initial()

if __name__ == '__main__':
    main()


#from ast import If
#from pprint import pp
#from typing import final
from doctest import master
from multiprocessing.reduction import duplicate
import re
import string
from struct import pack
from unittest import result
from tokens import tokens
from tkinter import *
from tkinter import ttk
#from Interfaz import datos
resultReservadas = []
resultCaracteresEspeciales = []
resultDelimitadores = []
resultIndefinidas = []
resultErrores = []
resultDigitos = []
listResultados = []

class analizador:

    tokens = tokens()

    def inicio_analizador(self, palabras):
        resultReservadas = []
        resultCaracteresEspeciales = []
        resultDelimitadores = []
        resultDigitos = []
        resultIndefinidas = []
        resultErrores = []
        print("--- Lexico ---")
        for i in tokens.reservadas:
            for j in palabras:
                if (i == j):
                    resultReservadas.append(i)
                    palabras.remove(i)
        for l in tokens.caracteres_especiales:
            for k in palabras:
                if (l == k):
                    resultCaracteresEspeciales.append(k)
                    palabras.remove(l)
        for t in tokens.delimitadores:
            for f in palabras:
                if (t == f):
                    resultDelimitadores.append(t)
                    palabras.remove(t)
        for g in range (len(palabras)):
            #dato = re.search("^[A-Za-z]+$*", palabras[g])
            dato = re.search("^[a-zA-Z][a-zA-Z]+$", palabras[g])
            if dato:
                resultIndefinidas.append(palabras[g])
            else:
                dato1 = re.search("^[0-9]+$", palabras[g])
                if dato1:
                    resultDigitos.append(palabras[g])
                else:
                    resultErrores.append(palabras[g])
        print("Token Reservadas: ",resultReservadas)
        print("Token Caracteres Especiales: ",resultCaracteresEspeciales)
        print("Token Delimitadores: ",resultDelimitadores)
        print("Token Indefinidas: ",resultIndefinidas)
        print("Token Digitos: ",resultDigitos)
        print("Errores: ",resultErrores)
        listResultados.append(resultReservadas)
        listResultados.append(resultCaracteresEspeciales)
        listResultados.append(resultDelimitadores)
        listResultados.append(resultIndefinidas)
        listResultados.append(resultDigitos)
        listResultados.append(resultErrores)
        return listResultados

    def funcAuxiliar(self, palabras):
        # se buscan y se agregan a una lista los terminales
        for i in tokens.reservadas:
            for j in palabras:
                if (i == j):
                    resultReservadas.append(i)

        for l in tokens.caracteres_especiales:
            for k in palabras:
                if (l == k):
                    resultCaracteresEspeciales.append(k)

        for t in tokens.delimitadores:
            for f in palabras:
                if (t == f):
                    resultDelimitadores.append(t)
        # evaluando la cantidad existentes
        c = 0
        s = 0
        p = 0
        d = 0
        cs = 0
        i = 0
        pa = 0
        pc = 0
        dp = 0
        up = 0
        for cantidadReservadas in resultReservadas:
            if cantidadReservadas == "class":
                print("encontro un class")
                c += 1
            if cantidadReservadas == "self":
                print("encontro un self")
                s += 1
            if cantidadReservadas == "print":
                print("encontro un print")
                p += 1
            if cantidadReservadas == "def":
                print("encontro un def")
                d += 1
        for cantidadCaracteres in resultCaracteresEspeciales:
            if cantidadCaracteres == "'":
                print("encontro un ' ")
                cs += 1
        for cantidadDelimitadores in resultDelimitadores:
            if cantidadDelimitadores == "=":
                print("encontro un = ")
                i += 1
            if cantidadDelimitadores == "(":
                print("encontro un ( ")
                pa += 1
            if cantidadDelimitadores == ")":
                print("encontro un ) ")
                pc += 1
            if cantidadDelimitadores == ":":
                print("encontro un : ")
                dp += 1
            if cantidadDelimitadores == ".":
                print("encontro un . ")
                up += 1
        if c == 1 and s == 1 and p == 1 and d == 1 and cs == 2 and i == 2 and pa == 5 and pc == 5 and dp == 1 and up == 1:
            print("CUMPLE")
            palabras.remove("class")
            palabras.remove("self")
            palabras.remove("def")
            palabras.remove("print")
            palabras.remove("'")
            palabras.remove("'")
            palabras.remove("=")
            palabras.remove("=")
            palabras.remove("(")
            palabras.remove("(")
            palabras.remove("(")
            palabras.remove("(")
            palabras.remove("(")
            palabras.remove(")")
            palabras.remove(")")
            palabras.remove(")")
            palabras.remove(")")
            palabras.remove(")")
            palabras.remove(":")
            palabras.remove(".")

            print(palabras)
        else:
            print("NO CUMPLE")
        #print("Existentes REPETIDOS:", existentes)
        #print("Reservadas: ",resultReservadas)
        #print("Caracteres especiales: ",resultCaracteresEspeciales)
        #print("Delimitadores: ",resultDelimitadores)
        #print(palabras)


#!/usr/bin/env python
#
# An example on how to read the YAML output from etisnoop
# Pipe etisnoop to this script
#
# License: public domain

import sys
import yaml

for frame in yaml.load_all(sys.stdin):
    print("FIGs in frame {}".format(frame['Frame']))
    for fib in frame['LIDATA']['FIC']:
        if fib['FIGs']:
            for fig in fib['FIGs']:
                print(" FIG " + fig['FIG'])


import sys
import time
import numpy as np

import torch
import torch.nn as nn

class RewardTracker:
    def __init__(self, writer, stop_reward, group_rewards=1):
        self.writer = writer
        self.stop_reward = stop_reward
        self.reward_buf = []
        self.steps_buf = []
        self.group_rewards = group_rewards

    def __enter__(self):
        self.ts = time.time()
        self.ts_frame = 0
        self.total_rewards = []
        self.total_steps = []
        return self

    def __exit__(self, *args):
        self.writer.close()

    def reward(self, reward_steps, frame, epsilon=None):
        reward, steps = reward_steps
        self.reward_buf.append(reward)
        self.steps_buf.append(steps)
        if len(self.reward_buf) < self.group_rewards:
            return False
        reward = np.mean(self.reward_buf)
        steps = np.mean(self.steps_buf)
        self.reward_buf.clear()
        self.steps_buf.clear()
        self.total_rewards.append(reward)
        self.total_steps.append(steps)
        speed = (frame - self.ts_frame) / (time.time() - self.ts)
        self.ts_frame = frame
        self.ts = time.time()
        mean_reward = np.mean(self.total_rewards[-100:])
        mean_steps = np.mean(self.total_steps[-100:])
        epsilon_str = "" if epsilon is None else ", eps %.2f" % epsilon
        print("%d: done %d games, mean reward %.3f, mean steps %.2f, speed %.2f f/s%s" % (
            frame, len(self.total_rewards)*self.group_rewards, mean_reward, mean_steps, speed, epsilon_str
        ))
        sys.stdout.flush()
        if epsilon is not None:
            self.writer.add_scalar("epsilon", epsilon, frame)
        self.writer.add_scalar("speed", speed, frame)
        self.writer.add_scalar("reward_100", mean_reward, frame)
        self.writer.add_scalar("reward", reward, frame)
        self.writer.add_scalar("steps_100", mean_steps, frame)
        self.writer.add_scalar("steps", steps, frame)
        if mean_reward > self.stop_reward:
            print("Solved in %d frames!" % frame)
            return True
        return False

def calc_values_of_states(states, net, device="cpu"):
    """    action_values_v = net(states_v)：這裡，模型net為給定的states_v預測每個可能動作的價值。

    best_action_values_v = action_values_v.max(1)[0]：接著，我們只考慮每個狀態的最佳動作價值，這是透過取每一行（代表每個狀態）的最大值來完成的。

    結果是所有狀態的最佳動作價值的均值。

    回答你的問題：“假設我部位完全沒有任何的變化下，收益為什麼會改變？”：

    即使部位方向不變，模型的權重和偏差是在訓練過程中不斷更新的。所以，當你使用同一組states重新評估模型時，你會得到不同的動作價值，因為模型已經學到了新的知識。

    動作價值的改變並不直接代表收益的改變。它只是模型對給定狀態應該採取何種動作的估計價值。當你在真實環境中執行這些動作時，真正的收益可能會與模型的估計有所不同。

    訓練過程中，模型試圖學習一個策略，使其預測的動作價值越來越接近真實價值。但這不代表模型總是正確的，只是說它試圖接近真實價值。

    所以，雖然部位方向不變，但模型的估計動作價值可能會變，這反映了模型在訓練過程中的學習進展。

    Args:
        states (_type_): _description_
        net (_type_): _description_
        device (str, optional): _description_. Defaults to "cpu".

    Returns:
        _type_: _description_
    """

    mean_vals = []
    for batch in np.array_split(states, 64):
        states_v = torch.tensor(batch).to(device)
        action_values_v = net(states_v)
        best_action_values_v = action_values_v.max(1)[0]
        mean_vals.append(best_action_values_v.mean().item())
    return np.mean(mean_vals)

def unpack_batch(batch):
    states, actions, rewards, dones, last_states = [], [], [], [], []
    for exp in batch:
        state = np.array(exp.state, copy=False)
        states.append(state)
        actions.append(exp.action)
        rewards.append(exp.reward)
        dones.append(exp.last_state is None)
        if exp.last_state is None:
            last_states.append(state)       # the result will be masked anyway
        else:
            last_states.append(np.array(exp.last_state, copy=False))
    return np.array(states, copy=False), np.array(actions), np.array(rewards, dtype=np.float32), \
           np.array(dones, dtype=np.uint8), np.array(last_states, copy=False)

def calc_loss(batch, net, tgt_net, gamma, device="cpu"):
    states, actions, rewards, dones, next_states = unpack_batch(batch)

    states_v = torch.tensor(states).to(device)
    next_states_v = torch.tensor(next_states).to(device)
    actions_v = torch.tensor(actions).to(device)
    rewards_v = torch.tensor(rewards).to(device)

    done_mask = torch.tensor(dones, dtype=torch.bool).to(device)

    state_action_values = net(states_v).gather(1, actions_v.unsqueeze(-1)).squeeze(-1)
    next_state_actions = net(next_states_v).max(1)[1]
    next_state_values = tgt_net(next_states_v).gather(1, next_state_actions.unsqueeze(-1)).squeeze(-1)
    next_state_values[done_mask] = 0.0

    expected_state_action_values = next_state_values.detach() * gamma + rewards_v
    return nn.MSELoss()(state_action_values, expected_state_action_values)


import xml.etree.ElementTree as ET
import time
import requests

class BaseAPIConnector(object):
    def __init__(self, user_agent='', verbose=False):

        self.user_agent = user_agent
        self.verbose = verbose

    def construct_url(self):
        return None

    def html_request(self):
        if self.user_agent == '':
            raise UserWarning('Please specify a user agent.')

        url = self.construct_url()
        if self.verbose:
            print(url)

        request = None
        exception_count = 0
        while exception_count < 10:
            try:
                request = requests.get(url, headers={'User-Agent': self.user_agent})
            except Exception as e:
                print("Exception '%s' while querying url: '%s', trying again..." % (e, url))
                time.sleep(10)
                exception_count += 1
            else:
                break

        return request

    def get_xml_from_url(self):
        try:
            return ET.fromstring(self.html_request().text)
        except:
            return None

    def get_json_from_url(self):
        return self.html_request().json()


from hashlib import sha1
import time
import json
from pulsar import get_actor, Future
from pulsar.apps.wsgi import WSGIServer, WsgiResponse, WsgiHandler
import aio_etcd as etcd
from jirachi.io.abstract import JirachiMonitor, JirachiMonitorNotFound
from pulsar.apps.wsgi import Router

__all__ = ['RemoteMonitorWSGI']

blueprint = Router('/')

@blueprint.router('/remote/<string:monitor>/event/<string:event>/', methods=['post'])
async def remote_wsgi(request):
    hash_code = sha1(str(time.time()).encode()).hexdigest()
    monitor_name = request.urlargs['monitor']
    event_name = request.urlargs['event']
    data = request.body_data

    actor = get_actor()
    monitor = actor.get_actor(monitor_name)
    if monitor:
        monitor.fire_event(event_name, data)
    elif not (actor.is_arbiter() or actor.is_monitor() and actor.monitor == monitor):
        actor.monitor.fire_event(event_name, msg=data)
    else:
        raise JirachiMonitorNotFound('Cant found Monitor %s' % monitor)
    return WsgiResponse(200, json.dumps({
        'successed': True,
        'token': hash_code
    }))

class RemoteMonitorWSGI(WSGIServer, JirachiMonitor):
    name = 'remote_monitor'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.cfg.callable = WsgiHandler((blueprint, ))
        if not hasattr(self.cfg, 'blacklist'):
            self.cfg.blacklist = []

    @staticmethod
    async def singlecast(msg, actor_name):
        actor = get_actor()
        if not actor.name == actor_name:
            actor = actor.get_actor(actor_name)
        if not actor and actor.monitor.name == actor.name:
            actor = actor.monitor
        actor.fire_event('singlecast', msg)
        actor.future = Future()
        return actor.future

    @staticmethod
    def event_test(msg):
        print('test event %s' % msg)

    async def monitor_start(self, monitor, exec=None):
        monitor.bind_event('test', self.event_test)
        if not hasattr(self.cfg, 'etcdconf'):
            monitor.etcd = etcd.Client()
        else:
            monitor.etcd = etcd.Client(**self.cfg.etcdconf)
        await super().monitor_start(monitor)

    async def worker_start(self, worker, *args, **kwargs):
        worker.bind_event('test', self.event_test)

    async def search_remote(self):
        pass

    async def sync_remote(self):
        pass


from binaryninja import *
import xxhash

################################################################################################################
#                                       MLIL Instruction                                                       #
################################################################################################################

class Neo4jInstruction:

    def __init__(self, instr: mediumlevelil.MediumLevelILInstruction, context, parent_type: str):
        self.instr = instr
        self.parent_type = parent_type
        self.operands = str(instr.operands)
        self.context = context

        if self.parent_type == 'BasicBlock':
            self.relationship_label = 'InstructionChain'
        else:
            self.relationship_label = 'NextInstruction'

        self.context.set_hash(self.instr_hash())

    def instr_hash(self):
        instruction_hash = xxhash.xxh64()
        instruction_hash.update(str(self.instr.operands) + str(self.instr.operation))

        return instruction_hash.hexdigest()

    def serialize(self):
        csv_template = {
            'mandatory_node_dict': {
                'HASH': self.context.SelfHASH,
                'LABEL': 'Instruction',
            },
            'mandatory_relationship_dict': {
                'START_ID': self.context.ParentHASH,
                'END_ID': self.context.SelfHASH,
                'TYPE': self.relationship_label,
                'StartNodeLabel': self.parent_type,
                'EndNodeLabel': 'Instruction',
                'AssemblyOffset': self.instr.address,
            },

            'mandatory_context_dict': self.context.get_context(),

            'node_attributes': {
            },
            'relationship_attributes': {
                'InstructionIndex': self.instr.instr_index,
                'PossibleValues': self.instr.possible_values.type.value,
                'VarsRead': [var.name for var in self.instr.vars_read],
                'VarsWritten': [var.name for var in self.instr.vars_written],
            },
        }
        return csv_template


from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys

from tkinter import messagebox
import time
import os

from App.Login import username, password, security_question

class Core:
    def __init__(self, file_path, driver):
        self.driver = driver
        self.file_path = file_path
        self.file_content = self.readFileContent()
        self.file_name = os.path.split(file_path)[1]

        self.IDs_list = []

        if self.file_content:
            try:
                self.collectCheckboxsIDs()
                self.selectCheckbox()
            except NoSuchElementException:
                self.driver.close()
                messagebox.showinfo(
                    "Bet Selector", "Loading took too much time or Not yet defined.")
        else:
            messagebox.showinfo(
                "Bet Selector", "Your Bet File '{}' is Empty.".format(self.file_name))

    def readFileContent(self):
        with open(self.file_path, 'r') as f:
            file_content = f.read().split('\n')
            file_content = list(filter(None, file_content))
            return file_content

    def collectCheckboxsIDs(self):
        checkboxs = self.driver.find_elements_by_class_name("checkbox")
        IDs_list = []
        for el in checkboxs:
            IDs_list.append(el.get_attribute('id'))

        IDs_list = [IDs_list[x:x+9] for x in range(0, len(IDs_list), 9)]

        IDs_list.insert(0, [])
        for lis in IDs_list:
            lis.insert(0, "")
        self.IDs_list = IDs_list

    def selectCheckbox(self):
        for line in self.file_content:
            line = line.split(',')
            row = 1
            for choice in line:
                i = 0
                while i < len(choice):
                    self.driver.find_element_by_id(
                        self.IDs_list[row][int(choice[i])]).click()
                    i += 1
                row += 1
            self.driver.find_element_by_xpath(
                '//*[@title="Add to Slip"]').click()


# need to implement CSRF
from rest_framework.authentication import CSRFCheck
from rest_framework_simplejwt.authentication import JWTAuthentication
from rest_framework import exceptions
from channels.db import database_sync_to_async
from server.settings import SIMPLE_JWT
from django.core.exceptions import ObjectDoesNotExist
from rest_framework_simplejwt.tokens import UntypedToken, TokenError
from urllib.parse import parse_qs
from server.settings import SIMPLE_JWT, SECRET_KEY
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AnonymousUser
import jwt

def enforce_csrf(request):
    check = CSRFCheck()
    check.process_request(request)
    reason = check.process_view(request, None, (), {})
    if reason:
        raise exceptions.PermissionDenied('CSRF failed: %s' % reason)

class CustomAuthentication(JWTAuthentication):
    def authenticate(self, request):
        header = self.get_header(request)

        if header is None:
            raw_token = request.COOKIES.get(SIMPLE_JWT['AUTH_COOKIE']) or None
        else:
            raw_token = self.get_raw_token(header)
        if raw_token is None:
            return None

        validated_token = self.get_validated_token(raw_token)
        enforce_csrf(request)
        return self.get_user(validated_token), validated_token

@database_sync_to_async
def get_user(user_id):
    try:
        return get_user_model().objects.get(pk=user_id)
    except ObjectDoesNotExist:
        return AnonymousUser()

class TokenAuthMiddleWare:
    def __init__(self, app):
        self.app = app

    async def __call__(self, scope, receive, send):
        # needs utf8 to decode from byte format b''
        user_id = -1
        try:
            raw_token = parse_qs(scope["query_string"].decode("utf8"))["token"][0]
            UntypedToken(raw_token)
            decode_token = jwt.decode(raw_token, SECRET_KEY, SIMPLE_JWT["ALGORITHM"])
            user_id = decode_token['user_id']
        except:
            print("Token is invalid")
        finally:
            user = await get_user(user_id)
            return await self.app(dict(scope, user=user), receive, send)


#!/usr/bin/env python
## Test an algorithm in real life
import sys
if sys.version_info[0] != 3 or sys.version_info[1] < 6:
  print("This script requires Python version >=3.6")
  sys.exit(1)

import algorithms
import datetime
import exchange
import pandas_market_calendars as mcal
import portfolioLive

## Main function
def main():
  today = datetime.date.today()
  nyse = mcal.get_calendar('NYSE')
  if len(nyse.valid_days(start_date=today, end_date=today)) == 0:
    print("Markets are closed today")
    return

  with open("symbolsPruned", "r", newline="\n") as file:
    symbols = file.read().strip().split("\n")

  exchange.init(
      symbols,
      period=1,
      historyDays=algorithms.KIPPslowDays)
  p = portfolioLive.Portfolio()

  algorithms.KIPP(symbols, p)

if __name__ == "__main__":
  main()


from osgeo import gdalnumeric
from osgeo import osr
from osgeo import gdal
from osgeo.gdal_array import *
from osgeo.gdalconst import *
from PIL import Image
import pylab as P
import os
import numpy as np
from IPython.core.debugger import set_trace

def readData(filename, ndtype=np.float64):
    '''
    z=readData('/path/to/file')
    '''
    if os.path.isfile(filename):
      return LoadFile(filename).astype(ndtype);
    else:
      return gdal.Open(filename, gdal.GA_ReadOnly).readAsArray()

def writeTiff(ary, coord, filename='kgiAlos.tif', rescale=None, format=gdal.GDT_Float64,lon=None, lat=None, nodata=None, grid=False, cog=False, srs_proj4='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', options=[], gcps=None):
    '''writeTiff(ary, geoTransform, filename='kgiAlos.tif', rescale=None, format=gdal.GDT_Float64 ,lon=None, lat=None):
    ary: 2D array.
    geoTransform: [top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution]
    rescale: [min max]: If given rescale ary values between min and max.

    If lon lat is specified set coord to None

    '''

    if coord is None and gcps is None:
        import scipy
        import scipy.linalg
        s=[sk//10 for sk in ary.shape]
        ary10=ary[::s[0],::s[1]];
        lon10=lon[::s[0],::s[1]];
        lat10=lat[::s[0],::s[1]];
        #P.figure()
        #P.scatter(lon10.ravel(), lat10.ravel(), 5, ary10.ravel(), edgecolors='none')
        A=np.ones([np.multiply(*ary10.shape),3])
        line,pixel=np.meshgrid(np.r_[0:ary.shape[0]:s[0]],np.r_[0:ary.shape[1]:s[1]])
        A[:,1]=pixel.ravel()
        A[:,2]=line.ravel()
        xlon=np.dot(scipy.linalg.pinv(A), lon10.ravel())
        xlat=np.dot(scipy.linalg.pinv(A), lat10.ravel())
        ##check flip flop
        #if xlon[1]<0: #flip lr
        #    ary=np.fliplr(ary)
        #
        #if xlat[2]>0: #flip ud
        #    ary=np.flipud(ary)
        coord=[xlon[0],xlon[2], xlon[1], xlat[0], xlat[2], xlat[1]];
        print(coord)
        #x=lon[0,0]
        #y=lat[0,0]
        #dx=lon[0,1]-lon[0,0]
        #dy=lat[1,0]-lat[0,0]
        #xrot=0.
        #yrot=0.
        #coord=[x,dx, xrot, y,yrot, dy]

        if grid:
            import scipy.interpolate
            LON,LAT=np.meshgrid(np.r_[lon.min():lon.max():abs(coord[1])], np.r_[lat.max():lat.min():-abs(coord[5])])
            #ary=P.griddata(lon.ravel(),lat.ravel(),ary.ravel(),LON,LAT);
            ary=scipy.interpolate.griddata(np.array([lon.ravel(),lat.ravel()]).T,ary.ravel(),(LON,LAT), method='cubic');
            coord=[LON[0,0],abs(coord[1]), 0, LAT[0,0], 0,-abs(coord[5])];
            print(coord)

    if rescale:
        import basic
        ary=basic.rescale(ary, rescale);

    # data exists in 'ary' with values range 0 - 255
    # Uncomment next line if ary[0][0] is upper-left corner
    #ary = numpy.flipup(ary)
    if ary.ndim==2:
      Ny, Nx = ary.shape
      Nb=1;
      #ds = driver.Create(filename, Nx, Ny, 1, gdal.GDT_Float64)
    elif ary.ndim==3:
      Ny,Nx,Nb = ary.shape #Nb: number of bands. #osgeo.gdal expects, (band, row, col), so this is a deviation from that.
    else:
      print("Input array has to be 2D or 3D.")
      return None
    driver = gdal.GetDriverByName("GTiff")
    if cog:
      options = ["TILED=YES","COMPRESS=LZW","INTERLEAVE=BAND","BIGTIFF=YES"]
    ds = driver.Create(filename, Nx, Ny, Nb, gdal.GDT_Float64, options)

    srs=osr.SpatialReference()
    srs.ImportFromProj4(srs_proj4)
    ds.SetProjection(srs.ExportToWkt() );
    #ds.SetGeoTransform( ... ) # define GeoTransform tuple
    # top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution
    if gcps is None:
      ds.SetGeoTransform( coord )
    else:
      if type(gcps[0])== gdal.GCP:
        ds.SetGCPs(gcps, srs.ExportToWkt())
      elif type(gcps[0])==np.int and len(gcps)==2 and lat is not None:
        gcps_list=create_gcp_list(lon,lat,np.zeros(lat.shape), gcp_count=[gcps[0], gcps[1]])
        ds.SetGCPs(gcp_list, srs.ExportToWkt())
      else:
        print('unsupported type of GCPs. Skipping.')
    if nodata is not None:
        ds.GetRasterBand(1).SetNoDataValue(nodata);
    if Nb==1:
      ds.GetRasterBand(1).WriteArray(ary)
    else:
      for b in range(Nb):
        ds.GetRasterBand(b+1).WriteArray(ary[:,:,b])
    # optimize for COG
    if cog:
      ds.BuildOverviews("NEAREST", [2, 4, 8, 16, 32, 64, 128, 256])

    ds = None
    print("File written to: " + filename);

def create_gcp_list(x,y,z,p=None, l=None,gcp_count=[2,2]):
    """create_gcp_list(x,y,z,p=None, l=None, gcp_count=[2,2])
    if xyz is in the same shape as image, uses gcp count to select a reasonable amount of gcps.
    if xyz is not in the same shape as image, p and l need to be provided to select the correct pixel and line.
    """
    gcp_list=[]
    if l is None or p is None:
      p=np.linspace(0,x.shape[0]-1, gcp_count[0]).astype(int)
      l=np.linspace(0,x.shape[1]-1, gcp_count[1]).astype(int)
      for pp in p:
        for ll in l:
          gcp=gdal.GCP(x[pp,ll], y[pp,ll], z[pp,ll], float(pp), float(ll))
          gcp_list.append(gcp)
    else:
      p=p.ravel().astype(float)
      l=l.ravel().astype(float)
      x=x.ravel()
      y=y.ravel()
      z=z.ravel()
      for k in range(l.size):
        gcp=gdal.GCP(x[k], y[k], z[k], p[k], l[k])
        gcp_list.append(gcp)
    return gcp_list

def writeAny(ary, coord, fileformat="GTiff", filename='kgiAlos.tif', rescale=None, format=gdal.GDT_Float64,lon=None, lat=None, nodata=None, grid=False, srs_proj4='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'):
    '''writeAny(ary, geoTransform, format="GTiff", filename='kgiAlos.tif', rescale=None, format=gdal.GDT_Float64 ,lon=None, lat=None):
    ary: 2D array.
    geoTransform: [top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution]
    format: "GTiff"
    rescale: [min max]: If given rescale ary values between min and max.

    If lon lat is specified set coord to None

    '''

    if coord is None:
        import scipy
        import scipy.linalg
        s=[sk//10 for sk in ary.shape]#BRANDON EDIT FOR COMPATIBILITY: changed / to // for python 3

        ary10=ary[::s[0],::s[1]];
        lon10=lon[::s[0],::s[1]];
        lat10=lat[::s[0],::s[1]];
        #P.figure()
        #P.scatter(lon10.ravel(), lat10.ravel(), 5, ary10.ravel(), edgecolors='none')
        A=np.ones([np.multiply(*ary10.shape),3])
        line,pixel=np.meshgrid(np.r_[0:ary.shape[0]:s[0]],np.r_[0:ary.shape[1]:s[1]])
        A[:,1]=pixel.ravel()
        A[:,2]=line.ravel()
        xlon=np.dot(scipy.linalg.pinv(A), lon10.ravel())
        xlat=np.dot(scipy.linalg.pinv(A), lat10.ravel())
        ##check flip flop
        #if xlon[1]<0: #flip lr
        #    ary=np.fliplr(ary)
        #
        #if xlat[2]>0: #flip ud
        #    ary=np.flipud(ary)
        coord=[xlon[0],xlon[2], xlon[1], xlat[0], xlat[2], xlat[1]];
        print(coord)
        #x=lon[0,0]
        #y=lat[0,0]
        #dx=lon[0,1]-lon[0,0]
        #dy=lat[1,0]-lat[0,0]
        #xrot=0.
        #yrot=0.
        #coord=[x,dx, xrot, y,yrot, dy]

        if grid:
            import scipy.interpolate
            LON,LAT=np.meshgrid(np.r_[lon.min():lon.max():abs(coord[1])], np.r_[lat.max():lat.min():-abs(coord[5])])
            #ary=P.griddata(lon.ravel(),lat.ravel(),ary.ravel(),LON,LAT);
            ary=scipy.interpolate.griddata(np.array([lon.ravel(),lat.ravel()]).T,ary.ravel(),(LON,LAT), method='cubic');
            coord=[LON[0,0],abs(coord[1]), 0, LAT[0,0], 0,-abs(coord[5])];
            print(coord)

    if rescale:
        import basic
        ary=basic.rescale(ary, rescale);

    # data exists in 'ary' with values range 0 - 255
    # Uncomment next line if ary[0][0] is upper-left corner
    #ary = numpy.flipup(ary)
    if ary.ndim ==2:
      Ny, Nx = ary.shape
      Nb = 1;
    elif ary.ndim==3:
      Ny,Nx,Nb=ary.shape
    else:
      print("Input array has to be 2D or 3D.")
      return None

    driver = gdal.GetDriverByName(fileformat)
    ds = driver.Create(filename, Nx, Ny, Nb, gdal.GDT_Float64)

    #ds.SetGeoTransform( ... ) # define GeoTransform tuple
    # top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution
    ds.SetGeoTransform( coord )
    srs=osr.SpatialReference()
    srs.ImportFromProj4(srs_proj4)
    ds.SetProjection(srs.ExportToWkt() );
    if nodata is not None:
        ds.GetRasterBand(1).SetNoDataValue(0);
    if Nb==1:
      ds.GetRasterBand(1).WriteArray(ary)
    else:
      for b in range(Nb):
        ds.GetRasterBand(b+1).WriteArray(ary[:,:,b])
    ds = None
    print("File written to: " + filename);

def writeCSV(ary, filename='gis_file.csv', geotransform=None, rescale=None, format="%f", lon=None, lat=None, nodata=None, grid=False, srs_proj4='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'):
    '''writeCSV(ary, geoTransform=None, format="GTiff", filename='gis_file.csv', rescale=None, format="%f" ,lon=None, lat=None):
    ary: 2D array.
    geoTransform: [top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution]
    format: "GTiff"
    rescale: [min max]: If given rescale ary values between min and max.

    '''

    if geotransform is None:
        import scipy
        import scipy.linalg
        s=[sk//10 for sk in ary.shape]
        ary10=ary[::s[0],::s[1]];
        lon10=lon[::s[0],::s[1]];
        lat10=lat[::s[0],::s[1]];
        #P.figure()
        #P.scatter(lon10.ravel(), lat10.ravel(), 5, ary10.ravel(), edgecolors='none')
        A=np.ones([np.multiply(*ary10.shape),3])
        line,pixel=np.meshgrid(np.r_[0:ary.shape[0]:s[0]],np.r_[0:ary.shape[1]:s[1]])
        A[:,1]=pixel.ravel()
        A[:,2]=line.ravel()
        xlon=np.dot(scipy.linalg.pinv(A), lon10.ravel())
        xlat=np.dot(scipy.linalg.pinv(A), lat10.ravel())
        ##check flip flop
        #if xlon[1]<0: #flip lr
        #    ary=np.fliplr(ary)
        #
        #if xlat[2]>0: #flip ud
        #    ary=np.flipud(ary)
        geotransform=[xlon[0],xlon[2], xlon[1], xlat[0], xlat[2], xlat[1]];
        print(geotransform)
        #x=lon[0,0]
        #y=lat[0,0]
        #dx=lon[0,1]-lon[0,0]
        #dy=lat[1,0]-lat[0,0]
        #xrot=0.
        #yrot=0.
        #coord=[x,dx, xrot, y,yrot, dy]

        if grid:
            import scipy.interpolate
            LON,LAT=np.meshgrid(np.r_[lon.min():lon.max():abs(coord[1])], np.r_[lat.max():lat.min():-abs(coord[5])])
            #ary=P.griddata(lon.ravel(),lat.ravel(),ary.ravel(),LON,LAT);
            ary=scipy.interpolate.griddata(np.array([lon.ravel(),lat.ravel()]).T,ary.ravel(),(LON,LAT), method='cubic');
            geotransform=[LON[0,0],abs(coord[1]), 0, LAT[0,0], 0,-abs(coord[5])];
            print(geotransform)
    else:
      y = np.linspace(1, ary.shape[0], ary.shape[0])
      x = np.linspace(1, ary.shape[1], ary.shape[1])
      Y,X=np.meshgrid(y,x ,indexing='ij')
      lon=geotransform[0]+geotransform[1]*X+Y*geotransform[2]
      lat=geotransform[3]+geotransform[4]*X+Y*geotransform[5]
    if rescale:
        import basic
        ary=basic.rescale(ary, rescale);

    # data exists in 'ary' with values range 0 - 255
    # Uncomment next line if ary[0][0] is upper-left corner
    #ary = numpy.flipup(ary)
    Ny, Nx = ary.shape
    item_length=Ny*Nx

    import csv
    lol=[lon.ravel(), lat.ravel(), ary.ravel()]
    with open(filename, 'wb') as test_file:
        file_writer = csv.writer(test_file)
        for i in range(item_length):
            file_writer.writerow([x[i] for x in lol])
    print("File written to: " + filename);

def readCoord(filename, srs_proj4=None, dtype=np.float64):
    '''
    lon,lat=lonlat('/path/to/file', srs_proj4=None)
    '''
    #http://stackoverflow.com/questions/2922532/obtain-latitude-and-longitude-from-a-geotiff-file
    # get the existing coordinate system
    xn,yn,xN,yN=corners(filename);

    ds = gdal.Open(filename)
    if srs_proj4 is None:
        old_cs=osr.SpatialReference()
        old_cs.ImportFromWkt(ds.GetProjectionRef())
    else:
        old_cs=osr.SpatialReference()
        old_cs.ImportFromProj4(srs_proj4);

    # create the new coordinate system
    wgs84_wkt = """
    GEOGCS["WGS 84",
        DATUM["WGS_1984",
            SPHEROID["WGS 84",6378137,298.257223563,
                AUTHORITY["EPSG","7030"]],
            AUTHORITY["EPSG","6326"]],
        PRIMEM["Greenwich",0,
            AUTHORITY["EPSG","8901"]],
        UNIT["degree",0.01745329251994328,
            AUTHORITY["EPSG","9122"]],
        AUTHORITY["EPSG","4326"]]"""
    new_cs = osr.SpatialReference()
    new_cs .ImportFromWkt(wgs84_wkt)
    # create a transform object to convert between coordinate systems
    transform = osr.CoordinateTransformation(old_cs,new_cs)

    #get the point to transform, pixel (0,0) in this case
    #width = ds.RasterXSize
    #height = ds.RasterYSize
    #gt = ds.GetGeoTransform()
    #minx = gt[0]
    #miny = gt[3] + width*gt[4] + height*gt[5]

    #get the coordinates in lat long
    #latlong = transform.TransformPoint(minx,miny)
    lonn,latn,z=transform.TransformPoint(xn,yn)
   # print(latn, lon)
    #lonN,latn,z=transform.TransformPoint(xN,yn)
    lonN,latN,z=transform.TransformPoint(xN,yN)
    lat=np.linspace(latn,latN,ds.RasterYSize).astype(dtype);
    lon=np.linspace(lonn,lonN,ds.RasterXSize).astype(dtype);
    LON,LAT=np.meshgrid(lon,lat);
    return LON, np.flipud(LAT);

def corners(filename):
    '''
    (minx,miny,maxx,maxy)=corners('/path/to/file')
    '''
    #http://stackoverflow.com/questions/2922532/obtain-latitude-and-longitude-from-a-geotiff-file
    ds = gdal.Open(filename)
    width = ds.RasterXSize
    height = ds.RasterYSize
    gt = ds.GetGeoTransform()
    minx = gt[0]
    miny = gt[3] + width*gt[4] + height*gt[5]
    maxx = gt[0] + width*gt[1] + height*gt[2]
    maxy = gt[3]
    return (minx,miny,maxx,maxy)

def bounding_box(filename):
    """
    ((lon1,lat1), (lon2,lat2), (lon3,lat3), (lon4,lat4))=bounding_box('/path/to/file')
    """
    gT=getGeoTransform(filename)
    width, height=get_size(filename)
    return (xy2coord(0,0,gT), xy2coord(width,0,gT), xy2coord(width, height,gT), xy2coord(0, height,gT))

def xy2coord(x,y,gT):
    '''
    lon,lat=xy2coord(x,y,geoTransform)
    projects pixel index to position based on geotransform.
    '''
    coord_x=gT[0] + x*gT[1] + y*gT[2]
    coord_y=gT[3] + x*gT[4] + y*gT[5]
    return coord_x, coord_y

def coord2xy(x,y,gT):
    '''
    x,y = coord2xy(lon, lat, geoTransform)
    calculates pixel index closest to the lon, lat.
    '''
    #ref: https://gis.stackexchange.com/questions/221292/retrieve-pixel-value-with-geographic-coordinate-as-input-with-gdal/221430
    xOrigin = gT[0]
    yOrigin = gT[3]
    pixelWidth = gT[1]
    pixelHeight = -gT[5]

    col = np.array((x - xOrigin) / pixelWidth).astype(int)
    row = np.array((yOrigin - y) / pixelHeight).astype(int)

    return row,col

def getGeoTransform(filename):
    '''
    [top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution]=getGeoTransform('/path/to/file')
    '''
    #http://stackoverflow.com/questions/2922532/obtain-latitude-and-longitude-from-a-geotiff-file
    ds = gdal.Open(filename)
    return ds.GetGeoTransform()
def get_size(filename):
    """(width, height) = get_size(filename)
    """
    ds = gdal.Open(filename)
    width = ds.RasterXSize
    height = ds.RasterYSize
    ds=None
    return (width, height)

def get_proj4(filename):
    ds=gdal.Open(filename)
    sr=gdal.osr.SpatialReference()
    sr.ImportFromWkt(ds.GetProjectionRef)
    return sr.ExportToProj4()

def transformPoint(x,y,z,s_srs='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', t_srs='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'):
    '''
    transformPoint(x,y,z,s_srs='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', t_srs='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs')

    Known Bugs: gdal transform may fail if a proj4 string can not be found for the EPSG or WKT formats.
    '''
    from .. import base

    srs_cs=osr.SpatialReference()
    if "EPSG" == s_srs[0:4]:
      srs_cs.ImportFromEPSG(int(s_srs.split(':')[1]));
    elif "GEOCCS" == s_srs[0:6]:
      srs_cs.ImportFromWkt(s_srs);
    else:
      srs_cs.ImportFromProj4(s_srs);

    trs_cs=osr.SpatialReference()
    if "EPSG" == t_srs[0:4]:
      trs_cs.ImportFromEPSG(int(t_srs.split(':')[1]));
    elif "GEOCCS" == t_srs[0:6]:
      trs_cs.ImportFromWkt(t_srs);
    else:
      trs_cs.ImportFromProj4(t_srs);
    transform = osr.CoordinateTransformation(srs_cs,trs_cs)

    if base.numel(x)>1:
      return [  transformPoint(x[k], y[k], z[k]) for k in range(base.numel(x))]
    else:
      try:
        return transform.TransformPoint((x,y,z));
      except:
        return transform.TransformPoint(x,y,z)

def rsat2_export(filename, export_filename=None, yclip=225):
    """ 'export_filename'=rsat2_export(filename, export_filename=None, yclip=225)
    """
    ds=gdal.Open(filename, gdal.GA_ReadOnly)
    w=ds.RasterXSize
    h=ds.RasterYSize
    data=10.*np.log10(abs(ds.ReadAsArray(ysize=ds.RasterYSize-yclip)))
    gT=ds.GetGeoTransform()
    if export_filename is None:
      timestr=''.join(ds.GetMetadata()['ACQUISITION_START_TIME'].split(".")[0].split(":"))
      export_filename='_'.join(filename.split(":")[0:2])+"_"+timestr+"_cog.tif"
    data[data==-np.inf]=np.nan
    data[data==np.inf]=np.nan #should not be necessary
    writeTiff(data, gT, filename=export_filename, cog=True, gcps=ds.GetGCPs(), nodata=np.nan)
    return export_filename

def clip_gT(gT, xmin, xmax, ymin, ymax, method='image'):
    '''calculate new geotransform for a clipped raster either using pixels or projected coordinates.
    clipped_gT=clip_gT(gT, xmin, xmax, ymin, ymax, method='image')
    method: 'image' | 'coord'
    '''
    if method == 'image':
      y,x=xy2coord(ymin, xmin, gT); #top left, reference, coordinate
    if method == 'coord':
      #find nearest pixel
      yi, xi = coord2xy(ymin, xmin, gT)
      #get pixel coordinate
      y,x=xy2coord(yi, xi, gT)
    gTc=list(gT)
    gTc[0]=y
    gTc[3]=x
    return tuple(gTc)

def auto_clip(arr, gT, no_data=np.nan):
    """automatically  remova the excess no-data pixels in raster. Similar to auto_clip in GIMP.
    cliipped_raster, clipped_gT = auto_clip(raster, geoTransform, no_data=np.nan)
    """
    if np.isnan(no_data):
      m=~np.isnan(arr)
    else:
      m= arr!=no_data
    data_cols = numpy.where(m.sum(0)  > 50)[0]
    data_rows = numpy.where(m.sum(1)  > 50)[0]
    gTc=clip_gT(gT, data_rows[0], data_rows[-1], data_cols[0], data_cols[-1])
    arrC=arr[data_rows[0]:data_rows[-1], data_cols[0]:data_cols[-1]]
    return arrC, gTc

def translate_gT(gT, x_offset, y_offset):
    '''gT_translated=translate_gT(gT, x_offset, y_offset)
    simply offsets the starting 0th and 3rd elements of geotransform accordingly.
    '''
    gTt=list(gT)
    gTt[0]=gTt[0]+x_offset
    gTt[3]=gTt[3]+y_offset
    return tuple(gTt)

def translate_tif(filename, x_offset, y_offset):
    arr=readData(filename)
    gT=getGeoTransform(filename)
    gTt=gsp.connectors.gdal.translate_gT(gT, x_offset, y_offset)
    writeTiff(arr, gTt, filename=filename[-4]+'_translated.tif')
    return filename[:-4]+'_translated.tif'

def auto_clip_tif(f, no_data=np.nan):
    print('Reading {}'.format(f))
    arr=readData(f)
    gT=getGeoTransform(f)
    if np.isnan(no_data):
      m=~np.isnan(arr)
    else:
      m= arr!=no_data
    data_cols = numpy.where(m.sum(0)  > 50)[0]
    data_rows = numpy.where(m.sum(1)  > 50)[0]
    gTc=clip_gT(gT, data_rows[0], data_rows[-1], data_cols[0], data_cols[-1])
    arrC=arr[data_rows[0]:data_rows[-1], data_cols[0]:data_cols[-1]]
    writeTiff(arrC, gTc, filename=f[:-4]+'_clipped.tif')

def distance_lat_lon(lat1, lon1, lat2, lon2):
    p = 0.017453292519943295
    a = 0.5 - np.cos((lat2-lat1)*p)/2 + np.cos(lat1*p)*np.cos(lat2*p) * (1-np.cos((lon2-lon1)*p)) / 2
    return 12742 * np.arcsin(np.sqrt(a))

def closest_lat_lon(lat_vector, lon_vector, lat_point, lon_point):
    """
     Find the closest index in a vector.
     index = closest_lat_lon(lat_vector, lon_vector, lat_point, lon_point)
    """
    return np.argmin(distance_lat_lon(lat_vector, lon_vector,lat_point,lon_point))
def get_point_value(filename, x,y,srs_proj4='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', band=1):
    """
    z=get_point_value(filename, x,y,srs_proj4='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs')
    """
    ds=gdal.Open(filename, gdal.GA_ReadOnly)
    w=ds.RasterXSize
    h=ds.RasterYSize
    gT=ds.GetGeoTransform()
    t_srs=get_proj4(filename)
    rb=ds.GetRasterBand(band)
    if t_srs != srs_proj4:
      x,y,z=transformPoint(x,y,z,s_srs=srs_proj4, t_srs=t_srs)
    cx,cy=coord2xy(x,y,gT)
    return rb.ReadAsArray(px,py,1,1)[0]


from django.core.exceptions import ValidationError
from rest_framework import serializers

from reviews.models import Category, Comment, Genre, Review, Title, User

class ConfirmationTokenSerializer(serializers.Serializer):
    """Serializing verification data to provide full user registration"""

    username = serializers.CharField(required=True)
    confirmation_code = serializers.CharField(required=True)

class RegistrationSerializer(serializers.Serializer):
    """Serializing data to provide a user creation"""

    email = serializers.EmailField(required=True)
    username = serializers.CharField(required=True)

    def validate(self, data):
        if data["username"] == "me":
            raise ValidationError("Пользователь не может иметь имя 'me'")
        return data

class UserSerializer(serializers.ModelSerializer):
    """Serializing data for work with user and his profile"""

    class Meta:
        model = User
        fields = (
            "username",
            "first_name",
            "last_name",
            "email",
            "bio",
            "role",
        )

class CategorySerializer(serializers.ModelSerializer):
    """Serializer for categories."""

    class Meta:
        model = Category
        fields = ("name", "slug")
        lookup_field = 'slug'

class GenreSerializer(serializers.ModelSerializer):
    """Serializer for genres."""

    class Meta:
        model = Genre
        fields = ("name", "slug")

class WriteTitleSerializer(serializers.ModelSerializer):
    """Serializer for write request for titles."""

    category = serializers.SlugRelatedField(
        slug_field="slug",
        queryset=Category.objects.all(),
    )
    genre = serializers.SlugRelatedField(
        many=True,
        slug_field="slug",
        queryset=Genre.objects.all(),
    )
    rating = serializers.SerializerMethodField()

    class Meta:
        model = Title
        fields = "__all__"

    def get_rating(self, obj):
        """Return 0 after creation."""
        return 0

class ReadTitleSerializer(serializers.ModelSerializer):
    """Serializer for read requests for titles."""

    genre = GenreSerializer(many=True)
    category = CategorySerializer()
    rating = serializers.SerializerMethodField()

    class Meta:
        model = Title
        fields = "__all__"
        read_only_fields = ("name", "year", "description", "genre", "category")

    def get_rating(self, obj):
        """Return object rating calculated in viewset."""
        return obj.rating

class ReviewSerializer(serializers.ModelSerializer):
    """Serializer for reviews."""
    score = serializers.IntegerField(max_value=10, min_value=0)
    author = serializers.SlugRelatedField(
        slug_field="username",
        read_only=True,
        default=serializers.CurrentUserDefault(),  # добавил новое
    )

    class Meta:
        model = Review
        fields = ("id", "text", "author", "score", "pub_date")

    def validate(self, attrs):
        """Check that each author can have only one review
        for particular title.
        """
        if not self.context["request"].method == "POST":
            return attrs
        if Review.objects.filter(
            title_id=self.context["view"].kwargs.get("title_id"),
            author=self.context["request"].user,
        ).exists():
            raise serializers.ValidationError(
                (
                    "Автор может оставлять ревью на каждое произведение "
                    "только один раз"
                )
            )
        return attrs

class CommentSerializer(serializers.ModelSerializer):
    """Serializer for comments."""

    author = serializers.SlugRelatedField(
        slug_field="username",
        read_only=True,
    )

    class Meta:
        model = Comment
        fields = ("id", "text", "author", "pub_date")


from pathlib import Path

import joblib

IS_KAGGLE = True

if IS_KAGGLE:
    DATA_DIR = Path("/kaggle/working/chap5-data")
    OUTPUT_DIR = Path("/kaggle/working/")
else:
    DATA_DIR = Path("../data")  # Path(os.getenv("QQP_DATA_DIR", "/data"))
    OUTPUT_DIR = Path("../outputs")

INPUT_DIR = DATA_DIR / "input"
TRAIN_CSV_PATH = INPUT_DIR / "train.csv"
TEST_CSV_PATH = INPUT_DIR / "test.csv"
EMBEDDING_DIR = DATA_DIR / "embeddings"
GLOVE_PATH = EMBEDDING_DIR / "glove.840B.300d.bin"

FEATURE_MEMORY = joblib.Memory(DATA_DIR / "cache")

SPLIT_RANDOM_SEED = 1
EPS = 1e-10
NUM_PROCESSES = 4  # int(os.getenv("NUM_PROCESSES", 1))
NUM_TRAIN_SAMPLES = 404290
NUM_TEST_SAMPLES = 2345796
NUM_DRYRUN_SAMPLES = 1000


from warnings import filters
from billing.billing.api.sales_invoice.create_sales_invoice import re_eveluate_sales_orders
# from billing.billing.utils.payment_notifications import get_party_phone
import frappe
from datetime import date
from frappe.utils.background_jobs import enqueue

from frappe.utils.data import nowdate

def calculate_age(birthDate):
    days_in_year = 365.2425
    age = int((date.today() - birthDate).days / days_in_year)
    return age

def lab_test_after_insert_hook(doc,state):
    patient = doc.get('patient')
    dob = frappe.db.get_value('Patient',{ 'name':patient }, 'dob')
    gender = frappe.db.get_value('Patient',{ 'name':patient }, 'gender')
    sex = frappe.db.get_value('Patient',{ 'name':patient }, 'sex')
    doc.patient_age = age_calc(dob,doc.name)
    doc.patient_sex = gender or sex
    doc.save(ignore_permissions=True)
    enqueue(method=append_same_category_tests,name=doc.get('name'), queue='short', timeout=600)
    enqueue(method=tetst_age_fix, queue='short', timeout=600)
    # test_sharing_sample_with = doc.get('share_sample_with')
    # frappe.msgprint("sharing sample  " + test_sharing_sample_with)
    # if test_sharing_sample_with:
    #     frappe.db.set_value('Lab Test', test_sharing_sample_with,{'share_sample_with': doc.name})
    # tests_sharing_sample =  frappe.db.get_all('Lab Test Sample Share',filters={'parent':doc.name},fields=['name','lab_test'])
    # if len(tests_sharing_sample)>0:
    #     for test in tests_sharing_sample:
    #         lab_test = frappe.get_doc('Lab Test',test['lab_test'])
    #         test_item = lab_test.append('lab_test_sample_share')
    #         test_item.lab_test = test['lab_test']
    #         lab_test.save(ignore_permissions=True)
    #     frappe.msgprint('updated related')

def age_calc(dob,lab_name=''):
    currentDate = date.today()#nowdate() #datetime.datetime.now()
    # dob = '2022-05-01'
    # deadline= dob #input ('Plz enter your date of birth (mm/dd/yyyy) ')
    deadlineDate= dob# datetime.datetime.strptime(deadline,'%Y-%m-%d')
    # print (type(deadlineDate))
    # print (type(currentDate))

    daysLeft = currentDate - deadlineDate
    # print(daysLeft)

    years = ((daysLeft.total_seconds())/(365.242*24*3600))
    yearsInt=int(years)

    months=(years-yearsInt)*12
    monthsInt=int(months)

    days=(months-monthsInt)*(365.242/12)
    daysInt=int(days)

    hours = (days-daysInt)*24
    hoursInt=int(hours)
    if yearsInt>0:
        # if yearsInt>100:
        #     enqueue(method=log_error ,lab_name=lab_name,dob=dob, queue='short', timeout=600)
        # # print("{0}Y".format(yearsInt))
        # else:
        return "{0}Y".format(yearsInt)
    if monthsInt>0:
        # print("{0}M".format(monthsInt))
        return "{0}M".format(monthsInt)
    if daysInt>0:
        # print("{0}D".format(daysInt))
        return "{0}D".format(daysInt)
    if hoursInt>0:
        # print("{0}H".format(hoursInt))
        return "{0}H".format(hoursInt)

# bench execute lims.doc_hooks.lab_test.age_test
def age_test():
    pats = frappe.get_all('Lab Test',fields=['name','patient'],filters={'patient':'1122083'})
    for p in pats:
        print(p['name'])
        dob = frappe.db.get_value('Patient',{ 'name': p['patient'] }, 'dob')
        # print(type(dob))
        print(str(dob))
        age = age_calc(dob,p['name'])
        print('age ',age)
        # frappe.db.set_value('Lab Test',p['name'],{'patient_age':age})

# bench execute lims.doc_hooks.lab_test.append_same_category_tests
@frappe.whitelist()
def append_same_category_tests(name):
    from mtrh_dev.mtrh_dev.utilities import get_link_to_form_new_tab
    # from clinical.hook.lab_test import get_sample_shares
    # name='3BQ'
    lab_doc = frappe.get_doc('Lab Test',name)
    sql="""select tlt.name,tlt.template,tlt.workflow_state,ltc.test_group,ltc.lab_test_template,tlt.patient from `tabLab Test` tlt RIGHT join `tabLab Test Codes` ltc on tlt.template=ltc.lab_test_template where  tlt.patient='{0}' and tlt.docstatus=0""".format(lab_doc.patient)
    # tlt.workflow_state='To Receive' and  and tlt.template='{1}' ,doc.template
    # print(sql)
    # link_arr=[]
    name_arr=[]
    res = frappe.db.sql(sql,as_dict=1)
    names = [x.name for x in res]
    for n in names:
        # link_arr.append(get_link_to_form_new_tab(doctype="Lab Test", name=n, label=n))
        name_arr.append({'name':get_link_to_form_new_tab(doctype="Lab Test", name=n, label=n) ,'Test':frappe.db.get_value('Lab Test',n,'template'),'workflow_state':frappe.db.get_value('Lab Test',n,'workflow_state')})
        update_sample_share(n,names)
    # frappe.msgprint(title='Labs Sharing Same Sample',msg=str(name_arr))
    get_sample_shares(name)
    return name_arr

def update_sample_share(name,names):
    lab_doc = frappe.get_doc('Lab Test',name)
    for n in names:
        if n!= lab_doc.get('name'):
            if not frappe.db.exists('Lab Test Sample Share',{'parent':name,'lab_test':n}):
                sample_share = lab_doc.append('lab_test_sample_share')
                sample_share.lab_test = n
                lab_doc.save(ignore_permissions=True)

def get_sample_shares(lab_name):
    tests_sharing_sample_child =  frappe.db.get_all('Lab Test Sample Share',filters={'lab_test':['IN',lab_name]},fields=['name','lab_test','parent'])
    tests_sharing_sample_parent =  frappe.db.get_all('Lab Test Sample Share',filters={'parent':lab_name},fields=['name','lab_test','parent'])
    tests_sharing_sample = tests_sharing_sample_parent or tests_sharing_sample_child
    test_names = []
    if len(tests_sharing_sample)>0:
        parent_test = tests_sharing_sample[0]['parent']
        tests_sharing_sample =  frappe.db.get_all('Lab Test Sample Share',filters={'parent':parent_test},fields=['name','lab_test'])
        for test in tests_sharing_sample:
            test_names.append(test['lab_test'])
        test_names.append(lab_name)
    else:
        test_names.append(lab_name)
    shares = list(dict.fromkeys(test_names))
    process_lab_array= frappe.db.get_all('Lab Test',filters={'name':['IN',shares]},fields=['processing_lab'])
    employee_array = frappe.db.get_all('Lab Test',filters={'name':['IN',shares]},fields=['employee'])
    process_lab=''
    employee=''
    # print(str(process_lab))
    # print(str(employee))
    for l in process_lab_array:
        if l.processing_lab:
            process_lab = l.processing_lab
    for e in employee_array:
        if e.employee:
            employee = e.employee
    for n in shares:
        print(n)
        # bulk_workflow_update(docname=n,process_lab=process_lab,employee=employee)

def bulk_workflow_update(docname,process_lab='',employee=''):
    from frappe.model.workflow import apply_workflow
    doc=frappe.get_doc('Lab Test',docname )#'IP'ß
    # print(get_sample_shares(doc.name))
    # actions=['Forward For Payment','Approve Payment','Send To Lab','Receive Lab Test','Forward For Verification']
    # state_action_dict=[
    #     { 'state':'Awaiting Checkin','action':actions[0]},
    #     { 'state':'Awaiting Payment','action':actions[1]},
    #     { 'state':'Awaiting Sampling','action':actions[2]},
    #     { 'state':'To receive','action':actions[3]},
    #     { 'state':'Processing','action':actions[4]},
    #     { 'state':'Awaiting Verification','action':actions[5]}
    # ]
    workflow_state = doc.get('workflow_state')
    if workflow_state=='Awaiting Checkin':
        apply_workflow(doc=doc, action="Forward For Payment")
    if workflow_state=='Awaiting Payment':
        # apply_workflow(doc=doc, action="Approve Payment")
        re_eveluate_sales_orders(patient_name=doc.patient,lab_name=doc.name)
        apply_workflow(doc=doc, action="Approve Payment")
    if workflow_state=='Awaiting Sampling':
        doc.processing_lab = process_lab
        doc.employee = employee
        doc.save(ignore_permissions=True)
        apply_workflow(doc=doc, action="Send To Lab")
    if workflow_state=='To receive':
        doc.processing_lab = process_lab
        doc.employee = employee
        doc.save(ignore_permissions=True)
        apply_workflow(doc=doc, action="Receive Lab Test")
    if workflow_state=='Processing':
        apply_workflow(doc=doc, action="Forward For Verification")
    if workflow_state=='Awaiting Verification':
        apply_workflow(doc=doc, action="Post Lab Test")

# bench execute lims.doc_hooks.lab_test.lab_clean

def lab_clean():
    sql = "select name,idx from `tabNormal Test Result` where parent='B73'"
    items=frappe.db.sql(sql,as_dict=1)
    count = 0
    for i in items:
        count +=1
        if count>1:
            sq= "delete from `tabNormal Test Result` where name='{0}'".format(i.name)
            frappe.db.sql(sq,as_dict=1)
            print(count)

# bench execute lims.doc_hooks.lab_test.comment_count
def comment_count(name='B73'):
    # return 1
    sqlc="select count(name) as cnt,reference_name from tabComment where reference_doctype='Sales Invoice' and reference_name is not null group by reference_name HAVING COUNT(name) > 15 order by reference_name"
    parents=frappe.db.sql(sqlc,as_dict=1)
    for p in parents:
        print('ref ',p.reference_name, '  ',p.cnt)
        sql = "select name,reference_doctype,reference_name from tabComment where reference_name='{0}'".format(p.reference_name)
        # print(sql)
        items=frappe.db.sql(sql,as_dict=1)
        count = 0
        for i in items:
            count +=1
            if count>1:
                print('item count ',count)
                sq= "delete from tabComment where name='{0}'".format(i.name)
                frappe.db.sql(sq,as_dict=1)
                # frappe.delete_doc('Comment',i.name)
                frappe.db.commit()
                # print(count)

 # bench execute lims.doc_hooks.lab_test.tetst_age_fix
def tetst_age_fix():
    sql = "select name,patient,docstatus from `tabLab Test` where patient_age is null;"
    labs = frappe.db.sql(sql,as_dict=1)
    for lab in labs:
        patient = lab.get('patient')
        # print(' patient ', patient)
        dob = frappe.db.get_value('Patient',{ 'name':patient }, 'dob')
        patient_age = age_calc(dob,lab.get('name'))
        # print(patient_age)
        up_sq = "update `tabLab Test` set patient_age ='{0}' where name='{1}';".format(patient_age,lab.get('name'))
        print(up_sq)
        frappe.db.sql(up_sq,as_dict=1)

def log_error(lab_name,dob):
    log  = frappe.new_doc('Lims Error Log')
    log.ordernumber  = lab_name
    log.log_number = ''
    log.unprocessed_result = str(dob)
    log.save(ignore_permissions=True)

# bench execute lims.doc_hooks.lab_test.patient_record_exist
def patient_record_exist():
    # labs = frappe.get_all("Lab Test",filters={'docstatus':1},fields=['name','patient'])
    labs = frappe.db.count('Lab Test', {'docstatus': 1})
    recs = frappe.db.count('Patient Medical Record', {'reference_doctype': 'Lab Test'})
    print('labs ',labs, ' recs ',recs)


from django.urls import path, include
from . import views
from rest_framework import routers

route = routers.DefaultRouter()
route.register("user", views.UserViewSet, basename='user')
route.register("tuyenxe", views.TuyenXeViewset, basename='tuyenxe')
route.register("chuyenxe", views.ChuyenXeViewset, basename='chuyenxe')
route.register("datve", views.DatVeViewset, basename='datve')
route.register(prefix='comments', viewset=views.CommentViewSet, basename='comment')
# route.register("thongke", views.ThongKeViewSet, basename='thongke')

urlpatterns = [
    path('', include(route.urls)),
]


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import re
import os
import copy
import yaml
from enum import Enum

from utils.util import mace_check
from utils.util import MaceLogger
from py_proto import mace_pb2

CPP_KEYWORDS = [
    'alignas', 'alignof', 'and', 'and_eq', 'asm', 'atomic_cancel',
    'atomic_commit', 'atomic_noexcept', 'auto', 'bitand', 'bitor',
    'bool', 'break', 'case', 'catch', 'char', 'char16_t', 'char32_t',
    'class', 'compl', 'concept', 'const', 'constexpr', 'const_cast',
    'continue', 'co_await', 'co_return', 'co_yield', 'decltype', 'default',
    'delete', 'do', 'double', 'dynamic_cast', 'else', 'enum', 'explicit',
    'export', 'extern', 'false', 'float', 'for', 'friend', 'goto', 'if',
    'import', 'inline', 'int', 'long', 'module', 'mutable', 'namespace',
    'new', 'noexcept', 'not', 'not_eq', 'nullptr', 'operator', 'or', 'or_eq',
    'private', 'protected', 'public', 'register', 'reinterpret_cast',
    'requires', 'return', 'short', 'signed', 'sizeof', 'static',
    'static_assert', 'static_cast', 'struct', 'switch', 'synchronized',
    'template', 'this', 'thread_local', 'throw', 'true', 'try', 'typedef',
    'typeid', 'typename', 'union', 'unsigned', 'using', 'virtual', 'void',
    'volatile', 'wchar_t', 'while', 'xor', 'xor_eq', 'override', 'final',
    'transaction_safe', 'transaction_safe_dynamic', 'if', 'elif', 'else',
    'endif', 'defined', 'ifdef', 'ifndef', 'define', 'undef', 'include',
    'line', 'error', 'pragma',
]

def sanitize_load(s):
    # do not let yaml parse ON/OFF to boolean
    for w in ["ON", "OFF", "on", "off"]:
        s = re.sub(r":\s+" + w + "$", r": '" + w + "'", s)

    # sub ${} to env value
    s = re.sub(r"\${(\w+)}", lambda x: os.environ[x.group(1)], s)
    return yaml.load(s)

def parse(path):
    with open(path) as f:
        config = sanitize_load(f.read())

    return config

def parse_device_info(path):
    conf = parse(path)
    return conf["devices"]

class ModelKeys(object):
    platform = "platform"
    runtime = "runtime"
    models = 'models'
    graph_optimize_options = "graph_optimize_options"
    input_tensors = "input_tensors"
    input_shapes = "input_shapes"
    input_data_types = "input_data_types"
    input_data_formats = "input_data_formats"
    input_ranges = "input_ranges"
    output_tensors = "output_tensors"
    output_shapes = "output_shapes"
    output_data_types = "output_data_types"
    output_data_formats = "output_data_formats"
    check_tensors = "check_tensors"
    check_shapes = "check_shapes"
    model_file_path = "model_file_path"
    model_sha256_checksum = "model_sha256_checksum"
    weight_file_path = "weight_file_path"
    weight_sha256_checksum = "weight_sha256_checksum"
    quantize_range_file = "quantize_range_file"
    quantize = "quantize"
    quantize_schema = "quantize_schema"
    quantize_large_weights = "quantize_large_weights"
    quantize_stat = "quantize_stat"
    change_concat_ranges = "change_concat_ranges"
    winograd = "winograd"
    cl_mem_type = "cl_mem_type"
    data_type = "data_type"
    subgraphs = "subgraphs"
    validation_inputs_data = "validation_inputs_data"

class DataFormat(Enum):
    NONE = 0
    NHWC = 1
    NCHW = 2
    HWIO = 100
    OIHW = 101
    HWOI = 102
    OHWI = 103
    AUTO = 1000

def parse_data_format(str):
    str = str.upper()
    mace_check(str in [e.name for e in DataFormat],
               "unknown data format %s" % str)
    return DataFormat[str]

class DeviceType(Enum):
    CPU = 0
    GPU = 2
    HEXAGON = 3
    HTA = 4
    APU = 5
    CPU_GPU = 100

DEVICE_MAP = {
    "cpu": DeviceType.CPU,
    "gpu": DeviceType.GPU,
    "hexagon": DeviceType.HEXAGON,
    "dsp": DeviceType.HEXAGON,
    "hta": DeviceType.HTA,
    "apu": DeviceType.APU,
    "cpu+gpu": DeviceType.CPU_GPU
}

def parse_device_type(str):
    mace_check(str in DEVICE_MAP, "unknown device %s" % str)
    return DEVICE_MAP[str]

class Platform(Enum):
    TENSORFLOW = 0
    CAFFE = 1
    ONNX = 2
    MEGENGINE = 3
    KERAS = 4
    PYTORCH = 5

def parse_platform(str):
    str = str.upper()
    mace_check(str in [e.name for e in Platform],
               "unknown platform %s" % str)
    return Platform[str]

DATA_TYPE_MAP = {
    'float32': mace_pb2.DT_FLOAT,
    'int32': mace_pb2.DT_INT32,
}

def parse_data_type(str):
    if str == "float32":
        return mace_pb2.DT_FLOAT
    if str == "float16":
        return mace_pb2.DT_FLOAT16
    elif str == "int32":
        return mace_pb2.DT_INT32
    else:
        mace_check(False, "data type %s not supported" % str)

def parse_internal_data_type(str):
    if str == 'fp32_fp32':
        return mace_pb2.DT_FLOAT
    elif str == 'bf16_fp32':
        return mace_pb2.DT_BFLOAT16
    elif str == 'fp16_fp16':
        return mace_pb2.DT_FLOAT16
    else:
        return mace_pb2.DT_HALF

def to_list(x):
    if isinstance(x, list):
        return x
    else:
        return [x]

def parse_int_array(xs):
    if len(xs) is 0:
        return [1]
    return [int(x) for x in xs.split(",")]

def parse_float_array(xs):
    return [float(x) for x in xs.split(",")]

def normalize_input_data_types(conf, input_count):
    default_input_dt = conf[ModelKeys.data_type]
    if default_input_dt == mace_pb2.DT_HALF:
        default_input_dt = mace_pb2.DT_FLOAT  # Compatible with old version
    conf_input_dts = to_list(conf.get(ModelKeys.input_data_types, []))
    if len(conf_input_dts) == 0:
        input_data_types = [default_input_dt]
    else:
        input_data_types = [parse_data_type(dt) for dt in conf_input_dts]

    if len(input_data_types) == 1 and input_count > 1:
        input_data_types = [input_data_types[0]] * input_count
    mace_check(len(input_data_types) == input_count,
               "the number of input_data_types should be "
               "the same as input tensors")
    conf[ModelKeys.input_data_types] = input_data_types

def normalize_output_data_types(conf, output_count):
    default_output_dt = conf[ModelKeys.data_type]
    if default_output_dt == mace_pb2.DT_HALF:
        default_output_dt = mace_pb2.DT_FLOAT  # Compatible with old version
    conf_output_dts = to_list(conf.get(ModelKeys.output_data_types, []))
    if len(conf_output_dts) == 0:
        output_data_types = [default_output_dt]
    else:
        output_data_types = [parse_data_type(dt) for dt in conf_output_dts]

    if len(output_data_types) == 1 and output_count > 1:
        output_data_types = [output_data_types[0]] * output_count
    mace_check(len(output_data_types) == output_count,
               "the number of output_data_types should be "
               "the same as output tensors")
    conf[ModelKeys.output_data_types] = output_data_types

def normalize_model_config(conf):
    conf = copy.deepcopy(conf)
    if ModelKeys.subgraphs in conf:
        subgraph = conf[ModelKeys.subgraphs][0]
        del conf[ModelKeys.subgraphs]
        conf.update(subgraph)

    conf[ModelKeys.platform] = parse_platform(conf[ModelKeys.platform])
    conf[ModelKeys.runtime] = parse_device_type(conf[ModelKeys.runtime])

    if ModelKeys.quantize in conf and conf[ModelKeys.quantize] == 1:
        conf[ModelKeys.data_type] = mace_pb2.DT_FLOAT
    else:
        if ModelKeys.data_type in conf:
            conf[ModelKeys.data_type] = parse_internal_data_type(
                conf[ModelKeys.data_type])
        else:
            conf[ModelKeys.data_type] = mace_pb2.DT_HALF

    # parse input
    conf[ModelKeys.input_tensors] = to_list(conf[ModelKeys.input_tensors])
    conf[ModelKeys.input_tensors] = [str(i) for i in
                                     conf[ModelKeys.input_tensors]]
    input_count = len(conf[ModelKeys.input_tensors])
    conf[ModelKeys.input_shapes] = [parse_int_array(shape) for shape in
                                    to_list(conf[ModelKeys.input_shapes])]
    mace_check(
        len(conf[ModelKeys.input_shapes]) == input_count,
        "input node count and shape count do not match")

    normalize_input_data_types(conf, input_count)

    input_data_formats = [parse_data_format(df) for df in
                          to_list(conf.get(ModelKeys.input_data_formats,
                                           ["NHWC"]))]
    if len(input_data_formats) == 1 and input_count > 1:
        input_data_formats = [input_data_formats[0]] * input_count
    mace_check(len(input_data_formats) == input_count,
               "the number of input_data_formats should be "
               "the same as input tensors")
    conf[ModelKeys.input_data_formats] = input_data_formats

    input_ranges = [parse_float_array(r) for r in
                    to_list(conf.get(ModelKeys.input_ranges,
                                     ["-1.0,1.0"]))]
    if len(input_ranges) == 1 and input_count > 1:
        input_ranges = [input_ranges[0]] * input_count
    mace_check(len(input_ranges) == input_count,
               "the number of input_ranges should be "
               "the same as input tensors")
    conf[ModelKeys.input_ranges] = input_ranges

    # parse output
    conf[ModelKeys.output_tensors] = to_list(conf[ModelKeys.output_tensors])
    conf[ModelKeys.output_tensors] = [str(i) for i in
                                      conf[ModelKeys.output_tensors]]
    output_count = len(conf[ModelKeys.output_tensors])
    conf[ModelKeys.output_shapes] = [parse_int_array(shape) for shape in
                                     to_list(conf[ModelKeys.output_shapes])]
    mace_check(len(conf[ModelKeys.output_tensors]) == output_count,
               "output node count and shape count do not match")

    normalize_output_data_types(conf, output_count)

    output_data_formats = [parse_data_format(df) for df in
                           to_list(conf.get(ModelKeys.output_data_formats,
                                            ["NHWC"]))]
    if len(output_data_formats) == 1 and output_count > 1:
        output_data_formats = [output_data_formats[0]] * output_count
    mace_check(len(output_data_formats) == output_count,
               "the number of output_data_formats should be "
               "the same as output tensors")
    conf[ModelKeys.output_data_formats] = output_data_formats

    if ModelKeys.check_tensors in conf:
        conf[ModelKeys.check_tensors] = to_list(conf[ModelKeys.check_tensors])
        conf[ModelKeys.check_shapes] = [parse_int_array(shape) for shape in
                                        to_list(conf[ModelKeys.check_shapes])]
        mace_check(len(conf[ModelKeys.check_tensors]) == len(
            conf[ModelKeys.check_shapes]),
                   "check tensors count and shape count do not match.")

    MaceLogger.summary(conf)

    return conf