import tulisdata

#Kamus:
'''
    type dataSaham : tuple =  (IdPemilik: integer,
                                IdPT: string,
                                Nilai : integer)
    data : SEQFILE of
        (*) row : dataSaham
        (1) mark
    rows : array of string
    length : int
    amountDataSaham : int
    arrayDataSaham : list of tuple
    fungsi  bubbleSort(array)
    fungsi roundDecimals
    prosedur calculateMean

'''

# Algoritma

def insertionSort(array, index):
    # Menukar data bila argumen Id Pemilik lebih lebih kecil dari index sebelumnya
    while (index >=1 and (array[index][0] < array[index-1][0])):
        temp = array[index]
        array[index] = array[index-1]
        array[index-1] = temp
        index = index -1
    return array

def calculateMean(array):

    index = 0
    # Menghitung rata-rata setiap jenis data yang sama
    while (array[index][0] != mark):
        sum = 0
        amount = 0
        currentID = array[index][0]
        while (array[index][0] == currentID):
            sum = sum + int(array[index][2])
            amount = amount +1
            index = index+1
        #mean = roundDecimals(sum/amount, 2)
        mean = sum/amount
        print(currentID+'='+str("{:.2f}".format(mean,2)))

# Menginput nama file dan isi data file
namafile = input()
tulisdata.TulisDataSaham(namafile)
mark = "99999999"

data = open(namafile)

rows = [row.rstrip() for row in data.readlines()]

if rows[0] == mark:
    print("File kosong")
else:
    # Mengukur panjang
    length = 0
    for i in rows:
        length +=1

    amountdataSaham = (length-1)//3
    arraydataSaham = [0 for i in range(amountdataSaham)]

    for i in range(amountdataSaham):
        dataSaham = (rows[i*3], rows[i*3+1], rows[i*3+2])
        arraydataSaham[i] = dataSaham
        arraydataSaham = insertionSort(arraydataSaham, i)

    arraydataSaham = arraydataSaham + [(mark, mark, mark)]      # Penambahan mark pada data array

    calculateMean(arraydataSaham)

data.close()


from collections import namedtuple
from collections import defaultdict
from pprint import pprint

import argparse
import csv
import json
import re
import subprocess

instruction_strings = []
sub_instruction_strings = []
meta_instructions = []
instructions_masks = []
specialization_params = {}
endloops = {}
patterns = []
meta_mapping = defaultdict(list)
implemented_meta = 0
implemented_insn = 0
system_insn = 0
float_insn = 0
implemented_vect = 0
vectorial_meta = 0
Constant = namedtuple('Constant', ['identifier', 'bits', 'signed',
                                   'multiple', 'pc_offset', 'ranges'])
Register = namedtuple('Register', ['identifier', 'bits', 'ranges',
                                   'dot_new', 'predicate'])
Range = namedtuple('Range', ['start', 'end'])

DECODER_HEADER = """#ifndef HEXAGON_DECODER_H
#define HEXAGON_DECODER_H

#include "exec/log.h"
#include "qemu-common.h"
#include "tcg.h"

#define LOG_DIS(...) qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__)
#define SET_USED_REG(reg_struct, reg) {\\
        if (is_conditional) \\
            reg_struct.conditional |= (uint64_t)1 << (reg); \\
        else \\
            reg_struct.written |= (uint64_t)1 << (reg); \\
        if (reg < 32) \\
            push_destination_reg(&(reg_struct.destination), reg); \\
}
#define SET_WRITTEN_PRE(dc, pre_index) dc->deps[dc->i].written |= (uint8_t)1 << (pre_index)
#define GET_WRITTEN_PRE(dc, pre_index) dc->deps[dc->i].written & (uint64_t)1 << (pre_index)
#define GET_WRITTEN_PREV_PRE(dc, pre_index)                      \\
        (dc->i == 0 ?                                            \\
        (dc->deps[1].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[2].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[3].written & (uint64_t)1 << (pre_index))       \\
        : dc->i == 1 ?                                           \\
        (dc->deps[0].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[2].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[3].written & (uint64_t)1 << (pre_index))       \\
        : dc->i == 2 ?                                           \\
        (dc->deps[0].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[1].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[3].written & (uint64_t)1 << (pre_index))       \\
        : dc->i == 3 ?                                           \\
        (dc->deps[0].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[1].written & (uint64_t)1 << (pre_index)) |     \\
        (dc->deps[2].written & (uint64_t)1 << (pre_index)) : -1 )
#define GET_WRITTEN_ANY_PRE(dc) \\
        (dc->deps[0].written & 0b1111) | \\
        (dc->deps[1].written & 0b1111) | \\
        (dc->deps[2].written & 0b1111) | \\
        (dc->deps[3].written & 0b1111)
#define SET_READ_PRE(dc, pre_index) dc->deps[dc->i].read |= (uint8_t)1 << (pre_index)
#define GET_USED_REG(reg_struct, reg) (reg_struct).written & (uint64_t)1 << (reg)
#define GET_COND_REG(reg_struct, reg) (reg_struct).conditional & (uint64_t)1 << (reg)
#define SET_BEGIN_COND() is_conditional = true;
#define SET_END_COND() is_conditional = false;
#define SET_JUMP_FLAG(dc) { \\
    switch (dc->jump_count) { \\
        case 0: SET_WRITTEN_PRE(dc, 4); break; \\
        case 1: SET_READ_PRE(dc, 4); break; \\
        default: assert(false && "More than two jumps in a packet!"); \\
    } \\
    dc->jump_count++; \\
}
#define ADD_IF_ZERO(x, y) {\\
        assert((x == 0 || y == 0) && "Overlapping instruction encodings!");\\
        x += y;\\
}
#define EXTRACT_P(index, dotnew) (dotnew) ? \\
        ((CR[P] & (0xff << 8 * index)) >> 8 * index) : \\
        ((CR_new[P] & (0xff << 8 * index)) >> 8 * index) : \\

#define EXTR(src, start, end) \\
(((src) >> (31 - end)) & ((1 << (end - start + 1)) - 1))
#define EXTR_2(src, start, end, start2, end2) \\
(EXTR(src, start, end) << (end2 - start2 + 1)) | EXTR(src, start2, end2)
#define EXTR_3(src, start, end, start2, end2, start3, end3) \\
(EXTR(src, start, end) << (end2 - start2 + end3 - start3 + 2)) |\\
(EXTR(src, start2, end2) << (end3 - start3 + 1)) | (EXTR(src, start3, end3));
#define EXTR_4(src, start, end, start2, end2, start3, end3, start4, end4) \\
EXTR(src, start, end) << (end2 - start2 + end3 - start3 + end4 - start4 + 3)\\
| (EXTR(src, start2, end2) << (end3 - start3 + end4 - start4 + 2))\\
| (EXTR(src, start3, end3) << (end4 - start4 + 1))\\
| EXTR(src, start4, end4)
#define EXTR_BITS(src, x) \\
((src >> x) & 1)
#define EXTR_BITS_2(src, x, y) \\
(EXTR_BITS(src, x) | (EXTR_BITS(src, y) << 1))
#define EXTR_BITS_3(src, x, y, z) \\
(EXTR_BITS_2(src, x, y) | (EXTR_BITS(src, z) << 2))
#define EXTR_BITS_4(src, x, y, z, a) \\
(EXTR_BITS_2(src, x, y) | (EXTR_BITS_2(src, z, a) << 2))
#define EXTR_BITS_5(src, x, y, z, a, b) \\
(EXTR_BITS_4(src, x, y, z, a) | (EXTR_BITS(src, b) << 4))
#define EXTR_BITS_6(src, x, y, z, a, b, c) \\
(EXTR_BITS_3(src, x, y, z) | (EXTR_BITS_3(src, a, b, c) << 3))\n

typedef struct destination_reg_node {
    int index;
    struct destination_reg_node *next;
} d_reg_node;
typedef d_reg_node * d_reg_list;

typedef struct regs {
    uint64_t written;
    uint64_t conditional;
    d_reg_list destination;
} regs_t;

typedef struct dep {
    TCGOp *begin;
    TCGOp *end;
    uint8_t written;
    uint8_t read;
} deps_t;

/* This is the state at translation time.  */
typedef struct DisasContext {
    HexagonCPU *cpu;
    target_ulong pc;
    target_ulong old_pc;
    target_ulong instruction_pc;
    target_ulong npc;

    int i;
    uint32_t ir;
    uint32_t const_ext;
    bool block_end;
    bool new_packet;
    bool extender_present;
    bool pc_written;
    bool duplex;
    bool endloop[2];
    bool is_pre_written;
    TCGOp *packet_first_op;
    int jump_count;
    regs_t regs;
    deps_t deps[4];
    deps_t * original[4];
    deps_t * ordered[4];

    struct TranslationBlock *tb;
} DisasContext;

extern TCGv GPR[32];
extern TCGv CR[32];
extern TCGv SR[64];
extern TCGv GPR_new[32];
extern TCGv CR_new[32];
extern TCGv PC_written;
extern TCGv SA[2];
extern TCGv LC[2];
extern TCGv LPCFG;

int get_destination_reg(regs_t regs, int t);
void push_destination_reg(d_reg_list* list, int index);
void register_dependency(int index, DisasContext *dc);
uint32_t decode(uint32_t ir);
uint32_t sub_decode(uint32_t ir);
regs_t execute(unsigned inst_id, DisasContext *dc);
regs_t sub_execute(unsigned inst_id, DisasContext *dc);
void endloop0(void);
void endloop01(void);
void endloop1(void);

"""
DECODER_INCLUDES = """#include "qemu/osdep.h"
#include "cpu.h"
#include "decoder.h"
#include "tcg-op.h"

static const int pred_whitening[4] = { 0xffffff00,
                                       0xffff00ff,
                                       0xff00ffff,
                                       0x00ffffff };

bool is_conditional = false;

int get_destination_reg(regs_t regs, int t) {
    d_reg_list reg_list = regs.destination;
    if (reg_list == NULL)
        assert(false && "Invalid .new instruction reference!");
    for(int i = 1; i < t; i++) {
        assert(reg_list != NULL && "Invalid .new instruction reference!");
        reg_list = reg_list->next;
    }
    return reg_list->index;
}

void push_destination_reg(d_reg_list* list, int index) {
    if (index == 29 || index == 30 || index == 31 ||
        (list != NULL && *list != NULL && (*list)->index == index)) {
        return;
    }
    d_reg_node *new_reg = (d_reg_node *)malloc(sizeof(d_reg_node));
    new_reg->next = *list;
    new_reg->index = index;
    *list = new_reg;
}

"""

def gen_macros():
    code = ""
    code += DECODER_INCLUDES
    with open(decoder_c, "w") as d:
        d.write(code)

def find_integer(string):
    for i, letter in enumerate(string):
        if not letter.isdigit():
            return i
    return len(string)

def extract_shift(string):
    if string[1:5] == ":<<N":
        return Constant(identifier="N", bits=32, signed=False,
                        multiple=1, pc_offset=False, ranges=[])
    else:
        return None

# Parse a constant from a tokenized string
def extract_const(string):
    # Extract features from constant placeholders
    identifier = "j" if string[1].islower() else "I"
    if string[1] in {"u", "U"}:
        # Unsigned N-bit immediate value
        signed = False
        pc_offset = False
    elif string[1] in {"s", "S"}:
        # Signed N-bit immediate value
        signed = True
        pc_offset = False
    elif string[1] in {"m", "M"}:
        # Signed N-bit immediate value
        signed = True
        pc_offset = False
    elif string[1] in {"r", "R"}:
        # Signed N-bit immediate value
        signed = True
        pc_offset = True
    elif string[1] == "#":
        # 32 bit value
        return Constant(bits=32)
    elif string[1] == "-" or string[1].isdigit():
        return None
    else:
        assert(False and "Constant placeholder parsing error!")

    bits_delimiter = find_integer(string[2:]) + 2
    bits = int(string[2:bits_delimiter])
    if len(string) > bits_delimiter and string[bits_delimiter] == ":":
        multiple_delimiter = find_integer(
                string[bits_delimiter + 1:]) + bits_delimiter + 1
        multiple = 2 ** int(string[bits_delimiter + 1:multiple_delimiter])
    else:
        multiple = 1
    return Constant(identifier=identifier, bits=bits, signed=signed,
                    multiple=multiple, pc_offset=pc_offset, ranges=[])

# Parse a register from a tokenized string
def extract_reg(string):
    # TODO: Handle 64bit composite registers
    if string[0] in {"R", "C", "P",
                     "M", "G", "S"}:
        if string[1] in {"e", "s", "d", "t", "u", "v", "x", "y"}:
            if len(string) >= 3 and string[2] == string[1]:
                return Register(identifier=string[1],
                                bits=64,
                                ranges=[],
                                dot_new=False,
                                predicate=string[0] == "P")
            return Register(identifier=string[1],
                            bits=32,
                            ranges=[],
                            dot_new=False,
                            predicate=string[0] == "P")
    elif string[0] in {"N"}:
        if string[1] in {"t"}:
            return Register(identifier=string[1],
                            bits=32,
                            ranges=[],
                            dot_new=True,
                            predicate=False)
        elif string[1] in {"s"}:
            return Register(identifier=string[1],
                            bits=32,
                            ranges=[],
                            dot_new=True,
                            predicate=False)

# Search operand from identifier
def find_op(identifier, operands):
    return [op for op in operands if op.identifier == identifier][0]

# Parse operands and compute format strings
def parse_op(inst_str):
    # Find operands order
    operands = []
    format_identifiers = []
    for i, letter in enumerate(inst_str):
        op = None
        # Signed and unsigned constants
        if letter == "#":
            op = extract_const(inst_str[i:])
        elif letter == "[":
            op = extract_shift(inst_str[i:])
        # Registers and predicates
        else:
            op = extract_reg(inst_str[i:])
        format_identifiers.append(op)
        if op not in operands:
            operands.append(op)
    identifiers = []
    for op in format_identifiers:
        if op is not None:
            if op.bits == 64:
                identifiers.append("(" + op.identifier + " + 1)")
                identifiers.append(op.identifier)
            else:
                identifiers.append(op.identifier)
    return ([op for op in operands if op is not None],
            identifiers)

# Operands extraction code generation
def gen_extract_op(inst_str, encoding, operands):
    code = ""
    identifiers = set([c.identifier
                       if type(c) in {Constant, Register}
                       else c
                       for c in operands])
    # Extract bit ranges from encoding scheme
    char, start = '', 0
    for i, letter in enumerate(encoding):
        if letter != char:
            if char in identifiers:
                find_op(char, operands).ranges.append(
                        Range(start=start, end=(i - 1)))
            start = i
            char = letter
        # Verify that no meaningful bits are left behind
        if letter not in {'0', '1', '-', 'P'}.union(set(identifiers)):
            print("Bit " + letter +
                  " was not parsed in instruction " + inst_str)

    if char in identifiers:
        find_op(char, operands).ranges.append(
                Range(start=start, end=(len(encoding) - 1)))

    # Verify encoding scheme coherence
    for op in operands:
        if type(op) == Constant and op.identifier != "N":
            sum = 0
            for r in op.ranges:
                sum += r.end - r.start + 1
            assert(sum == op.bits and "Non-coherent encoding scheme!")

    # Patch Nt encoding ranges
    for op in operands:
        if type(op) == Register and op.dot_new:
            old_range = op.ranges[-1]
            op.ranges[-1] = Range(start=old_range.start, end=old_range.end - 1)

    # Constand extender is not present, apply multiplier
    ext_imm = extendable_index(inst_str)
    constant_index = 0
    # Emit code for each operand
    for i, op in enumerate(operands):
        multiple = ""
        if len(op.ranges) == 1:
            code += "raw_value = EXTR(dc->ir, {}, {});\n".format(
                    op.ranges[0].start,
                    op.ranges[0].end)
        elif len(op.ranges) == 2:
            code += "raw_value = EXTR_2(dc->ir, {}, {}, {}, {});\n".format(
                            op.ranges[0].start,
                            op.ranges[0].end,
                            op.ranges[1].start,
                            op.ranges[1].end)
        elif len(op.ranges) == 3:
            code += "raw_value = EXTR_3(dc->ir, {}, {}, {}, {}, {}, {});"\
                    "\n".format(
                            op.ranges[0].start,
                            op.ranges[0].end,
                            op.ranges[1].start,
                            op.ranges[1].end,
                            op.ranges[2].start,
                            op.ranges[2].end)
        elif len(op.ranges) == 4:
            code += "raw_value = EXTR_4(dc->ir, {}, {}, {}, {}, {}, {}, {},"\
                    " {});\n".format(
                            op.ranges[0].start,
                            op.ranges[0].end,
                            op.ranges[1].start,
                            op.ranges[1].end,
                            op.ranges[2].start,
                            op.ranges[2].end,
                            op.ranges[3].start,
                            op.ranges[3].end)
        if type(op) == Constant:
            if op.multiple != 0:
                multiple = " * "+str(op.multiple)
            if op.signed:
                extended = "((0xffffffff >> {bits}) << {bits}) | "\
                        "raw_value".format(bits=op.bits)
                code += "uint32_t {op_name} = ((raw_value >> ({bits} - 1))"\
                        " ? ({extended}) : raw_value);".format(
                                op_name=op.identifier,
                                bits=op.bits,
                                extended=extended)
                # If is extendable emit conditional multiplier
                if constant_index == ext_imm:
                    code += "{op_name} = (dc->extender_present ? "\
                            "{op_name} : ({op_name}{multiple}));\n".format(
                                    op_name=op.identifier,
                                    multiple=multiple)
                # Otherwise just multiply
                else:
                    code += "{op_name} = {op_name}{multiple};\n".format(
                                    op_name=op.identifier,
                                    multiple=multiple)

            else:
                code += "uint32_t {op_name} = "\
                        "raw_value;".format(
                                op_name=op.identifier,
                                index=i,
                                bits=op.bits)
                # If is extendable emit conditional multiplier
                if constant_index == ext_imm:
                    code += "{op_name} = (dc->extender_present ? "\
                            "{op_name} : ({op_name}{multiple}));\n".format(
                                    op_name=op.identifier,
                                    multiple=multiple)
                # Otherwise just multiply
                else:
                    code += "{op_name} = {op_name}{multiple};\n".format(
                                    op_name=op.identifier,
                                    multiple=multiple)
            constant_index += 1
        else:
            code += "uint32_t "+op.identifier+" = raw_value;"
    return code

def extendable_index(inst_str):
    inst_str = inst_str.replace(" ", "")
    extendables = []
    with open(const_ext_csv) as f:
        lines = csv.reader(f)
        for line in lines:
            extendables.append(line)
    indexes, expressions = zip(*extendables)
    regexes = map(to_extender_regex, expressions)
    patterns = map(re.compile, regexes)
    for i, pattern in enumerate(patterns):
        match = pattern.match(inst_str)
        if match is not None:
            return int(indexes[i])
    return None

# Function execution code generation
def gen_execute():
    global system_insn
    code = "regs_t execute(unsigned inst_id, DisasContext *dc) {"\
           "uint32_t raw_value = 0;"\
           "regs_t regs = { 0 };"\
           "switch (inst_id) {"
    encodings = parse_encodings(instructions_csv)
    # Count SYSTEM instructions
    for e in encodings:
        if e[0:4] == ['0', '1', '1', '0']:
            system_insn += 1
    for inst_id, inst_str in enumerate(instruction_strings):
        operands, format_identifiers = parse_op(inst_str)
        identifiers = [op.identifier for op in operands]
        code += "case {}:\n{{".format(inst_id)
        code += gen_extract_op(inst_str, encodings[inst_id], operands)
        for op in operands:
            if type(op) == Register and op.dot_new:
                code += "{id} = get_destination_reg(dc->regs, {id});\n".format(
                        id=op.identifier)
            if type(op) == Register and not op.dot_new and not op.predicate:
                sum = 0
                for r in op.ranges:
                    sum += r.end - r.start + 1
                    if sum < 5:
                        code += "if (" + op.identifier + " >= 8)\n"
                        code += op.identifier + " += 8;\n"
        # If present, apply constant extender value
        ext_imm = extendable_index(inst_str)
        if ext_imm is not None:
            constants = [op for op in operands if type(op) == Constant]
            identifier = constants[ext_imm].identifier
            code += "if (dc->extender_present) {\n"
            code += identifier + " &= 0x3f;\n"
            code += identifier + " |= dc->const_ext;\n"
            code += "}\n"
        # if inst_id == 233:
        #    pprint(inst_str)
        #    pprint(ext_imm)
        # Print LOG_DIS disassembly using format.h
        code += ("LOG_DIS({}" +
                 ", {}" * len(format_identifiers) +
                 ");\n").format(to_format_string(inst_str), *format_identifiers)
        # Call the correct semantics function
        code += gen_function_call(inst_str, identifiers)
        code += "break;\n}\n"

    code += 'default: assert(false && '\
            '"Decoding not implemented for this instruction");'

    code += "}\nreturn regs;\n}\n\n"

    with open(decoder_c, "a") as d:
        d.write(code)

# Translate regex groups to function parameters
def get_params(pattern):
    flags = {}
    params = []
    # Each group becomes a parameter
    for k, v in pattern.groupindex.items():
        flags[v - 1] = k
    for i in range(len(flags)):
        params.append(flags[i])
    return params

# Invoke semantics compiler to fill function body
def gen_function_body(pattern_index):
    global implemented_meta
    global implemented_insn
    global implemented_vect
    qemu_code = ""
    qemu_code += "regs_t regs = { 0 };\n"
    instruction_code = meta_instructions[pattern_index]["code"]
    # Patch missing semicolons
    instruction_code = instruction_code.replace(" if", "; if")
    # Parse memory operation size
    mem_args = []
    mem_size = re.findall(r'mem(.*)\(', meta_instructions[pattern_index]["str"])
    assert(len(mem_size) < 2 and "More than one memory operation!")
    if len(mem_size) == 1:
        mem_size = mem_size[0]
        if "u" in mem_size:
            mem_args.append("-u")
        if "b" in mem_size:
            mem_args.append("-b")
        elif "h" in mem_size:
            mem_args.append("-h")
        elif "w" in mem_size:
            mem_args.append("-w")
        elif "d" in mem_size:
            mem_args.append("-d")
    # Parse comparison signedness
    cmp_sign = re.findall(r'cmp.(?:gt|ge|lt|le)(.)\(',
                          meta_instructions[pattern_index]["str"])
    if len(cmp_sign) == 1 and "u" in cmp_sign:
        mem_args.append("-u")
    # Parse jump flag
    jumps = re.findall(r'jump', meta_instructions[pattern_index]["str"])
    if len(jumps) > 0:
        mem_args.append("-j")
    # Parse stop instruction
    if meta_instructions[pattern_index]["str"] == "stop(Rs)":
        mem_args.append("-s")
    args = [semantics_path] + mem_args
    proc = subprocess.Popen(" ".join(args),
                            shell=True,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            stdin=subprocess.PIPE)
    proc.stdin.write(instruction_code.encode("utf-8"))
    proc.stdin.close()
    proc.wait()
    # Check bison exit code
    if proc.returncode == 0:
        qemu_code += proc.stdout.read().decode("utf-8")
        implemented_meta += 1
        implemented_insn += len(meta_mapping[pattern_index])
        if "=v" in meta_instructions[pattern_index]["str"]:
            implemented_vect += 1
    else:
        qemu_code += 'assert(false && "Instruction not implemented!");\n'
        # pprint(meta_instructions[pattern_index]["code"])
    qemu_code += "return regs;"
    return qemu_code

# Generate functions signatures
def gen_functions():
    code = ""
    regexes = map(to_regex, meta_instructions)
    patterns = map(re.compile, regexes)
    # Map meta-instructions into effective instructions
    for pattern_index, pattern in enumerate(patterns):
        meta_instruction = meta_instructions[pattern_index]["str"]
        meta_instruction = re.sub(r"\[(:<<N)\]", r"\[:<<N\]", meta_instruction)
        meta_instruction = re.sub(r"\[(:<<1)\]", r"\[:<<N\]", meta_instruction)
        operands, _ = parse_op(meta_instruction)
        identifiers = [op.identifier for op in operands]
        params = get_params(pattern)
        # Add function string comment
        code += "/* " + meta_instructions[pattern_index]["str"] + " */\n"
        code += "/* " + meta_instructions[pattern_index]["code"] + " */\n"
        # Generate function signature
        if len(identifiers) + len(params) == 0:
            code += "regs_t function_{}(DisasContext *dc) {{".format(pattern_index)
        else:
            function_str = ("regs_t function_{}(DisasContext *dc, " +
                            ", ".join(["uint32_t {}"] * len(identifiers) +
                                      ["bool {}"] * len(params)) + "){{\n")
            code += function_str.format(pattern_index,
                                        *identifiers,
                                        *params)
        code += gen_function_body(pattern_index)
        code += "}\n"
    with open(decoder_c, "a") as d:
        d.write(code)
    print("{}/{} meta instructions are vectorial!".format(vectorial_meta, len(meta_instructions)))
    print("{}/{} meta instructions have been implemented!".format(implemented_meta, len(meta_instructions)))
    print("{}/{} vectorial meta instructions have been implemented!".format(implemented_vect, vectorial_meta))
    print("{}/{} instructions have been implemented!".format(implemented_insn, len(instruction_strings) - system_insn - float_insn))

# Match instructions corresponding to a meta-instruction
def gen_pattern_matching(candidates, instruction_masks):
    if len(candidates) == 0:
        code = 'assert(false && '\
                '"Invalid instruction encoding!");'
        return code
    elif len(candidates) == 1:
        return "return {};".format(candidates[0])
    else:
        code = ""
        for candidate in candidates:
            code += "ADD_IF_ZERO(inst_index, "\
                    "(((ir & {}) ^ {}) == 0) * {});\n".format(
                        hex(int(instruction_masks[candidate][1], 2)),
                        hex(int(instruction_masks[candidate][0], 2)),
                        candidate)
        code += "return inst_index;\n"
        return code

# Generate switch case for decoding
def explore_switch(decode, instruction_masks):
    # Recursively generate switch cases
    n_bits = len(decode["bits"])
    code = ("switch (" +
            "EXTR_BITS_{}(".format(n_bits) +
            ("ir"+", {}"*n_bits+")) {{").format(*decode["bits"]))
    if "options" in decode.keys():
        for option in decode["options"]:
            code += "case {}: ".format(hex(int(option, 2)))
            code += explore_switch(decode["options"][option],
                                   instruction_masks)
            code += "break;"
        code += 'default: assert(false && '\
                '"Invalid instruction encoding!");'
    elif "instructions" in decode.keys():
        for instruction in decode["instructions"]:
            code += "case {}:".format(hex(int(instruction, 2)))
            code += gen_pattern_matching(decode["instructions"][instruction],
                                         instruction_masks)
            code += "break;"
        code += 'default: assert(false && '\
                '"Invalid instruction encoding!");'
    else:
        assert(False and "Unable to generate parsing code for:")
        pprint(decode)
    code += "}"
    return code

def gen_decoder_switch():
    with open(instruction_decoding_json) as f:
        decoding_json = json.loads(f.read())
    decode = decoding_json["decode"]
    instruction_masks = decoding_json["instructions"]

    code = "uint32_t decode(uint32_t ir) {\n"
    code += "uint64_t inst_index = 0;"
    code += explore_switch(decode, instruction_masks)
    code += "}"
    with open(decoder_c, "a") as d:
        d.write(code)

# Translate instruction string into a format string
def to_format_string(inst_str):
    code = ""
    code += "    "
    code += '"'
    format_string = re.sub("Nt", "Rt", inst_str)
    format_string = re.sub("([RCPMGNS])[esdtuvxy]{2}",
                           '\g<1>%" PRIu32 ":%" PRIu32 "', format_string)
    format_string = re.sub("([RCPMGNS])[esdtuvxy]",
                           '\g<1>%" PRIu32 "', format_string)
    format_string = re.sub("\[:<<N\]", '[:<<%" PRIi32 "]', format_string)
    format_string = re.sub("#[uU][0-9]+:[0-9]+", '#%" PRIu32 "', format_string)
    format_string = re.sub("#[uU][0-9]+", '#%" PRIu32 "', format_string)
    format_string = re.sub("#[sS][0-9]+:[0-9]+", '#%" PRIi32 "', format_string)
    format_string = re.sub("#[sS][0-9]+", '#%" PRIi32 "', format_string)
    format_string = re.sub("#m[0-9]+", '#%" PRIi32 "', format_string)
    format_string = re.sub("#r[0-9]+", '#%" PRIi32 "', format_string)
    format_string = re.sub("##", '#%" PRIi32 "', format_string)
    code += format_string
    code += '"\n'
    return code

# Translate a meta-instruction string into a regex
def to_regex(meta_instruction):
    regex = meta_instruction["str"]
    regex = regex.replace("(", "\(")
    regex = regex.replace(")", "\)")
    regex = regex.replace("*", "\*")
    regex = regex.replace("?", "\?")
    regex = regex.replace("^", "\^")
    regex = regex.replace("\n", "")
    regex = regex.replace(" ", "")
    regex = re.sub(r"\[(:<<1)\]", r"(?P<opt_shift>\1)?", regex)
    regex = re.sub(r"\[(:<<N)\]", r"\[:<<N\]", regex)
    regex = re.sub(r"\[(:<<1)\]", r"\[:<<N\]", regex)
    regex = re.sub(r"\[([&|]+)\]", r"(?P<and_or>[\1])", regex)
    regex = re.sub(r"\[([+-]+)\]", r"(?P<plus_minus>[\1])", regex)
    regex = re.sub(r"\[([HL]+)\]", r"(?P<high_low0>[\1])", regex, count=1)
    regex = re.sub(r"(?<!>)\[([HL]+)\]", r"(?P<high_low1>[\1])", regex, count=1)
    regex = re.sub(r"\[([01]+)\]", r"(?P<zero_one0>[01])", regex, count=1)
    regex = re.sub(r"(?<!>)\[([01]+)\]", r"(?P<zero_one1>[01])", regex, count=1)
    regex = re.sub(r"\[!\]", r"(?P<not0>[!]*)?", regex, count=1)
    regex = re.sub(r"(?<!>)\[!\]", r"(?P<not1>[!]*)", regex, count=1)
    regex = re.sub(r"\[(:sat)\]", r"(?P<sat>\1)?", regex)
    regex = re.sub(r"\[(:rnd)\]", r"(?P<rnd>\1)?", regex)
    regex = re.sub(r"\[(.new)\]", r"(?P<new>\1)?", regex)
    regex = re.sub(r":<hint>", r"(?P<hint>\:n?t)?", regex)
    regex = regex.replace("+", "\+")
    regex = regex.replace("|", "\|")
    regex = regex.replace(".", "\.")
    regex = "^" + regex + "$"
    return regex

# Translate a constant extender string into a regex
def to_extender_regex(meta_instruction):
    regex = meta_instruction
    regex = regex.replace("(", "\(")
    regex = regex.replace(")", "\)")
    regex = regex.replace("?", "\?")
    regex = regex.replace("Rd[d]", "Rdd?")
    regex = regex.replace("[.new]", r"(.new)?")
    regex = regex.replace("{", "(?:")
    regex = regex.replace("}", ")")
    regex = regex.replace("*", "\*")
    regex = regex.replace("^", "\^")
    regex = regex.replace("\n", "")
    regex = regex.replace(" ", "")
    regex = regex.replace("+", "\+")
    regex = regex.replace(".", "\.")
    regex = re.sub(r"\[!\]", r"!?", regex, count=1)
    regex = regex.replace(",", "\,")
    regex = regex.replace("hint", "n?t")
    regex = regex.replace("xx", "(?:gt|ge|lt|le|eq)u?")
    regex = regex.replace("target", "#[rRuUsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("#u6", "#[uUsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("##u32", "#[uUsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("##U32", "#[uUsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("##s32", "#[rRsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("##S32", "#[rRsS][0-9]+(?:\:?[0-9]+)?")
    regex = regex.replace("Pt", "P[a-z]")
    regex = regex.replace("add/sub/and/or", "(?:add|sub|and|or)")
    regex = regex.replace("asl/asr/lsr", "(?:asl|asr|lsr)")
    # pprint(regex)
    return regex

def parse_instructions(filename):
    global float_insn
    instruction_strings = []
    # Extract effective instructions from instructions.csv
    with open(filename) as f:
        inst_reader = csv.reader(f, delimiter=",", quotechar='"')
        for inst_id, inst_str in enumerate(inst_reader):
            instruction_strings.append(inst_str[-1])
            # Count floating point instructions
            if "=sf" in inst_str[-1]:
                float_insn += 1
    return instruction_strings

def parse_encodings(filename):
    instruction_strings = []
    # Extract effective instructions from instructions.csv
    with open(filename) as f:
        inst_reader = csv.reader(f, delimiter=",", quotechar='"')
        for inst_id, inst_str in enumerate(inst_reader):
            instruction_strings.append(inst_str[:32])
    return instruction_strings

def parse_meta_instructions():
    global endloops
    global vectorial_meta
    meta_instructions = []
    with open(meta_instructions_csv) as inst:
        insts = csv.reader(inst)
        instruction = {}
        for inst in insts:
            if inst[0] in {"endloop0", "endloop1", "endloop01"}:
                endloops[inst[0]] = inst[1]
                continue
            # Exclude syntactic sugar
            if "Assembler" in inst[1]:
                continue
            if len(inst[0]) != 0:
                meta_instructions.append(instruction)
                instruction = {}
                instruction["str"] = inst[0]
                if "=v" in instruction["str"]:
                    vectorial_meta += 1
                if len(inst[1]) == 0:
                    instruction["code"] = "NOP;"
                else:
                    instruction["code"] = inst[1]
            else:
                instruction["code"] += "\n"+inst[1]
        meta_instructions.append(instruction)
        del meta_instructions[0]
    return meta_instructions

def gen_function_call(inst_str, identifiers):
    inst_str = inst_str.replace(" ", "")
    arguments = identifiers
    pattern_id = None
    matched = False
    for i, pattern in enumerate(patterns):
        match = pattern.match(inst_str)
        if match is not None:
            matched = True
            for group in match.groups():
                arguments.append("true" if group in {"&", "+", "1", "!", "H",
                                                     ":sat", ":rnd", ".new",
                                                     "t"} else "false")
            pattern_id = i
            break
    if matched:
        meta_mapping[pattern_id].append(inst_str)
        return ("regs = function_{}(dc" +
                ", {}" * len(identifiers) +
                ");\n").format(pattern_id, *arguments)
    else:
        return ('assert(false && "This instruction is not implemented!");')

def gen_sub_decoder():
    global duplex_masks
    with open(sub_instruction_decoding_json) as f:
        decoding_json = json.loads(f.read())
    decode = decoding_json["decode"]
    duplex_masks = decoding_json["instructions"]

    code = "uint32_t sub_decode(uint32_t ir) {\n"
    code += "uint64_t inst_index = 0;"
    code += explore_switch(decode, duplex_masks)
    code += "}"
    with open(decoder_c, "a") as d:
        d.write(code)

# Function execution code generation
def gen_sub_execute():
    code = "regs_t sub_execute(unsigned inst_id, DisasContext *dc) {"\
           "uint32_t raw_value = 0;"\
           "regs_t regs = { 0 };"\
           "switch (inst_id) {"
    encodings = parse_encodings(sub_instructions_csv)
    for inst_id, inst_str in enumerate(sub_instruction_strings):
        operands, format_identifiers = parse_op(inst_str)
        identifiers = [op.identifier for op in operands]
        code += "case {}:\n{{".format(inst_id)
        # Implement weird operator encoding
        code += gen_extract_op(inst_str, encodings[inst_id], operands)
        for op in operands:
            if type(op) == Register:
                if op.bits == 64:
                    code += op.identifier + " *= 2;\n"
                code += "if (" + op.identifier + " >= 8)\n"
                code += op.identifier + " += 8;\n"
        # If present, apply constant extender value
        ext_imm = extendable_index(inst_str)
        if ext_imm is not None:
            constants = [op for op in operands if type(op) == Constant]
            identifier = constants[ext_imm].identifier
            code += "if (dc->extender_present) {\n"
            code += identifier + " &= 0x3f;\n"
            code += identifier + " |= dc->const_ext;\n"
            code += "dc->extender_present = false;\n"
            code += "dc->const_ext = 0;\n"
            code += "}\n"
        # Print LOG_DIS disassembly using format.h
        code += ("LOG_DIS({}" +
                 ", {}" * len(format_identifiers) +
                 ");\n").format(to_format_string(inst_str), *format_identifiers)
        # Call the correct semantics function
        code += gen_function_call(inst_str, identifiers)
        code += "break;\n}\n"

    code += 'default: assert(false && '\
            '"Decoding not implemented for this instruction");'

    code += "}\nreturn regs;\n}\n\n"

    with open(decoder_c, "a") as d:
        d.write(code)

def gen_endloop():
    code = ""
    for name, pseudocode in endloops.items():
        code += "void "+name+"(void)\n"
        code += "{\n"
        proc = subprocess.Popen([semantics_path,
                                 '-t'],
                                stdout=subprocess.PIPE,
                                stdin=subprocess.PIPE)
        proc.stdin.write(pseudocode.encode("utf-8"))
        proc.stdin.close()
        proc.wait()
        assert(proc.returncode == 0 and "Unhandled endloop instruction!")
        code += proc.stdout.read().decode("utf-8")
        code += "}\n"
    with open(decoder_c, "a") as d:
        d.write(code)

def analyse_inst_mapping():
    no_match_count = 0
    regexes = map(to_regex, meta_instructions)
    patterns = list(map(re.compile, regexes))
    for i, inst_str in enumerate(instruction_strings):
        # Patch instruction strings to make them more easily parsable
        inst_str = inst_str.replace(" ", "")
        no_match = True
        for pattern in patterns:
            match = pattern.match(inst_str)
            if match is not None:
                no_match = False
        if no_match:
            no_match_count += 1
            assert(inst_str.split("=")[1][0] == "v" and "Non-vectorial "
                   "instruction has no corresponding meta-instruction!")
    print("{}/{} instructions did not match any meta "
          "instruction!".format(no_match_count, len(instruction_strings)))

def gen_decoder_header():
    multi_reference_count = 0
    decoder_header = DECODER_HEADER
    regexes = map(to_regex, meta_instructions)
    patterns = map(re.compile, regexes)
    # Map meta-instructions into effective instructions
    for pattern_index, pattern in enumerate(patterns):
        meta_instruction = meta_instructions[pattern_index]["str"]
        meta_instruction = re.sub(r"\[(:<<N)\]", r"\[:<<N\]", meta_instruction)
        meta_instruction = re.sub(r"\[(:<<1)\]", r"\[:<<N\]", meta_instruction)
        operands, _ = parse_op(meta_instruction)
        identifiers = [op.identifier for op in operands]
        flags = {}
        params = []
        # Each group becomes a parameter
        for k, v in pattern.groupindex.items():
            flags[v - 1] = k
        for i in range(len(flags)):
            params.append(flags[i])
        # Generate function signature
        if len(identifiers) + len(params) == 0:
            decoder_header += "regs_t function_{}(DisasContext *dc);\n".format(pattern_index)
        else:
            function_str = ("regs_t function_{}(DisasContext *dc, " +
                            ", ".join(["uint32_t {}"] * len(identifiers) +
                                      ["bool {}"] * len(params)) + ");\n")
            decoder_header += function_str.format(pattern_index,
                                                  *identifiers,
                                                  *params)
    print("{}/{} instructions were referenced multiple "
          "times!".format(multi_reference_count, len(instruction_strings)))
    decoder_header += "\n#endif"
    with open(decoder_h, "w") as f:
        f.write(decoder_header)

def indent():
    # Optionally, indent
    try:
        subprocess.run("indent -linux " + decoder_c, shell=True, check=True)
    except:
        pass

def main():
    parser = argparse.ArgumentParser()
    # Support programs
    parser.add_argument("semantics", metavar="SEMANTICS")

    # Data extracted from ISA reference manual
    parser.add_argument("meta_instructions_csv", metavar="META_INSTRUCTION_CSV")
    parser.add_argument("instructions_csv", metavar="INSTRUCTION_CSV")
    parser.add_argument("sub_instructions_csv", metavar="SUB_INSTRUCTION_CSV")
    parser.add_argument("const_ext_csv", metavar="CONST_EXT_CSV")

    # Generated by best-decoding
    parser.add_argument("instruction_decoding_json", metavar="INSTRUCTION_DECODING_JSON")
    parser.add_argument("sub_instruction_decoding_json", metavar="SUB_INSTRUCTION_DECODING_JSON")

    # Output files
    parser.add_argument("decoder_c", metavar="DECODER_C")
    parser.add_argument("decoder_h", metavar="DECODER_H")

    args = parser.parse_args()

    global semantics_path
    global meta_instructions_csv
    global instructions_csv
    global sub_instructions_csv
    global const_ext_csv
    global instruction_decoding_json
    global sub_instruction_decoding_json
    global decoder_c
    global decoder_h
    semantics_path = args.semantics
    meta_instructions_csv = args.meta_instructions_csv
    instructions_csv = args.instructions_csv
    sub_instructions_csv = args.sub_instructions_csv
    const_ext_csv = args.const_ext_csv
    instruction_decoding_json = args.instruction_decoding_json
    sub_instruction_decoding_json = args.sub_instruction_decoding_json
    decoder_c = args.decoder_c
    decoder_h = args.decoder_h

    global meta_instructions
    global instruction_strings
    global sub_instruction_strings
    global patterns
    # Parse instructions and meta-instructions
    meta_instructions = parse_meta_instructions()
    instruction_strings = parse_instructions(instructions_csv)
    sub_instruction_strings = parse_instructions(sub_instructions_csv)
    regexes = map(to_regex, meta_instructions)
    patterns = list(map(re.compile, regexes))
    analyse_inst_mapping()
    # decoder.h
    gen_decoder_header()
    # decoder.c
    gen_macros()
    gen_decoder_switch()
    gen_execute()
    gen_functions()
    # Duplex instructions
    gen_sub_decoder()
    gen_sub_execute()
    gen_endloop()
    # auto-indent
    indent()

if __name__ == "__main__":
    main()


from urllib.request import urlopen
from linkFinder import linkFinder
from mkdirectory import *
import re
import ssl

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

class spider:

    # class variables shared among all instances
	projectName = ''
	baseURL = ''
	domainName= ''
	queueFile = ''
	crawledFile = ''
	notLinksFile = ''
	cannotOpenFile = ''
	queue = set()
	crawled = set()
	notLinks = set()
	cannotOpen = set()

	def __init__(self,projectName,baseURL,domainName):
		spider.projectName = projectName
		spider.baseURL = baseURL
		spider.domainName = domainName
		spider.queueFile = spider.projectName + '/queue.txt'
		spider.crawledFile = spider.projectName + '/crawled.txt'
		spider.notLinksFile = spider.projectName + '/notLinks.txt'
		spider.cannotOpenFile = spider.projectName + '/cannotOpen.txt'

		self.boot()
		self.crawlPage('First Spider',spider.baseURL)

	@staticmethod
	def boot():
        	createProjectDir(spider.projectName)
       		createDataFiles(spider.projectName, spider.baseURL)
        	spider.queue=fileToSet(spider.queueFile)
        	spider.crawled=fileToSet(spider.crawledFile)
        	spider.notLinks = fileToSet(spider.notLinksFile)
        	spider.cannotOpen = fileToSet(spider.cannotOpenFile)

	@staticmethod
	def crawlPage(threadName,pageURL):
		ignore = ["calendar","events","mailto","day","month","year","webmail"]
		i=0
		for item in ignore:
			if item in pageURL.lower():
				i += 1
				#print (i)
		if (i==0) and (pageURL not in spider.crawled):
			print(threadName+' crawling '+pageURL)
			print('Queue '+str(len(spider.queue))+' | Crawled '+ str(len(spider.crawled)))
			spider.addLinksToQueue(spider.gatherLinks(pageURL))
			spider.queue.remove(pageURL)
			spider.crawled.add(pageURL)
			spider.updateFiles()

	@staticmethod
	def gatherLinks(pageURL):
		htmlString = ''
		try:
			response = urlopen(pageURL)
            		#print (response.getheader('Content-Type'))
			if 'text/html' in response.getheader('Content-Type'):
				htmlBytes = response.read()
                #print('a')
				htmlString = htmlBytes.decode('utf-8')
                #print('d')
			finder = linkFinder(spider.baseURL,pageURL)
			finder.feed(htmlString)

		except:
			try:
				response=urlopen(pageURL)
				if 'text/html' in response.getheader('Content-Type'):
					htmlBytes = response.read()
					htmlString = htmlBytes.decode('iso-8859-1')
				finder = linkFinder(spider.baseURL,pageURL)
				finder.feed(htmlString)
			except Exception as e:
				try:
					response = urlopen(pageURL)
					blah = response.getheader("Content-Type") +' ' +  pageURL
					spider.notLinks.add(blah)
					return set()
				except:
					print(e)
					print('Cannot open '+ pageURL)
					asd = str(e) + ' ' + pageURL
					spider.cannotOpen.add(asd)
					return set()
		return finder.pageLinks()

	@staticmethod
	def addLinksToQueue(linkSet):
		for url in linkSet:
			if url in spider.queue:
				continue
			if url in spider.crawled:
				continue
			if url in spider.notLinks:
				continue
			if url in spider.cannotOpen:
				continue
			if spider.domainName not in url:
				continue
			spider.queue.add(url)

	@staticmethod
	def updateFiles():
		setToFile(spider.queue,spider.queueFile)
		setToFile(spider.crawled,spider.crawledFile)
		setToFile(spider.notLinks,spider.notLinksFile)
		setToFile(spider.cannotOpen,spider.cannotOpenFile)


# import logging
# import logging.config

import logging
from os import path

# log_file_path = path.join("/home/q123/Desktop/explo/logging.conf")
# logging.config.fileConfig(log_file_path)
logger = logging.getLogger("ShapeLog."+__name__)

from collections import OrderedDict
from copy import deepcopy
from itertools import chain
from typing import Callable, List, Optional, Tuple, Union

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import Identity, Linear, Sequential

class MLP(torch.nn.Module):

    """Implements an MLP that is differentiable w.r.t to parameters (when they are fed by BO)
    """

    def __init__(
                self,
                input_shape,output_shape,
                Ls: List[int],
                add_bias: bool,
                nonlinearity: Optional[Callable] = None,
                **kwargs
                ):

        """Inits MLP with the provided weights
        Note the MLP can support batches of weights """

        super(MLP, self).__init__()

        # if input_shape is not None:
        #     Ls = [input_shape,output_shape]

        weight_sizes  = [(in_size,out_size)
                                for in_size, out_size in zip(Ls[:-1], Ls[1:])]
        n_layers = len(weight_sizes)

        len_params = sum(
            [
                (in_size + 1 * add_bias) * out_size
                for in_size, out_size in zip(Ls[:-1], Ls[1:])
            ]
        )

        n_actions = Ls[-1]

        if nonlinearity is None:
            nonlinearity = torch.nn.Tanh()
            #nonlinearity = torch.nn.functional.tanh()
            #nonlinearity = torch.nn.Identity()

        self.__dict__.update(locals())

        self.register_parameter("default_weights",nn.Parameter(torch.zeros((1,self.len_params)))) ## maybe add device later

    def create_weights(self,params):

        weights,biases = [],[]

        start,end = (0,0)

        for (in_size,out_size) in self.weight_sizes:

            start = deepcopy(end)
            end   = deepcopy(start)  + (in_size * out_size)
            weight = params[...,start:end].reshape(*params.shape[:-1],out_size,in_size)

            if self.add_bias:

                bias = params[...,end:end+out_size].reshape(*params.shape[:-1],out_size)
                end = deepcopy(end) + out_size

            else :

                bias = torch.zeros(*params.shape[:-1],out_size)

            weights.append(weight) ## add transpose or dim error
            biases.append(bias)

            #print(f'weight.shape {weight.shape} bias.shape {bias.shape}')

        return weights,biases

    def forward(self,states,params=None):

        #logger.debug(f'params {params.shape} states {states.shape}')

        if params is None :

            params = self.default_weights.data

        weights,biases = self.create_weights(params)
        outputs = states.T

        for i,(w,b) in enumerate(
                            zip(weights,biases)
                            ):

            w_tmp = w @ outputs
            b_tmp = b.unsqueeze(-1).expand_as(w_tmp)
            outputs =  w_tmp + b_tmp

        ## no nonlinearity for last layer
            # if (i+1) < (self.n_layers) :
            #     outputs = self.nonlinearity(outputs)

            outputs = self.nonlinearity(outputs) ###remove

        logger.debug(f'MLP : actions {outputs.shape}')
        return outputs

    def predict(self,states,*args,**kwargs):

        return self.forward(self.default_weights,states)

class ActorNetwork(nn.Module):

    def __init__(self,
                Ls: List[int],
                add_bias: bool = False,
                nonlinearity = None,
                **kwargs,
                ):
        super(ActorNetwork, self).__init__()

        self.weight_sizes  = [(in_size,out_size)
                                for in_size, out_size in zip(Ls[:-1], Ls[1:])]

        self.n_layers = len(self.weight_sizes)

        self.len_params = sum(
            [
                (in_size + 1 * add_bias) * out_size
                for in_size, out_size in zip(Ls[:-1], Ls[1:])
            ]
        )

        self.n_actions = Ls[-1]

        self.nonlinearity = torch.nn.Identity()  if nonlinearity is None else nonlinearity

        self.add_bias = add_bias

        self.layers = self.build_layers()

    def build_layers(self):

        dct = OrderedDict(
                            ('layer'+str(i),nn.Linear(sizes[0],sizes[1],bias=self.add_bias))
                            for i,sizes in enumerate(self.weight_sizes)
                        )

        layers = nn.Sequential(dct)

        return layers

    @property
    def n_params(self):

        n = 0
        for p in self.parameters():
            p_shape = torch.tensor(p.shape)
            n += torch.prod(p_shape,0)

        return n

    @property
    def device(self):
        device = next(self.parameters()).device
        return device

    def get_params(self):

        vector_params = []

        for p in self.parameters():

            vector_params.append(p.data.flatten())

        return torch.cat(vector_params)

    def set_params(self,new_params):

        with torch.no_grad():

            idx = 0
            for param in self.parameters():
                    weights = param.data
                    weights_shape = torch.tensor(weights.shape)
                    n_steps = torch.prod(weights_shape,0)
                    new_param = new_params[idx:idx+n_steps].reshape(*weights_shape)
                    param.data = new_param
                    idx += n_steps

    def add_noise(self,noise):

        with torch.no_grad():

            if not torch.is_tensor(noise):
                noise = torch.tensor(noise,device=self.device)

            self.set_params(self.get_params()+noise)

    def forward(self, states):

        outputs = states

        for i,layer in enumerate(self.layers):

            outputs = layer(outputs)

            if (i+1) < (self.n_layers) :

                outputs = self.nonlinearity(outputs)

        return outputs

    def super_forward(self,params,states):

        self.set_params(params)
        a = self(states)

        return a


import os
import json
import sys
import warnings

import click
import cv2
import tqdm
from loguru import logger

from utils.feature_extractor import RootSift
from utils.result_filter import RansacGeometricVerification
from utils.candidate_retriever import BagOfWordsTopKRetriever
from utils.image_preprocessor import ImageScaler

warnings.filterwarnings("ignore")
logger.remove()
logger.add(sys.stderr, level="INFO")

SIFT_NUM_FEATURES = 711
N_DOCS = 15000
N_TERMS = 1000000
TOP_K = 100
MIN_INTERSECTING_VISUAL_WORDS = 2

@click.command()
@click.option('--data_root_path', required=True, type=str, help='The root path to the benchmark data')
@click.option('--object_type', required=True, help='Which object type to evaluate for')
@click.option('--search_artifacts_name',
              type=click.Choice(['all', 'oods']),
              help='The name of folder that contains search artifacts')
@click.option('--n_terms', default=1000000, help='Number of total terms / visual words')
@click.option('--n_docs', default=15000, help='Number of documents used in IDF calculation')
@click.option('--opencv_threads', default=8, help='Number of threads for running opencv')
def main(data_root_path: str, object_type: str, search_artifacts_name: str, n_terms: int, n_docs: int, opencv_threads: int):
    query_image_list_file = os.path.join(data_root_path, 'metadata', f'{object_type}_query.ndjson')
    query_image_info = list(map(json.loads, open(query_image_list_file)))
    query_image_paths = [info['image'] for info in query_image_info]
    query_image_paths = [os.path.join(data_root_path, img) for img in query_image_paths]
    logger.info(f'Number of query images: {len(query_image_paths)}')

    cv2.setNumThreads(opencv_threads)

    config = {
        "SIFT_NUM_FEATURES": "711",
        "FAISS_OMP_THREADS": "30",
        "N_DOCS": n_docs,
        "N_TERMS": n_terms,
        "TOP_K": "10",
        "MIN_INTERSECTING_VISUAL_WORDS": "2",
        "DESCRIPTORS_COMPUTE_STRATEGY": "use_matched_words",
        "ACTIVATE_HOMOGRAPHY_CHECKER": "True",
        "ACTIVATE_BOUNDING_POLYGON_CHECKER": "True",
        "CODEBOOK_PATH": "build_codebook/artifacts/codebook.index",
        "DOCUMENT_FREQUENCY_PATH": "build_codebook/artifacts/document_frequency.npy",
        "TFIDF_MATRIX_PATH": f"build_index/{search_artifacts_name}/tfidf_matrix.npz",
        "TFIDF_MATRIX_IMAGES_PATH": f"build_index/{search_artifacts_name}/tfidf_matrix-image_urls.txt",
        "HISTOGRAMS_PATH": f"build_index/{search_artifacts_name}/histograms.pkl",
        "INVERTED_INDEX_PATH": f"build_index/{search_artifacts_name}/inverted_index.pkl",
        "LABELS_PATH": f"build_index/{search_artifacts_name}/labels.ndjson",
        "METADATA_PATH": f"build_index/{search_artifacts_name}/metadata.pkl",
    }
    retrieval_results_file = f'retrieval_results/{object_type}_{search_artifacts_name}.ndjson'
    os.makedirs(os.path.dirname(retrieval_results_file), exist_ok='True')
    save_fp = open(retrieval_results_file, 'w')

    image_preprocessor = ImageScaler(config)
    feature_extractor = RootSift(config)
    candidate_retriever = BagOfWordsTopKRetriever(config)
    result_filter = RansacGeometricVerification(config)

    query_id = 1
    candidate_scores = []
    candidate_ids = []
    for query_image_path in tqdm.tqdm(query_image_paths):
        image_data = image_preprocessor.process(query_image_path)
        features = feature_extractor.extract(image_data)
        candidates = candidate_retriever.retrieve(features)
        results = result_filter.filter(image_data, candidates)

        result_info = {}
        result_info['query_id'] = query_id
        result_info['candidate_db_ids_before_reranking'] = [int(r.label) - 1 for r in candidates]
        result_info['global_scores_before_reranking'] = [float(r.features['similarity']) for r in candidates]
        result_info['candidate_db_ids'] = [int(r.label) - 1 for r in results]
        result_info['global_scores'] = [float(r.features['similarity']) for r in results]
        result_info['num_inlier'] = [int(r.features['num_inliers']) for r in results]
        save_fp.write(f'{json.dumps(result_info)}\n')

        query_id += 1

        candidate_scores.append(result_info['global_scores'])
        candidate_ids.append(result_info['candidate_db_ids'])

    save_fp.close()

if __name__ == '__main__':
    main()


import torch.nn as nn

class Generator(nn.Module):
    '''
        Args:
            nz: Input noise dimension
            nd: Output image dimension
            nc: Number of channels in output image
    '''
    def __init__(self, nz, nd, nc):
        super(Generator, self).__init__()
        self.main = nn.Sequential(
            # input is Z, going into a convolution
            nn.ConvTranspose2d(nz, nd * 8, 4, 1, 0, bias=False),
            nn.BatchNorm2d(nd * 8),
            nn.ReLU(True),
            # state size. (ngf*8) x 4 x 4
            nn.ConvTranspose2d(nd * 8, nd * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(nd * 4),
            nn.ReLU(True),
            # state size. (ngf*4) x 8 x 8
            nn.ConvTranspose2d(nd * 4, nd * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(nd * 2),
            nn.ReLU(True),
            # state size. (ngf*2) x 16 x 16
            nn.ConvTranspose2d(nd * 2, nd, 4, 2, 1, bias=False),
            nn.BatchNorm2d(nd),
            nn.ReLU(True),
            # state size. (ngf) x 32 x 32
            nn.ConvTranspose2d(nd, nc, 4, 2, 1, bias=False),
            nn.Tanh()
            # state size. (nc) x 64 x 64
        )

    def forward(self, input):
        return self.main(input)


from multiprocessing.pool import ThreadPool
from bs4 import BeautifulSoup

import requests
from time import sleep

import pandas as pd
from xml.etree import ElementTree

import epo_ops
import yaml
from tqdm import tqdm

# Create an EPO patent based upon the search results as parameters.
# Return a dictionary of EPO Patents
def create_epo_patent(client, search_responses_document_ids, search_responses_document_countries,
                      search_responses_document_kinds):
    epo_patents = {}
    patent_inventors = []
    patent_applicants = []

    # Attempt to attain published data on a specific patent
    # Due to API restrictions, it can result in an HTTP error.
    # If so, delay and reattempt.
    try:
        response = client.published_data(  # Retrieve bibliography data
            reference_type='publication',  # publication, application, priority
            input=epo_ops.models.Docdb(search_responses_document_ids, search_responses_document_countries,
                                       search_responses_document_kinds),  # original, docdb, epodoc
            endpoint='biblio',  # optional, defaults to biblio in case of published_data
            constituents=[]  # optional, list of constituents
        )
    except requests.exceptions.HTTPError as e:
        print("Attempt to retrieve bibliography data failed. \nReason: {}.\nReattempting.".format(e))
        sleep(0.1)
        response = client.published_data(  # Retrieve bibliography data
            reference_type='publication',  # publication, application, priority
            input=epo_ops.models.Docdb(search_responses_document_ids, search_responses_document_countries,
                                       search_responses_document_kinds),  # original, docdb, epodoc
            endpoint='biblio',  # optional, defaults to biblio in case of published_data
            constituents=[]  # optional, list of constituents
        )
    contents = response.text

    '''
        Using BeautifulSoup, parse the resulting xml response from OPS for the key information
        required for each patent, mainly the patent applicants, inventors, title and abstract text.
    '''
    soup = BeautifulSoup(contents, "lxml")

    applicants = soup.findAll("applicants", {"data-format": "original"})
    for applicant in applicants:
        patent_applicants.append(applicant.get_text())

    inventors = soup.findAll("inventor", {"data-format": "original"})

    for inventor in inventors:
        patent_inventors.append(inventor.get_text())

    title = soup.find('invention-title')
    patent_title = title.get_text()

    abstract = soup.find('abstract')
    if abstract is None:
        patent_abstract = ""
    else:
        patent_abstract = abstract.get_text()

    epo_patents['Applicants'] = patent_applicants
    epo_patents['Inventor Names'] = patent_inventors
    epo_patents['Patent Title'] = patent_title
    epo_patents['Patent Abstract'] = patent_abstract

    return epo_patents

# Get patent IDs for specific search term.
def get_epo_patent_ids_for_search_term(client, term, range_begin, range_end):
    search_responses_document_ids = []
    search_responses_document_countries = []
    search_responses_document_kinds = []
    try:
        response = client.published_data_search(cql='ti=' + term, range_begin=range_begin, range_end=range_end,
                                                constituents=None)
    except requests.exceptions.HTTPError as e:
        print("Attempt to retrieve published data failed.\nReason: {}.\nReattempting.".format(e))
        sleep(0.1)
        response = client.published_data_search(cql='ti=' + term, range_begin=range_begin, range_end=range_end,
                                                constituents=None)
    tree = ElementTree.fromstring(response.content)

    '''
        Parse resulting xml into specific lists, i.e. document-number, country of origin and the type of document.
    '''

    for doc_id in tree.iter('{http://www.epo.org/exchange}doc-number'):
        search_responses_document_ids.append(doc_id.text)

    for country in tree.iter('{http://www.epo.org/exchange}country'):
        search_responses_document_countries.append(country.text)

    for kind in tree.iter('{http://www.epo.org/exchange}kind'):
        search_responses_document_kinds.append(kind.text)

    return search_responses_document_ids, search_responses_document_countries, search_responses_document_kinds

# For each search term, retrieve the top N patents for each term where N is range max.
# NOTE: Due to restrictions from OPS, range_max cannot exceed 2000
# OPS Key and Secret can be generated from developers.epo.org
def get_epo_patents_for_search_term(search_term):
    client = epo_ops.Client(key='XXXXX', secret='XXXXX')  # Instantiate client

    search_term_epo_patent_ids = []
    search_term_epo_patent_country_origin = []
    search_term_epo_patent_type = []
    search_results = {}

    range_begin = 1
    range_max = 500

    while range_begin <= range_max:
        epo_patent_ids, epo_patent_countries, epo_patent_types = get_epo_patent_ids_for_search_term(client, search_term,
                                                                                range_begin,
                                                                                (range_begin + 99))
        if not epo_patent_ids or not epo_patent_countries or not epo_patent_types:
            continue
        else:
            search_term_epo_patent_ids.extend(epo_patent_ids)
            search_term_epo_patent_country_origin.extend(epo_patent_countries)
            search_term_epo_patent_type.extend(epo_patent_types)
        range_begin += 100

    # Create search text as dict. Applicants (Companies typically), inventor names, Title, abstract

    for i in range(0, len(search_term_epo_patent_ids)):
        if search_term_epo_patent_ids[i] is None or search_term_epo_patent_country_origin[i] is None or \
                search_term_epo_patent_type[i] is None:
            continue
        else:
            search_results[search_term_epo_patent_ids[i]] = create_epo_patent(client, search_term_epo_patent_ids[i],
                                                                              search_term_epo_patent_country_origin[i],
                                                                              search_term_epo_patent_type[i])
    return search_results

def imap_unordered_bar(func, args, n_processes=4):
    p = ThreadPool(n_processes)
    res_list = []
    with tqdm(total=len(args)) as pbar:
        for i, res in tqdm(enumerate(p.imap_unordered(func, args))):
            pbar.update()
            res_list.append(res)
    pbar.close()
    p.close()
    p.join()
    return res_list

if __name__ == "__main__":
    with open("../EPO/Config/cfg_search_terms.yml", 'r') as ymlfile:
        cfg_search_terms = yaml.load(ymlfile)

    search_terms = cfg_search_terms["search_terms"]["categories"]
    search_terms_epo_patents = imap_unordered_bar(get_epo_patents_for_search_term, search_terms)

    df = pd.DataFrame(search_terms_epo_patents)
    df.to_csv("../EPO/EPOPatents.csv")


from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import State, StatesGroup
import aiogram.utils.markdown as fmt

from config import dp, bot, URL_WEB_APP, admin_usertag
from keyboards import keys
import db
from text_config.texts import step_2

class Making_user(StatesGroup):
	waiting_for_user_name = State()
	waiting_for_user_stack = State()
	waiting_for_user_id = State()

@dp.message_handler(text='Регистрация🔐', state="*")
async def muser(message: types.Message):
		print(db.if_exists_create(message.chat.id))
		if db.if_exists_create(message.chat.id):
			await bot.send_message(message.chat.id,  f'Ты не можешь создать аккаунт с данного устройства, его id, уже есть в БД.\nВспомни пароль или свяжись с администратором - {admin_usertag}', parse_mode=types.ParseMode.HTML)
		else:
			await bot.send_message(message.chat.id,  f'Привет {message.from_user.first_name} \nВведите Ваш Никнейм, он будет виден пользователям и в вашей группе', parse_mode=types.ParseMode.HTML)
			await Making_user.waiting_for_user_name.set()

@dp.message_handler(state=Making_user.waiting_for_user_name, content_types=types.ContentTypes.TEXT)
async def user_name(message: types.Message, state: FSMContext):
	await state.update_data(user_name=message.text)
	await Making_user.next()
	mes = await bot.send_message(message.chat.id, f"{message.text.capitalize()},\n"+step_2, reply_markup=keys.skip())
	await state.update_data(mes=mes)

@dp.message_handler(state=Making_user.waiting_for_user_stack, content_types=types.ContentTypes.TEXT)
async def user_stack(message: types.Message, state: FSMContext):
	if "text" in message:
		await state.update_data(stack=message.text)
	else:
		await state.update_data(stack=None)
	user_data = await state.get_data()

	mes = await bot.send_message(user_data['mes'].chat.id, f"{user_data['user_name']}  - Ваш Никнейм\nCоздать профиль?", reply_markup=keys.step_2())
	await state.update_data(mes=mes)
	await Making_user.next()

#INLINE_CALLBACK_PART
@dp.callback_query_handler(lambda c: c.data == 'skip', state = Making_user.waiting_for_user_stack)
async def cancel_handler(message: types.Message, state: FSMContext):
	user_data = await state.get_data()
	await bot.delete_message(user_data['mes'].chat.id, user_data['mes'].message_id)
	await user_stack(message, state)

@dp.callback_query_handler(lambda c: c.data == 'yes_create', state=Making_user.waiting_for_user_id)
async def cancel_handler(message: types.Message, state: FSMContext):
	user_data = await state.get_data()

	db.create_user(user_data['mes']['chat']['id'], user_data['user_name'], user_data['stack'])
	await bot.delete_message(user_data['mes']['chat']['id'], user_data['mes'].message_id)
	await bot.send_message(user_data['mes'].chat.id, 'Поздравляю, вы создали нового пользователя', reply_markup=keys.key_main_menu(), disable_web_page_preview=True)
	await state.finish()

@dp.callback_query_handler(lambda c: c.data == 'cancel', state = Making_user.waiting_for_user_id)
async def cancel_handler(message: types.Message, state: FSMContext):
	user_data = await state.get_data()
	await bot.delete_message(user_data['mes']['chat']['id'], user_data['mes'].message_id)
	await state.finish()


from unittest import skipUnless, skipIf

from django import VERSION
from django.contrib.auth.models import User
from django.core.exceptions import MultipleObjectsReturned
from django.db.models.manager import Manager
from django.test.testcases import TestCase

from moderation.managers import ModerationObjectsManager
from moderation.models import ModeratedObject
from moderation.moderator import GenericModerator
from tests.models import ModelWithSlugField2, ModelWithVisibilityField, UserProfile
from tests.utils import setup_moderation, teardown_moderation

class ModerationObjectsManagerTestCase(TestCase):
    fixtures = ['test_users.json', 'test_moderation.json']

    def setUp(self):

        self.user = User.objects.get(username='moderator')
        self.profile = UserProfile.objects.get(user__username='moderator')

        class UserProfileModerator(GenericModerator):
            visibility_column = 'is_public'

        self.moderation = setup_moderation(
            [UserProfile, (ModelWithVisibilityField, UserProfileModerator)]
        )

    def tearDown(self):
        teardown_moderation()

    def test_moderation_objects_manager(self):
        ManagerClass = ModerationObjectsManager()(Manager)

        self.assertEqual(
            str(ManagerClass.__bases__),
            "(<class 'moderation.managers.ModerationObjectsManager'>"
            ", <class 'django.db.models.manager.Manager'>)",
        )

    def test_filter_moderated_objects_returns_empty_queryset(self):
        """Test filter_moderated_objects returns empty queryset
        for object that has moderated object"""

        ManagerClass = ModerationObjectsManager()(Manager)
        manager = ManagerClass()
        manager.model = UserProfile

        query_set = UserProfile._default_manager.all()
        moderated_object = ModeratedObject(content_object=self.profile)
        moderated_object.save()

        self.assertEqual(str(list(manager.filter_moderated_objects(query_set))), "[]")

    def test_filter_moderated_objects_returns_object(self):
        """Test if filter_moderated_objects return object when object
        doesn't have moderated object or deserialized object is <> object"""
        moderated_object = ModeratedObject(content_object=self.profile)
        moderated_object.save()
        moderated_object.approve()

        self.profile.description = "New"
        self.profile.save()

        self.assertEqual(
            str(list(UserProfile.objects.all())),
            '[<UserProfile: moderator - http://www.google.com>]',
        )

    def test_exclude_objs_by_visibility_col(self):
        ManagerClass = ModerationObjectsManager()(Manager)
        manager = ManagerClass()
        manager.model = ModelWithVisibilityField

        ModelWithVisibilityField(test='test 1').save()
        ModelWithVisibilityField(test='test 2', is_public=True).save()

        query_set = ModelWithVisibilityField.objects.all()

        query_set = manager.exclude_objs_by_visibility_col(query_set)

        self.assertEqual(
            str(list(query_set)),
            "[<ModelWithVisibilityField: test 2 - is public True>]",
        )

class ModeratedObjectManagerTestCase(TestCase):
    fixtures = ['test_users.json']

    def setUp(self):
        self.moderation = setup_moderation([UserProfile, ModelWithSlugField2])

        self.user = User.objects.get(username='admin')

    def tearDown(self):
        teardown_moderation()

    def test_objects_with_same_object_id_django(self):
        model1 = ModelWithSlugField2(slug='test')
        model1.save()

        model2 = UserProfile(
            description='Profile for new user',
            url='http://www.yahoo.com',
            user=User.objects.get(username='user1'),
        )

        model2.save()

        self.assertRaises(
            MultipleObjectsReturned, ModeratedObject.objects.get, object_pk=model2.pk
        )

        moderated_obj1 = ModeratedObject.objects.get_for_instance(model1)
        moderated_obj2 = ModeratedObject.objects.get_for_instance(model2)

        self.assertEqual(
            repr(moderated_obj2), '<ModeratedObject: user1 - http://www.yahoo.com>'
        )
        if VERSION < (2, 0):
            self.assertEqual(
                repr(moderated_obj1), "<ModeratedObject: ModelWithSlugField2 object>"
            )
        else:
            self.assertEqual(
                repr(moderated_obj1),
                "<ModeratedObject: ModelWithSlugField2 object (1)>",
            )

    def test_instance_with_many_moderations(self):
        # Register a moderator that keeps the history
        # un-register UserProfile
        from moderation import moderation

        class KeepHistoryModerator(GenericModerator):
            keep_history = True

        moderation.unregister(UserProfile)
        moderation.register(UserProfile, KeepHistoryModerator)

        profile = UserProfile(
            description='Profile for new user',
            url='http://www.yahoo.com',
            user=self.user,
        )
        profile.save()

        # There should only be one ModeratedObject
        self.assertEqual(1, ModeratedObject.objects.count())

        profile.url = 'http://www.google.com'
        profile.save()

        # Should now be two
        self.assertEqual(2, ModeratedObject.objects.count())

        # Getting for instance should return the one with the google url
        moderated_object = ModeratedObject.objects.get_for_instance(profile)
        self.assertEqual(profile.url, moderated_object.changed_object.url)

        # Get the first object, and does it have the yahoo address
        moderated_object_pk1 = ModeratedObject.objects.get(pk=1)
        self.assertEqual(
            'http://www.yahoo.com', moderated_object_pk1.changed_object.url
        )


with open("input.txt", "r") as f:
    depths = [int(x.strip()) for x in f.readlines()]

# part one

increasing = 0
previous_n = None
for n in depths:
    if previous_n:
        if n > previous_n:
            increasing += 1
    previous_n = n

print(increasing)

# part two

last_depth = len(depths) - 2

increasing = 0
previous_sum = None
for i in range(last_depth):
    current_sum = sum(depths[i : i + 3])
    if previous_sum:
        if current_sum > previous_sum:
            increasing += 1
    previous_sum = current_sum

print(increasing)


## load the data from mit-bih-arrhythmia
import numpy as np
import pandas as pd
import wfdb
import matplotlib.pyplot as plt

## split the raw data into 10 secs for each sample in dataset
def data_prepare(raw_datapath, save_path):
    """
    split the raw data into dataset (each sample for 10 secs)
    :param raw_datapath: the path to the raw data from mit-bih
    :param save_path: the path to save the ecg and labels (ex: "Data")
    :return: ecgs and labels
    """
    ## data path
    data_path = raw_datapath

    ## data lists

    # pts = ['100', '104', '108', '113', '117', '122', '201', '207', '212', '217', '222', '231',
    #        '101', '105', '109', '114', '118', '123', '202', '208', '213', '219', '223', '232',
    #        '102', '106', '111', '115', '119', '124', '203', '209', '214', '220', '228', '233',
    #        '103', '107', '112', '116', '121', '200', '205', '210', '215', '221', '230', '234']
    pts = ["100"]
    ## map the ~19 classes to 5 classes
    ## according to the paper https://arxiv.org/pdf/1805.00794.pdf
    mapping = {'N':0, 'L':0, 'R':0, 'e':0, 'j':0, 'B':0, # N = 0
               'A':1, 'a':1, 'J':1, 'S':1, # S = 1
               'V':2, 'E':2, 'r':2, 'n':2, # V = 2
               'F':3, # F = 3
               '/':4, 'f':4, 'Q':4, '?':4} # Q = 4
    ignore = ['+', '!', '[', ']', 'x', '~', '|', '"']

    secs = 5
    sample_rates = 360
    num_cols = 2 * secs * sample_rates
    ecg = np.zeros((1, num_cols))
    label = np.zeros((1, 1))

    for file in pts:
        # first load the simple file
        # load the ecg signal file
        x = wfdb.rdsamp(data_path + file)
        ecg_signal = x[0][:, 0]

        # vertify frequency ist 360
        # assert x[1]['fs'] == 360, 'sample freq is not 360'

        # load the annotations (in the symbol is the classes)
        annotation = wfdb.rdann(data_path + file, 'atr')
        atr_symbol = annotation.symbol
        atr_samples = annotation.sample

        classes = []
        new_samples = []
        idx = 0
        for i in atr_symbol:
            if i in ignore:
                idx += 1
                continue
            classes.append(mapping[i])
            new_samples.append(atr_samples[idx])
            idx += 1

        # make a df
        df_ann = pd.DataFrame({"atr_symbol" : classes,
                               "atr_samples" : new_samples})
        num_rows = len(df_ann)

        X = np.zeros((num_rows, num_cols))
        Y = np.zeros((num_rows, 1))
        max_row = 0

        for i, j in zip(new_samples, classes):
            left = max([0, (i - secs * sample_rates)])
            right = min([len(ecg_signal), (i + secs * sample_rates)])
            signal = ecg_signal[left:right]

            if len(signal) == num_cols:
                X[max_row, :] = signal
                Y[max_row, :] = j
                max_row += 1
            else:
                X[max_row, :len(signal)] = signal
                Y[max_row, :] = j
                max_row += 1
        # merge all the signal and label in two arrays
        ecg = np.concatenate((ecg, X), axis= 0)
        label = np.concatenate((label, Y), axis= 0)

    # data normalization between 0 and 1
    data_min = np.min(ecg, axis= 1).reshape(-1, 1)
    data_max = np.max(ecg, axis = 1).reshape(-1, 1)
    ecg = (ecg - data_min)/(data_max - data_min)

    # not using the first row
    ecg = ecg[1:, :]
    label = label[1:, :]
    ecg = pd.DataFrame(ecg)
    label = pd.DataFrame(label)
    ecg_path = save_path + "/ecg_signal.csv"
    label_path = save_path + "/label.csv"
    ecg.to_csv(ecg_path, index= None, header= None)
    label.to_csv(label_path, index= None, header= None)
    return ecg, label

# if __name__ == "__main__":
#     ecg, label = data_prepare("mit-bih-arrhythmia-database-1.0.0/", "Data")

## plot the ecg
# time = np.arange(len(ecg_signal))
# plt.figure()
# plt.plot(time[0:3600], ecg_signal[0:3600], '-')
# plt.show()


from helpers import *

@contextual
def test_create_composite(ctx):
    from melkman.db.composite import Composite
    cc = Composite.create(ctx)
    cc.save()

@contextual
def test_composites_by_sub(ctx):
    from melkman.db.bucket import NewsBucket
    from melkman.db.composite import Composite, view_composites_by_subscription

    c1 = Composite.create(ctx)
    c2 = Composite.create(ctx)

    bucket1 = NewsBucket.create(ctx)
    bucket1.save()

    bucket2 = NewsBucket.create(ctx)
    bucket2.save()

    bucket3 = NewsBucket.create(ctx)
    bucket3.save()

    c1.subscribe(bucket1)
    c1.save()

    c2.subscribe(bucket1)
    c2.subscribe(bucket2)
    c2.save()

    count = 0
    seen = set()
    for r in view_composites_by_subscription(ctx.db, include_docs=True, startkey=bucket1.id, endkey=bucket1.id):
        comp = Composite.wrap(r.doc)
        seen.add(comp.id)
        count += 1
    assert count == 2
    assert c1.id in seen
    assert c2.id in seen

    count = 0
    seen = set()
    for r in view_composites_by_subscription(ctx.db, include_docs=True, startkey=bucket2.id, endkey=bucket2.id):
        comp = Composite.from_doc(r.doc, ctx)
        seen.add(comp.id)
        count += 1
    assert count == 1
    assert c2.id in seen

    for r in view_composites_by_subscription(ctx.db, include_docs=True, startkey=bucket3.id, endkey=bucket3.id):
        assert False, 'unexpected subscription'

@contextual
def test_composite_subs_by_title(ctx):
    from melkman.db.bucket import NewsBucket
    from melkman.db.composite import Composite, view_composite_subscriptions_by_title
    from random import shuffle

    cc = Composite.create(ctx)

    buckets = []
    for i in range(10):
        bucket = NewsBucket.create(ctx)
        bucket.title = 'bucket %d' % i
        bucket.save()
        buckets.append(bucket)

    shuffled_buckets = list(buckets)
    shuffle(shuffled_buckets)

    for bucket in shuffled_buckets:
        cc.subscribe(bucket)
    cc.save()

    # should come out in alphabetical order

    for i, row in enumerate(view_composite_subscriptions_by_title(ctx.db, startkey=[cc.id, None], endkey=[cc.id, {}])):
        assert row.value['bucket_id'] == buckets[i].id
    assert i + 1 == len(buckets)

@contextual
def test_composite_filtered_update(ctx):
    from melkman.db.composite import Composite
    from random import shuffle

    cc = Composite.create(ctx)
    # a filter stack that accepts only things with the
    # word tortoise in the title, or is tagged tortoise
    cc.filters.append({'op': 'match_title',
                       'config': {'values': ['tortoise'],
                                  'match_type': 'substring'},
                       'action': 'accept'})
    cc.filters.append({'op': 'match_tag',
                      'config': {'values': ['tortoise']},
                      'action': 'accept'})
    cc.filters.append({'op': 'match_all',
                       'config': {},
                       'action': 'reject'})

    ok_items = [dummy_news_item({'title': "The tortoise and the O'Hare"}),
                dummy_news_item({'details': {'tags': [{'label': 'tortoise'}, {'label': 'shells'}]}})]

    not_ok_items = [dummy_news_item({'title': 'Jellybirds'}),
                    dummy_news_item({'details': {'tags': [{'label': 'hare'}, {'label': 'shells'}]}})]

    all_items = []
    all_items += ok_items
    all_items += not_ok_items
    shuffle(all_items)

    cc.filtered_update(all_items)

    for item in ok_items:
        assert cc.has_news_item(item)

    for item in not_ok_items:
        assert not cc.has_news_item(item)

    cc.save()
    cc.reload()

    for item in ok_items:
        assert cc.has_news_item(item)

    for item in not_ok_items:
        assert not cc.has_news_item(item)


import json
import uuid

from nifiapi.properties import PropertyDescriptor
from nifiapi.properties import StandardValidators
from nifiapi.properties import ExpressionLanguageScope
from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult

import pinecone

class PutPineconeVectorEmbedding(FlowFileTransform):
    class Java:
        implements = ['org.apache.nifi.python.processor.FlowFileTransform']
    class ProcessorDetails:
        dependencies = ['langchain', 'openai', 'pinecone-client','tiktoken']
        version = '0.0.1-SNAPSHOT'
        description = 'Upserts vector text embeddings into Pinecone with the configured index. Expected format is a list of json elements with text, embeddign and metadata'
        tags = [ 'Pinecone', 'AI, ''OpenAI',  'Vector Database', 'Embeddings' ]

    def __init__(self, **kwargs):
        # Build Property Descriptors

        self.pinecone_api_key = PropertyDescriptor(
            name="Pinecone API Key",
            description="The API key to connect to the Pinecone",
            required = True,
            sensitive = True
        )
        self.pinecone_environment_name = PropertyDescriptor(
            name="Pinecone Env Name",
            description="The environment that the Pinecone index is located.",
            required = True,
            sensitive = True
        )

        self.pinecone_index_name = PropertyDescriptor(
            name="Index Name",
            description="The Pinecone index to store the embeddings.",
            required = True,
        )

        self.pinecone_namespace = PropertyDescriptor(
            name="Namepace",
            description="The  namespace within the index to store the vector embeddings.",
            required = True,
        )

        self.batch_size = PropertyDescriptor(
            name="Batch Size",
            default_value = "100",
            description="The number of text docs to batch up before upserting into Pinecone",
            required=True,
        )

        self.descriptors = [self.pinecone_api_key, self.pinecone_environment_name, self.pinecone_index_name, self.pinecone_namespace, self.batch_size]
        # self.descriptors = [self.vector_embedding_model_service]

    def onScheduled(self, context):
        self.logger.info("Initializing Pinecone Index")

        # INitialize Pinecone and get the indecx we will be upserting into. Assume the index already exists
        pinecone_api_key =  context.getProperty(self.pinecone_api_key.name).getValue()
        pinecone_environment = context.getProperty(self.pinecone_environment_name.name).getValue()
        pinecone_index = context.getProperty(self.pinecone_index_name.name).getValue()

        pinecone.init(
            api_key=pinecone_api_key,
            environment=pinecone_environment
        )
        self.pinecone_index = pinecone.Index(index_name=pinecone_index)

    def transform(self, context, flowFile):
        self.logger.info("Inside transform of PutPineconeVectorEmbedding..")

        # Convert the json embedding string into list of text embedding documents that contain text, embedding and metadata for each text chunk
        chunked_docs_string = flowFile.getContentsAsBytes().decode('utf-8')
        chunk_docs_json_list_deserialized = json.loads(chunked_docs_string)

        # Store the text, embedding and metadata in seperate lists which we will use to batch up and insert into pinecone
        texts = []
        embeddings = []
        metadatas = []
        for doc_dict in chunk_docs_json_list_deserialized:
            texts.append(doc_dict["text"])
            embeddings.append(doc_dict['embedding'])
            metadatas.append(doc_dict['metadata'])

        batch_size = context.getProperty(self.batch_size.name).asInteger()
        namespace = context.getProperty(self.pinecone_namespace.name).getValue()

        # Create the List of tuples to insert into pinecone where tuple is string Id, tuple 2 is list of floats, and tuple 3 is metadata dictionary
        vector_ids = []
        for i in range(0, len(texts), batch_size):
            # set end position of batch
            i_end = min(i + batch_size, len(texts))
            # get batch of texts and ids
            lines_batch = texts[i:i_end]
            # create ids
            ids_batch = [str(uuid.uuid4()) for n in range(i, i_end)]
            vector_ids.extend(ids_batch)
            # get batch of embeddings
            embeddings_batch = embeddings[i:i_end]
            # prep metadata and upsert batch
            metadata_batch = metadatas[i:i_end]

            for j, line in enumerate(lines_batch):
                metadata_batch[j]["text"] = line
            to_upsert = zip(ids_batch, embeddings_batch, metadata_batch)

            # upsert to Pinecone
            self.pinecone_index.upsert(vectors=list(to_upsert), namespace=namespace)

        # Return a list of Ids for each embedding inserted
        vector_ids_json_string = json.dumps(vector_ids)
        self.logger.info(str(len(vector_ids)) + " vectors were inserted wtih the following ids: " + vector_ids_json_string)

        return FlowFileTransformResult(relationship="success", contents=vector_ids_json_string)

    def getPropertyDescriptors(self):
        return self.descriptors


def isPrime(n):
    if n == 2 or n == 3:
        return True
    else:
        for i in range(2,int(n**0.5)+1):
            if n % i == 0:
                if i == n//i:
                    return True
                return False
    return True

T = int(input())
for test_case in range(1,T+1):
    N = int(input())
    y = 2
    x = y+N
    while True:
        if not isPrime(x) and not isPrime(y):
            print(f'#{test_case} {x} {y}')
            break
        x+=1
        y+=1


from rest_framework import permissions
from .models import Place

class CraftShakeCounterPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        print(request.user.is_staff)
        if (request.user and request.user.is_staff):
            return True

class CraftShakeCustomerPermissions(permissions.BasePermission):

    def has_object_permission(self, request, view, obj):
        permitted_methods = ('GET', 'HEAD', 'OPTIONS')
        if (request.method in permitted_methods):
            print(request.user)
            # print('---------------------------------------------s')
            # print(obj.users.filter(id=request.user.id))
            user = obj.users.filter(id=request.user.id)
            if (user or request.user.is_staff and request.method in permitted_methods):
                print(request.method )
                return True


import numpy as np
import wave
import scipy.signal as signal
import matplotlib.pyplot as plt
import lpc

def enframe(signal, nw, inc, winfunc):
    '''将音频信号转化为帧。
    参数含义：
    signal:原始音频型号
    nw:每一帧的长度(这里指采样点的长度，即采样频率乘以时间间隔)
    inc:相邻帧的间隔（同上定义）
    '''
    signal_length=len(signal) #信号总长度
    if signal_length<=nw: #若信号长度小于一个帧的长度，则帧数定义为1
        nf=1
    else: #否则，计算帧的总长度
        nf=int(np.ceil((1.0*signal_length-nw+inc)/inc))
    pad_length=int((nf-1)*inc+nw) #所有帧加起来总的铺平后的长度
    zeros=np.zeros((pad_length-signal_length,)) #不够的长度使用0填补，类似于FFT中的扩充数组操作
    pad_signal=np.concatenate((signal,zeros)) #填补后的信号记为pad_signal
    indices=np.tile(np.arange(0,nw),(nf,1))+np.tile(np.arange(0,nf*inc,inc),(nw,1)).T  #相当于对所有帧的时间点进行抽取，得到nf*nw长度的矩阵[0-160],[40-200]...
    indices=np.array(indices,dtype=np.int32) #将indices转化为矩阵
    frames=pad_signal[indices] #得到帧信号
    win=np.tile(winfunc,(nf,1))  #window窗函数，这里默认取1
    return frames*win,nf   #返回帧信号矩阵

def wavread(filename):

    f = wave.open(filename, 'rb')
    params = f.getparams()
    nchannels, sampwidth, framerate, nframes = params[:4]
    strData = f.readframes(nframes)  # 读取音频，字符串格式
    waveData = np.fromstring(strData, dtype=np.int16)  # 将字符串转化为int
    f.close()
    waveData = waveData * 1.0 / (max(abs(waveData)))  # wave幅值归一化
    #画图
    time = np.arange(0, nframes) * (1.0 / framerate)
    plt.figure()
    plt.xlabel("Time(s)")
    plt.ylabel("Amplitude")
    plt.title("Single channel wavedata")
    plt.plot(time, waveData)
    plt.show()
    waveData = np.reshape(waveData, [nframes, nchannels]).T

    return waveData

def calEnergy(Frame,nf,nw):
    '''
    计算每一帧的能量
    :param Frame: 要计算的帧数组
    :param nf: 数组行，多少个帧
    :param nw: 数组列，每个帧多少个采样点
    :return: 返回所有帧的能量，一个列表
    '''
    result=[]
    if nf==1:
        sum = 0
        for j in range(nw):
            sum = sum + Frame[j] * Frame[j]
        result.append(sum)
    else:
        for i in range(nf):
            sum=0
            for j in range(nw):
                sum=sum+Frame[i][j]*Frame[i][j]
            result.append(sum)
    return result

def calzhuo(Energy,nf):
    '''
    计算出浊音的索引
    :param Energy: 能量数组
    :param nf: 多少帧
    :return: 返回每一个浊音段对应的帧的索引
    '''
    result=[]
    cache=[]
    for i in range(nf):
        if Energy[i]>0.1:
            cache.append(i)
            if Energy[i+1]<0.1 and Energy[i+2]<0.1:
                result.append(cache)
                cache=[]
    return result

def calxiangguan(Frame,i,nw):
    '''
    计算一帧自相关
    :param Frame: 加窗后的帧
    :param i: 对应帧的索引
    :param nw: 该帧有多少
    :return: 返回该帧的基因频率
    '''
    sum=0
    result=[]
    result2=[]
    for k in range(0,100):
        for j in range(nw-k):
            sum=sum+Frame[i][j]*Frame[i][j+k]
        result.append(sum)
    #归一化
    for l in range(len(result)):
        result2.append(result[l]/result[0])
    return 8000/result2.index(max(result2[-80:]))

filename = '01234.wav'
data = wavread(filename)
nw = 160  #窗大小
inc = 40 #窗偏移
winfunc = signal.hamming(nw) #汉明窗
Frame,nf = enframe(data[0], nw, inc, winfunc) #分帧加窗
Energy=calEnergy(Frame,nf,nw) #计算每一帧的能量

#画出每一帧的能量图
x = np.arange(1,len(Energy)+1)
plt.figure()
plt.title(u'每一帧的能量')
plt.xlabel(u'帧数')
plt.ylabel(u'能量')
plt.plot(x,Energy)
plt.show()

#计算浊音对应的帧
zhuo=calzhuo(Energy,nf)
print(zhuo)

#计算浊音每一帧的基音频率
xiangguanindex=[]
for i in range(5):
    for j in zhuo[i]:
        xiangguanindexi=calxiangguan(Frame,j,nw)
        xiangguanindex.append(xiangguanindexi)
    xiangguanindex.append(0 )
#画图
x = np.arange(1,len(xiangguanindex)+1)
plt.figure()
plt.title(u'浊音每一帧的基音频率')
plt.ylabel(u'基音频率')
plt.xlabel(u'每一部分的帧')
plt.plot(x,xiangguanindex)
plt.show()

#求预测增益
ans=[]
for i in zhuo[0]:
    A,E,K=lpc.levinson_1d(Frame[i],10)
    pre= signal.lfilter(A,[1],Frame[i])
    err=Frame[i]-pre
    erreng=calEnergy(err,1,160)
    result=Energy[i]/erreng
    ans.append(result)
    if i==180 :
        plt.figure()
        plt.subplot(2,1,1)
        x = np.arange(1, len(Frame[180]) + 1)
        plt.plot(x, Frame[180])
        plt.title(u'原始180帧')
        plt.subplot(2, 1, 2)
        x = np.arange(1, len(pre) + 1)
        plt.plot(x, pre)
        plt.title(u'预测180帧')
#画图
x = np.arange(1,len(ans)+1)
plt.figure()
plt.title(u'数字0的预测增益')
plt.plot(x,ans)
plt.show()


import numpy as np
import cv2
img=np.zeros((420,680,3), np.uint8)+255 			#创建一幅白色图像
def draw(event,x,y,flag,param):				#定义鼠标回调函数
    if event==cv2.EVENT_LBUTTONDBLCLK:
       cv2.circle(img,(x,y),20,(255,0,0),-1)			#双击鼠标左键时画圆
    elif event==cv2.EVENT_RBUTTONDBLCLK:
       cv2.rectangle(img,(x,y),(x+20,y+20),(0,0,255),-1)		#双击鼠标右键时画矩形
cv2.namedWindow('drawing')				#命名图像窗口
cv2.setMouseCallback('drawing',draw)				#为窗口绑定回调函数
while(True):
    cv2.imshow('drawing',img)           				#显示图像
    k = cv2.waitKey(1)
    if k == 27:                         				#按【Esc】键时结束循环
        break
cv2.destroyAllWindows()


#!/usr/bin/python3

"""
Write a program that takes a list of numbers (for example, a = [5, 10, 15, 20, 25]) and
 makes a new list of only the first and last elements of the given list"""

import example7
out_list = []

x_list = example7.randomList(10,100,5)
print(x_list)

out_list.append(x_list[0])
out_list.append(x_list[(len(x_list)-1)])

print (out_list)


import time as t
PrivateKey1=eval(input("Choose one:"))
PrivateKey2=eval(input("Choose another:"))
ClockNumber=eval(input("The size of the clock:"))
BaseNumber=eval(input("Base:"))

def getPPN(P1,P2,CN,BN):
    PPN1=((BN)**(P1))%CN
    PPN2=((BN)**(P2))%CN
    return PPN1,PPN2

def getShared(PPN,PK,CN):
    Remain=PPN%CN
    i=1
    while i < PK:
        Remain=(PPN*Remain)%CN
        i+=1
    return Remain
start=t.clock()
PPN1,PPN2=getPPN(PrivateKey1,PrivateKey2,ClockNumber,BaseNumber)
timeneed=t.clock()-start
print("PPN1,PPN2:",PPN1,PPN2)
print("timeneed:",round(timeneed,4))
print(getShared(PPN1,PrivateKey2,ClockNumber))
print(getShared(PPN2,PrivateKey1,ClockNumber))


import unittest, tempfile, subprocess, os, shutil
from SmileTrain import uc2denovo, util
from SmileTrain.test import fake_fh

class TestMissedLabels(unittest.TestCase):
    '''tests for missing_labels'''
    def test_correct(self):
        '''should parse uc file and get missing labels'''
        uc_lines = ["H       43107   253     100.0   +       0       0       526I253M617I    seq1;counts=617550      181719", "H       42869   252     100.0   +       0       0       526I252M625I    seq24;counts=296908     182945", "N       *       *       *       .       *       *       *       seq12996;counts=2       *"]
        labels = uc2denovo.missed_labels(uc_lines)
        self.assertEqual(labels, ['seq12996;counts=2'])

class TestMatchingFastaEntries(unittest.TestCase):
    '''tests for matching_fasta_entries'''
    def test_correct(self):
        '''should collect only the matching entries'''
        old_fasta = fake_fh([">a", "AAA", ">b", "TTT", ">c", "GGG"])
        labels = ["a", "c"]
        expected_new_fasta = ">a\nAAA\n>c\nGGG\n"

        new_fasta = uc2denovo.matching_fasta_entries(labels, old_fasta)
        self.assertEqual([new_fasta[0].id, str(new_fasta[0].seq), new_fasta[1].id, str(new_fasta[1].seq)], ['a', 'AAA', 'c', 'GGG'])


# Python Exam exercise 2

def main ():
	number = check_input()
	result = count_random(number)
	answer = check_if_prime(result)
	print("Number", result, answer)

# -------------------------------------------------------------

def check_input():
	number_tested = 0
	valid_number = 0
	while valid_number == 0:

		number_tested = int(input('Give a number between 10-30 or 45-50: '))

		if (number_tested >= 10 and number_tested <=30) or (number_tested >=45 and number_tested <=50) :
			valid_number = 1
		else:
			print("Try again, this number is not valid")
			valid_number =0

	return number_tested

# -------------------------------------------------------------

def count_random(dividend):
	import random as random
	random_number = int(round(random.uniform(20, 999)))
	division = int(round(random_number / dividend))
	return division

# -------------------------------------------------------------

def check_if_prime(my_number):
	if my_number > 1:
		for n in range(2, my_number):
			if(my_number % n) == 0:
				return (f'{my_number} is not a prime number')
			else:
				return (f'{my_number} is a prime number')

main()