from django.shortcuts import render
from django.conf import settings
from django.contrib.auth.decorators import login_required

# import models
from apps.core.models.message_block import MessageBlock
from apps.core.models.todo_action_status import TodoActionStatus
from apps.core.models.todo import Todo
from apps.core.models.summary_log_end_users import SummaryLogEndUsers

# import views
from apps.core.views.vendor_common.login_user_info import *

@login_required
def index(request):
    user_obj = get_login_user_objects(request)
    vendor_branch_id = user_obj["vendor_branch"].id
    vendor_id = user_obj["vendor_branch"].vendor.id

    """ Analytics Index """
    BG_COLOR_PRESET_LIST = ["#1FB6E9", "#F15424", "#29C4A9", "#F6FB4E", "#242D3C", "#E2DADB", "#C52F72"]
    BG_PROGRESS_BAR_TYPE_LIST = ["bg-danger", "bg-info", "bg-success", "bg-primary", "bg-warning"]

    user_obj = get_login_user_objects(request)

    # Total Conversation Rate
    try:
        sql_total_conversion_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/total_conversation_rate.sql"
        sql_total_conversion_base = open(sql_total_conversion_file, 'r').read()
        sql_total_conversion = sql_total_conversion_base.format(vendor_id=str(vendor_id))
        print("sql_total_conversion ====================")
        print(sql_total_conversion)

        summary_log_end_user = SummaryLogEndUsers.objects.raw(sql_total_conversion)[0]
        total_conversion_rate = round(summary_log_end_user.conversion_rate, 2)
    except Exception as e:
        total_conversion_rate = 0

    # Total Visitors
    try:
        sql_total_visitor_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/total_visitors.sql"
        sql_total_visitor_base = open(sql_total_visitor_file, 'r').read()
        sql_total_visitor = sql_total_visitor_base.format(vendor_id=str(vendor_id))
        print("sql_total_visitor ====================")
        print(sql_total_visitor)
        summary_log_end_user = SummaryLogEndUsers.objects.raw(sql_total_visitor)[0]
        total_visitors = summary_log_end_user.visitor_cnt
    except Exception as e:
        total_visitors = 0
    print("★")
    print(total_visitors)

    # get Todos numbers
    todo_action_status = TodoActionStatus.objects.filter(is_delete=False).exclude(name='DONE')
    todo_action_status_list = []
    for status in todo_action_status:
        todo_action_status_list.append(status.id)

    todos = Todo.objects.filter(vendor_branch_id=user_obj["vendor_branch"].id, todo_action_status_id__in=todo_action_status_list, is_delete=False)
    todocount = todos.count()

    # Weekly Conversion
    sql_weekly_conversion_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/weekly_conversion_rate.sql"
    sql_weekly_conversion_base = open(sql_weekly_conversion_file, 'r').read()
    sql_weekly_conversion = sql_weekly_conversion_base.format(vendor_id=str(vendor_id))
    print("sql_weekly_conversion ====================")
    print(sql_weekly_conversion)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_weekly_conversion)

    label_list = []
    data_list = []
    bg_color_list = []
    days_list = ["SUN", "MON", "TUE", "WEB", "THU", "FRI", "SAT"]

    # Should be 7 elements
    match_flg = 0
    for i in range(0, 7):
        label_list.append("\"" + days_list[i] + "\"")
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[i] + "\"")

        for cv in summary_log_end_users:
            match_flg = 0
            if i == int(cv.log_day):
                if cv.conversion_rate is not None:
                    data_list.append(cv.conversion_rate)
                else:
                    data_list.append(0)
                match_flg = 1
                break

        if match_flg == 0:
            data_list.append(0)

        print("data_list --------")
        print(data_list)

    w_cv_label_list_string = ','.join(map(str, label_list))
    w_cv_data_list_string = ','.join(map(str, data_list))
    w_cv_bg_color_list_string = ','.join(map(str, bg_color_list))

    w_cv_rate_chart = {
        "labels": w_cv_label_list_string,
        "data": w_cv_data_list_string,
        "bg_color": w_cv_bg_color_list_string,
    }

    # Weekly Visitors
    sql_weekly_visitors_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/weekly_visitors.sql"
    sql_weekly_visitors_base = open(sql_weekly_visitors_file, 'r').read()
    sql_weekly_visitors = sql_weekly_visitors_base.format(vendor_id=str(vendor_id))
    print("sql_weekly_visitors ====================")
    print(sql_weekly_visitors)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_weekly_visitors)

    label_list = []
    data_list = []
    bg_color_list = []
    days_list = ["SUN", "MON", "TUE", "WEB", "THU", "FRI", "SAT"]

    # Should be 7 elements
    match_flg = 0
    for i in range(0,7):
        label_list.append("\"" + days_list[i] + "\"")
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[i] + "\"")

        for visitor in summary_log_end_users:
            match_flg = 0
            if i == int(visitor.log_day):
                data_list.append(visitor.weekly_visitor_cnt)
                match_flg = 1
                break

        if match_flg == 0:
            data_list.append(0)

    w_visitor_label_list_string = ','.join(map(str, label_list))
    w_visitor_data_list_string = ','.join(map(str, data_list))
    w_visitor_bg_color_list_string = ','.join(map(str, bg_color_list))

    w_visitor_rate_chart = {
        "labels": w_visitor_label_list_string,
        "data": w_visitor_data_list_string,
        "bg_color": w_visitor_bg_color_list_string,
    }

    # Device Rate > 1: mobile 2: tablet  3: pc  4: other
    sql_device_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/device_rate.sql"
    sql_device_rate_base = open(sql_device_file, 'r').read()
    sql_device_rate = sql_device_rate_base.format(vendor_id=str(vendor_id))
    print("sql_device_rate ====================")
    print(sql_device_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_device_rate)
    label_list = []
    data_list = []
    bg_color_list = []
    for device in summary_log_end_users:
        if device.device_type == 1:
            label_list.append("\"" + "Mobile" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[0] + "\"")

        elif device.device_type == 2:
            label_list.append("\"" + "Tablet" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[1] + "\"")

        elif device.device_type == 3:
            label_list.append("\"" + "PC" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[2] + "\"")

        else:
            label_list.append("\"" + "Other" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[3] + "\"")

        data_list.append(device.os_family_cnt)

    # Convert List to String
    device_label_list_string = ','.join(map(str, label_list))
    device_bg_color_list_string = ','.join(map(str, bg_color_list))
    device_data_list_string = ','.join(map(str, data_list))

    device_rate_chart = {
        "labels": device_label_list_string,
        "data": device_data_list_string,
        "bg_color": device_bg_color_list_string,
    }
    print("device_rate_chart ==== ")
    print(device_rate_chart)

    # OS Rate
    sql_os_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/os_rate.sql"
    sql_os_rate_base = open(sql_os_file, 'r').read()
    sql_os_rate = sql_os_rate_base.format(vendor_id=str(vendor_id))
    print("sql_os_rate_base ====================")
    print(sql_os_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_os_rate)
    label_list = []
    data_list = []
    bg_color_list = []

    for i, os in enumerate(summary_log_end_users):
        label_list.append("\"" + os.os_family + "\"")
        data_list.append(os.os_family_cnt)
        bg_color_index = i % 6
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[bg_color_index] + "\"")

    # Convert List to String
    os_label_list_string = ','.join(map(str, label_list))
    os_data_list_string = ','.join(map(str, data_list))
    os_bg_color_list_string = ','.join(map(str, bg_color_list))

    os_rate_chart = {
        "labels": os_label_list_string,
        "data": os_data_list_string,
        "bg_color": os_bg_color_list_string,
    }

    print("os_rate_chart ==== ")
    print(os_rate_chart)

    # Browser Rate
    sql_browser_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/browser_rate.sql"
    sql_browser_rate_base = open(sql_browser_file, 'r').read()
    sql_browser_rate = sql_browser_rate_base.format(vendor_id=str(vendor_id))
    print("sql_browser_rate ====================")
    print(sql_browser_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_browser_rate)
    label_list = []
    data_list = []
    bg_color_list = []

    for i, browser in enumerate(summary_log_end_users):
        label_list.append("\"" + browser.browser_family + "\"")
        data_list.append(browser.browser_family_cnt)
        bg_color_index = i % 6
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[bg_color_index] + "\"")

    # Convert List to String
    browser_label_list_string = ','.join(map(str, label_list))
    browser_data_list_string = ','.join(map(str, data_list))
    browser_bg_color_list_string = ','.join(map(str, bg_color_list))

    browser_rate_chart = {
        "labels": browser_label_list_string,
        "data": browser_data_list_string,
        "bg_color": browser_bg_color_list_string,
    }

    print("browser_rate_chart ==== ")
    print(browser_rate_chart)

    # Conversation_Percent
    sql_conversation_rate_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/conversation_rate.sql"
    sql_conversation_rate_base = open(sql_conversation_rate_file, 'r').read()

    conversation_bar_list = []
    message_blocks = MessageBlock.objects.filter(vendor_branch_id=vendor_branch_id, is_delete=False).all()
    for message_block in message_blocks:
        tmp_bar_dict = {
            "message_block_name": message_block.admin_text,
            "progress_numerator_list": [],
            "progress_denominator_list": []
        }

        sql_conversation_rate = sql_conversation_rate_base.format(vendor_id=str(vendor_id), message_block_id=str(message_block.id))
        print("debug: sql_conversation_rate")
        print(sql_conversation_rate)
        summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_conversation_rate)

        for cv in summary_log_end_users:
            tmp_bar_dict["progress_numerator_list"].append(int(cv.progress_numerator))
            tmp_bar_dict["progress_denominator_list"].append(int(cv.progress_denominator))

        conversation_bar_list.append(tmp_bar_dict)

    print("conversation_bar_list =========")
    print(conversation_bar_list)

    conversation_rate_list = []
    for i, cv in enumerate(conversation_bar_list):
        if len(cv["progress_denominator_list"]) == 0:
            percentage = 0
        else:
            percentage = (sum(cv["progress_numerator_list"]) / sum(cv["progress_denominator_list"])) * 100

        tmp_dict = {
            "message_block_name": cv["message_block_name"],
            "bg_type": BG_PROGRESS_BAR_TYPE_LIST[i % len(BG_PROGRESS_BAR_TYPE_LIST)],
            "percentage": str(int(percentage))
        }

        conversation_rate_list.append(tmp_dict)

    print("conversation_rate_list")
    print(conversation_rate_list)

    context = {
        "title": "Analytics",
        "total_conversion_rate": total_conversion_rate,
        "total_visitors": total_visitors,
        "todocount" : todocount,
        "todos": todos,
        "w_cv_rate_chart": w_cv_rate_chart,
        "w_visitor_rate_chart": w_visitor_rate_chart,
        "device_rate_chart": device_rate_chart,
        "os_rate_chart": os_rate_chart,
        "browser_rate_chart": browser_rate_chart,
        "conversation_rate_list": conversation_rate_list,
        "namespace": user_obj["service_namespace"]
    }

    return render(request, "vendor_contactchat/statistics.html", context)


"""
http://docs.aws.amazon.com/amazondynamodb/latest/gettingstartedguide/GettingStarted.Python.03.html
"""

from __future__ import print_function # Python 2/3 compatibility
import boto3
import json
import decimal

# Helper class to convert a DynamoDB item to JSON.
class DecimalEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, decimal.Decimal):
            if o % 1 > 0:
                return float(o)
            else:
                return int(o)
        return super(DecimalEncoder, self).default(o)

dynamodb = boto3.resource('dynamodb',region_name='us-east-1')

table = dynamodb.Table('users')

### put item
id = "The Big New Movie"
email = "given.family@sungardas.com"
family_name = "Family"
given_name = "Given"
name = "Given Family"

response = table.put_item(
   Item={
        'id': id,
        'email': email,
        'family_name': family_name,
        'given_name': given_name,
        'name': name
    }
)

print("PutItem succeeded:")
print(json.dumps(response, indent=4, cls=DecimalEncoder))

### get Item

from boto3.dynamodb.conditions import Key, Attr
from botocore.exceptions import ClientError

try:
    response = table.get_item(
        Key={
            'id': id
        }
    )
except ClientError as e:
    print(e.response['Error']['Message'])
else:
    item = response['Item']
    print("GetItem succeeded:")
    print(json.dumps(item, indent=4, cls=DecimalEncoder))

### update item

try:
    response = table.update_item(
        Key={
                'id': id
        },
        UpdateExpression="set family_name = :fn, given_name=:gn, #name=:n",
        ExpressionAttributeNames={
            '#name': 'name'
        },
        ExpressionAttributeValues={
            ':fn': "Family2",
            ':gn': "Given2",
            ':n': "Given2 Family2"
        },
        #ConditionExpression="size(info.actors) > :num",
        #ExpressionAttributeValues={
        #    ':num': 3
        #},
        ReturnValues="UPDATED_NEW"
    )
except ClientError as e:
    print(e.response['Error']['Message'])
else:
    print("UpdateItem succeeded:")
    print(json.dumps(response, indent=4, cls=DecimalEncoder))

### delete item

try:
    response = table.delete_item(
        Key={
                'id': id
        },
        #ConditionExpression="family_name <> :val",
        #ExpressionAttributeValues= {
        #    ":val": "Family"
        #}
    )
except ClientError as e:
    if e.response['Error']['Code'] == "ConditionalCheckFailedException":
        print(e.response['Error']['Message'])
    else:
        raise
else:
    print("DeleteItem succeeded:")
    print(json.dumps(response, indent=4, cls=DecimalEncoder))


from utils import read_file

values = read_file(8, str, False)

grid = []

for current_tree_height in values:
    grid.append(list(current_tree_height))

visible = 0

def get_column(matrix, i):
    return [_row[i] for _row in matrix]

for row in range(len(grid)):
    for column in range(len(grid[row])):
        # Ignore Edges
        if row == 0 or row == len(grid) - 1 or column == 0 or column == len(grid[row]) - 1:
            visible += 1
            continue
        # Current tree height to compare to
        current_tree_height = grid[row][column]
        current_column = get_column(grid, column)
        current_row = grid[row]
        up = current_column[0:row]
        down = current_column[row + 1:]
        left = current_row[0:column]
        right = current_row[column + 1:]
        from_left = len([tree for tree in left if tree >= current_tree_height]) == 0
        from_right = len([tree for tree in right if tree >= current_tree_height]) == 0
        from_up = len([tree for tree in up if tree >= current_tree_height]) == 0
        from_down = len([tree for tree in down if tree >= current_tree_height]) == 0
        if from_up or from_down or from_left or from_right:
            visible += 1

print("Part 1: " + str(visible))
# Part 1: 1809

scenic_scores = []

for row in range(len(grid)):
    for column in range(len(grid[row])):
        # Ignore Edges
        if row == 0 or row == len(grid) - 1 or column == 0 or column == len(grid[row]) - 1:
            continue
        current_tree_height = grid[row][column]
        current_column = get_column(grid, column)
        current_row = grid[row]
        up = current_column[0:row]
        down = current_column[row + 1:]
        left = current_row[0:column]
        right = current_row[column + 1:]

        up_score, down_score, left_score, right_score = 0, 0, 0, 0
        for i in reversed(range(len(up))):
            up_score += 1
            if current_tree_height <= up[i]:
                break

        for i in range(len(down)):
            down_score += 1
            if current_tree_height <= down[i]:
                break

        for i in reversed(range(len(left))):
            left_score += 1
            if current_tree_height <= left[i]:
                break

        for i in range(len(right)):
            right_score += 1
            if current_tree_height <= right[i]:
                break

        scenic_score = up_score * left_score * right_score * down_score
        scenic_scores.append(scenic_score)

print("Part 2: " + str(max(scenic_scores)))
# Part 2: 479400


import os
import sys
import traceback
import pdb
import re
import numpy as np
import pandas as pd
from math import log
from collections import Counter
from functools import reduce
from operator import add
from multiprocessing import Pool

# directory name: {dataset}_{search-l}_{type}_{# of the shard}
# usage "python anss_jh_breakdown-parser.py [log directory] [# of the thread] [query depth] [# of the query]"

OUTLIER_THRESHOLD = 50000000
# OUTLIER_THRESHOLD = 20000000

DIRECTORY = sys.argv[1]
if DIRECTORY[-1] == '/':
    DIRECTORY = DIRECTORY[:-1]
THREAD_NUM = int(sys.argv[2])
QUERY_DEPTH = int(sys.argv[3])
QUERY_NUM = int(sys.argv[4])
# NDP_NUM = int(sys.argv[4])
if len(sys.argv) == 7:
    TYPE = sys.argv[5]
else:
    TYPE = ""

FILE = 'debug.log'
TERMINAL_FILE = 'system.terminal'
POSTFIX = "-BREAKDOWN"

TYPE_LIST = [
    'disk',
    'distributed',
    'base',
    'ndp',
    'cache',
    'nearest',
    'infinite',
]

NDP_LIST = [
    'ndp',
    'cache',
    'nearest',
]

COLUMNS = [
    "update",
    "calculation",
    "graph",
    "query vector",
    "sq entry set",
    "doorbell",
    "sq dma",
    "cq dma",
    "polling",
    "accumulation",
    "merge",
    "network",
    "total",
    "submission",
]

RDMA_PATTERN = r'([\d \+]* \d*)(?= B)'
RDMA_REGEX = re.compile(RDMA_PATTERN)

ACCURACY_PATTERN = r'([0-9]*[.]?[0-9]+) ([+-]?([0-9]*[.])?[0-9]+) [+-]?([0-9]*[.])?[0-9]+ [+-]?([0-9]*[.])?[0-9]+'
ACCURACY_REGEX = re.compile(ACCURACY_PATTERN)

DATASET_PATTERN = r'(?<=datasetDir: ")[\s\S]*\/(\w*)(?=")'
DATASET_REGEX = re.compile(DATASET_PATTERN)

def getInitDict():
    _tmp_dict = {}
    for column in COLUMNS:
        _tmp_dict[column] = 0
    return _tmp_dict

OUT_COLUMNS = ["dataset", "search-L", "type"]
OUT_COLUMNS.extend(COLUMNS)
OUT_COLUMNS.extend(["end-to-end", "accuracy", "num query", "num ndp"])

MAGIC_WORDS = ['P', 'Q', 'm5_print']

INITIAL_VALUE = -1

### point ###
# host
SET_START = 'set_start'
QUERY_SET = 'query_set'
DISTANCE_START = 'distance_start'
SEND_DOORBELL_DONE = 'send_doorbell_done'
POLLING_END_DONE = 'polling_end_done'
DISTANCE_END = 'distance_end'
UPDATE_END = 'update_end'
TRAVERSE_END = 'traverse_end'
SET_END = 'set_end'
# device
SEND_DOORBELL = 'send_doorbell'
RECV_DOORBELL = 'recv_doorbell'
RECV_SQ = 'recv_sq'
GET_QUERY_VECTOR_START = 'get_query_vector_start'
COMPUTATION_START = 'computation_start'
ALLOCATE_UNIT = 'allocate_unit'
DEALLOCATE_UNIT = 'deallocate_unit'
COMPUTATION_END = 'computation_end'
CQ_DMA_END = 'cq_dma_end'
POLLING_END = 'polling_end'
# distributed
DISTRIBUTED_MASTER_SEND = 'distributed_master_send'
DISTRIBUTED_SLAVE_RECV = 'distributed_slave_recv'
DISTRIBUTED_SLAVE_SEND = 'distributed_slave_send'
DISTRIBUTED_MASTER_RECV = 'distributed_master_recv'
DISTRIBUTED_POSTPROCESSING = 'distributed_postprocessing'
DISTRIBUTED_MERGE_END = 'distributed_end'

DISTRIBUTED_POINT_LIST = [
    DISTRIBUTED_MASTER_SEND,
    DISTRIBUTED_SLAVE_RECV,
    DISTRIBUTED_SLAVE_SEND,
    DISTRIBUTED_MASTER_RECV,
    DISTRIBUTED_POSTPROCESSING,
    DISTRIBUTED_MERGE_END,
]

class OneQuery:
    def __init__(self, type, thread_id, query_index, ndp_num=0, interface="dmaone"):
        self.type = type
        self.thread_id = thread_id
        self.query_index = query_index
        self.ndp_num = ndp_num
        self.interface = interface

        self.doorbell_counter = 0
        self.first_doorbell = 0
        self.cq_dma_counter = 0
        self.polling_counter = 0
        self.timestamps = {
            SET_START: 0,
            QUERY_SET: 0,
            DISTANCE_START: 0,
            SEND_DOORBELL: 0,
            SEND_DOORBELL_DONE: 0,
            RECV_DOORBELL: 0,
            RECV_SQ: 0,
            GET_QUERY_VECTOR_START: 0,
            COMPUTATION_START: 0,
            COMPUTATION_END: 0,
            CQ_DMA_END: 0,
            POLLING_END: 0,
            POLLING_END_DONE: 0,
            DISTANCE_END: 0,
            UPDATE_END: 0,
            TRAVERSE_END: 0,
            SET_END: 0,
        }
        self.ticks = getInitDict()

    def one_hop_reset(self):
        self.doorbell_counter = 0
        self.first_doorbell = 0
        self.cq_dma_counter = 0
        self.polling_counter = 0
        self.timestamps = {
            SET_START: 0,
            QUERY_SET: 0,
            DISTANCE_START: 0,
            SEND_DOORBELL: 0,
            SEND_DOORBELL_DONE: 0,
            RECV_DOORBELL: 0,
            RECV_SQ: 0,
            GET_QUERY_VECTOR_START: 0,
            COMPUTATION_START: 0,
            COMPUTATION_END: 0,
            CQ_DMA_END: 0,
            POLLING_END: 0,
            POLLING_END_DONE: 0,
            DISTANCE_END: 0,
            UPDATE_END: 0,
            TRAVERSE_END: 0,
            SET_END: 0,
        }

    def query_reset(self):
        self.one_hop_reset()
        self.ticks = getInitDict()

    def update_ticks(self, timestamp):
        if self.type in NDP_LIST:
            if self.timestamps[SET_START] != 0:
                assert self.timestamps[QUERY_SET] != 0
                if self.interface == 'dmaone' or self.interface == 'dma':
                    assert self.timestamps[GET_QUERY_VECTOR_START] != 0
                    self.ticks["query vector"] += self.timestamps[COMPUTATION_START] - self.timestamps[GET_QUERY_VECTOR_START]
                self.ticks["query vector"] += self.timestamps[QUERY_SET] - self.timestamps[SET_START]

                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[QUERY_SET]
                self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]
            elif self.timestamps[TRAVERSE_END] == 0:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[UPDATE_END]
                self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]
                # self.ticks["submission"] += self.timestamps[SEND_DOORBELL_DONE] - self.timestamps[DISTANCE_START]
            else:
                self.ticks["graph"] += self.timestamps[TRAVERSE_END] - self.timestamps[UPDATE_END]
                self.ticks["update"] += timestamp - self.timestamps[TRAVERSE_END]

            if self.interface == 'dmaone':
                if  self.timestamps[DISTANCE_START] < self.first_doorbell:
                    self.ticks["sq entry set"] += self.first_doorbell - self.timestamps[DISTANCE_START]
                    self.ticks["doorbell"] += self.timestamps[RECV_DOORBELL] - self.first_doorbell
            else:
                self.ticks["sq entry set"] += self.timestamps[SEND_DOORBELL] - self.timestamps[DISTANCE_START]
                self.ticks["doorbell"] += self.timestamps[RECV_DOORBELL] - self.timestamps[SEND_DOORBELL]

            if self.interface != 'mmio':
                self.ticks["sq dma"] += self.timestamps[RECV_SQ] - self.timestamps[RECV_DOORBELL]

            self.ticks["calculation"] += self.timestamps[COMPUTATION_END] - self.timestamps[COMPUTATION_START]

            if self.cq_dma_counter != self.ndp_num:
                self.ticks["cq dma"] += self.timestamps[POLLING_END] - self.timestamps[COMPUTATION_END]
            else:
                self.ticks["cq dma"] += min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]) - self.timestamps[COMPUTATION_END]
                if (self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]) < OUTLIER_THRESHOLD):
                    self.ticks["polling"] += self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END])
                if OUTLIER_THRESHOLD < self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]):
                    print(f'{timestamp}: {self.timestamps[POLLING_END] - self.timestamps[CQ_DMA_END]}')

            if self.timestamps[DISTANCE_START] < self.first_doorbell:
                self.ticks["accumulation"] += self.timestamps[DISTANCE_END] - self.timestamps[POLLING_END]
        else:
            if self.timestamps[SET_START] != 0:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[SET_START]
            else:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[UPDATE_END]
            self.ticks["calculation"] += self.timestamps[DISTANCE_END] - self.timestamps[DISTANCE_START]
            self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]

        self.one_hop_reset()

    def error(self, log_line):
        print(log_line)
        traceback.print_tb()
        exit(1)

    def insert_timestamp(self, point, timestamp, log_line, row_list):
        if point == UPDATE_END or point == SET_END:
            self.update_ticks(timestamp)

        # increment doorbell counter
        if point == SEND_DOORBELL:
            if self.doorbell_counter == 0:
                self.first_doorbell = timestamp
            self.doorbell_counter += 1

        # increment CQ DMA counter
        if point == CQ_DMA_END and \
           self.timestamps[COMPUTATION_END] != 0 :
            self.cq_dma_counter += 1

        if point == POLLING_END:
            self.polling_counter += 1
        if point == TRAVERSE_END:
            self.polling_counter += self.ndp_num

        # insert timestamp
        self.timestamps[point] = timestamp

        if point == SET_END:
            self.ticks['total'] = sum(self.ticks.values())
            row_list.append(self.ticks)
            self.query_reset()

def update_distributed_tick(timestamp, point, distributed, network_memcpy, network_update, network_merge):
    if point == DISTRIBUTED_POSTPROCESSING:
        network_memcpy += distributed[DISTRIBUTED_SLAVE_RECV] - distributed[DISTRIBUTED_MASTER_SEND]
        network_memcpy += distributed[DISTRIBUTED_MASTER_RECV] - distributed[DISTRIBUTED_SLAVE_SEND]
        network_update += timestamp - distributed[DISTRIBUTED_MASTER_RECV]
    elif point == DISTRIBUTED_MERGE_END:
        network_merge += timestamp - distributed[DISTRIBUTED_POSTPROCESSING]
    else:
        print("Wrong point")
        exit(1)

    return (network_memcpy, network_update, network_merge)

def parsing(file_full_name):
    path = file_full_name[0]
    file = file_full_name[1]

    setting = path.split('/')[-1]
    if TYPE:
        type = TYPE
    else:
        type = setting.split('_')[-2]

    if type not in TYPE_LIST:
        print(f"wrong type: {type}")
        exit(1)
    elif type == 'nearest':
        query_depth = QUERY_DEPTH
    else:
        query_depth = 1

    if type in NDP_LIST:
        ndp_num = int(setting.split('_')[-1])
    elif type == "distributed":
        ndp_num = int(setting.split('_')[-1])
    else:
        ndp_num = 0

    # get end-to-end latency
    start_tick = 0
    end_tick = 0
    end_to_end = 0
    is_first = True

    dataset = ''
    accuracy = 0
    search_l = 0

    queries = []
    for i in range(THREAD_NUM):
        queries.append([])
        for j in range(query_depth):
            queries[i].append(OneQuery(type, i, j, ndp_num))

    # distributed
    distributed = {
        DISTRIBUTED_MASTER_SEND: 0,
        DISTRIBUTED_SLAVE_RECV: 0,
        DISTRIBUTED_SLAVE_SEND: 0,
        DISTRIBUTED_MASTER_RECV: 0,
        DISTRIBUTED_POSTPROCESSING: 0,
        DISTRIBUTED_MERGE_END: 0,
    }
    distributed_merge = 0
    distributed_update = 0
    network_memcpy = 0
    network_link = 0

    row_list = []
    output = setting + POSTFIX

    print(f'{path}/{file}')
    f = open(f'{path}/{file}')
    lines = f.read().splitlines()
    for line in lines:
        row = line.split(': ')
        magic_word = row[2]
        if magic_word not in MAGIC_WORDS:
            continue

        # distributed
        if magic_word == 'm5_print':
            rdma_matching = RDMA_REGEX.search(line)
            # 3.7 us per 512 bytes
            network_link += (eval(rdma_matching.group()) / 512 * 3700000)
            continue

        tick = int(row[0])
        point = row[-1]
        # dev_index = int(row[-2])
        query_index = int(row[-3])
        thread_id = int(row[-4])

        # print(line)

        # distributed
        if point in DISTRIBUTED_POINT_LIST:
            if point == DISTRIBUTED_POSTPROCESSING or point == DISTRIBUTED_MERGE_END:
                (network_memcpy, distributed_update, distributed_merge) = update_distributed_tick(tick, point, distributed, network_memcpy, distributed_update, distributed_merge)
            distributed[point] = tick
            continue

        queries[thread_id][query_index].insert_timestamp(point, tick, line, row_list)

        if point == SET_START and is_first:
            start_tick = tick
            is_first = False

        if point == SET_END:
            end_tick = tick

    end_to_end = end_tick - start_tick

    # get accuracy
    # get search-l
    terminal_file = open(os.path.join(path, TERMINAL_FILE))
    lines = terminal_file.read().splitlines()
    for line in lines:
        dataset_match = DATASET_REGEX.search(line)
        if dataset_match:
            dataset = dataset_match.group(1)
        accuracy_match = ACCURACY_REGEX.search(line)
        if accuracy_match:
            accuracy = float(accuracy_match.group(2))
            search_l = int(accuracy_match.group(1))

    df = pd.DataFrame(row_list, columns=COLUMNS).fillna(0).astype(int)

    # distributed
    if type == 'distributed':
        end_to_end += distributed_update + network_memcpy
        end_to_end //= ndp_num
        end_to_end += network_link + distributed_merge
        distributed_merge //= QUERY_NUM
        distributed_update //= (ndp_num * QUERY_NUM)
        network_memcpy //= (ndp_num * QUERY_NUM)
        network_link //= QUERY_NUM
        df['total'] += distributed_update + network_memcpy + network_link
        df['merge'] = distributed_merge + distributed_update
        df['network'] = network_memcpy + network_link

    # generate mean
    mean = df.mean().values.tolist()
    mean = [dataset, search_l, type] + mean
    mean.extend([end_to_end, accuracy, QUERY_NUM, ndp_num])

    df.to_csv(f'{DIRECTORY}/{output}.csv', index=False)
    f.close()

    return mean

if __name__ == '__main__':
    # pool input
    file_full_names = []
    for (path, dir, files) in os.walk(DIRECTORY):
        for file in files:
            if file != FILE:
                continue
            file_full_names.append((path, file))

    # multiprocessing
    with Pool(1) as p:
        means = p.map(parsing, file_full_names)

    output = DIRECTORY.split('/')[-1] + POSTFIX
    df = pd.DataFrame(means, columns=OUT_COLUMNS).fillna(0)
    df['search-L'] = pd.to_numeric(df['search-L'])
    df.sort_values(["type", "search-L"],
        axis=0,
        ascending=[True, True],
        inplace=True)
    print(df)

    df.to_csv(f'{DIRECTORY}/{output}.csv', index=False)
    # df.to_excel(f'{DIRECTORY}/{output}.xlsx', index=False)


import glob
import json
import os
import argparse
import time
import re
import pickle

# Libs
from tqdm import tqdm
from xmltodict import unparse
import xml.etree.ElementTree as ET
import sys
import numpy as np
import pandas as pd
from multiprocessing import Pool

import matplotlib

# If you get _tkinter.TclError: no display name and no $DISPLAY environment variable use
# matplotlib.use('Agg') instead
matplotlib.use('TkAgg')

from six import BytesIO

import matplotlib.pyplot as plt
from PIL import Image, ImageDraw, ImageFont

import tensorflow as tf
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as viz_utils

# Own modules
import bbox_utils as bbox
import image_utils as im
from datetime import datetime

__author__ = 'Alexander Wendt'
__copyright__ = 'Copyright 2021, Christian Doppler Laboratory for ' \
                'Embedded Machine Learning'
__credits__ = ['']
__license__ = 'ISC'
__version__ = '0.1.0'
__maintainer__ = 'Alexander Wendt'
__email__ = 'alexander.wendt@tuwien.ac.at'
__status__ = 'Experimental'

parser = argparse.ArgumentParser(description='Google Tensorflow Detection API 2.0 Inferrer')
# parser.add_argument("-p", '--model_path', default='pre-trained-models/efficientdet_d5_coco17_tpu-32/saved_model/',
#                    help='Saved model path', required=False)
parser.add_argument("-i", '--image_dir', default='images/inference',
                    help='Image directory', required=False)
parser.add_argument("-l", '--labelmap', default='annotations/mscoco_label_map.pbtxt.txt',
                    help='Labelmap path', required=False)
parser.add_argument("-d", '--detections_file', default='detections.csv',
                    help='TF2 Object Detection API saved inference file as csv.', required=False)
parser.add_argument("-s", '--min_score', default=0.5, type=float,
                    help='Max score of detection box to save the image.', required=False)

# parser.add_argument("-r", '--run_detection', default=False,
#                    help='Run detection or load saved detection model', required=False, type=bool)
# parser.add_argument("-x", '--xml_dir', default=None,
#                    help='Source of XML files. '
#                         'If run_detection is True, xml files are saved here. '
#                         'If run detection is False, XML files are loaded from here. '
#                         'If run_detection is True and value is None, no XMLs are saved.', required=False, type=str)
# parser.add_argument("-vis", '--run_visualization', default=False,
#                    help='Run image visualization', required=False, type=bool)
parser.add_argument("-o", '--output_dir', default="detection_images", help='Result directory for images. ',
                    required=False)
# parser.add_argument("-lat", '--latency_out', default="latency.csv", help='Output path for latencies file, which is '
#                                                                         'appended or created new. ',
#                    required=False)

# parser.add_argument("-m", '--model_name', default="Model", type=str,
#                    help='Model name for collecting model data.', required=False)
# parser.add_argument("-hw", '--hardware_name', default="Hardware", type=str,
#                    help='Hardware name collecting statistical data.', required=False)

args = parser.parse_args()

def load_model(model_path):
    '''
    Load tensorflow model

    :param model_path:
    :return:
    '''

    print("Start model loading from path ", model_path)
    tf.keras.backend.clear_session()
    start_time = time.time()
    detect_fn = tf.saved_model.load(model_path)
    end_time = time.time()
    elapsed_time = end_time - start_time
    print('Finished. Elapsed time: {:.0f}s'.format(elapsed_time))

    return detect_fn

def create_single_imagedict(source, image_name):
    '''

    :type

    '''
    image_dict = {}
    image_path = os.path.join(source, image_name)
    image_np = bbox.load_image_into_numpy_array(image_path)
    input_tensor = np.expand_dims(image_np, 0)
    image_dict[image_name] = (image_np, input_tensor)
    return image_dict

def detect_image(detect_fn, image_path, min_score):
    '''

    :param detect_fn:
    :param image_dict:

    :return:
    '''
    elapsed = []
    detection_dict = dict()

    # print("Start detection")
    # for image_name in image_list:
    # Load image
    # image_path = os.path.join(image_dir, image_name)
    # Convert image to array
    print("Process ", image_path)
    image_np = im.load_image_into_numpy_array(image_path)

    # Make image tensor of it
    input_tensor = np.expand_dims(image_np, 0)

    # Infer
    start_time = time.time()
    detections = detect_fn(input_tensor)
    end_time = time.time()
    latency = end_time - start_time
    # elapsed.append(latency)

    image_filename = os.path.basename(image_path)

    print("Inference time {} : {}s".format(image_filename, latency))

    # Process detections
    boxes = detections['detection_boxes'][0].numpy()
    classes = detections['detection_classes'][0].numpy().astype(np.int32)
    scores = detections['detection_scores'][0].numpy()

    return image_filename, image_np, boxes, classes, scores, latency

def convert_reduced_detections_to_df(image_filename, image_np, boxes, classes, scores, min_score=0.8):
    image_width = image_np.shape[1]
    image_height = image_np.shape[0]

    column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax', 'score']
    xml_df = pd.DataFrame(columns=column_name)

    for i in range(scores.shape[0]):
        if min_score <= scores[i]:
            ymin, xmin, ymax, xmax = tuple(boxes[i].tolist())
            content = [image_filename, image_width, image_height,
                       classes[i], xmin, ymin, xmax, ymax, scores[i]]
            xml_df = xml_df.append(pd.DataFrame([content], columns=column_name))

    return xml_df

def plot_image(image, title=None):
    ax = plt.subplot(111)
    ax.tick_params(labelbottom=False, labelleft=False)
    if title:
        plt.title(title, fontsize=40)
    plt.imshow(image)

    plt.axis('off')
    plt.tight_layout()

    return plt.gcf()

def infer_images(detections_file, image_dir, labelmap, min_score, output_dir):
    '''

    '''
    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)
        print("Created ", output_dir)

    # Load inference images
    print("Loading images from ", image_dir)
    image_list = im.get_images_name(image_dir)

    # Load label path
    print("Loading labelmap from ", labelmap)
    category_index = label_map_util.create_category_index_from_labelmap(os.path.abspath(labelmap))

    # Load stored XML Files
    print("Loading saved Detections files from ealier inferences from ", detections_file)
    data = pd.read_csv(detections_file, sep=';').set_index('filename')

    # Define scores and latencies
    # detection_scores = pd.DataFrame(columns=['filename', 'width', 'height', 'class', 'xmin',
    #
    # 'ymin', 'xmax', 'ymax', 'score'])
    # Process each image
    for image_name in image_list:

        print("Load xml data for ", image_name)
        if isinstance(data.loc[image_name], pd.Series):
            subdata = pd.DataFrame([data.loc[image_name]])
        else:
            subdata = data.loc[image_name]
        #classes = np.array(data.loc[image_name].shape[0])
        boxes = np.zeros([subdata.shape[0], 4])
        classes = np.zeros([subdata.shape[0]]).astype('int')

        if 'score' in subdata.columns and subdata['score'][0] is not None:
            scores = np.zeros([subdata.shape[0]])
        else:
            scores = None

        for i in range(subdata.shape[0]):
            boxes[i][0] = subdata['ymin'][i]
            boxes[i][1] = subdata['xmin'][i]
            boxes[i][2] = subdata['ymax'][i]
            boxes[i][3] = subdata['xmax'][i]

            if 'score' in data.columns and subdata['score'][i] is not None:
                scores[i] = subdata['score'][i]

            #class_index = [category_index[j + 1].get('id') for j in range(len(category_index)) if
            #               category_index[j + 1].get('name') == subdata['class'][i]][0]

            classes[i] = subdata['class'][i]

        #if classes.size == 1:  # If only one detection
        #    classes = classes.reshape(-1)
        #    np.vstack(classes, 1)
        #scores = np.array(data.loc[image_name]['score'])
        #if scores.size == 1:  # If only one detection
        #    scores = scores.reshape(-1)
        #    np.vstack(scores, 0)
        #boxes = np.array(data.loc[image_name][['ymin', 'xmin', 'ymax', 'xmax']])
        #if boxes.size==4:
        #    boxes = boxes.reshape(1, -1)
        #    np.vstack(boxes, [0, 0, 0, 0])
        image_filename = image_name
        image_np = im.load_image_into_numpy_array(os.path.join(image_dir, image_name))

        # If output directory is provided, visualize and save image
        image = bbox.visualize_image(image_name, image_np, scores, boxes, classes, category_index, min_score=min_score)

        plt.gcf()
        new_image_path = os.path.join(output_dir, os.path.splitext(image_filename)[0] + "_det" + ".png")
        print("Save image {} to {}".format(image_filename, new_image_path))

        fig = plot_image(image)
        plt.savefig(new_image_path)

if __name__ == "__main__":
    infer_images(args.detections_file, args.image_dir, args.labelmap, args.min_score, args.output_dir)

    print("=== Program end ===")


from typing import List, Optional
from db.database import SessionLocal
from db import models
import logging

class NoTeamException(Exception):
    pass

class TeamRepository:

    def __init__(self):
        self.db = SessionLocal()

    def create_team(self, name: str, country: str, cur_pos_league: str) -> models.Team:
        new_team = models.Team(name=name, country=country,
                               cur_pos_league=cur_pos_league)
        self.db.add(new_team)
        self.db.commit()
        self.db.refresh(new_team)
        return new_team

    def delete_team(self, id: int) -> int:
        team = self.db.query(models.Team).filter(
            models.Team.id == id).delete(synchronize_session=False)
        self.db.commit()
        logging.debug('TEAMS FOUND: ',  team)
        if not team:
            raise NoTeamException()
        return team

    def get_teams(self, skip: int, limit: Optional[int] = None) -> List[models.Team]:
        teams = self.db.query(models.Team).offset(skip).limit(limit)
        return teams

    def get_team(self, id: int) -> models.Team:
        team = self.db.query(models.Team).filter(models.Team.id == id).first()
        if not team:
            raise NoTeamException()
        return team

    def update_team(self, id: int, name: str, country: str, cur_pos_league: str) -> models.Team:
        team = self.db.query(models.Team)\
            .filter(models.Team.id == id)\
            .update({models.Team.name: name if name else models.Team.name,
                    models.Team.country: country if country else models.Team.country,
                    models.Team.cur_pos_league: cur_pos_league if cur_pos_league else models.Team.cur_pos_league},
                    synchronize_session=False)
        self.db.commit()
        logging.debug('TEAMS FOUND: ',  team)
        if not team:
            raise NoTeamException()
        return self.db.query(models.Team).filter(models.Team.id == id).first()


from datetime import datetime
from functools import lru_cache

import requests
from fastapi import Depends, FastAPI, HTTPException
from fastapi.responses import RedirectResponse
from pydantic import BaseModel, BaseSettings

app = FastAPI(docs_url=None, redoc_url=None)

class Labels(BaseModel):
    alertname: str
    prometheus: str | None = None
    severity: str | None = None

class Annotations(BaseModel):
    description: str
    runbook_url: str
    summary: str

class SingleAlert(BaseModel):
    status: str
    labels: Labels
    annotations: Annotations
    startsAt: datetime | None = None
    endsAt: datetime | None = None
    generatorURL: str
    fingerprint: str | None = None

class Alerts(BaseModel):
    receiver: str
    status: str
    alerts: list[SingleAlert]
    groupLabels: Labels
    commonLabels: Labels
    commonAnnotations: Annotations
    externalURL: str
    version: int
    groupKey: str
    truncatedAlerts: int | None = None

class Settings(BaseSettings):
    env_name: str = "Local"
    kuma_url: str = "http://127.0.0.1:3001"
    kuma_tokens: list[str] = ["AAaaAaAaaa"]

    class Config:
        env_file = ".env"

@lru_cache()
def get_settings():
    return Settings()

# Incoming alerts debugging (POST request body printing)
# NOTE: Need Request in fastapi import
# @app.post("/{token}")
# async def inspect(token: str, r: Request, s: Settings = Depends(get_settings)):
#     body = await r.json()
#     print(body)
#     return body

@app.get("/", response_class=RedirectResponse, status_code=302)
def redirect():
    return "https://github.com/hadret/forwardly"

@app.post("/{token}")
def forward(am: Alerts, token: str, settings: Settings = Depends(get_settings)):
    if token in settings.kuma_tokens:
        kuma = requests.get(f"{settings.kuma_url}/{token}")
        return am, kuma.json()
    else:
        raise HTTPException(status_code=401, detail="Unauthorized")


#!/bin/env python3
import itertools

def invalid_preamble_number(numbers, preamble):
    for p in range(preamble, len(numbers)):
        if not any(a + b == numbers[p] for a, b in itertools.combinations(numbers[p - preamble:p], 2)):
            yield numbers[p]

def contiguous_set_code(numbers, total):
    num_set = []
    num_set_sum = 0
    for n in numbers:
        num_set += [n]
        num_set_sum += n
        while num_set_sum > total:
            num_set_sum -= num_set.pop(0)
        if num_set_sum == total:
            return min(num_set) + max(num_set)
    return 0

def main():
    with open("input.txt") as fd:
        numbers = [int(line.strip()) for line in fd.readlines()]

    part1 = next(invalid_preamble_number(numbers, 25))
    print("Part One", part1)

    part2 = contiguous_set_code(numbers, part1)
    print("Part Two", part2)

main()


from django.urls import path
from .views import home, send_invitation, accept_invitation, reject_invitation
from django.contrib.auth.views import LoginView, LogoutView

urlpatterns = [
    path('', home, name = 'player_home'),
    path('login/', LoginView.as_view(template_name='player/login_form.html'), name='player_login'),
    path('logout/', LogoutView.as_view(), name='player_logout'),
    path('send_invite/', send_invitation, name='send_invitation'),
    path('accept/<str:username>', accept_invitation, name='accept_invitation'),
    path('reject/<str:username>', reject_invitation, name='reject_invitation'),
]


from mrjob.job import MRJob
from mrjob.step import MRStep
import re
WORD_RE = re.compile(r"[\w]+")

# clean reference frame input, save it to a list
f = open('myframe.txt', 'r')
x = f.readlines()
f.close()
total = []
for i in range(len(x)):
	temp = x[i].split(',')
	for t in temp:
		t = t.strip()
		t = t.strip('\n')
		total.append(t)

REFERENCE = total

class MRCount(MRJob):
	def mapper(self, _, line):
		'''
		This function goes through each line in the text files, counts words
		and loop over the reference frame and records the co-occurence time
		of the word in reference frame and the keyword in the text file. It
		yields a list, which has one more elements than the reference frame
		(the word counts).
		'''
		newline = WORD_RE.findall(line)
		for word in newline:
			rvlist = [0] * (len(REFERENCE)+1)
			rvlist[0] = 1
			for ind in range(len(REFERENCE)):
				if (REFERENCE[ind] in newline) and (REFERENCE[ind]!=word):
					rvlist[ind+1] = 1

			yield word, rvlist

	def combiner(self, word, countstep1):
		'''
		Sum up the list with same key word.
		:param word: word in text file.
		:param countstep1: the list generated in the mapper.
		:return: a list.
		'''
		sumstep1 = list(countstep1)
		yield word,[sum(i) for i in zip(*sumstep1)]

	def reducer_pre(self, word, countstep2):
		'''
		Sum up the list with same key word.
		:param word: word in text file.
		:param countstep2: the list combined in the combiner.
		:return: a list.
		'''
		sumstep2 = list(countstep2)
		yield word,[sum(i) for i in zip(*sumstep2)]

	def reducer_final(self, word, allnum):
		'''
		Divide the second-to-last elements in the list by the first
		element. Yield a vector with the same length of reference frame for
		each unique word in the text file.
		:param word: unique word in the text file.
		:param allnum: the list generated in the first step.
		:return: a list.
		'''
		final = []
		mycount = list(allnum)[0]
		for i in range(len(mycount)):
			if i > 0:
				final.append(mycount[i]/mycount[0])
		yield word, final

	def steps(self):
		return[
		MRStep(
			mapper = self.mapper,
			combiner = self.combiner,
			reducer = self.reducer_pre),
		MRStep(reducer = self.reducer_final)]

if __name__ == '__main__':
  MRCount.run()


from matplotlib.collections import PolyCollection
from matplotlib import cm
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import nanopores.geometries.pughpore as pughpore
import nanopores
from  scipy.interpolate import LinearNDInterpolator
import numpy as np
from nanopores.tools import fields
import folders
from mirror import xf, Felx, Fely, Felz, Fdragx, Fdragy, Fdragz, Fx, Fy, Fz, Jf, Ja
xf=np.array(xf)
f=LinearNDInterpolator(xf,Ja)

fig = plt.figure()
ax = fig.gca(projection='3d')

up = nanopores.user_params(pughpore.params, k=3)

R = up.R
H = up.H
l0 = up.l0
l1 = up.l1
l2 = up.l2
l3 = up.l3
l4 = up.l4
hpore = up.hpore
hmem = up.hmem
h2 = up.h2
h1 = up.h1
h4 = up.h4
rMolecule = up.rMolecule
eps = 0.1
r = rMolecule + eps

verts=[]
xs = [l3*.5,R*.5,R*.5,l4*.5,l4*.5,l0*.5,l0*.5,l1*.5,l1*.5,l2*.5,l2*.5,l3*.5]
xsm = [-xs[i] for i in range(len(xs))]
ys = [-hpore*.5,-hpore*.5,-hpore*.5+hmem,-hpore*.5+hmem,-hpore*.5+h4,-hpore*.5+h4,hpore*.5,hpore*.5,hpore*.5-h1,hpore*.5-h1,hpore*.5-h2,hpore*.5-h2]
verts.append(list(zip(xs,ys)))
verts.append(list(zip(xsm,ys)))

poly = PolyCollection(verts)
poly.set_alpha(.3)

ax.set_xlim3d(-15, 15)
ax.set_ylim3d(-15, 15)
ax.set_zlim3d(1.1e-9,1.3e-9)
lx=80
ly=160
X_=np.linspace(-R*.5,R*.5,lx)
Y_=np.linspace(-hpore*.5,hpore*.5,ly)
#X_=np.linspace(-(l3*.5-r),l3*.5-r)
X, Y = np.meshgrid(X_,Y_)
#np.save('X',X)
#np.save('Y',Y)
#ZJ=np.array([[f.__call__(np.array([X_[i],0.,Y_[j]]))[0] for i in range(X_.shape[0])] for j in range(Y_.shape[0])])
#np.save('J_int_shrink',ZJ)
ZJ=np.load('J_int.npy')
surf=ax.plot_surface(X,Y,ZJ, rstride=1, cstride=1, cmap=cm.viridis, linewidth=0., alpha=1.)
fig.colorbar(surf, shrink=0.5, aspect=5)

ax.add_collection3d(poly, zs=[0.,0.], zdir='z')

ax.view_init(23.,-32.)
#ax.view_init(90.,-90.)
plt.tight_layout()
plt.show()


from django.urls import path

from .views import add_contact, contacts_list, edit_contact, delete_contact

urlpatterns = [
    path('add/', add_contact, name='add_contact'),
    path('edit/<int:id>/', edit_contact, name='edit_contact'),
    path('delete/<int:id>/', delete_contact, name='delete_contact'),
    path('', contacts_list, name='contacts_list')
]


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from .views import ChurroViewSet, submit_survey, submit_career, submit_contact, submit_order, home, submit_booking

router = DefaultRouter()
router.register(r'churros', ChurroViewSet)

urlpatterns = [
    path('', include(router.urls)),
    path('submit-survey/', submit_survey, name='submit_survey'),
    path('submit-career/', submit_career, name='submit_career'),
    path('submit-contact/', submit_contact, name='submit_contact'),
    path('submit-order/', submit_order, name='submit_order'),
    path('submit-booking/', submit_booking, name='submit_booking'),
    path('home/', home, name='home'),  # Assuming you want a separate endpoint for home
]


import os

from nose.tools import assert_equal, with_setup

from mod import f

def f_setup():
    files = os.listdir('.')
    if 'no.txt' in files:
        os.remove('no.txt')
    if 'yes.txt' in files:
        os.remove('yes.txt')

def f_teardown():
    files = os.listdir('.')
    if 'yes.txt' in files:
        os.remove('yes.txt')

@with_setup(setup=f_setup, teardown=f_teardown)
def test_f():
    exp = 42
    f()
    with open('yes.txt', 'r') as fhandle:
        obs = int(fhandle.read())
    assert_equal(exp, obs)


import jax.numpy as jnp
import jax
from jax import device_put
from jax import numpy as np
from bench.core.executer import Executer

def box_area(boxes):
    return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])

def box_iou(boxes1, boxes2):

    area1 = box_area(boxes1)
    area2 = box_area(boxes2)

    lt = np.maximum(boxes1[:, None, :2], boxes2[:, :2])  # [N,M,2]
    rb = np.minimum(boxes1[:, None, 2:], boxes2[:, 2:])  # [N,M,2]

    wh = jax.numpy.clip(rb - lt, 1.0)  # [N,M,2]
    inter = wh[:, :, 0] * wh[:, :, 1]  # [N,M]

    union = area1[:, None] + area2 - inter

    iou = inter / union
    return iou, union

def generalized_box_iou(boxes1, boxes2):
    """
    Generalized IoU from https://giou.stanford.edu/

    The boxes should be in [x0, y0, x1, y1] format

    Returns a [N, M] pairwise matrix, where N = len(boxes1)
    and M = len(boxes2)
    """
    # degenerate boxes gives inf / nan results
    # so do an early check
    assert (boxes2[:, 2:] >= boxes2[:, :2]).all()
    iou, union = box_iou(boxes1, boxes2)

    lt = np.minimum(boxes1[:, None, :2], boxes2[:, :2])
    rb = np.maximum(boxes1[:, None, 2:], boxes2[:, 2:])

    wh = jax.numpy.clip(rb - lt, 1.0)  # [N,M,2]  # [N,M,2]
    area = wh[:, :, 0] * wh[:, :, 1]

    return iou - (area - union) / area

def args_adaptor(np_args):
    boxes1 = device_put(np_args[0])
    boxes2 = device_put(np_args[1])
    return [boxes1, boxes2]

def executer_creator():
    return Executer(generalized_box_iou, args_adaptor)


import numpy as np
import matplotlib.pyplot as plt
import pickle
# import required libraries
from scipy.stats import norm
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb

#   https://www.askpython.com/python/normal-distribution
# creating the dataset
# data = {'C':20, 'C++':15, 'Java':30,
#         'Python':35}
# courses = list(data.keys())
# values = list(data.values())
with open("stc.pkl", "rb") as f:
    score = pickle.load(f)
# courses = []
# values = []
mean, std = 0,0
max1,min1 = 0,0
for k,v in score.items():
    # print(v)
    x = np.array(v)
    v = x.astype(np.float)
    # for js in v:
    #     courses

    # courses = v
    # values =
    #Calculate mean and Standard deviation.
    # print(v)
    mean = np.mean(v)
    std = np.std(v)
    max1 = np.amax(v)
    min1 = np.amin(v)
    # break

    # Creating the distribution
    # data = np.arange(1,10,0.01)
    print(min1,max1,mean,std)
    data = np.arange(min1,max1, 0.01)
    pdf = norm.pdf(data , loc = mean , scale = std )

    #Visualizing the distribution

    sb.set_style('whitegrid')
    sb.lineplot(data, pdf , color = 'black')
    plt.xlabel('Score')
    plt.ylabel('Probability Density')
    plt.title(k[3:])

    # fig = plt.figure(figsize = (10, 5))

    # # creating the bar plot
    # plt.bar(courses, values, color ='maroon',
    #         width = 0.4)

    # plt.xlabel("Courses offered")
    # plt.ylabel("No. of students enrolled")
    # plt.title("Students enrolled in different courses")
    # # plt.show()
    plt.savefig("fig/{}.png".format(k[3:]))
    plt.clf()


import os
import speech_recognition as sr

def transcribe_audio(audio_file):
    recognizer = sr.Recognizer()

    with sr.AudioFile(audio_file) as source:
        audio = recognizer.record(source)

    try:
        transcript = recognizer.recognize_google(audio, language="es-ES")
        return transcript
    except sr.UnknownValueError:
        return "No se pudo transcribir el audio"
    except sr.RequestError:
        return "Error en la solicitud al servicio de reconocimiento de voz"

# Ruta al archivo de audio
ruta_carpeta = "C:/Users/User/Documents/coding/xd/audios"
carpetaDeAudios = os.listdir("C:/Users/User/Documents/coding/xd/audios")
contador = 1
for file in carpetaDeAudios:
    audio_file = os.path.join(ruta_carpeta, file)
    transcript = transcribe_audio(audio_file)
    name_audio = "audio" + str(contador)
    output_file_path = f"C:/Users/User/Documents/coding/xd/transcipciones/{name_audio}.txt"
    with open(output_file_path, "w") as file:
        file.write(transcript)
    contador += 1
    print("Transcripción guardada correctamente.")


from __future__ import (absolute_import, division,
                        print_function, unicode_literals, )
from future.builtins import range

import numpy as np
import tt
from tt.optimize import tt_min
import copy
import scipy

import ttrecipes as tr

def search_subspace(t, Xs, Ys, tsq=None, tmin=None, tmax=None, hinv=True, vinv=True, dims=None, eps=1e-3):
    """
    Find a region that bests match a given template (both have to be axis-aligned).

    :param Xs: a list of M ndarrays (each of dimension M) encoding the positions to be searched. Note: non-rectangular regions will be extended to their axis-aligned bounding box
    :param Ys: an ndarray of dimension M containing the values to be searched. Must have the same size as the elements of Xs
    :param hinv: whether to search with invariance to horizontal translation (default: True). If True, the values in Xs must lie between 0 and 1
    :param vinv: whether to search with invariance to vertical translation (default: True). If True, the values in Ys must lie between 0 and 1
    :param dims: the dimensions along which the subspace should be searched. Must be a list with M elements. If None (default), the best tuple of M dimensions will be chosen
    :return: match: the center of the best-matching window; dims: the dimensions along which the best subspace was found; borders: a list of M pairs
     (start, end) with the coordinates for the region

    TODO: support non-invariances
    TODO: support search in 2D and more
    TODO: use masks

    """

    N = t.d
    M = Ys.ndim
    if M != 1:
        raise NotImplementedError("Only 1D search is supported for now")
    if hinv == False or vinv == False:
        raise NotImplementedError
    assert len(Xs) == M
    assert np.all([X.ndim == M for X in Xs])
    if hinv:
        assert np.all([np.all(X >= 0) for X in Xs])
        assert np.all([np.all(X <= 1) for X in Xs])
    if vinv:
        assert np.all(Ys >= 0)
        assert np.all(Ys <= 1)
    if dims is not None:
        assert len(dims) == M

    if dims == None:
        dims = np.arange(N)
    inds = np.argsort(Xs[0]) # Sort the values, in case they are reversed
    Xs[0] = Xs[0][inds]
    Ys = Ys[inds]
    record_value = float('inf')

    for dim in dims:
        Xs_adj = (Xs[0]-np.min(Xs[0])-1e-9)*t.n[dim]  # Xs adjusted to this dimension's grid ticks
        Ys_adj = scipy.interpolate.interp1d(Xs_adj, Ys)(np.arange(np.floor(Xs_adj[-1])+1))  # Adjusted Ys
        Ys_adj = Ys_adj*(tmax - tmin) + tmin  # Ys are now mapped to the tensor output's range
        Ys_adj -= np.mean(Ys_adj)  # Vertical invariance

        # Compute the loss tensor: for each position, the squared norm (dot product of itself) of a windowed fiber
        # consisting of the difference between the original fiber (x minus its mean t, for vertical invariance) and
        # the vertically-centered template fiber y (with zero mean). We use the expansion:
        # (x - t - y)*(x - t - y) = x*x - 2*x*t - 2*x*y + 2*t*y + t*t + y*y = x*x - t*t - 2*x*y + y*y
        # where we have used that mean(y) = 0

        if tsq is None:  # This part may be reused by subsequent queries, so we memoize it
            tsq = tt.multifuncrs2([t], lambda x: x**2, eps=eps, verb=False)

        box_filter = np.ones([1, Ys_adj.size, 1]) / Ys_adj.size  # To average TT cores along their 2nd dimension

        cores = tt.vector.to_list(tsq)
        cores[dim] = scipy.signal.convolve(cores[dim], box_filter, mode='valid')
        tsquarebox = tt.vector.from_list(cores)

        cores = tt.vector.to_list(t)
        cores[dim] = scipy.signal.convolve(cores[dim], box_filter, mode='valid')
        tbox = tt.vector.from_list(cores)
        tboxsquare = tt.multifuncrs2([tbox], lambda x: x**2, eps=eps, verb=False)

        cores = tt.vector.to_list(t)
        cores[dim] = scipy.signal.convolve(cores[dim], np.reshape(Ys_adj[::-1], [1, Ys_adj.size, 1]), mode='valid')
        tconv = tt.vector.from_list(cores)

        tfinal = Ys_adj.size*tsquarebox - Ys_adj.size*tboxsquare - 2*tconv + np.dot(Ys_adj,Ys_adj)*tt.ones(tsquarebox.n)

        # Find an approximate minimum
        value, match = tt_min.min_tens(tfinal, rmax=5, nswp=5, verb=False)

        if value < record_value:
            record_value = value
            record_match = [int(m) for m in match]
            record_match[dim] += len(Ys_adj)/2  # Sum half the window size to get the final focus point
            record_dim = [dim]
            record_borders = [[record_match[dim] - len(Ys_adj)/2, record_match[dim]+len(Ys_adj)/2-1]]

    return record_match, record_dim, record_borders

def best_subspace(t, ndim=1, target='max', mode='cross', eps=1e-6, verbose=False, **kwargs):
    """
    Find an axis-aligned subspace of a certain dimensionality that has the highest/lowest variance. Example applications:
    - In visualization, to find interesting subspaces
    - In factor fixing (sensitivity analysis), to find the set of parameters (and their values) that will minimize the uncertainty of a model

    TODO: only uniform independently distributed inputs are supported now

    :param t: a TT
    :param ndim: dimensionality of the subspace sought (default is 2)
    :param target: if 'max' (default), the highest variance will be sought; if 'min', the lowest one
    :param mode: 'cross' (default) or 'kronecker'
    :param verbose:
    :param kwargs: arguments for the cross-approximation
    :return: (a) a list of indices, with slice(None) in the free subspace's dimensions, and (b) the variance of that subspace

    """

    assert mode in ('cross', 'kronecker')
    assert target in ('max', 'min')

    # Build up a tensor that contains variances of all possible subspaces of any dimensionality, using the formula E(X^2) - E(X)^2
    if mode == 'cross':
        cores = tt.vector.to_list(tt.multifuncrs2([t], lambda x: x ** 2, eps=eps, verb=verbose, **kwargs))
    else:
        cores = tt.vector.to_list((t*t).round(0))
    cores = [np.concatenate([np.mean(core, axis=1, keepdims=True), core], axis=1) for core in cores]
    part1 = tt.vector.from_list(cores)  # E(X^2)

    cores = tt.vector.to_list(t)
    cores = [np.concatenate([np.mean(core, axis=1, keepdims=True), core], axis=1) for core in cores]
    part2 = tt.vector.from_list(cores)
    if mode == 'cross':
        part2 = tt.multifuncrs2([part2], lambda x: x ** 2, eps=eps, verb=verbose, **kwargs)
    else:
        part2 = (part2*part2).round(0)  # E(X)^2

    variances = (part1 - part2).round(0)

    # Filter out encoded subspaces that do not have the target dimensionality
    mask = tt.vector.to_list(tr.core.hamming_eq_mask(t.d, t.d-ndim))
    mask = [np.concatenate([core[:, 0:1, :], np.repeat(core[:, 1:, :], sh, axis=1)], axis=1) for core, sh in zip(mask, t.n)]
    mask = tt.vector.from_list(mask)

    # Find and return the best candidate
    if target == 'max':
        prod = tt.vector.round(variances*mask, eps=eps)
        val, point = tt_min.min_tens(-prod, verb=verbose)
        val = -val
    else:
        shift = -1e3*tt_min.min_tens(-variances, verb=False, rmax=1)[0]
        variances_shifted = variances - tt.vector.from_list([np.ones([1, sh+1, 1]) for sh in t.n])*shift
        val, point = tt_min.min_tens(variances_shifted*mask, verb=verbose)
        val += shift
    nones = np.where(np.array(point) == 0)[0]
    point = [p - 1 for p in point]
    for i in nones:
        point[i] = slice(None)
    return point, val

def moments(t, modes, order, centered=False, normalized=False, keepdims=False, eps=1e-3, verbose=False, **kwargs):
    """
    Given an N-dimensional TT and a list of M modes, returns a TT of dimension N - M that contains the k-th order moments along these modes

    :param t: a TT
    :param modes: a list of M integers
    :param order: an integer
    :param centered: if True the moments will be computed about their mean. Default is False
    :param normalized: if True the moments will be divided by sigma^order. Default is False
    :param eps: accuracy for cross-approximation (default is 1e-3)
    :return: a TT of dimension N - M

    """

    N = t.d
    assert np.all(0 <= np.array(modes))
    assert np.all(np.array(modes) < N)
    if not hasattr(modes, '__len__'):
        modes = [modes]
    assert len(modes) == len(set(modes))  # Modes may not be repeated
    assert 1 <= len(modes) <= N

    if centered or normalized:
        central_cores = []
        cores = tt.vector.to_list(t)
        for n in range(N):
            if n in modes:
                central_cores.append(np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1))
            else:
                central_cores.append(cores[n])
        central = t - tt.vector.from_list(central_cores)
    if centered:
        if order == 1:
            moments = copy.deepcopy(central)
        else:
            moments = tt.multifuncrs2([central], lambda x: x**order, eps=eps, verb=verbose, **kwargs)
    else:
        if order == 1:
            moments = copy.deepcopy(t)
        else:
            moments = tt.multifuncrs2([t], lambda x: x**order, eps=eps, verb=verbose, **kwargs)
    cores = tt.vector.to_list(moments)
    for mode in modes:
        cores[mode] = np.mean(cores[mode], axis=1, keepdims=True)
    moments = tt.vector.from_list(cores)
    if normalized:
        central = tt.multifuncrs2([central], lambda x: x**2, eps=eps, verb=verbose, **kwargs)
        cores = tt.vector.to_list(central)
        for mode in modes:
            cores[mode] = np.mean(cores[mode], axis=1, keepdims=True)
        variances = tt.vector.from_list(cores)
        moments = tt.multifuncrs2([moments, variances], lambda x: x[:, 0] / (x[:, 1] ** (order/2.)), eps=eps, verb=verbose, **kwargs)
    if not keepdims:
        moments = tr.core.squeeze(moments, modes=modes)
    return moments

def means(t, modes, **kwargs):
    """
    Convenience function for the first moment (see :func: `moments`)
    """

    return moments(t, modes, order=1, centered=False, normalized=False, **kwargs)

def variances(t, modes, **kwargs):
    """
    Convenience function for the second centered unnormalized moment (see :func: `moments`)
    """

    return moments(t, modes, order=2, centered=True, normalized=False, **kwargs)

# 0, 2
# 0 1 2 3 4
# 2 1 0 3 4

# 0 1
# 1 0

# 0 1 2 3 4 5
# 1 0 2 3 4 5
# 2 0 1 3 4 5
# 3 0 1 2 4 5
# 4 0 1 2 3 5
# 5 0 1 2 3 4

# 0 1 2 3 4 5
# 1 0 2 3 4 5
# 2 1 0 3 4 5
# 3 1 2 0 4 5
# 4 1 2 3 0 5
# 5 1 2 3 4 0

# 1 0 2 3 4 5
# 0 1 2 3 4 5
# 0 2 1 3 4 5
# 0 3 2 1 4 5
# 0 4 2 3 1 5
# 0 5 1 2 3 4

# 0 1 2 :
# 0 : 2 3
# t1 x y z : s
# s2 x y z t :
# x y z t1 s
# x y z t s2

# 0 - 1
# 1 0 -

# 0 1 2
# 1 0 2
# 2 0 1

# f(x1, r, s) = f(x2, r, s) forall r, s
# t(x1, r, s) = f(r, x2, s) forall r, s

# 0 - 1 2
# 1 0 - 2
# 2 0 1 -

# x, r, s, t
# f(x1, r, s, t) = f(x2, r, s, t) forall r, s, t
# f(x1, s, t) = f(x2, r, t)
# t(x1, s, t) = t(r, x2, t) forall r, s, t

# f(x1, x1, x2, t) = f(x2, x1, x2, t)

# x 1 2
# 0 y 2

# 0 1 2 3 4
# 2 1 0 3 4
# 0 1 2 3 4

def pca_ensemble(t, modes, degree=3, orthogonal=False, anchor_mean=True, eps=1e-3, verbose=False):
    """
    Given a tensor and a list of modes, consider the space of subtensors obtained
    by fixing these modes. Find a PCA basis for that space and return the coefficients
    for each subtensor

    :param t: a TT
    :param modes: the ensemble modes. Can also be a list of lists; in that case
    the ensembles will share bases
    :param degree: how large a PCA basis to take (default is 3)
    :param orthogonal:
    :param anchor_mean:
    :param eps: relative tolerance while permuting modes. Default is 1e-3
    :param verbose:
    :return: (ensemble, preserved):
        - ensemble: a TT of size M1 x ... x Mk x `degree`, where M1, etc. are the sizes of `t` along `modes`
        - preserved: the fraction (between 0 and 1; higher is better) of norm preserved by the embedding

    """

    N = t.d
    if verbose:
        print('Computing PCA ensemble...')
    if not hasattr(modes, '__len__'):
        modes = [modes]
    modesarray = np.array(modes)
    single = False
    if modesarray.ndim == 1:
        single = True
        modesarray = np.array([modesarray])
    if not all([len(modesarray[m]) == len(modesarray[0]) for m in range(len(modesarray))]):
        raise ValueError("All mode lists must have the same length")
    # modesarray = np.array([[3, 6], [1, 4]])
    # Group target modes next to each other
    first = int(np.round(np.mean(modesarray.flatten())-modesarray.shape[1]/2.))
    # print(np.rint(0.50000))

    import matplotlib.pyplot as plt
    plt.figure()
    # plt.plot(t.full()[20, :] - np.mean(t.full()[20, :]))
    # plt.plot(t.full()[:, 4] - np.mean(t.full()[:, 4]))
    # plt.imshow(t.full(), origin='lower')
    # print(t.full()[17, :])
    # print(t.full()[:, 4])
    # print(t.full()[17, :] - np.mean(t.full()[17, :]))
    # print(t.full()[:, 4] - np.mean(t.full()[:, 4]))
    # plt.show()
    # assert 0

    print(modesarray, first)
    stack = []
    for modes in modesarray:
        if len(modes) != len(np.unique(modes)):
            raise ValueError('Modes may not be repeated')
        assert 0 <= np.min(modes)
        assert np.max(modes) < N

        if verbose:
            print('\tTransposing tensor... 0', end='')
        # part1 = np.arange(first)
        all1 = list(range(first))
        all2 = list(range(first, N))
        part1 = np.setdiff1d(all1, modes)
        part2 = np.setdiff1d(all2, modes)
        print('We now transpose to', list(part1)+list(modes)+list(part2))
        ttransp = tr.core.transpose(t, list(part1)+list(modes)+list(part2), eps=eps)
        for m in modes:
            if m in all1:
                all1[all1.index(m)] = -1
            if m in all2:
                all2[all2.index(m)] = -1
        all1.append(-2)  # Extra dimension to stack the tensors
        print(all1, all2)
        converted = np.array(all1+list(modes)+all2)
        assert first == np.where(converted == -2)[0][0]
        # shape = t.n[modes]
        idx = np.where(converted < 0)[0]
        shape = np.ones_like(idx)
        shape[np.where(converted[idx] == -1)[0]] = t.n[modes]
        # shape[np.where(converted[idx] == -1)[0]] = 1
        print('idx = {}, shape = {}'.format(idx, shape))
        # idx1 = []
        # idx2 = np.setdiff1d(np.arange(first, N), part2)
        # print('idx1:', idx1)
        # print('idx2:', idx2)
        ttransp = tr.core.insert_dummies(ttransp, idx, shape=shape)
        print('shape:', shape)
        # print(ttransp)
        # print(ttransp[[0, 0, 0, 0]+[0]*6])
        # print(ttransp[[0, 0, 0, 1]+[0]*6])
        # print(ttransp[[0, 0, 0, 2]+[0]*6])
        # print(ttransp)
        #
        # print((1./np.prod(t.n[modes])))
        # assert 0
        # ttransp *= (np.prod(t.n[modes]))
        # cores = tt.vector.to_list(ttransp)
        # cores[2] /= 32
        # ttransp = tt.vector.from_list(cores)
        stack.append(ttransp)
    stack = tr.core.concatenate(stack, axis=first, eps=eps)

    # first = first-1
    last = first+len(modes)+1
    # print(first, last)
    # print(t, first, last)
    # assert 0
    # print(stack)
    # print(stack[0, 15, 0, 25])
    # print(t[15, 25])
    # print(stack[0, 15, 0, 15])
    # print(t[15, 15])
    # print(stack[0, 13, 0, 25])
    # print(t[13, 25])
    # print(stack[1, 25, 12, 25])
    # print(t[12, 25])
    # blah = stack.full()
    # blah = blah - np.mean(blah, axis=(2, 3), keepdims=True)
    # print(np.mean(blah, axis=(2, 3), keepdims=True).shape)
    # print(blah.shape, np.linalg.norm(blah))
    # print(first, last)
    # print(modesarray)
    t = stack
    # assert 0
    # print('*******')
    # print(t)
    # print(tt.vector.round(t, eps=eps))
    # # print(tt.vector.norm(t))
    # assert 0

    if anchor_mean:
        # We now project each vector on the subspace spanned by the constant vector
        # This is achieved by subtracting each vector's within-mean
        cores = copy.deepcopy(tt.vector.to_list(t))
        for n in range(t.d):
            if n < first or n >= last:
                cores[n] = np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1)
        withinmeanrep = tt.vector.from_list(cores)
        print("Let's do means")
        print('t=', t)
        print('modes =', list(range(first))+list(range(last, t.d)))
        withinmean = tr.core.means(t, modes=list(range(first))+list(range(last, t.d)))
        t = tt.vector.round(t-withinmeanrep, eps=eps)
        degree -= 1

    # print(tt.vector.norm(t))
    # assert 0
    # print(withinmean)
    # assert 0
    # withinmeanrep2 = withinmeanrep - tr.core.constant_tt(shape=withinmean.n, fill=tr.core.mean(withinmeanrep))
    withinmean2 = withinmean - tr.core.constant_tt(shape=withinmean.n, fill=tr.core.mean(withinmean))
    # sumsq = np.sum(withinmean2)
    # print(withinmean2)
    # sumsq1 = tr.core.variances(withinmean2, modes=np.arange(len(modes))).full() #* np.prod(t.n)
    # sumsq1 = np.asscalar(sumsq1)
    sumsq1 = np.asscalar(np.sum(withinmean2.full()**2))# * np.prod(t.n[np.delete(np.arange(N), modes)])
    # targetsq1 = np.var(withinmean.full())*np.prod(t.n[modes])

    sumsq1 = np.sqrt(sumsq1)
    # targetsq1 = np.sqrt(targetsq1)
    print('sumsq1 = {}'.format(sumsq1))
    # print('sumsq1 = {}, targetsq1 = {}, sumsq1/targetsq1 = {}'.format(sumsq1, targetsq1, sumsq1/targetsq1))

    # print(np.sqrt(sumsq1/targetsq1))
    # print(tt.vector.norm(withinmean2)**2)
    # sumsq = np.sum(withinmean2.full()**2)
    # print(sumsq)
    # assert 0

    # Subtract the cross-mean (mean of the collection of vectors)
    cores = copy.deepcopy(tt.vector.to_list(t))
    for n in range(first, last):
        cores[n] = np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1)
    crossmean = tt.vector.from_list(cores)
    t = tt.vector.round(t-crossmean, eps=eps)

    # sumsq2 = tt.vector.norm(t) / np.sqrt(np.prod(t.n[np.delete(np.arange(N), modes)]))
    # sumsq2 = tt.vector.norm(t) / np.sqrt(np.prod(t.n[np.delete(np.arange(t.d), [0, 1])]))
    sumsq2 = tt.vector.norm(t) / np.sqrt(np.prod(t.n[np.arange(first)])*np.prod(t.n[np.arange(last, t.d)]))
    # sumsq2 = np.sum(tr.core.variances(t, modes=list(range(final_pos))+list(range(final_pos+len(modes), N))).full())\
    #          * np.prod(t.n[modes])
    # sumsq2 = np.sqrt(sumsq2)
    print('sumsq2 = {}'.format(sumsq2))
    # t = t*(sumsq2 / sumsq1 * targetsq1)
    # print('Now 0:', tt.vector.norm(t))
    # t *= (targetsq1 / sumsq1)
    # print('Now 1:', tt.vector.norm(t))
    # assert 0
    # print(t)

    # Orthogonalize to the left and right of these modes
    cores = tt.vector.to_list(t)
    if verbose:
        print('\n\tOrthogonalization...')
    for i in range(first):
        tr.core.left_orthogonalize(cores, i)
    for i in range(t.d-1, last-1, -1):
        tr.core.right_orthogonalize(cores, i)

    # Cut out the relevant sequence of modes
    cores = cores[first:last]
    # print(first, last, len(cores))
    # assert 0

    # Convert the sequence into a TT (convert ranks to spatial modes)
    cores = [np.eye(cores[0].shape[0])[np.newaxis, :, :]] + cores + [np.eye(cores[-1].shape[2])[:, :, np.newaxis]]
    t = tt.vector.from_list(cores)
    # sumsq2 = np.sum(t.full()**2)/(64**(N-2))
    # print(sumsq2)
    # print(t)
    # assert 0

    # Put the rank modes together at the very right
    if verbose:
        print('\tPutting rank modes together...')
    t = tr.core.shift_mode(t, 0, t.d-2, eps=eps)
    cores = tt.vector.to_list(t)

    # Merge the two rank dimensions into one
    cores[-2] = np.einsum('iaj,jbk->ikab', cores[-2], cores[-1])
    del cores[-1]
    cores[-1] = np.reshape(cores[-1], [-1, cores[-1].shape[2]*cores[-1].shape[3], 1])
    t = tt.vector.from_list(cores)

    # Subtract the cross-mean (mean of the collection of vectors)
    # print('t1:', t)
    # cores = [np.repeat(np.mean(c, axis=1, keepdims=True), c.shape[1], axis=1) for c in cores[:-1]] + [cores[-1]]
    # crossmean = tt.vector.from_list(cores)
    # t = tt.vector.round(t-crossmean, eps=eps)
    # print('t2:', t)
    print('Norma ara:', tt.vector.norm(t))
    norm = tt.vector.norm(t)
    if norm > 0:
        t *= (sumsq2 / norm)
    print('Norma ara:', tt.vector.norm(t))
    # print(t)

    # Compute the truncated PCA of the space spanned by the rank modes
    if verbose:
        print('\tPCA compression...')
    cores = tt.vector.to_list(t)
    tr.core.orthogonalize(cores, t.d-1)

    print('********')
    t = tt.vector.from_list(cores)
    print(t)
    for i in range(32):
        print(i, tt.vector.norm(t[0, i, :] - t[1, 24, :]))
    # assert 0
    # sumsq2 = np.sum(cores[-1]**2)
    # sumsq2 = np.sum(tr.core.variances(tt.vector.from_list(cores), modes=[0, 1]).full())
    # print(tt.vector.from_list(cores).full().shape)
    # sumsq2 = np.sum(np.sum(tt.vector.from_list(cores).full()**2, axis=-1)) / np.prod(tt.vector.from_list(cores).n)
    # print(tr.core.means(tt.vector.from_list(cores), modes=[0, 1]).full())
    # print(sumsq2)
    # assert 0
    cores[-1] = cores[-1][..., 0]
    svd = np.linalg.svd(cores[-1], full_matrices=False)
    cores[-1] = svd[0][:, :degree]
    if not orthogonal:
        cores[-1] = cores[-1].dot(np.diag(svd[1][:degree]))
    cores[-1] = cores[-1][:, :, np.newaxis]
    if cores[-1].shape[1] < degree:  # Degenerate case: add a slice of zeros
        cores[-1] = np.concatenate([cores[-1], np.zeros([cores[-1].shape[0], degree-cores[-1].shape[1],
                                                         cores[-1].shape[2]])], axis=1)
    t = tt.vector.from_list(cores)

    # print('********')
    # t = tt.vector.from_list(cores)
    # print(t)
    # for i in range(32):
    #     print(i, tt.vector.norm(t[0, i, :] - t[1, 24, :]))
    # assert 0
    # print('Norma ara:', tt.vector.norm(t))
    # t *= (1./tt.vector.norm(t))
    # assert 0

    # print('Now:', np.sqrt(np.var(t.full())))
    if anchor_mean:
        withinmean = tt.vector.from_list(tt.vector.to_list(withinmean) + [np.ones([1, 1, 1])])
        # print('**********')
        # print(withinmean, t)
        # print(tt.vector.norm(withinmean))
        # print(tt.vector.norm(t))
        # assert 0
        t = tr.core.concatenate([withinmean, t], axis=withinmean.d-1, eps=eps)

    if verbose:
        print()
    print(t.full())
    # assert 0
    # print(tt.vector.norm(t))
    print()
    # print(np.sum(t.full()**2))
    # print(t)
    tfull = t.full()
    # print(print(np.mean(np.mean(t.full(), axis=0), axis=0)))
    print('Obtained:')
    print(np.linalg.norm(tfull[..., 0]-np.mean(tfull[..., 0])))
    print(np.linalg.norm(withinmean2.full()))
    obtained = np.linalg.norm(tfull[..., 1:]-np.mean(tfull[..., 1:]))
    print(obtained)
    print()
    preserved = (sumsq1**2 + obtained**2) / (sumsq1**2 + sumsq2**2)
    print('This embedding preserves {:g}% of the original norm'.format(preserved*100))
    # print(np.sqrt(np.var(t[..., 0])))
    # print(np.sqrt(np.var(t[..., 1:])))
    # print(t[0, :, 0].full().flatten()[:10])
    # print(t[0, :, 1].full().flatten()[:10])
    # print(t[0, :, 2].full().flatten()[:10])
    # assert 0
    tlist = []
    for n in range(t.n[0]):
        tlist.append(t[[n] + [slice(None)]*(t.d-1)])
    if single:
        tlist = tlist[0]
    print()
    print('***************** tlist:', tlist)
    # assert 0
    return tlist, preserved

def pca_ensemble2(t, modes, degree=3, orthogonal=False, anchor_mean=True, eps=1e-3, verbose=False):
    """
    Given a tensor and a list of modes, consider the space of subtensors obtained
    by fixing these modes. Find a PCA basis for that space and return the coefficients
    for each subtensor

    :param t: a TT
    :param modes: the ensemble modes
    :param degree: how large a PCA basis to take (default is 3)
    :param orthogonal:
    :param anchor_mean:
    :param eps: relative tolerance while permuting modes. Default is 1e-3
    :param verbose:
    :return: (ensemble, preserved):
        - ensemble: a TT of size M1 x ... x Mk x `degree`, where M1, etc. are the sizes of `t` along `modes`
        - preserved: the fraction (between 0 and 1; higher is better) of norm preserved by the embedding
        - residual: the tensor containing the trailing R-degree principal components

    """

    # modes = [3, 6]
    # N = t.d
    # part1 = np.arange(modes[0])
    # part2 = np.setdiff1d(np.arange(modes[0], N), modes)
    # # print('**************', modes, ' **** ', list(part1)+list(modes)+list(part2))
    # order = list(part1)+list(modes)+list(part2)
    # idx = np.empty(len(order))
    # idx[order] = np.arange(len(order))
    # d = t.d
    # print(order)
    # assert 0
    # tr.core.transpose(t, order)
    # assert 0

    N = t.d
    shape = t.n
    if not hasattr(modes, '__len__'):
        modes = [modes]
    modesarray = np.array(modes)
    if modesarray.ndim == 1:
        modesarray = np.array([modesarray])
    if not all([len(modesarray[m]) == len(modesarray[0]) for m in range(len(modesarray))]):
        raise ValueError("All mode lists must have the same length")
    for modes in modesarray:
        if len(modes) != len(np.unique(modes)):
            raise ValueError('Modes may not be repeated')
        assert 0 <= np.min(modes)
        assert np.max(modes) < N

        if verbose:
            print('Computing PCA ensemble...')

        # Group target modes next to each other
        if verbose:
            print('\tTransposing tensor... 0', end='')
        # idx = list(range(N))
        # def list_shift(idx, mode, shift):
        #     assert mode + shift >= 0
        #     assert mode + shift < len(idx)
        #     idx.insert(mode+shift, int(idx[mode]))
        #     del idx[mode]
        # pivot = modes[0]  # We will organize modes from left-to-right after this one
        # for i in range(1, len(modes)):
        #     source = idx.index(modes[i])
        #     target = idx.index(pivot) + i
        #     if source < target:
        #         target -= 1
        #     t = tr.core.shift_mode(t, source, target-source, eps=eps)
        #     list_shift(idx, source, target-source)
        #     if verbose:
        #         print(' {}'.format(i), end='')
        # final_pos = idx.index(pivot)

        # part1 = np.arange(modes[0])
        part1 = np.setdiff1d(np.arange(modes[0]), modes)
        part2 = np.setdiff1d(np.arange(modes[0], N), modes)
        print('**************', modes, ' **** ', list(part1)+list(modes)+list(part2))
        order = list(part1)+list(modes)+list(part2)
        # t = tt.core.tools.permute(t, list(part1)+list(modes)+list(part2), eps=eps)
        t = tr.core.transpose(t, order, eps=eps)
        print(t)
        # assert 0
        # final_pos = modes[0]
        final_pos = order.index(modes[0])
        # final_pos = min(modes)

        if anchor_mean:
            # We now project each vector on the subspace spanned by the constant vector
            # This is achieved by subtracting each vector's within-mean
            cores = copy.deepcopy(tt.vector.to_list(t))
            for n in range(N):
                if n < final_pos or n >= final_pos+len(modes):
                    cores[n] = np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1)
            withinmeanrep = tt.vector.from_list(cores)
            withinmean = tr.core.means(t, modes=list(range(final_pos))+list(range(final_pos+len(modes), N)))
            t = tt.vector.round(t-withinmeanrep, eps=eps)
            degree -= 1

        # withinmeanrep2 = withinmeanrep - tr.core.constant_tt(shape=withinmean.n, fill=tr.core.mean(withinmeanrep))
        withinmean2 = withinmean - tr.core.constant_tt(shape=withinmean.n, fill=tr.core.mean(withinmean))
        # sumsq = np.sum(withinmean2)
        # print(withinmean2)
        # sumsq1 = tr.core.variances(withinmean2, modes=np.arange(len(modes))).full() #* np.prod(t.n)
        # sumsq1 = np.asscalar(sumsq1)
        sumsq1 = np.asscalar(np.sum(withinmean2.full()**2))# * np.prod(t.n[np.delete(np.arange(N), modes)])
        # targetsq1 = np.var(withinmean.full())*np.prod(t.n[modes])

        sumsq1 = np.sqrt(sumsq1)
        # targetsq1 = np.sqrt(targetsq1)
        print('sumsq1 = {}'.format(sumsq1))
        # print('sumsq1 = {}, targetsq1 = {}, sumsq1/targetsq1 = {}'.format(sumsq1, targetsq1, sumsq1/targetsq1))

        # print(np.sqrt(sumsq1/targetsq1))
        # print(tt.vector.norm(withinmean2)**2)
        # sumsq = np.sum(withinmean2.full()**2)
        # print(sumsq)
        # print(modes)
        # assert 0

        # Subtract the cross-mean (mean of the collection of vectors)
        cores = copy.deepcopy(tt.vector.to_list(t))
        print(t, final_pos, final_pos+len(modes))
        for n in range(final_pos, final_pos+len(modes)):
            cores[n] = np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1)
        crossmean = tt.vector.from_list(cores)
        t = tt.vector.round(t-crossmean, eps=eps)

        sumsq2 = tt.vector.norm(t) / np.sqrt(np.prod(shape[np.delete(np.arange(N), modes)]))
        # sumsq2 = np.sum(tr.core.variances(t, modes=list(range(final_pos))+list(range(final_pos+len(modes), N))).full())\
        #          * np.prod(t.n[modes])
        # sumsq2 = np.sqrt(sumsq2)
        print('sumsq2 = {}'.format(sumsq2))
        # t = t*(sumsq2 / sumsq1 * targetsq1)
        # print('Now 0:', tt.vector.norm(t))
        # t *= (targetsq1 / sumsq1)
        # print('Now 1:', tt.vector.norm(t))
        # assert 0
        # print(t)

        # Orthogonalize to the left and right of these modes
        cores = tt.vector.to_list(t)
        if verbose:
            print('\n\tOrthogonalization...')
        for i in range(final_pos):
            tr.core.left_orthogonalize(cores, i)
        for i in range(N-1, final_pos+len(modes)-1, -1):
            tr.core.right_orthogonalize(cores, i)

        # Cut out the relevant sequence of modes
        cores = cores[final_pos:final_pos+len(modes)]

        # Convert the sequence into a TT (convert ranks to spatial modes)
        cores = [np.eye(cores[0].shape[0])[np.newaxis, :, :]] + cores + [np.eye(cores[-1].shape[2])[:, :, np.newaxis]]
        t = tt.vector.from_list(cores)
        # sumsq2 = np.sum(t.full()**2)/(64**(N-2))
        # print(sumsq2)
        # assert 0

        # Put the rank modes together at the very right
        if verbose:
            print('\tPutting rank modes together...')
        t = tr.core.shift_mode(t, 0, t.d-2, eps=eps)
        cores = tt.vector.to_list(t)

        # Merge the two rank dimensions into one
        cores[-2] = np.einsum('iaj,jbk->ikab', cores[-2], cores[-1])
        del cores[-1]
        cores[-1] = np.reshape(cores[-1], [-1, cores[-1].shape[2]*cores[-1].shape[3], 1])
        t = tt.vector.from_list(cores)

        # Subtract the cross-mean (mean of the collection of vectors)
        # print('t1:', t)
        # cores = [np.repeat(np.mean(c, axis=1, keepdims=True), c.shape[1], axis=1) for c in cores[:-1]] + [cores[-1]]
        # crossmean = tt.vector.from_list(cores)
        # t = tt.vector.round(t-crossmean, eps=eps)
        # print('t2:', t)
        print('Norma ara:', tt.vector.norm(t))
        norm = tt.vector.norm(t)
        if norm > 0:
            t *= (sumsq2 / norm)
        print('Norma ara:', tt.vector.norm(t))
        print(t)

        # Compute the truncated PCA of the space spanned by the rank modes
        if verbose:
            print('\tPCA compression...')
        cores = tt.vector.to_list(t)
        tr.core.orthogonalize(cores, t.d-1)
        # sumsq2 = np.sum(cores[-1]**2)
        # sumsq2 = np.sum(tr.core.variances(tt.vector.from_list(cores), modes=[0, 1]).full())
        # print(tt.vector.from_list(cores).full().shape)
        # sumsq2 = np.sum(np.sum(tt.vector.from_list(cores).full()**2, axis=-1)) / np.prod(tt.vector.from_list(cores).n)
        # print(tr.core.means(tt.vector.from_list(cores), modes=[0, 1]).full())
        # print(sumsq2)
        # assert 0
        cores[-1] = cores[-1][..., 0]
        svd = np.linalg.svd(cores[-1], full_matrices=False)

        # Compute error
        cores2 = copy.deepcopy(cores)
        cores2[-1] = svd[0][:, degree:].dot(np.diag(svd[1])[degree:])[:, :, np.newaxis]
        error = tt.vector.from_list(cores2)

        cores[-1] = svd[0][:, :degree]
        if not orthogonal:
            cores[-1] = cores[-1].dot(np.diag(svd[1][:degree]))
        cores[-1] = cores[-1][:, :, np.newaxis]
        if cores[-1].shape[1] < degree:  # Degenerate case: add a slice of zeros
            cores[-1] = np.concatenate([cores[-1], np.zeros([cores[-1].shape[0], degree-cores[-1].shape[1],
                                                             cores[-1].shape[2]])], axis=1)
        t = tt.vector.from_list(cores)

        # print('Norma ara:', tt.vector.norm(t))
        # t *= (1./tt.vector.norm(t))
        # assert 0

        # print('Now:', np.sqrt(np.var(t.full())))
        if anchor_mean:
            withinmean = tt.vector.from_list(tt.vector.to_list(withinmean) + [np.ones([1, 1, 1])])
            t = tr.core.concatenate([withinmean, t], axis=len(modes), eps=eps)

        if verbose:
            print()

    # print(tt.vector.norm(t))
    print()
    # print(np.sum(t.full()**2))
    # print(t)
    tfull = t.full()
    # print(print(np.mean(np.mean(t.full(), axis=0), axis=0)))
    print('Obtained:')
    print(np.linalg.norm(tfull[..., 0]-np.mean(tfull[..., 0])))
    print(np.linalg.norm(withinmean2.full()))
    obtained = np.linalg.norm(tfull[..., 1:]-np.mean(tfull[..., 1:]))
    print(obtained)
    print()
    preserved = (sumsq1**2 + obtained**2) / (sumsq1**2 + sumsq2**2)
    print('This embedding preserves {:g}% of the original norm'.format(preserved*100))
    # print(np.sqrt(np.var(t[..., 0])))
    # print(np.sqrt(np.var(t[..., 1:])))
    # assert 0
    return t, preserved, error

# def pca_ensemble(t, modes, degree=3, orthogonal=False, anchor_mean=True, eps=1e-3, verbose=False):
#     """
#     Given a tensor and a list of modes, consider the space of subtensors obtained
#     by fixing these modes. Find a PCA basis for that space and return the coefficients
#     for each subtensor
#
#     :param t: a TT
#     :param modes: the ensemble modes
#     :param degree: how large a PCA basis to take (default is 3)
#     :param orthogonal:
#     :param anchor_mean:
#     :param eps: relative tolerance while permuting modes. Default is 1e-3
#     :param verbose:
#     :return: M1 x ... x Mk x degree, where M1, etc. are the sizes of `t` along `modes`
#
#     """
#
#     N = t.d
#     if not hasattr(modes, '__len__'):
#         modes = [modes]
#     if len(modes) != len(np.unique(modes)):
#         raise ValueError('Modes may not be repeated')
#     assert 0 <= np.min(modes)
#     assert np.max(modes) < N
#
#     if verbose:
#         print('Computing PCA ensemble...')
#
#     # Group target modes next to each other
#     if verbose:
#         print('\tTransposing tensor... 0', end='')
#     idx = list(range(N))
#     def list_shift(idx, mode, shift):
#         assert mode + shift >= 0
#         assert mode + shift < len(idx)
#         tmp = idx[mode]
#         del idx[mode]
#         if shift < 0:
#             idx.insert(mode+shift, tmp)
#         else:
#             idx.insert(mode+shift, tmp)
#     pivot = modes[0]  # We will organize modes from left-to-right after this one
#     for i in range(1, len(modes)):
#         source = idx.index(modes[i])
#         target = idx.index(pivot) + i
#         if source < target:
#             target -= 1
#         t = tr.core.shift_mode(t, source, target-source, eps=eps)
#         list_shift(idx, source, target-source)
#         if verbose:
#             print(' {}'.format(i), end='')
#     final_pos = idx.index(pivot)
#
#     if anchor_mean:
#         # We now project each vector on the subspace spanned by the constant vector
#         # This is achieved by subtracting each vector's within-mean
#         cores = copy.deepcopy(tt.vector.to_list(t))
#         for n in range(N):
#             if n < final_pos or n >= final_pos+len(modes):
#                 cores[n] = np.repeat(np.mean(cores[n], axis=1, keepdims=True), cores[n].shape[1], axis=1)
#         tmeanrep = tt.vector.from_list(cores)
#         withinmean = tr.core.means(t, modes=list(range(final_pos))+list(range(final_pos+len(modes), N)))
#         t = tt.vector.round(t-tmeanrep, eps=eps)
#         degree -= 1
#
#     # Orthogonalize to the left and right of these modes
#     cores = tt.vector.to_list(t)
#     if verbose:
#         print('\n\tOrthogonalization...')
#     for i in range(final_pos):
#         tr.core.left_orthogonalize(cores, i)
#     for i in range(N-1, final_pos+len(modes)-1, -1):
#         tr.core.right_orthogonalize(cores, i)
#
#     # Cut out the relevant sequence of modes
#     cores = cores[final_pos:final_pos+len(modes)]
#
#     # Convert the sequence into a TT (convert ranks to spatial modes)
#     cores = [np.eye(cores[0].shape[0])[np.newaxis, :, :]] + cores + [np.eye(cores[-1].shape[2])[:, :, np.newaxis]]
#     t = tt.vector.from_list(cores)
#
#     # Put the rank modes together at the very right
#     if verbose:
#         print('\tPutting rank modes together...')
#     t = tr.core.shift_mode(t, 0, t.d-2, eps=eps)
#     cores = tt.vector.to_list(t)
#
#     # Merge the two rank dimensions into one
#     cores[-2] = np.einsum('iaj,jbk->ikab', cores[-2], cores[-1])
#     del cores[-1]
#     cores[-1] = np.reshape(cores[-1], [-1, cores[-1].shape[2]*cores[-1].shape[3], 1])
#
#     # Subtract the cross-mean (mean of the collection of vectors)
#     t = tt.vector.from_list(cores)
#     cores = [np.repeat(np.mean(c, axis=1, keepdims=True), c.shape[1], axis=1) for c in cores[:-1]] + [cores[-1]]
#     crossmean = tt.vector.from_list(cores)
#     t = tt.vector.round(t-crossmean, eps=eps)
#
#     # Compute the truncated PCA of the space spanned by the rank modes
#     if verbose:
#         print('\tPCA compression...')
#     cores = tt.vector.to_list(t)
#     tr.core.orthogonalize(cores, t.d-1)
#     cores[-1] = cores[-1][..., 0]
#     svd = np.linalg.svd(cores[-1], full_matrices=False)
#     cores[-1] = svd[0][:, :degree]
#     if not orthogonal:
#         cores[-1] = cores[-1].dot(np.diag(svd[1][:degree]))
#     cores[-1] = cores[-1][:, :, np.newaxis]
#     if cores[-1].shape[1] < degree:  # Degenerate case: add a slice of zeros
#         cores[-1] = np.concatenate([cores[-1], np.zeros([cores[-1].shape[0], degree-cores[-1].shape[1],
#                                                          cores[-1].shape[2]])], axis=1)
#     t = tt.vector.from_list(cores)
#
#     if anchor_mean:
#         withinmean = tt.vector.from_list(tt.vector.to_list(withinmean) + [np.ones([1, 1, 1])])
#         t = tr.core.concatenate([withinmean, t], axis=len(modes), eps=eps)
#
#     if verbose:
#         print()
#
#     return t

# def pca_ensemble(t, modes, degree=3, eps='same', verbose=False):
#     """
#     Given a tensor and a list of modes, consider the space of subtensors obtained
#     by fixing these modes. Find a PCA basis for that space and return the coefficients
#     for each subtensor
#
#     :param t: a TT
#     :param modes: the ensemble modes
#     :param degree: how large a PCA basis to take (default is 3)
#     :param eps: relative tolerance while permuting modes. Default behavior is to
#     preserve the original number of ranks
#     :param verbose:
#     :return: M1 x ... x Mk x degree, where M1, etc. are the sizes of `t` along `modes`
#
#     """
#
#     N = t.d
#
#     if verbose:
#         print('Computing PCA ensemble...')
#
#     # Group target modes next to each other
#     if verbose:
#         print('\tTransposing tensor... 0', end='')
#     modes = np.sort(modes)
#     for i in range(1, len(modes)):
#         t = tr.core.shift_mode(t, modes[i], modes[0] + i - modes[i], eps=eps)
#         if verbose:
#             print(' {}'.format(i), end='')
#     cores = tt.vector.to_list(t)
#
#     # Orthogonalize to the left and right of these modes
#     if verbose:
#         print('\n\tOrthogonalization...')
#     for i in range(modes[0]):
#         tr.core.left_orthogonalize(cores, i)
#     for i in range(N-1, modes[0]+len(modes)-1, -1):
#         tr.core.right_orthogonalize(cores, i)
#
#     # Decompress (end ranks are not 1 in general)
#     if verbose:
#         print('\tMerging spatial dimensions...')
#     subtensors = tt.vector.from_list(cores[modes[0]:modes[0]+len(modes)])
#     stack = tr.core.full(subtensors, keep_end_ranks=True)
#
#     # Merge all spatial dimensions in one
#     stack = np.transpose(stack, [0, stack.ndim-1] + list(range(1, stack.ndim-1)))
#     stack = np.reshape(stack, [stack.shape[0]*stack.shape[1], -1])
#
#     # Compress (PCA) along the spatial dimension
#     if verbose:
#         print('\tPCA compression...')
#     svd = np.linalg.svd(stack, full_matrices=False)
#     ensemble = np.diag(svd[1][:degree]).dot(svd[2][:degree, :]).T
#
#     # Reorganize the ensemble as expected
#     ensemble = np.reshape(ensemble, list(subtensors.n) + [degree])
#
#     if verbose:
#         print('\tDone')
#
#     return ensemble


import sys
from collections import deque
sys.setrecursionlimit(100000)

def SNS(n_nodes, myInput, a, b):
    '''
    엘리스친구의 친구관계가 myInput으로 주어지고, 사용자 a, b가 주어질 때 둘 사이의 촌수를 반환합니다.
    '''
    if a == b : return 0

    graph = [[] for i in range(n_nodes)]
    m_edges = len(myInput)

    for line in myInput:
        graph[line[0]].append(line[1])
        graph[line[1]].append(line[0])

    queue = deque([a])
    visit = [0] * n_nodes

    visit[a] = 1

    while queue:
        cur = queue.popleft()
        for node in graph[cur]:
            if visit[node] == 0:
                visit[node] = visit[cur] + 1
                queue.append(node)
    return visit[b] -1 if visit[b] > 0 else -1

def main():
    '''
    아래 코드를 수정하지 마시오.
    '''

    n_nodes, m_edges = input().split()
    n_nodes = int(n_nodes)
    m_edges = int(m_edges)

    a, b = input().split()
    a = int(a)
    b = int(b)

    myInput = []

    for i in range(m_edges) :
        line = [int(x) for x in input().split()]
        myInput.append(line)

    print(SNS(n_nodes,myInput,a,b))

if __name__ == "__main__":
    main()