import common
import management_roles
import role
import scanner_stack_roles

def create_scanner_stack_resources(context):
    prefix = common.get_prefix(context, 'scanner')

    properties = context.properties

    project_id = context.env['project']
    region = properties['region']
    management_service_account_id = properties['managementServiceAccountID']

    scanner_topic_name = f'{prefix}-scanner-topic'
    scanner_topic = {
        'name': scanner_topic_name,
        'type': 'pubsub.v1.topic',
        'properties': {
            'name': f"projects/{project_id}/topics/{scanner_topic_name}",
            'topic': scanner_topic_name
        },
        'accessControl': {
            'gcpIamPolicy': {
                'bindings': [
                    {
                        'role': role.get_role_name_ref(project_id, management_roles.PUBSUB_MANAGEMENT_ROLE),
                        'members': [
                            f"serviceAccount:{management_service_account_id}"
                        ]
                    },
                    {
                        'role': role.get_role_name_ref(project_id, management_roles.PUBSUB_IAM_MANAGEMENT_ROLE),
                        'members': [
                            f"serviceAccount:{management_service_account_id}"
                        ]
                    },
                    {
                        'role': 'roles/pubsub.publisher',
                        'members': [
                            f'serviceAccount:$(ref.{prefix}-scanner-service-account.email)',
                        ]
                    }
                ]
            }
        }
    }

    pattern_updater_topic_name = f'{prefix}-pattern-updater-topic'
    pattern_updater_topic = {
        'name': pattern_updater_topic_name,
        'type': 'pubsub.v1.topic',
        'properties': {
            'name': f"projects/{project_id}/topics/{pattern_updater_topic_name}",
            'topic': pattern_updater_topic_name
        }
    }

    pattern_update_bucket_name = f'{prefix}-pattern-update-bucket'
    pattern_update_bucket= {
        'name': pattern_update_bucket_name,
        'type': 'gcp-types/storage-v1:buckets',
        'properties': {
            'name': pattern_update_bucket_name,
            'iamConfiguration': {
                'uniformBucketLevelAccess' : {
                    'enabled': True
                }
            }
        }
    }

    pattern_update_bucket_scanner_role_binding = {
        'name': 'pattern-update-bucket-scanner-role-binding',
        'type': 'gcp-types/storage-v1:virtual.buckets.iamMemberBinding',
        'properties':
            {
                'bucket': f"$(ref.{pattern_update_bucket['name']}.name)",
                'role': role.get_role_name_ref(project_id, scanner_stack_roles.GET_PATTERN_UPDATE_SCANNER_ROLE),
                'member': f'serviceAccount:$(ref.{prefix}-scanner-service-account.email)'
            }
    }

    pattern_update_bucket_role_binding = {
        'name': 'pattern-update-bucket-role-binding',
        'type': 'gcp-types/storage-v1:virtual.buckets.iamMemberBinding',
        'properties':
            {
                'bucket': f"$(ref.{pattern_update_bucket['name']}.name)",
                'role': role.get_role_name_ref(project_id, scanner_stack_roles.PATTERN_UPDATE_ROLE),
                'member': f'serviceAccount:$(ref.{prefix}-pattern-updater-service-account.email)'
            }
    }

    pattern_updater = {
        'name': f'{prefix}-pattern-updater',
        'type': 'gcp-types/cloudfunctions-v1:projects.locations.functions',
        'properties': {
            'parent': f'projects/{project_id}/locations/{region}',
            'function': f'{prefix}-pattern-updater',
            'entryPoint': 'main',
            'serviceAccountEmail': f'$(ref.{prefix}-pattern-updater-service-account.email)',
            'sourceArchiveUrl': f"gs://{properties['artifactBucket']}/gcp-scanner-pattern-updater.zip",
            'runtime': 'python38',
            'availableMemoryMb': 2048,
            'timeout': '120s',
            'environmentVariables': {
                'LD_LIBRARY_PATH': './:./lib',
                'PATTERN_UPDATE_BUCKET': pattern_update_bucket['name']
            },
            'eventTrigger': {
                'eventType': 'providers/cloud.pubsub/eventTypes/topic.publish',
                'resource': f"$(ref.{pattern_updater_topic_name}.name)",
                'failurePolicy': {
                    'retry': {}
                }
            }
        }
    }

    pattern_updater_scheduler = {
        'name': f'{prefix}-scheduler',
        'type': 'gcp-types/cloudscheduler-v1:projects.locations.jobs',
        'properties': {
            'parent': f'projects/{project_id}/locations/{region}',
            'name': f'{prefix}-pattern-update',
            'description': 'pattern update',
            'schedule': "0 * * * *",
            'timeZone': 'UTC',
            'pubsubTarget': {
                'topicName': f"projects/{project_id}/topics/{pattern_updater_topic['name']}",
                'data': 'UGF0dGVybiBVcGRhdGU=',
            }
        },
        'metadata': {
            'dependsOn': [pattern_updater_topic['name']]
        }
    }

    scanner = {
        'name': f'{prefix}-scanner',
        # https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions
        'type': 'gcp-types/cloudfunctions-v1:projects.locations.functions',
        'properties': {
            'parent': f'projects/{project_id}/locations/{region}',
            'function': f'{prefix}-scanner',
            'entryPoint': 'main',
            'sourceArchiveUrl': f"gs://{properties['artifactBucket']}/gcp-scanner.zip",
            'serviceAccountEmail': f'$(ref.{prefix}-scanner-service-account.email)',
            'runtime': 'python38',
            'availableMemoryMb': 2048,
            'timeout': '120s',
            'environmentVariables': {
                'LD_LIBRARY_PATH': '/workspace:/workspace/lib',
                'PATTERN_PATH': './patterns',
                'PROJECT_ID': project_id,
                'REGION': region,
                'DEPLOYMENT_NAME': properties['deploymentName'],
                'PATTERN_UPDATE_BUCKET': pattern_update_bucket['name']
            },
            'secretEnvironmentVariables': [
                {
                    'key': 'SCANNER_SECRETS',
                    'projectId': project_id,
                    'secret': f'{properties["scannerSecretsName"]}',
                    'version': 'latest'
                }
            ],
            'eventTrigger': {
                'eventType': 'providers/cloud.pubsub/eventTypes/topic.publish',
                'resource': f"projects/{project_id}/topics/{scanner_topic['name']}",
                'failurePolicy': {
                    'retry': {}
                }
            }
        },
        'metadata': {
            'dependsOn': [scanner_topic['name']]
        }
    }

    scanner_management_account_role_binding = {
        'name': 'scanner-management-account-role-binding',
        'type': 'gcp-types/cloudfunctions-v1:virtual.projects.locations.functions.iamMemberBinding',
        'properties': {
            'resource': f"$(ref.{scanner['name']}.name)",
            'role': role.get_role_name_ref(project_id, management_roles.CLOUD_FUNCTION_MANAGEMENT_ROLE),
            'member': f'serviceAccount:{management_service_account_id}'
        }
    }

    pattern_updater_management_account_role_binding = {
        'name': 'pattern-updater-management-account-role-binding',
        'type': 'gcp-types/cloudfunctions-v1:virtual.projects.locations.functions.iamMemberBinding',
        'properties': {
            'resource': f"$(ref.{pattern_updater['name']}.name)",
            'role': role.get_role_name_ref(project_id, management_roles.CLOUD_FUNCTION_MANAGEMENT_ROLE),
            'member': f'serviceAccount:{management_service_account_id}'
        }
    }

    scanner_topic_dlt_name = f'{prefix}-scanner-topic-dlt'
    scanner_topic_dlt = {
        'name': scanner_topic_dlt_name,
        'type': 'pubsub.v1.topic',
        'properties': {
            'name': f"projects/{project_id}/topics/{scanner_topic_dlt_name}",
            'topic': scanner_topic_dlt_name
        },
        'accessControl': {
            'gcpIamPolicy': {
                'bindings': [
                    {
                        'role': role.get_role_name_ref(project_id, management_roles.PUBSUB_MANAGEMENT_ROLE),
                        'members': [
                            f"serviceAccount:{management_service_account_id}"
                        ]
                    },
                    {
                        'role': role.get_role_name_ref(project_id, management_roles.PUBSUB_IAM_MANAGEMENT_ROLE),
                        'members': [
                            f"serviceAccount:{management_service_account_id}"
                        ]
                    },
                    {
                        'role': 'roles/pubsub.publisher',
                        'members': [
                            f"serviceAccount:service-{context.env['project_number']}@gcp-sa-pubsub.iam.gserviceaccount.com",
                        ]
                    }
                ]
            }
        }
    }

    scanner_dlt = {
        'name': f'{prefix}-scanner-dlt',
        # https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions
        'type': 'gcp-types/cloudfunctions-v1:projects.locations.functions',
        'properties': {
            'parent': f'projects/{project_id}/locations/{region}',
            'function': f'{prefix}-scanner-dlt',
            'entryPoint': 'main',
            'sourceArchiveUrl': f"gs://{properties['artifactBucket']}/gcp-scanner-dlt.zip",
            'serviceAccountEmail': f'$(ref.{prefix}-scanner-service-account.email)',
            'runtime': 'python38',
            'eventTrigger': {
                'eventType': 'providers/cloud.pubsub/eventTypes/topic.publish',
                'resource': f"projects/{project_id}/topics/{scanner_topic_dlt['name']}",
            },
            'environmentVariables': {}
        },
        'metadata': {
            'dependsOn': [scanner_topic_dlt['name']]
        }
    }

    scanner_dlt_management_account_role_binding = {
        'name': 'scanner-dlt-management-account-role-binding',
        'type': 'gcp-types/cloudfunctions-v1:virtual.projects.locations.functions.iamMemberBinding',
        'properties': {
            'resource': f"$(ref.{scanner_dlt['name']}.name)",
            'role': role.get_role_name_ref(project_id, management_roles.CLOUD_FUNCTION_MANAGEMENT_ROLE),
            'member': f'serviceAccount:{management_service_account_id}'
        }
    }

    scanner_stack_function_update_role_binding = {
        'name':'scanner-stack-function-update-role-binding',
        'type': 'gcp-types/cloudresourcemanager-v1:virtual.projects.iamMemberBinding',
        'properties': {
            'resource': project_id,
            'role': role.get_role_name_ref(project_id, management_roles.SOURCE_CODE_SET_ROLE),
            'member': f'serviceAccount:{management_service_account_id}'
        }
    }

    resources = [
        scanner,
        scanner_dlt,
        scanner_topic,
        scanner_topic_dlt,
        scanner_management_account_role_binding,
        scanner_dlt_management_account_role_binding,
        pattern_updater_topic,
        pattern_update_bucket,
        pattern_updater,
        pattern_updater_scheduler,
        pattern_update_bucket_scanner_role_binding,
        pattern_update_bucket_role_binding,
        pattern_updater_management_account_role_binding
    ]

    if properties['functionAutoUpdate']:
        resources.append(scanner_stack_function_update_role_binding)

    outputs = [{
        'name': 'scannerTopic',
        'value': scanner_topic['name']
    },{
        'name': 'scannerTopicDLT',
        'value': scanner_topic_dlt['name']
    },{
        'name': 'scannerProjectID',
        'value': project_id
    },{
        'name': 'scannerFunctionName',
        'value': '$(ref.{}.name)'.format(scanner['name'])
    },{
        'name': 'patternUpdaterFunctionName',
        'value': '$(ref.{}.name)'.format(pattern_updater['name'])
    },{
        'name': 'scannerDLTFunctionName',
        'value': '$(ref.{}.name)'.format(scanner_dlt['name'])
    },{
        'name': 'region',
        'value': region
    },{
        'name': 'scannerSecretsName',
        'value': properties['scannerSecretsName']
    },{
        'name': 'patternUpdateSchedulerJobName',
        'value': pattern_updater_scheduler['properties']['name']
    },{
        'name': 'patternUpdateBucket',
        'value': '$(ref.{}.name)'.format(pattern_update_bucket['name'])
    },{
        'name': 'functionAutoUpdate',
        'value': properties['functionAutoUpdate']
    }]
    return (resources, outputs)

def generate_config(context):
    """ Entry point for the deployment resources. """

    resources, outputs = create_scanner_stack_resources(context)

    return {
        'resources': resources,
        'outputs': outputs
    }


### Python script to find flags for Linux chmod cmd ###
import angr, claripy, monkeyhex, sys

class hook_getopt(angr.SimProcedure):
    def run(self):
        print('Hooking...')
        return SYMB

def solve():

    global SYMB

    # Set load options and load binary
    loadOpt = {'main_opts':{'arch':'x86'}}
    proj = angr.Project('chmod', load_options = loadOpt, auto_load_libs = False)

    '''
    # Set up the hook
    proj.hook_symbol('getopt_long', hook_getopt())
    '''

    # Find execution start point using CFG
    cfg = proj.analyses.CFGFast()
    for addr in list(cfg.kb.functions):
        if cfg.kb.functions[addr].name == 'getopt_long':
            GETOPT_ADDR = addr
            break
    TARGET_ADDR = cfg.get_any_node(GETOPT_ADDR).predecessors[1].addr

    # Explore the binary
    argv = ['chmod', claripy.BVS('flags', 8)]
    state = proj.factory.blank_state(args = argv, remove_options =
            {angr.options.LAZY_SOLVES}, addr = TARGET_ADDR)
    state.add_constraints(argv[1] >= ' ')
    state.add_constraints(argv[1] <= '~')
    SYMB = argv[1]

    sim = proj.factory.simgr(state)
    print("exploring with cmdline args %s from %s" % (argv[1], sim.active))

    while len(sim.active) == 1:
        sim.step()

    print(sim)
    '''
    for step in sim.unconstrained[0].history.descriptions:
        print(step)
    '''

if __name__ == '__main__':
    solve()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('eatery_profile', '0003_specials_normal_price'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='reviews',
            name='stars_given',
        ),
    ]


import pygame
import random
import math

from src.game_settings import GAME_BOUNDARY_X, GAME_BOUNDARY_Y

class Ant:
    def __init__(self, x, y, width, height, color):
        self.rect = pygame.Rect(x, y, width, height)
        self.color = color
        self.is_facing = "up"
        self.moved_times = 0
        self.direction = 0
        self.speed = 1
        self.change_direction_freq = random.randint(10, 100)
        self.randomize_direction()

    def rect(self):
        return self.rect

    def draw(self, screen):
        pygame.draw.rect(screen, self.color, self.rect)

    def move(self):
        if self.is_facing == "up":
            self.rect.y -= self.speed
        elif self.is_facing == "down":
            self.rect.y += self.speed
        elif self.is_facing == "left":
            self.rect.x -= self.speed
        elif self.is_facing == "right":
            self.rect.x += self.speed

    def calc_next_position(self):
        direction_rad = math.radians(self.direction)

        delta_x = self.speed * math.cos(direction_rad)
        delta_y = self.speed * math.sin(direction_rad)

        next_x = self.rect.x + delta_x
        next_y = self.rect.y + delta_y

        return round(next_x), round(next_y)

    def moveTo(self, x, y):
        if self.would_hit_boundary(x, y):
            self.randomize_direction()
            self.moved_times = 0
            return
        else:
            self.rect.x = x
            self.rect.y = y

    def would_hit_boundary(self, x, y):
        if x < 0 or x > GAME_BOUNDARY_X - self.rect.width:
            return True
        if y < 0 or y > GAME_BOUNDARY_Y - self.rect.height:
            return True
        return False

    def randomize_direction(self):
        random_direction = random.randint(0, 360)
        self.direction = random_direction

    def update(self):
        (next_position_x, next_position_y) = self.calc_next_position()
        self.moveTo(next_position_x, next_position_y)
        self.moved_times += 1
        if self.moved_times == self.change_direction_freq:
            self.randomize_direction()
            self.moved_times = 0


"""
deepdataspace.server.resources.api_v1.object_confirm

RESTful API for object confirm.
"""

import logging
import time

from deepdataspace.constants import ErrCode
from deepdataspace.constants import DatasetStatus
from deepdataspace.model import DataSet
from deepdataspace.model import Label
from deepdataspace.model import Object
from deepdataspace.model.image import Image
from deepdataspace.utils.http import Argument
from deepdataspace.utils.http import BaseAPIView
from deepdataspace.utils.http import format_response
from deepdataspace.utils.http import parse_arguments
from deepdataspace.utils.http import raise_exception

logger = logging.getLogger("django")

class ObjectConfirmView(BaseAPIView):
    """
    - POST /api/v1/object_confirm
    """

    post_args = [
        Argument("dataset_id", str, Argument.JSON, required=True),
        Argument("label_id", str, Argument.JSON, required=True),
        Argument("image_id", int, Argument.JSON, required=True),
        Argument("confirm", Argument.Choice([1, 2]), Argument.JSON, required=True),
        Argument("objects", list, Argument.JSON, required=False),
    ]

    def post(self, request):
        """
        Confirm a label set of an image.

        - POST /api/v1/object_confirm
        """

        dataset_id, label_id, image_id, confirm, objects = parse_arguments(request, self.post_args)

        dataset = DataSet.find_one({"id": dataset_id})
        if dataset is None:
            raise_exception(ErrCode.DatasetNotFound, f"dataset_id[{dataset_id}] not found")
        if dataset.status in DatasetStatus.DontRead_:
            raise_exception(404, f"dataset_id[{dataset_id}] is in status [{dataset.status}] now, try again later")

        label = Label.find_one({"id": label_id, "dataset_id": dataset_id})
        if label is None:
            raise_exception(ErrCode.DatasetLabelNotFound, f"label_id[{label_id}] not found")

        image = Image(dataset_id).find_one({"id": image_id})
        if image is None:
            raise_exception(ErrCode.DatasetImageNotFound, f"image_id[{image_id}] not found")

        new_objects = []
        for idx, obj in enumerate(objects):
            if obj["label_id"] != label_id:
                continue

            try:
                obj = Object.from_dict(obj)
                obj.compare_result = {}
                obj.matched_det_idx = None
            except Exception as err:
                logger.warning(err)
                raise_exception(ErrCode.AnnotationFormatError, f"objects[{idx}] data structure mismatch")
            new_objects.append(obj)

        saving_objects = [o for o in image.objects if o.label_id != label_id]
        saving_objects.extend(new_objects)
        image.objects = saving_objects

        label_confirm = image.label_confirm
        confirm_ts = int(time.time())
        label_confirm[label_id] = {"confirm": confirm, "confirm_ts": confirm_ts}
        image.save()

        return format_response({"confirm": confirm, "confirm_ts": confirm_ts})


# Mission to Mars web scraping app calling on news, images, weather, facts, and hemisphere information for Mars
from bs4 import BeautifulSoup
import requests
import pymongo
from splinter import Browser
import pandas as pd
import time
import re

# Initialize PyMongo to work with MongoDBs
conn = 'mongodb://localhost:27017'
client = pymongo.MongoClient(conn)

def init_browser():

    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {"executable_path": "chromedriver.exe"}
    return Browser("chrome", **executable_path, headless=False)

def scrape():
    browser = init_browser()

    #Retrieve NANA Mars News

    NASA_url = 'https://mars.nasa.gov/news/'

    Mars_news = requests.get(NASA_url)

    NASA_soup = BeautifulSoup(Mars_news.text, 'html.parser')

    news_story = NASA_soup.find('div', class_='slide')

    #Latest Mars News Story Title
    news_title = news_story.find('div', class_='content_title').find('a').text.strip()
    #Latest Mars News Story Blurb
    news_content = news_story.find('div', class_='rollover_description_inner').text.strip()

    #Retrieve JPL Mars Space Featured Image
    JPL_url = 'https://www.jpl.nasa.gov/'
    browser.visit(JPL_url)
    time.sleep(5)
    JPL_html = browser.html

    JPL_soup = BeautifulSoup(JPL_html, 'html.parser')

    image = JPL_soup.find('article', class_='slide-1')
    #Featured Image URL
    featured_url = image.img['src']

    #Retrieve Mars Weather information
    weather_url = 'https://twitter.com/marswxreport?lang=en'

    browser.visit(weather_url)

    time.sleep(5)

    weather_html = browser.html

    # weather_soup = BeautifulSoup(weather_html, 'html.parser')

    #Latest Mars weather information
    weather = re.findall(r'(InSight sol [\d]{3} \([\d]{4}-[\d]{2}-[\d]{2}\) low -[\d]{1,3}\.\d.C \(-[\d]{1,3}.\d.F\) high -[\d]{1,3}.\d.C \([\d]{1,3}\.\d.F\)\nwinds from the [\w]{1,3} at [\d]{1,3}\.\d m/s \([\d]{1,3}\.\d mph\) gusting to [\d]{1,3}\.\d m/s \([\d]{1,3}\.\d mph\)\npressure at [\d]{1,3}\.[\d]{2} hPa)', weather_html)
    mars_weather = weather[0]

    #Retrieve Mars Facts
    facts_url = 'https://space-facts.com/mars/'

    tables = pd.read_html(facts_url)

    Mars_df = tables[0]
    Mars_df.set_index([0])
    Mars_df = Mars_df.rename(columns={0:'', 1:'Measurement'})
    Mars_df.set_index([""], inplace=True)
    Mars_table = Mars_df.to_html(classes='table table-borderless table-hover table-dark')
    #Mars_table = Mars_html_table.replace('\n', '')
    #Mars_table = Mars_df.to_html()
    #Mars_table = 'Mars_table.html'

    #Retrieve Mars Hemisphere data
    hemisphere_image_urls = []

    hem_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(hem_url)
    time.sleep(5)
    hem_html = browser.html
    hem_soup = BeautifulSoup(hem_html, 'html.parser')
    tag = list(hem_soup.find_all('h3'))

    for x in range(0,4):
        browser.find_by_text(tag[x].text.strip()).click()

        html = browser.html
        soup = BeautifulSoup(html, 'html.parser')

        image = soup.find('div', class_='downloads').find('ul').find('li')
        image_url = image.find('a')
        title = soup.find('h2', class_='title')

        hemisphere_image_urls.append({'title':title.text.strip(), 'image_url':image_url["href"]})

        browser.back()

    Mars_data = {'News':[news_title,
                        news_content],
                'Image': featured_url,
                'Weather': mars_weather,
                'Facts': Mars_table,
                'Hemispheres': hemisphere_image_urls}

    # Close the browser after scraping
    browser.quit()

    return Mars_data


import numpy as np
import pandas as pd
import collections
from math import floor
from scipy.sparse import issparse

def balanced_newick_tree(num_taxa):
    if num_taxa%2 != 0: raise ValueError("There is no balanced tree on {num_taxa} taxa. Please specify an even number.")
    if num_taxa == 2: return "(0,1);"
    def _balanced_newick_subtree(nt, left=False):
        if nt == 2:
            return "(_,_)"
        elif nt==3:
            return "((_,_),_)" if left else "(_,(_,_))"
        else:
            if nt%2==0:
                return f"({_balanced_newick_subtree(nt/2, True)},{_balanced_newick_subtree(nt/2)})"
            else:
                return f"({_balanced_newick_subtree(floor(nt/2) + int(left), True)},{_balanced_newick_subtree(floor(nt/2) + int(not left))})"
    newick_string = f"({_balanced_newick_subtree(num_taxa/2, True)},{_balanced_newick_subtree(num_taxa/2)});"
    for i in range(0, num_taxa):
        leaf_name = str(np.base_repr(i, base=max(i+1,2))) if num_taxa <= 36 else f't{str(i)}'
        newick_string = newick_string.replace('_', leaf_name, 1)
    return newick_string

def get_balance(s, asTuple=False):
    """Returns a string formatted 'X|X' which describes the balance of a given split string"""
    s = s.split("|")
    if not asTuple:
        return str(len(s[0])) + '|' + str(len(s[1]))
    else:
        return (len(s[0]), len(s[1]))

def is_sparse(matrix):
    return issparse(matrix)

def frob_norm(matrix, data_table=None):
    """Calculates the Frobenius Norm for a given matrix"""
    if data_table is not None:
        return sum(val**2 for _, val in data_table.itertuples(index=False))**(1/2)
    if is_sparse(matrix):
        return sum(matrix[i,j]**2 for i, j in zip(*matrix.nonzero()))**(1/2)
    else:
        return np.sqrt(sum(val**2 for val in np.nditer(matrix)))

def make_substitution_matrix(subs_prob, k):
    matrix = []
    for i in range(k):
        matrix.append([1-subs_prob if j==i else subs_prob/(k-1) for j in range(k)])
    return matrix

def all_splits(num_taxa, trivial=False, only_balance=None, randomise=False):
    """Generates all splits as string-representations

    Args:
        num_taxa: The number of taxa on the tree.
        trivial: Whether or not to calculate trivial splits, default True.
        only_trivial: Whether to ONLY create trivial splits, default False.

    Returns:
        A list of string-representations of splits (using '|'-notation)
    """
    if num_taxa > 35:
        raise ValueError("Cannot generate all splits for more than 35 taxa. Please create an NXTree and use the NXTree all_splits function instead.")
    k = only_balance
    n = num_taxa
    taxa_string = "".join(np.base_repr(i, base=n) for i in range(n))
    r = 0 if trivial else 1
    loop_over = range(r, 2**(n-1) - r)
    if randomise:
        import random
        loop_over = [i for i in loop_over]
        random.shuffle(loop_over)
    for i in loop_over:
        template = format(i, f'0{n}b')
        if not only_balance or sum(int(b) for b in template) in [only_balance, n-only_balance]:
            if r < sum(int(b) for b in template) < n-r:
                left  = ""
                right = ""
                for t, b in zip(taxa_string, template):
                    if b == '0':
                        left += t
                    else:
                        right += t
                if '0' in right:
                    left, right = right, left
                yield f'{left}|{right}'
###
# Functions for reading in a sequence alignment

def read_alignment_from_file(pathToFile, check=True):
    file = open(pathToFile, 'r')
    alignment = collections.OrderedDict()  # Keeps insertion order
    currentSeq = ""
    for line in file:
        if '>' in line:
            currentSeq = line.replace('>', '').replace('\n', '')
        else:
            if currentSeq in alignment:
                alignment[currentSeq] += line.replace('\n', '')
            else:
                alignment[currentSeq] = line.replace('\n', '')
    if check:
        return alignment if __valid_alignment(alignment) else None
    else:
        return alignment

def __valid_alignment(alignmentDict):
    ordered = isinstance(alignmentDict, collections.OrderedDict)  # Must be ordered to avoid incorrect patterns
    sameLength = len(set(len(value) for key, value in alignmentDict.items())) == 1  # Must be same length
    return sameLength and ordered

def get_pattern_counts(alignment, asNumpyArray=False):
    patterns = {}
    sequences = list(alignment.values())
    sequenceLength = len(sequences[0])
    usableSequenceLength = 0
    for i in range(sequenceLength):
        patternAtSitei = ''.join(s[i] for s in sequences).upper()
        if all(c in ('A', 'C', 'G', 'T') for c in patternAtSitei):  # Need to handle U (let U = T)
            usableSequenceLength += 1
            if patternAtSitei in patterns:
                patterns[patternAtSitei] += 1
            else:
                patterns[patternAtSitei] = 1

    if asNumpyArray:
        patterns = np.array([[key, val] for key, val in patterns.items()])

    return patterns, usableSequenceLength

def pattern_counts_to_probs(patterns, seqLen):
    newCounts = [float(float(count) / seqLen) for count in patterns[:, 1]]
    patternList = patterns[:, 0]
    return np.array([patternList, newCounts]).transpose()

def pattern_probs_from_alignment(pathToFile, check=True):
    alignment = read_alignment_from_file(pathToFile, check)
    counts, sequenceLength = get_pattern_counts(alignment, True)
    probs = pattern_counts_to_probs(counts, sequenceLength)
    probs = pd.DataFrame(probs, index=probs[:, 0])
    probs[[1]] = probs[[1]].astype("float")
    return probs, len(alignment), sequenceLength

def scores_to_weights(scores):
    scores = [s if s != 0 else 10 ** (-100) for s in scores]
    inverses = [1 / s for s in scores]
    weights = [i / sum(inverses) for i in inverses]
    return weights

def split_equivalence_classes(splits, group):
    """Breaks up list of splits into equivalence classes under the canonical action of the specified group."""
    def __action(perm, split):
        index_of_sep = split.index('|')
        split = split.replace('|', '')
        split = split = ''.join(str(perm(int(s))) for s in split)
        split = sorted(["".join(sorted(split[:index_of_sep])), "".join(sorted(split[index_of_sep:]))])
        split = f'{split[0]}|{split[1]}'
        return split
    def __orbit(split, group):
        orbit = set()
        for g in group.elements:
            orbit.add(__action(g, split))
        return frozenset(orbit)
    def __orbits(splits, group):
        orbits = set()
        for split in splits:
            orbits.add(__orbit(split, group))
        return sorted(list(set(orbit) for orbit in orbits), key=len)
    return __orbits(splits, group)

def normalised(scores):
    scores = scores.copy()
    if type(scores) is dict:
        minimum = min(scores.values())
        maximum = max(scores.values())
        for key, value in scores.items():
            scores[key] = (value - minimum)/(maximum - minimum)
    elif type(scores) is list:
        minimum = min(scores)
        maximum = max(scores)
        scores = [(score-minimum)/(maximum-minimum) for score in scores]
    return scores

def check_splits_representatives(split_reps, orbits):
    represented_orbits = set()
    for orbit in orbits:
        for split in split_reps:
            if split in orbit:
                represented_orbits.add(frozenset(orbit))
    return len(represented_orbits) == len(orbits)

# Function for scaling the S part of the hard-coded H matrix
def scaled_h_matrix(_lambda):
    if _lambda == None:
        return None, None
    if _lambda == "identity":
        return np.matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [1, 1, 1, 1]]), "identity"
    H = np.matrix([[1, -1], [1, 1]])
    H = np.kron(H, H)
    S = np.empty((0, 4))
    for row in H:
        if len(set(np.asarray(row)[0])) != 1:
            S = np.append(S, row, axis=0)
    S = _lambda * S
    H = np.append(S, [[1, 1, 1, 1]], axis=0)
    return (H, _lambda)


from datetime import datetime
import logging
from mogo import Model, Field

def safe_int(value):
    try:
        return int(value)
    except (ValueError, TypeError):
        return 0

class Sequence(Model):

    @staticmethod
    def get_next(name):
        return Sequence._get_collection().find_and_modify({'_id': name}, {'$inc': {'seq': 1}}, new=True, upsert=True)['seq']

class Settings(Model):
    # id              = models.AutoField(primary_key=True)
    var_name        = Field(required=True)
    var_value       = Field()
    date_created    = Field(datetime, default=datetime.utcnow)
    last_modified   = Field(datetime, default=datetime.utcnow)

    @staticmethod
    def set(name, value):
        var = Settings.search(var_name=name).first()
        if var:
            old_value = var.var_value
            if value != old_value:
                var.var_value = value
                var.last_modified = datetime.utcnow()
                var.save()
            return old_value
        else:
            Settings.create(var_name=name, var_value=value)

    @classmethod
    def get_str(cls, name, default=None):
        var = Settings.search(var_name=name).first()
        if var:
            return var.var_value
        return default

    @staticmethod
    def get_bool(name, default = None):
        value = Settings.get_str(name, default)
        if isinstance(value, str):
            if value.lower() == 'true' or safe_int(value) != 0:
                return True
            return False
        return value and int(value) != 0

    @staticmethod
    def get_int(name, default = None):
        var = Settings.search(var_name=name).first()
        if var:
            return safe_int(var.var_value)
        return default

    @staticmethod
    def get_long(name, default = None):
        var = Settings.search(var_name=name).first()
        if var:
            return int(var.var_value)
        return default

    @staticmethod
    def get_float(name, default = None):
        var = Settings.search(var_name=name).first()
        if var:
            return float(var.var_value)
        return default

    @staticmethod
    def get_datetime(name, default=None):
        var = Settings.search(var_name=name).first()
        if var:
            try:
                return datetime.strptime(var.var_value, "%Y-%m-%dT%H:%M:%S.%f")
            except Exception as ex:
                logging.exception("Error parsing date [%s]", var.var_value)
        return default


import random
from os import makedirs, path
from shutil import rmtree
from tqdm import tqdm
from glob import glob
from PIL import Image
import torch
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
import torchvision.transforms as transforms
import numpy as np
import configparser

WEBCAM_DIR = "webcam_dataset"
SAMPLED_DATASET = "resized-sampled-dataset"
OUTPUT_DIRECTORY = "dataset"
TRAIN_RATIO = 0.8 # range from 0 to 1
SEED = 27
NUM_SAMPLE = 2500 # -1 for all samples, > 0 for sampling data
RESIZED_SHAPE = (64,64)
classes = ['incorrect_mask', 'no_mask', 'mask']

def process_image(images, target_dir=OUTPUT_DIRECTORY):
    if not path.exists(path.join(target_dir,'train',_class)):
        makedirs(path.join(target_dir,'train',_class))
    if not path.exists(path.join(target_dir,'val',_class)):
        makedirs(path.join(target_dir,'val',_class))

    # randomize
    random.seed(SEED)
    random.shuffle(images)
    if NUM_SAMPLE > 0:
        images = images[:NUM_SAMPLE]

    # Copy files to the output directory
    positive_sample = int(TRAIN_RATIO * len(images))
    for img in tqdm(images):
        im_pil = Image.open(img)
        imResize = im_pil.resize(RESIZED_SHAPE, Image.ANTIALIAS)
        file_name = path.split(img)[1]
        if img in images[:positive_sample]:
            imResize.save(path.join(target_dir,'train',_class,file_name), 'JPEG', quality=90)
        else:
            imResize.save(path.join(target_dir,'val',_class,file_name), 'JPEG', quality=90)

# Remove existing dataset dir
if path.exists(path.join(OUTPUT_DIRECTORY,'train')):
    rmtree(path.join(OUTPUT_DIRECTORY,'train'))
if path.exists(path.join(OUTPUT_DIRECTORY,'val')):
    rmtree(path.join(OUTPUT_DIRECTORY,'val'))
makedirs(path.join(OUTPUT_DIRECTORY,'train'))
makedirs(path.join(OUTPUT_DIRECTORY,'val'))

for _class in classes:
    print('Processing: {}'.format(_class))
    #======================Processing Webcam Dataset=========================
    images = glob(path.join(WEBCAM_DIR,_class,'*'))
    process_image(images=images, target_dir=OUTPUT_DIRECTORY)

    #======================Processing Public Dataset=========================
    images_train = glob(path.join(SAMPLED_DATASET,'train',_class,'*'))
    images_val = glob(path.join(SAMPLED_DATASET,'val',_class,'*'))
    process_image(images=images_train+images_val, target_dir=OUTPUT_DIRECTORY)

# Getting Mean and Std
print('\nGetting mean and std . . .')
train_set = ImageFolder(root=OUTPUT_DIRECTORY, transform=transforms.ToTensor())
data = torch.cat([d[0] for d in DataLoader(train_set)])
mean, std = data.mean(dim=[0, 2, 3]), data.std(dim=[0, 2, 3])
print('mean: {}\nstd: {}\n'.format(list(np.array(mean)),list(np.array(std))))

# Update config.ini
filename = 'config.ini'
config = configparser.ConfigParser()
config.read(filename)
config.set('LIST','MEAN',str(list(np.array(mean))))
config.set('LIST','STD',str(list(np.array(std))))
with open(filename, 'w') as configfile:
    config.write(configfile)
print('Config.ini is updated!\n')


import torch
import pandas as pd
from torch.utils.data import Dataset
from transformers import BertTokenizerFast

class ToxicCommentsDataset(Dataset):
    """Custom Dataset

    :returns: dictionary
    :rtype: Dict
    """

    def __init__(
            self,
            data: pd.DataFrame,
            target_columns: list,
            tokenizer: BertTokenizerFast,
            max_token_len: int = 128
    ):
        self.data = data
        self.target_columns = target_columns
        self.tokenizer = tokenizer
        self.max_token_len = max_token_len

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index: int):
        data_row = self.data.iloc[index]
        _id = data_row['id']
        comment_text = data_row.comment_text
        labels = data_row[self.target_columns]

        encoding = self.tokenizer.encode_plus(
            comment_text,
            max_length=self.max_token_len,
            padding='max_length',
            truncation=True,
            add_special_tokens=True,  # [CLS] & [SEP]
            return_token_type_ids=False,
            return_attention_mask=True,  # attention_mask
            return_tensors='pt',
        )

        return dict(
            _id=_id,
            comment_text=comment_text,
            input_ids=encoding['input_ids'].flatten(),
            attention_mask=encoding['attention_mask'].flatten(),
            labels=torch.FloatTensor(labels)
        )


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 15 11:44:28 2020

Importing decision_tree_functions to perform classification o regression tasks based on the user's specificaion.

@author: aless
"""

import numpy as np
import pandas as pd
import os
import random

import matplotlib.pyplot as plt

import decision_tree_functions as dtf

######################################
#     REGRESSION PLOT FUNCTIONS      #
######################################

def create_plot(df, tree, title):
    """
    DESCRIPTION:
        Scatter plot of actual versus predicted values

    """
    predictions = df.apply(dtf.predict_example, args=(tree,), axis=1)
    actual = df['label']

    plt.scatter(actual, predictions)
    plt.grid()

def plot_target_vs_feature(df, tree, feature, label_name):
    """
    DESCRIPTION:
        Creates a plot of the target variable versus a specified feature.
        The target is plotted both the actul and the predicted

    PRECONDITION:
        The target vaiable in the input dataframe is named 'label'

    """
    predictions = df.apply(dtf.predict_example, args=(tree,), axis=1)
    actual = df['label']

    fig, ax = plt.subplots()

    ax.scatter(df[feature], actual, label='actual')
    ax.scatter(df[feature], predictions, label='predicted')
    plt.grid()
    plt.xlabel(feature)
    plt.ylabel(label_name)
    plt.legend()

##########################
#     INPUT SECTION      #
##########################

data_path = r'C:\Users\aless\Desktop\AI\CORSI\GustoMSC\JB117'
data_file = 'JB117_wd_20_30_40m.pkl'

# 'sig_roll', 'sig_pitch', 'wd',
features = ['sig_z_all_legs', 'sig_roll', 'sig_pitch', 'wd', 'UF_MPM']        # list of features to keep. Some features may not be necessary
target_name = 'UF_MPM'                 # name of the taget variable. It will be changed to 'label' (required by the imported functions)

is_ml_task_classification = True    # True if 'classification', False if 'regression'

test_size = 0.3                     # can be either an integer or a percentage
# random.seed(0)

###########################
#    DATA PREPARATION     #
###########################

df_input = pd.read_pickle(os.path.join(data_path, data_file))
# df_corr = df.corr()

# df = pd.read_csv(os.path.join(data_path, data_file))
# df = df.drop('Id', axis=1)
random.seed(0)
df = df_input[features]
df = df.rename(columns={target_name: 'label'})          # decision_tree_functions.py requires the target name to be 'label'

train_df, test_df = dtf.train_test_split(df, test_size)

###########################
# DECISION TREE ALGORITHM #
###########################

# #sub_tree = {question: [yes_answer, no_answer]}
if is_ml_task_classification:                   # Classification
    tree = dtf.decision_tree_algorithm(train_df, ml_task="classification", max_depth=3)
else:
    train_df, test_df = dtf.train_test_split(df, test_size)
    tree = dtf.decision_tree_algorithm(train_df, ml_task="regression", max_depth=3)
    r_squared = dtf.calculate_r_squared(test_df, tree)
    print('r_squared: ',  r_squared)
    #create_plot(test_df, tree, title="Test Data")
    plot_target_vs_feature(test_df, tree, 'sig_z_all_legs', 'Unity Check [-]')
    plot_target_vs_feature(test_df, tree, 'sig_roll', 'Unity Check [-]')
    plot_target_vs_feature(test_df, tree, 'sig_pitch', 'Unity Check [-]')

    plot_target_vs_feature(test_df, tree, 'wd', 'Unity Check [-]')

    y_pred = test_df.apply(dtf.predict_example, args=(tree,), axis=1)
    y_test = test_df['label']

    from sklearn import metrics
    print('MAE:', metrics.mean_absolute_error(y_test, y_pred))
    print('MSE:', metrics.mean_squared_error(y_test, y_pred))
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))

    # plot_df = pd.DataFrame({"actual": actual, "predictions": predictions})

    # plot_df.plot(figsize=(18, 5), title=title)


from __future__ import print_function
import numpy as np
import time
import sys
import traceback
import math
# from spacy import en
import threading
# from multiprocessing.pool import ThreadPool
import Queue
import tensorflow as tf
from itertools import compress
import cPickle as pickle

from collections import defaultdict, Counter
from lib.data.dataset_utils import print_words_with_scores, print_words_with_labels
import pdb

import logging
logger = logging.getLogger('root')

QUERY_STOP_WORDS = []
LUCENE_SPECIAL_CHARS = u"+-&|!(){}[]^\"~*?:\\"
OTHER_SPECIAL_CHARS = u"/',.@#$%"
NEG_SAMPLE_W_REVIEW = False
NEG_SAMPLE_W_NONE_ANSWER = True

data_queue = Queue.Queue(5)
rslt_queue = Queue.Queue(5)
eval_lock = threading.Lock()

def np_softmax(x, axis=0):
  """Compute softmax values for each sets of scores in x."""
  e_x = np.exp(x - np.max(x, axis=axis, keepdims=True))
  return e_x / e_x.sum(axis=axis, keepdims=True)

def f1_score(prec, reca):
  prec_x_reca = prec * reca
  if prec_x_reca == 0.0:
    f1 = 0.
  else:
    f1 = 2. * prec_x_reca / (prec + reca)
  return f1

def rerank_performance(vals, batchid, stats, at_n):
  true_answer_id = vals['data_sample_id'][batchid]
  cand_answer_id = vals['non_answer_data_sample_id'][batchid]
  pairwise_match_scores = vals['neg_pairwise_match_scores_normalized'][batchid]
  semantic_match_scores = vals['neg_semantic_match_scores_normalized'][batchid]
  ensemble_match_scores = vals['neg_ensemble_match_scores'][batchid]
  assert 'pairwise_MAP_better_cnt_at_{}'.format(at_n) in stats
  assert 'pairwise_MAP_worse_cnt_at_{}'.format(at_n) in stats
  assert 'pairwise_MAP_equal_cnt_at_{}'.format(at_n) in stats
  assert 'semantic_MAP_better_cnt_at_{}'.format(at_n) in stats
  assert 'semantic_MAP_worse_cnt_at_{}'.format(at_n) in stats
  assert 'semantic_MAP_equal_cnt_at_{}'.format(at_n) in stats
  assert 'ensemble_MAP_better_cnt_at_{}'.format(at_n) in stats
  assert 'ensemble_MAP_worse_cnt_at_{}'.format(at_n) in stats
  assert 'ensemble_MAP_equal_cnt_at_{}'.format(at_n) in stats
  assert 'MAP_na_cnt_at_{}'.format(at_n) in stats
  if len(cand_answer_id) < at_n:
    logger.warning("Candidate answer number {} is less than top-{}".format(len(cand_answer_id), at_n))

  cand_answer_id_at_n = cand_answer_id[: at_n]
  pairwise_match_scores_at_n = pairwise_match_scores[: at_n]
  semantic_match_scores_at_n = semantic_match_scores[: at_n]
  ensemble_match_scores_at_n = ensemble_match_scores[: at_n]
  error_sample_ids = set([])

  if true_answer_id in cand_answer_id_at_n:
    # CANDIDATE MAP
    true_answer_location = np.where(cand_answer_id_at_n == true_answer_id)[0][0]
    candidate_MAP = 1.0 / (1.0 + true_answer_location)
    # PAIRWISE MATCH MAP
    cnt = 0.0
    for _, s in enumerate(pairwise_match_scores_at_n):
      if pairwise_match_scores_at_n[true_answer_location] < s:
        cnt += 1.0
        error_sample_ids.add(_)
    pairwise_MAP = 1.0 / (1.0 + cnt)
    # SEMANTIC MATCH MAP
    cnt = 0.0
    for _, s in enumerate(semantic_match_scores_at_n):
      if semantic_match_scores_at_n[true_answer_location] < s:
        cnt += 1.0
        error_sample_ids.add(_)
    semantic_MAP = 1.0 / (1.0 + cnt)
    # ENSEMBLE MATCH MAP
    cnt = 0.0
    for _, s in enumerate(ensemble_match_scores_at_n):
      if ensemble_match_scores_at_n[true_answer_location] < s:
        cnt += 1.0
        error_sample_ids.add(_)
    ensemble_MAP = 1.0 / (1.0 + cnt)
    na_cnt = 0.0
  else:
    candidate_MAP = 0.0
    pairwise_MAP = 0.0
    semantic_MAP = 0.0
    ensemble_MAP = 0.0
    na_cnt = 1.0
    true_answer_location = None

  return na_cnt, candidate_MAP, pairwise_MAP, semantic_MAP, ensemble_MAP, \
      true_answer_location, error_sample_ids

def run_graph(session,
              model,
              handle,
              timer,
              is_training,
              print_qualitative,
              minibatch_max_num):
  cnt = 0
  while True:
    cnt += 1
    if minibatch_max_num is not None and cnt > minibatch_max_num:
      rslt_queue.put('DATA_END')
      break
    try:
      start_time = time.time()
      # Fetch results
      fetches = {
          'data_sample_id': model.data_sample_id,
          'non_answer_data_sample_id': model.non_answer_data_sample_id,
          'l2_regu_loss': model.l2_regu_loss,
          'focus_match_loss': model.focus_match_loss,
          'pairwise_match_loss': model.pairwise_match_loss,
          # 'semantic_match_loss': model.semantic_match_loss,
          'answer_focus_classify_accuracy': model.answer_focus_classify_accuracy,
          'answer_pairwise_classify_accuracy': model.answer_pairwise_classify_accuracy,
          # 'answer_semantic_classify_accuracy': model.answer_semantic_classify_accuracy,
          # 'answer_ensemble_classify_accuracy': model.answer_ensemble_classify_accuracy,
          'merged_summary_op': model.merged_summary_op,
          'global_step': model.global_step,
          'question_focus_attn': model.question_focus_attn}
      # fetches['question_focus_loss'] = model.question_focus_loss
      # fetches['question_focus_prec'] = model.question_focus_prec
      # fetches['question_focus_reca'] = model.question_focus_reca
      # fetches['question_focus_f1'] = model.question_focus_f1
      if print_qualitative:
        fetches['question_focus_binary_pred'] = model.question_focus_binary_pred
        fetches['pairwise_match_logits_2d'] = model.pairwise_match_logits_2d
        fetches['pairwise_match_logits_diff'] = model.pairwise_match_logits_diff
        # fetches['answer_attn'] = model.answer_attn
        # fetches['neg_pairwise_match_scores_normalized'] = model.neg_pairwise_match_scores_normalized
        fetches['neg_pairwise_match_scores_normalized'] = model.non_answer_pairwise_classify_logits
        # fetches['neg_semantic_match_scores_normalized'] = model.neg_semantic_match_scores_normalized
        # fetches['neg_ensemble_match_scores'] = model.neg_ensemble_match_scores
        fetches['question_exact_match'] = model.question_exact_match

      if is_training:
        fetches['eval_op'] = model.train_op
      # Feed inputs
      feed = {model.handle: handle}
      try:
        vals = session.run(fetches=fetches, feed_dict=feed)
      except ValueError:
        excinfo = sys.exc_info()
        traceback.print_exception(*excinfo)

      timer.append(time.time() - start_time)
      # pdb.set_trace()
      rslt_queue.put(vals)

      # if True:  # TO DEBUG DATASET SAMPLING
      #   for id, neg_ids in zip(vals['data_sample_id'], vals['non_answer_data_sample_id']):
      #     print('{0:3d} : {1}'.format(id, ' '.join(map(str, neg_ids))))

    except tf.errors.OutOfRangeError:
      rslt_queue.put('DATA_END')
      break

def eval_stat(stats,
              display_sentence,
              get_label,
              timer,
              qsize,
              summary_writer,
              query_save_file,
              eval_rerank,
              at_n_list):
  step = 0
  while True:
    qsize.append(rslt_queue.qsize())
    rslt = rslt_queue.get()
    if rslt == "DATA_END":
      rslt_queue.put("DATA_END")
      return
    else:
      start_time = time.time()
      vals = rslt

      # SUM UP METRIC VALUES
      with eval_lock:
        stats["batch"] += 1
        stats["l2_regu_losses"] += vals["l2_regu_loss"]
        # stats["question_focus_losses"] += vals["question_focus_loss"]
        # stats["question_focus_prec"] += vals["question_focus_prec"]
        # stats["question_focus_reca"] += vals["question_focus_reca"]
        # stats["question_focus_f1"] += vals["question_focus_f1"]
        stats["focus_match_losses"] += vals["focus_match_loss"]
        stats["pairwise_match_losses"] += vals["pairwise_match_loss"]
        # stats["semantic_match_losses"] += vals["semantic_match_loss"]
        stats["answer_focus_classify_accuracy"] += vals["answer_focus_classify_accuracy"]
        stats["answer_pairwise_classify_accuracy"] += vals["answer_pairwise_classify_accuracy"]
        # stats["answer_semantic_classify_accuracy"] += vals["answer_semantic_classify_accuracy"]
        # stats["answer_ensemble_classify_accuracy"] += vals["answer_ensemble_classify_accuracy"]
        if summary_writer is not None:
          summary_writer.add_summary(vals['merged_summary_op'], vals['global_step'])
        step += 1

      # SAVE QUERY
      if query_save_file is not None:
        pickle.dump((vals['data_sample_id'], vals['question_focus_attn']), query_save_file)

      # EVALUATE SEMANTIC RERANK
      candidate_MAPs, pairwise_MAPs, semantic_MAPs, ensemble_MAPs = [], [], [], []
      true_answer_location, error_sample_ids = [], []
      if eval_rerank:
        batch_size = len(vals['data_sample_id'])
        for batchid in xrange(batch_size):
          candidate_MAPs.append([])
          pairwise_MAPs.append([])
          semantic_MAPs.append([])
          ensemble_MAPs.append([])
          for _, at_n in enumerate(at_n_list):
            na_cnt, candidate_MAP, pairwise_MAP, semantic_MAP, ensemble_MAP, \
                true_ans_loc, err_ids = \
                    rerank_performance(vals, batchid, stats, at_n)
            candidate_MAPs[-1].append(candidate_MAP)
            pairwise_MAPs[-1].append(pairwise_MAP)
            semantic_MAPs[-1].append(semantic_MAP)
            ensemble_MAPs[-1].append(ensemble_MAP)
            # SAVE RESULTS
            with eval_lock:
              stats['MAP_na_cnt_at_{}'.format(at_n)] += na_cnt
              if na_cnt == 0.0:
                if pairwise_MAP > candidate_MAP:
                  stats['pairwise_MAP_better_cnt_at_{}'.format(at_n)] += 1.0
                elif pairwise_MAP < candidate_MAP:
                  stats['pairwise_MAP_worse_cnt_at_{}'.format(at_n)] += 1.0
                else:
                  stats['pairwise_MAP_equal_cnt_at_{}'.format(at_n)] += 1.0
                if semantic_MAP > candidate_MAP:
                  stats['semantic_MAP_better_cnt_at_{}'.format(at_n)] += 1.0
                elif semantic_MAP < candidate_MAP:
                  stats['semantic_MAP_worse_cnt_at_{}'.format(at_n)] += 1.0
                else:
                  stats['semantic_MAP_equal_cnt_at_{}'.format(at_n)] += 1.0
                if ensemble_MAP > candidate_MAP:
                  stats['ensemble_MAP_better_cnt_at_{}'.format(at_n)] += 1.0
                elif ensemble_MAP < candidate_MAP:
                  stats['ensemble_MAP_worse_cnt_at_{}'.format(at_n)] += 1.0
                else:
                  stats['ensemble_MAP_equal_cnt_at_{}'.format(at_n)] += 1.0
                stats['candidate_MAP_at_{}'.format(at_n)].append(candidate_MAP)
                stats['pairwise_match_MAP_at_{}'.format(at_n)].append(pairwise_MAP)
                stats['semantic_match_MAP_at_{}'.format(at_n)].append(semantic_MAP)
                stats['ensemble_match_MAP_at_{}'.format(at_n)].append(ensemble_MAP)
              stats['candidate_MAP_at_{}_all'.format(at_n)].append(candidate_MAP)
              stats['pairwise_match_MAP_at_{}_all'.format(at_n)].append(pairwise_MAP)
              stats['semantic_match_MAP_at_{}_all'.format(at_n)].append(semantic_MAP)
              stats['ensemble_match_MAP_at_{}_all'.format(at_n)].append(ensemble_MAP)
          true_answer_location.append(true_ans_loc)
          error_sample_ids.append(err_ids)

      # DISPLAY QUALITATIVE RESULTS
      # if display_sentence is not None and get_label is not None:
      if display_sentence is not None:
        # if mode == 'question_focus':
        #   question = display_sentence('question', vals['data_sample_id'], exclude_pad=False)
        #   question_label = get_label('question_focus', vals['data_sample_id'])
        #   answer = display_sentence('answer', vals['data_sample_id'], exclude_pad=False)
        #   answer_label = get_label('answer_focus', vals['data_sample_id'])
        question = display_sentence('question', vals['data_sample_id'], exclude_pad=False)
        answer = display_sentence('answer', vals['data_sample_id'], exclude_pad=False)
        # question_label = get_label('question_focus', vals['data_sample_id'])
        # answer_label = get_label('answer_focus', vals['data_sample_id'])
        question_wo_pad = display_sentence('question', vals['data_sample_id'], exclude_pad=True)
        answer_wo_pad = display_sentence('answer', vals['data_sample_id'], exclude_pad=True)

        for batchid, (q, a) in enumerate(zip(question, answer)):
          try:
            # logger.debug('\n\n')
            # q_and_s = print_words_with_scores(q,
            #                                   vals['question_exact_match'][batchid],
            #                                   # vals['question_focus_binary_pred'][batchid],
            #                                   vals['question_focus_attn'][batchid])
            # logger.debug('[Question]:\t{}'.format(q_and_s))
            # # q_and_s = print_words_with_scores(q,
            # #                                   question_label[batchid],
            # #                                   vals['question_focus_binary_pred'][batchid],
            # #                                   vals['question_focus_attn'][batchid])
            # # logger.debug('[Question]:\t{}'.format(q_and_s))
            # # a_and_s = print_words_with_scores(a,
            # #                                   # answer_label[batchid],
            # #                                   vals['answer_attn'][batchid])
            # a_and_s = a
            # logger.debug('[Answer]:\t{}'.format(a_and_s))
            # # PRINT QUESTION QUALITATIVE
            # pw_scores = vals['pairwise_match_logits_2d'][batchid]
            # logits_diff = vals['pairwise_match_logits_diff'][batchid]  # FIXME
            # top_a_word_ids_match_q = np.flip(np.argsort(pw_scores, axis=1), axis=1)
            # # for i, (qw, l, ids) in enumerate(zip(question[batchid],
            # #                                      question_label[batchid],
            # #                                      top_a_word_ids_match_q)):
            # for i, (qw, ids) in enumerate(zip(question[batchid],
            #                                   top_a_word_ids_match_q)):
            #   if qw == '<pad>': break
            #   pw_match_info = '{0: <20} :\t{1: 5.4f} * {2:5.4f} = {3: 5.4f} |'.format(
            #       qw,
            #       vals['question_focus_attn'][batchid, i],
            #       logits_diff[i],
            #       vals['question_focus_attn'][batchid, i] * logits_diff[i])
            #   for _ in xrange(5):
            #     pw_match_info += '{0: <10} {1: <5.4f} | '.format(answer[batchid][ids[_]],
            #                                                      pw_scores[i, ids[_]])
            #   logger.debug(pw_match_info)
            # # # PRINT ANSWER PAIRWISE QUALITATIVE
            # # logger.debug('----- -----')
            # # top_q_word_ids_match_a = np.flip(np.argsort(pw_scores, axis=0), axis=0)
            # # for i, (aw, l, ids) in enumerate(zip(answer[batchid],
            # #                                      answer_label[batchid],
            # #                                      np.transpose(top_q_word_ids_match_a))):
            # #   if aw == '<pad>': break
            # #   pw_match_info = '{0: <20} :\t|'.format(aw + ('*' if l else ''))
            # #   for _ in xrange(5):
            # #     pw_match_info += '{0: <10} {1: <5.4f} | '.format(question[batchid][ids[_]],
            # #                                                      pw_scores[ids[_], i])
            # #   logger.debug(pw_match_info)

            # # # PRINT MORE ANSWER PAIRWISE QUALITATIVE
            # # logger.debug('----- -----')
            # # pw_scores_weighted = pw_scores * \
            # #                      np.expand_dims(vals['question_focus_attn'][batchid], axis=1)
            # # top_q_word_ids_match_a = np.flip(np.argsort(pw_scores_weighted, axis=0), axis=0)
            # # for i, (aw, l, ids) in enumerate(zip(answer[batchid],
            # #                                      answer_label[batchid],
            # #                                      np.transpose(top_q_word_ids_match_a))):
            # #   if aw == '<pad>': break
            # #   pw_match_info = '{0: <20} :\t|'.format(aw + ('*' if l else ''))
            # #   for _ in xrange(5):
            # #     pw_match_info += '{0: <10} {1: <5.4f} | '.format(question[batchid][ids[_]],
            # #                                                      pw_scores_weighted[ids[_], i])
            # #   logger.debug(pw_match_info)

            logger.debug('[Question]:\t{}'.format(q))
            logger.debug('[Answer]:\t{}'.format(a))
            pw_scores = vals['pairwise_match_logits_2d'][batchid]
            qlen = len(question_wo_pad[batchid])
            alen = len(answer_wo_pad[batchid])
            pw_scores = pw_scores[:qlen, :alen] + 10.0
            # pw_scores_normalized = pw_scores / pw_scores.sum(axis=1, keepdims=True)
            pw_scores_normalized = np_softmax(pw_scores, axis=1)

            # PRINT PAIRWISE INFORMATION (HEATMAP IF VISUALIZED)
            # pdb.set_trace()
            display_str = "\n{0:10} ".format("")
            for qw in question_wo_pad[batchid]:
              display_str += "{0:>10} ".format(qw[:10])
            display_str += "\n"
            for i, aw in enumerate(answer_wo_pad[batchid]):
              display_str += "{0:>10} ".format(aw[:10])
              for j in xrange(len(question_wo_pad[batchid])):
                display_str += "{0:10.5f} ".format(pw_scores_normalized[j, i])
              display_str += "\n"
            logger.debug(display_str)

            # PRINT RANKING QUALITATIVE
            if eval_rerank:
              non_answers = display_sentence('answer', vals['non_answer_data_sample_id'][batchid],
                                             exclude_pad=True)
              cnt = 0
              # for _, (na, s1, s2, s3) in enumerate(zip(
              #     non_answers,
              #     vals['neg_pairwise_match_scores_normalized'][batchid],
              #     vals['neg_semantic_match_scores_normalized'][batchid],
              #     vals['neg_ensemble_match_scores'][batchid]
              #     )):
              for _, (na, s1) in enumerate(zip(non_answers, vals['neg_pairwise_match_scores_normalized'][batchid])):
                if _ > 10 and \
                   _ != true_answer_location[batchid] and \
                   _ not in error_sample_ids[batchid]:
                  continue
                # logger.debug('{0:.5f} {1:.5f} {2:.5f} | {3}'.format(s1, s2, s3, u' '.join(na)))
                logger.debug('{0:.5f} | {3}'.format(s1, u' '.join(na)))

              for _, at_n in enumerate(at_n_list):
                # logger.debug('MAP at {0}: '.format(at_n) +
                #              'candidate {0:.5f}; pairwise {1:.5f}; semantic {2:.5f}; ensemble {3:.5f}'.format(
                #                  candidate_MAPs[batchid][_], pairwise_MAPs[batchid][_],
                #                  semantic_MAPs[batchid][_], ensemble_MAPs[batchid][_]))
                logger.debug('MAP at {0}: '.format(at_n) +
                             'candidate {0:.5f}; pairwise {1:.5f}'.format(
                                 candidate_MAPs[batchid][_], pairwise_MAPs[batchid][_])),

          except UnicodeError:
            excinfo = sys.exc_info()
            logger.debug('Unicode error: {}'.format(excinfo[1]))
          except ValueError:
            excinfo = sys.exc_info()
            traceback.print_exception(*excinfo)
            pdb.set_trace()
          except IndexError:
            excinfo = sys.exc_info()
            traceback.print_exception(*excinfo)
            pdb.set_trace()
          except:
            traceback.print_exc()
            pdb.set_trace()

      timer.append(time.time() - start_time)

def run_epoch(session,
              model,
              data_handle,  # return a data generator
              display_sentence=None,
              get_label=None,
              is_training=False,
              query_save_file=None,
              summary_writer=None,
              minibatch_max_num=None,
              eval_rerank=False,
              at_n_list=None):
  """Runs the model on the given data."""
  start_time = time.time()
  stats = {"batch": 0.0,
           "l2_regu_losses": 0.0,
           "question_focus_losses": 0.0,
           "question_focus_prec": 0.0,
           "question_focus_reca": 0.0,
           "question_focus_f1": 0.0,
           #
           "focus_match_losses": 0.0,
           "pairwise_match_losses": 0.0,
           "semantic_match_losses": 0.0,
           "answer_focus_classify_accuracy": 0.0,
           "answer_pairwise_classify_accuracy": 0.0,
           "answer_semantic_classify_accuracy": 0.0,
           "answer_ensemble_classify_accuracy": 0.0}
  if eval_rerank:
    assert at_n_list
    for at_n in at_n_list:
      stats["candidate_MAP_at_{}".format(at_n)] = []
      stats["pairwise_match_MAP_at_{}".format(at_n)] = []
      stats["semantic_match_MAP_at_{}".format(at_n)] = []
      stats["ensemble_match_MAP_at_{}".format(at_n)] = []
      stats["candidate_MAP_at_{}_all".format(at_n)] = []
      stats["pairwise_match_MAP_at_{}_all".format(at_n)] = []
      stats["semantic_match_MAP_at_{}_all".format(at_n)] = []
      stats["ensemble_match_MAP_at_{}_all".format(at_n)] = []
      stats["pairwise_MAP_better_cnt_at_{}".format(at_n)] = 0.0
      stats["pairwise_MAP_worse_cnt_at_{}".format(at_n)] = 0.0
      stats["pairwise_MAP_equal_cnt_at_{}".format(at_n)] = 0.0
      stats["semantic_MAP_better_cnt_at_{}".format(at_n)] = 0.0
      stats["semantic_MAP_worse_cnt_at_{}".format(at_n)] = 0.0
      stats["semantic_MAP_equal_cnt_at_{}".format(at_n)] = 0.0
      stats["ensemble_MAP_better_cnt_at_{}".format(at_n)] = 0.0
      stats["ensemble_MAP_worse_cnt_at_{}".format(at_n)] = 0.0
      stats["ensemble_MAP_equal_cnt_at_{}".format(at_n)] = 0.0
      stats["MAP_na_cnt_at_{}".format(at_n)] = 0.0

  run_graph_timer, eval_stat_timers, eval_stat_qsizes = [], [], []

  # Run tensorflow graph in a separate thread
  # if display_sentence is None or get_label is None:
  if display_sentence is None:
    print_qualitative = False
  else:
    print_qualitative = True
  run_graph_thread = threading.Thread(
      target=run_graph,
      args=(session,
            model,
            data_handle,
            run_graph_timer,
            is_training,
            print_qualitative,
            minibatch_max_num
            ))
  run_graph_thread.start()

  # Evaluation in other threads
  if display_sentence is not None:  # print qualitative results
    eval_thread_num = 1
  else:
    eval_thread_num = 4
  eval_stat_threads = []
  for _ in xrange(eval_thread_num):
    eval_stat_timers.append([])
    eval_stat_qsizes.append([])
    eval_stat_threads.append(threading.Thread(
        target=eval_stat,
        args=(stats,
              display_sentence,
              get_label,
              eval_stat_timers[_],
              eval_stat_qsizes[_],
              summary_writer,
              query_save_file,
              eval_rerank,
              at_n_list)))
  for eval_stat_thread in eval_stat_threads:
    eval_stat_thread.start()

  # Threading barrier
  run_graph_thread.join()
  for eval_stat_thread in eval_stat_threads:
    eval_stat_thread.join()
  assert (rslt_queue.qsize() == 1)
  rslt = rslt_queue.get()
  assert (rslt == "DATA_END")

  eval_stat_timer = [t for est in eval_stat_timers for t in est]
  eval_stat_qsize = [q for esq in eval_stat_qsizes for q in esq]

  logger.info("Time : %.0f seconds" % (time.time() - start_time))
  assert (len(run_graph_timer) == len(eval_stat_timer))
  logger.info("Time run graph : %f" % (np.mean(run_graph_timer)))
  logger.info("Time eval stat : %f, qsize : %.3f" % (np.mean(eval_stat_timer), np.mean(eval_stat_qsize)))
  rslt = {
      "l2_regu_loss": (stats["l2_regu_losses"] / stats["batch"]),
      "question_focus_loss": (stats["question_focus_losses"] / stats["batch"]),
      "question_focus_prec": (stats["question_focus_prec"] / stats["batch"]),
      "question_focus_reca": (stats["question_focus_reca"] / stats["batch"]),
      "question_focus_f1": (stats["question_focus_f1"] / stats["batch"]),
      #
      "focus_match_loss": (stats["focus_match_losses"] / stats["batch"]),
      "pairwise_match_loss": (stats["pairwise_match_losses"] / stats["batch"]),
      "semantic_match_loss": (stats["semantic_match_losses"] / stats["batch"]),
      "answer_focus_classify_accuracy": (stats["answer_focus_classify_accuracy"] / stats["batch"]),
      "answer_pairwise_classify_accuracy": (stats["answer_pairwise_classify_accuracy"] / stats["batch"]),
      "answer_semantic_classify_accuracy": (stats["answer_semantic_classify_accuracy"] / stats["batch"]),
      "answer_ensemble_classify_accuracy": (stats["answer_ensemble_classify_accuracy"] / stats["batch"])}

  # The following metrics evaluated how the reranking answer candidates improve the final results
  if eval_rerank:
    for at_n in at_n_list:
      rslt["candidate_MAP_at_{}".format(at_n)] = np.mean(stats["candidate_MAP_at_{}".format(at_n)])
      rslt["pairwise_match_MAP_at_{}".format(at_n)] = np.mean(stats["pairwise_match_MAP_at_{}".format(at_n)])
      rslt["semantic_match_MAP_at_{}".format(at_n)] = np.mean(stats["semantic_match_MAP_at_{}".format(at_n)])
      rslt["ensemble_match_MAP_at_{}".format(at_n)] = np.mean(stats["ensemble_match_MAP_at_{}".format(at_n)])
      rslt["candidate_MAP_at_{}_all".format(at_n)] = np.mean(stats["candidate_MAP_at_{}_all".format(at_n)])
      rslt["pairwise_match_MAP_at_{}_all".format(at_n)] = np.mean(stats["pairwise_match_MAP_at_{}_all".format(at_n)])
      rslt["semantic_match_MAP_at_{}_all".format(at_n)] = np.mean(stats["semantic_match_MAP_at_{}_all".format(at_n)])
      rslt["ensemble_match_MAP_at_{}_all".format(at_n)] = np.mean(stats["ensemble_match_MAP_at_{}_all".format(at_n)])
      rslt["pairwise_MAP_better_cnt_at_{}".format(at_n)] = stats["pairwise_MAP_better_cnt_at_{}".format(at_n)]
      rslt["pairwise_MAP_worse_cnt_at_{}".format(at_n)] = stats["pairwise_MAP_worse_cnt_at_{}".format(at_n)]
      rslt["pairwise_MAP_equal_cnt_at_{}".format(at_n)] = stats["pairwise_MAP_equal_cnt_at_{}".format(at_n)]
      rslt["semantic_MAP_better_cnt_at_{}".format(at_n)] = stats["semantic_MAP_better_cnt_at_{}".format(at_n)]
      rslt["semantic_MAP_worse_cnt_at_{}".format(at_n)] = stats["semantic_MAP_worse_cnt_at_{}".format(at_n)]
      rslt["semantic_MAP_equal_cnt_at_{}".format(at_n)] = stats["semantic_MAP_equal_cnt_at_{}".format(at_n)]
      rslt["ensemble_MAP_better_cnt_at_{}".format(at_n)] = stats["ensemble_MAP_better_cnt_at_{}".format(at_n)]
      rslt["ensemble_MAP_worse_cnt_at_{}".format(at_n)] = stats["ensemble_MAP_worse_cnt_at_{}".format(at_n)]
      rslt["ensemble_MAP_equal_cnt_at_{}".format(at_n)] = stats["ensemble_MAP_equal_cnt_at_{}".format(at_n)]
      rslt["MAP_na_cnt_at_{}".format(at_n)] = stats["MAP_na_cnt_at_{}".format(at_n)]

  return rslt


"""
pho.py
A collection of little useful screen scraping utilities.
"""
from functools import wraps
from time import sleep, time
from random import Random, seed
import requests
from bs4 import BeautifulSoup

def randomsleep(func):
    """
    Intentionally delays return of a function for a random number of seconds.
    :param func:
    :return:
    """

    @wraps(func)
    def wrapped_func(*args):
        result = func(*args)
        random = Random()
        random.seed(time())
        MINTIME, MAXTIME = 3, 8
        sleep_time = random.randrange(MINTIME, MAXTIME)
        print("CALLED {0} WITH ARGS {1} SLEEPING {2} seconds".format(func.__name__, str(args), sleep_time))
        sleep(sleep_time)
        return result
    return wrapped_func

def download_html_files(list_of_links, target_folder):
    """
    Downloads the files associated with a list of links to
    a local folder.
    note: There is an intentional random delay between downloads so as not to do rapid fire hammering of the
    target server.
    :param list_of_links: URL addresses to the intended files.
    :param target_folder: The path to the local folder to save files to
    :return: Nothing.
    """
    import re

    def _setup_file_path(lnk):

        new_file_name = lnk.split("/")[-1]
        file_extension_match = r"\.\w+"
        if re.search(file_extension_match, lnk ):
            new_file_name = re.sub(file_extension_match, "", new_file_name)

        new_file_name += ".html"

        return "{0}/{1}".format(target_folder, new_file_name)

    @randomsleep
    def _download(lnk, target_path):
        import requests
        import os

        res = requests.get(lnk)
        raw_data = res.text
        file_mode = "wt" if os.path.exists(target_folder) else "xt"
        with open(target_path, file_mode, encoding="utf-8") as file_ob:
            file_ob.write(raw_data)

    for link in list_of_links:
        targ_path = _setup_file_path(link)
        _download(link, targ_path)

def make_soup(fname):
    """
    Create a simple beautiful soup object out of the filename
    """
    if fname.startswith("http"):
        return BeautifulSoup(requests.get(fname).text.encode("utf-8"))
    with open(fname, "rt", encoding="utf-8") as file_ob:
        soup = BeautifulSoup(file_ob)
        return soup

def soup_line(dir_name, *exclusions):
    """
     Pair up soups with the files they're based on.
    :param dir_name: Directory with the html files needed.
    :param exclusions: Don't include these files.
    :return: A tuple of soup file name pairs.
    """
    import os

    from collections import namedtuple
    SoupFilePair = namedtuple("SoupFileName", ["soup", "file_name"])

    def is_valid_file(fpath):
        if os.path.isfile(fpath):
            if fpath.endswith(".html"):
                if fpath not in exclusions:
                    return True
        else:
            return False

    for fname in os.listdir(dir_name):
        file_path = "{0}/{1}".format(dir_name, fname)
        if is_valid_file(file_path):
            with open(file_path, "rt", encoding="utf-8") as file_ob:
                soup = BeautifulSoup(file_ob.read().strip())
                yield SoupFilePair(soup, fname)

if __name__ == '__main__':
    ###########################
    # TEST RUN CODE           #
    ##########################
    """
    download_html_files downloads a list of files based on urls specified in a list.
    It's set up with a random sleeper timer so not to alarm server admins and getting your ip address blocked.
    """
    target_urls =  ["http://www.reddit.com/r/python", "https://news.ycombinator.com"]
    download_html_files(target_urls, "sources")

    """
    soup_line is a generator function that makes Beautiful soup objects out of files in the directory specified.
    Each turn of the generator brings a new soup object along with the name of the file used to create it.
    The user can include files to NOT include.  For example, maybe you want category web pages but NOT the original
    category page they're all associated with.
    """
    for soup, file_name in soup_line("sources"):
        print(file_name, soup("title"))

    """
    The make_soup utility is a convenience function that can work with either a local file or a url.
    """
    second_soup = make_soup("sources/python.html")
    print(second_soup("title"))
    third_soup = make_soup("http://httpbin.org")
    print(third_soup("title"))


from ursina import *
from ursina.prefabs.first_person_controller import FirstPersonController

oyna = Ursina()
osmon_texture = load_texture('assets/skybox.png')

class Voxel(Button):
    def init(self, position=(0, 0, 0)):
        super().init__(
            position=position,
            model='cube',
            origin_y=0.5,
            texture='white_cube',
            color=color.color(0, 0, random.uniform(0.9, 1)),
            scale=0.5,
        )

    def input(self, key):
        if self.hovered:
            if key == 'left mouse down':
                voxel = Voxel(position=self.position + mouse.normal)
        elif key == 'right mouse down':
            destroy(self)

        def init(self):
            super().init(
                parent=scene)

class osmon(Entity):
        model = 'sphere'
        texture = osmon_texture
        double_sided = True

for x in range(20):
    for z in range(20):
        voxel = Voxel(position=(x, 0, z))

scale = 150
oyinchi = FirstPersonController()
osmon = osmon()
oyna.run()


import asyncio
from collections import defaultdict
from typing import Awaitable, Optional
from nonebot import on_message, on_request, get_bot, logger, get_driver
from nonebot.typing import T_State
from nonebot.rule import keyword, to_me, Rule
from nonebot.adapters import Bot, Event
from nonebot.adapters.onebot.v11 import GroupMessageEvent, GroupRequestEvent
from nonebot.adapters.onebot.v11 import MessageSegment, Message, permission, GroupMessageEvent
from nonebot.permission import Permission
from src.common.config import BotConfig, GroupConfig

import random
import time
# from .pseudorandom import roulette_randomizer

roulette_status = defaultdict(int)  # 0 关闭 1 开启
roulette_time = defaultdict(int)
roulette_count = defaultdict(int)
timeout = 300
roulette_player = defaultdict(list)
role_cache = defaultdict(lambda: defaultdict(str))

async def am_I_admin(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    info = await get_bot(str(event.self_id)).call_api('get_group_member_info', **{
        'user_id': event.self_id,
        'group_id': event.group_id
    })
    role = info['role']
    role_cache[event.self_id][event.group_id] = role
    return role == 'admin' or role == 'owner'

async def am_I_admin_by_cache(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    role = role_cache[event.self_id][event.group_id]
    return role == 'admin' or role == 'owner'

def can_roulette_start(group_id: int) -> bool:
    if roulette_status[group_id] == 0 or time.time() - roulette_time[group_id] > timeout:
        return True

    return False

async def participate_in_roulette(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    '''
    牛牛自己是否参与轮盘
    '''
    if BotConfig(event.self_id, event.group_id).drunkenness() <= 0:
        return False

    if GroupConfig(event.group_id).roulette_mode() == 1:
        # 没法禁言自己
        return False

    # 群主退不了群（除非解散），所以群主牛牛不参与游戏
    if role_cache[event.self_id][event.group_id] == 'owner':
        return False

    return random.random() < 0.1667

async def roulette(messagae_handle, bot: Bot, event: GroupMessageEvent, state: T_State):
    rand = random.randint(1, 6)
    logger.info('Roulette rand: {}'.format(rand))
    roulette_status[event.group_id] = rand
    roulette_count[event.group_id] = 0
    roulette_time[event.group_id] = time.time()
    partin = await participate_in_roulette(bot, event, state)
    if partin:
        roulette_player[event.group_id] = [event.self_id, event.user_id, ]
    else:
        roulette_player[event.group_id] = [event.user_id, ]

    mode = GroupConfig(event.group_id).roulette_mode()
    if mode == 0:
        type_msg = '踢出群聊'
    elif mode == 1:
        type_msg = '禁言'
    await messagae_handle.finish(
        f'这是一把充满荣耀与死亡的左轮手枪，六个弹槽只有一颗子弹，中弹的那个人将会被{type_msg}。勇敢的战士们啊，扣动你们的扳机吧！')

async def is_roulette_type_msg(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    if event.get_plaintext().strip() in ['牛牛轮盘踢人', '牛牛轮盘禁言', '牛牛踢人轮盘', '牛牛禁言轮盘']:
        if can_roulette_start(event.group_id):
            admin = await am_I_admin(bot, event, state)
            return admin
    return False

async def is_config_admin(event: GroupMessageEvent) -> bool:
    return BotConfig(event.self_id).is_admin_of_bot(event.user_id)

IsAdmin = permission.GROUP_OWNER | permission.GROUP_ADMIN | Permission(
    is_config_admin)

roulette_type_msg = on_message(
    priority=5,
    block=True,
    rule=Rule(is_roulette_type_msg),
    permission=IsAdmin
)

@roulette_type_msg.handle()
async def _(bot: Bot, event: GroupMessageEvent, state: T_State):
    plaintext = event.get_plaintext().strip()
    if '踢人' in plaintext:
        mode = 0
    elif '禁言' in plaintext:
        mode = 1
    GroupConfig(event.group_id).set_roulette_mode(mode)

    await roulette(roulette_type_msg, bot, event, state)

async def is_roulette_msg(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    if event.get_plaintext().strip() in ['牛牛轮盘']:
        if can_roulette_start(event.group_id):
            admin = await am_I_admin(bot, event, state)
            return admin

    return False

roulette_msg = on_message(
    priority=5,
    block=True,
    rule=Rule(is_roulette_msg),
    permission=permission.GROUP
)

@roulette_msg.handle()
async def _(bot: Bot, event: GroupMessageEvent, state: T_State):
    await roulette(roulette_msg, bot, event, state)

async def is_shot_msg(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    if roulette_status[event.group_id] != 0 and event.get_plaintext().strip() == '牛牛开枪':
        admin = await am_I_admin_by_cache(bot, event, state)
        return admin

    return False

kicked_users = defaultdict(set)

async def shot(self_id: int, user_id: int, group_id: int) -> Optional[Awaitable[None]]:
    mode = GroupConfig(group_id).roulette_mode()
    self_role = role_cache[self_id][group_id]

    if self_id == user_id:
        if mode == 0:   # 踢人
            if self_role == 'owner':      # 牛牛是群主不能退群，不然群就解散了
                return None

            async def group_leave():
                await get_bot(str(self_id)).call_api('set_group_leave', **{
                    'group_id': group_id
                })
            return group_leave
        elif mode == 1:   # 牛牛没法禁言自己
            return None

    user_info = await get_bot(str(self_id)).call_api('get_group_member_info', **{
        'user_id': user_id,
        'group_id': group_id
    })
    user_role = user_info['role']

    if user_role == "owner":
        return None
    elif user_role == "admin" and self_role != "owner":
        return None

    if mode == 0:   # 踢人
        async def group_kick():
            kicked_users[group_id].add(user_id)
            await get_bot(str(self_id)).call_api('set_group_kick', **{
                'user_id': user_id,
                'group_id': group_id
            })
        return group_kick

    elif mode == 1:           # 禁言
        async def group_ban():
            await get_bot(str(self_id)).call_api('set_group_ban', **{
                'user_id': user_id,
                'group_id': group_id,
                'duration': random.randint(5, 20) * 60
            })
        return group_ban

shot_msg = on_message(
    priority=5,
    block=True,
    rule=Rule(is_shot_msg),
    permission=permission.GROUP
)

shot_text = [
    '无需退路。',
    '英雄们啊，为这最强大的信念，请站在我们这边。',
    '颤抖吧，在真正的勇敢面前。',
    '哭嚎吧，为你们不堪一击的信念。',
    '现在可没有后悔的余地了。',
    "你将在此跪拜。"]

@shot_msg.handle()
async def _(bot: Bot, event: GroupMessageEvent, state: T_State):
    roulette_status[event.group_id] -= 1
    roulette_count[event.group_id] += 1
    shot_msg_count = roulette_count[event.group_id]
    roulette_time[event.group_id] = time.time()
    roulette_player[event.group_id].append(event.user_id)

    if shot_msg_count == 6 and random.random() < 0.125:
        roulette_status[event.group_id] = 0
        roulette_player[event.group_id] = []
        await roulette_msg.finish('我的手中的这把武器，找了无数工匠都难以修缮如新。不......不该如此......')

    elif roulette_status[event.group_id] > 0:
        await roulette_msg.finish(shot_text[shot_msg_count - 1] + f"( {shot_msg_count} / 6 )")

    roulette_status[event.group_id] = 0

    async def let_the_bullets_fly():
        await asyncio.sleep(random.randint(5, 20))

    if BotConfig(event.self_id, event.group_id).drunkenness() <= 0:
        roulette_player[event.group_id] = []
        shot_awaitable = await shot(event.self_id, event.user_id, event.group_id)
        if shot_awaitable:
            reply_msg = MessageSegment.text('米诺斯英雄们的故事......有喜剧，便也会有悲剧。舍弃了荣耀，') + MessageSegment.at(
                event.user_id) + MessageSegment.text('选择回归平凡......')
            await roulette_msg.send(reply_msg)
            await let_the_bullets_fly()
            await shot_awaitable()
        else:
            reply_msg = '听啊，悲鸣停止了。这是幸福的和平到来前的宁静。'
            await roulette_msg.finish(reply_msg)

    else:
        player = roulette_player[event.group_id]
        rand_list = player[-random.randint(1, min(len(player), 6)):][::-1]
        roulette_player[event.group_id] = []
        shot_awaitable_list = []
        for user_id in rand_list:
            shot_awaitable = await shot(event.self_id, user_id, event.group_id)
            if not shot_awaitable:
                continue

            shot_awaitable_list.append(shot_awaitable)

            reply_msg = MessageSegment.text('米诺斯英雄们的故事......有喜剧，便也会有悲剧。舍弃了荣耀，') + MessageSegment.at(
                user_id) + MessageSegment.text(f'选择回归平凡...... ( {len(shot_awaitable_list)} / 6 )')
            await roulette_msg.send(reply_msg)

        if not shot_awaitable_list:
            return

        await let_the_bullets_fly()
        for shot_awaitable in shot_awaitable_list:
            await shot_awaitable()

request_cmd = on_request(
    priority=15,
    block=False,
)

@request_cmd.handle()
async def _(bot: Bot, event: GroupRequestEvent, state: T_State):
    if event.sub_type == 'add' and event.user_id in kicked_users[event.group_id]:
        kicked_users[event.group_id].remove(event.user_id)
        await event.approve(bot)

async def is_drink_msg(bot: Bot, event: GroupMessageEvent, state: T_State) -> bool:
    if roulette_status[event.group_id] != 0 and event.get_plaintext().strip() in ['牛牛喝酒', '牛牛干杯', '牛牛继续喝']:
        admin = await am_I_admin_by_cache(bot, event, state)
        return admin

    return False

drink_msg = on_message(
    priority=4,
    block=False,
    rule=Rule(is_drink_msg),
    permission=permission.GROUP
)

@drink_msg.handle()
async def _(bot: Bot, event: GroupMessageEvent, state: T_State):
    roulette_player[event.group_id].append(event.user_id)


import os
import cv2
import numpy as np
from imgaug import augmenters as iaa

_rotation_plus = iaa.Affine(rotate=3)
_rotation_minus = iaa.Affine(rotate=-3)
_translate_plus = iaa.Affine(translate_percent={"x": 0, "y": 0.2})
_translate_minus = iaa.Affine(translate_percent={"x": 0, "y": -0.2})
_scale_up = iaa.Affine(scale=1.02, order=[0, 1])
_scale_down = iaa.Affine(scale=0.8, order=[0, 1])

_brightness = iaa.Multiply(0.6)
_dropout = iaa.Dropout(p=0.09, per_channel=True)
_gaussian_noise = iaa.AdditiveGaussianNoise(scale=30, per_channel=True)
_gaussian_blur = iaa.GaussianBlur(sigma=(1.5, 1.6))
_pepper = iaa.Pepper(0.05)
_hue_and_saturation = iaa.AddToHueAndSaturation((-25, 15))
_contrast = iaa.ContrastNormalization((0.4, 0.7))

def _apply_perspective(img):
    rows, cols, ch = img.shape

    param = 0.10 * rows

    pts1 = np.float32([[0, 0], [cols, 0], [cols, rows], [0, rows]])
    pts2 = np.float32([[param, param], [cols - param, param], [cols, rows - param], [0, rows - param]])

    M = cv2.getPerspectiveTransform(pts1, pts2)
    dst = cv2.warpPerspective(img, M, (cols, rows))

    return dst

_transformations = [
    iaa.Noop(),

    iaa.OneOf([
        _rotation_plus,
        _rotation_minus,
        _gaussian_blur,
        _brightness

    ]),

    #_translate_plus,

    #iaa.Sequential([
    #    _gaussian_noise,
    #
    #]),

    #_brightness,

    #iaa.Sequential([
    #    _hue_and_saturation,
    #    iaa.Sometimes(0.8, _gaussian_noise)
    #]),
    #iaa.Sequential([
    #    iaa.Sometimes(0.6, _scale_down),
    #    _translate_minus
    #], random_order=False),

    #iaa.Sequential([
    #    iaa.Sometimes(0.6, _scale_up),
    #    _translate_minus
    #], random_order=False),

    #iaa.Sequential([
    #    _contrast,
    #    iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=1.2))
    #])
]

def _augment_image(img, results):
    for t in _transformations:
        if isinstance(t, iaa.Augmenter):
            results.append(t.augment_image(img))
        else:
            results.append(t(img))

def resize_with_padding(img, size):
    width, height = img.shape[1], img.shape[0]

    target_h, target_w = size

    if width > height:
        ratio = float(target_w) / float(width)

        if height * ratio <= target_h:
            padding = int((target_h - ratio * height) / 2) + 1
            top = padding
            bottom = padding
            left = 0
            right = 0
            shape = (target_w, int(ratio * height))
            offset = (0, padding)
        else:
            ratio = float(target_h) / float(height)
            padding = int((target_w - ratio * width) / 2) + 1
            top = 0
            bottom = 0
            left = padding
            right = padding
            shape = (int(width * ratio), target_h)
            offset = (padding, 0)

    else:
        ratio = float(target_h) / float(height)

        if width * ratio <= target_w:
            padding = int((target_w - ratio * width) / 2) + 1
            top = 0
            bottom = 0
            left = padding
            right = padding
            shape = (int(ratio * width), target_h,)
            offset = (padding, 0)
        else:
            ratio = float(target_w) / float(width)
            padding = int((target_h - ratio * height) / 2) + 1
            top = padding
            bottom = padding
            left = 0
            right = 0
            shape = (target_w, int(ratio * height))
            offset = (padding, 0)

    resized_img = cv2.resize(img, dsize=shape, interpolation=cv2.INTER_AREA)
    padded = cv2.copyMakeBorder(resized_img, top, bottom, left, right, cv2.BORDER_CONSTANT)

    padded = padded[0:target_h, 0:target_w, :]

    return padded, ratio, offset

def document_imgaug( x , y ):

    x_generator = []
    y_generator = []
    # print('Aug generator')
    for i, img_array in enumerate(x):
        #img_array = np.asarray(img_array)
        #img_array = img_array.astype('float32')

        if(img_array.shape[2] == 3):
            img = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
        else:
            img = img_array

        _augment_image(img, x_generator)

        for c in range(0, len(_transformations)):
            y_generator.append(y[i])
        # print(i)
        # print(i,'  --  ', y[i],'  --  ',len(x_generator))
    # print('Aug generated')

    return np.array(x_generator), np.array(y_generator)