# Used many of the code from: https://github.com/kingosticks/mopidy-tunein/blob/master/mopidy_tunein/tunein.py

try:
    import xml.etree.cElementTree as elementtree
except ImportError:
    import xml.etree.ElementTree as elementtree

import re
import urlparse
from urllib import urlencode
from Helper import Helper
from Factory import StreamModelFactory
from aplayer import Player
from apis import Streams

import json
#import Config

def _make_stream_models(results):

    streams = []
    for item in results:
        stream = StreamModelFactory.createFromTunein(item)
        streams.append(stream)
    return streams

def search(searchterm):
    results = singelton.search(searchterm);
    return _make_stream_models(results)

def categories(categorie):
    results = singelton.categories(categorie)
    return results

def stations(id):
    results = singelton.stations(id)
    return _make_stream_models(results)

def play(item):
    model = StreamModelFactory.createFromStreamJson(item)
    parseUrl = True
    tuneurls = singelton.tune(model.Id, parseUrl)
    if len(tuneurls) > 0:
        model.Stream = tuneurls[0];
        _play(model)

def _play(s):
    #Player.on_media_end = on_media_end
    Player.play_next = on_next
    Player.play_prev = on_prev
    #Player.on_pause = on_pause

    Player.currentlyPlaying = {}
    Player.currentlyPlaying['webpath'] = s.Stream
    Player.currentlyPlaying['name'] = s.Name
    Player.currentlyPlaying['title'] = s.Name
    Player.currentlyPlaying['cover'] = s.Image
    Player.currentlyPlaying['type'] = Player.streamType
    Player.playPath(s.Stream)

def on_next():
    #do nothing
    next = True

def on_prev():
    #do nothing
    prev = True

def save(item):
    json_item = json.loads(item)
    model = StreamModelFactory.createFromStreamJson(json_item)
    tuneurls = singelton.tune(model.Id, True)
    if len(tuneurls) > 0:
        model.Stream = tuneurls[0];
    Streams.writeModelToFile(model)

tuneInUri = 'http://opml.radiotime.com/'
tuneInBrowseUri = tuneInUri + 'Browse.ashx'

def parse_m3u(url):

    urls = []
    m3u_content = Helper.downloadString(url)

    stream = Helper.parsem3u(m3u_content)
    if stream:
        urls.append(stream)
    return urls
    # Copied from mopidy.audio.playlists
    # Mopidy version expects a header but it's not always present
    #for line in data.readlines():
    #    if not line.startswith('#') and line.strip():
    #        yield line.strip()

def parse_pls(url):
    urls = []
    pls_content = Helper.downloadString(url)

    stream = Helper.parsePls(pls_content)
    if stream:
        urls.append(stream)
    return urls

'''
    # Copied from mopidy.audio.playlists
    try:
        print(data)
        cp = configparser.RawConfigParser()
        cp.readfp(data)
    except configparser.Error:
        return

    for section in cp.sections():
        if section.lower() != 'playlist':
            continue
        for i in xrange(cp.getint(section, 'numberofentries')):
            try:
                # TODO: Remove this horrible hack
                if cp.get(section, 'length%d' % (i+1)) == '-1':
                    yield cp.get(section, 'file%d' % (i+1))
            except configparser.NoOptionError:
                yield cp.get(section, 'file%d' % (i+1))
'''

def fix_asf_uri(uri):
    return re.sub(r'http://(.+\?mswmext=\.asf)', r'mms://\1', uri, re.I)

def parse_old_asx(data):
    print("parse old asx")
    try:
        cp = configparser.RawConfigParser()
        cp.readfp(data)
    except configparser.Error:
        return
    for section in cp.sections():
        if section.lower() != 'reference':
            continue
        for option in cp.options(section):
            if option.lower().startswith('ref'):
                uri = cp.get(section, option).lower()
                yield fix_asf_uri(uri)

def parse_new_asx(data):
    print("parse new asx")
    # Copied from mopidy.audio.playlists
    try:
        for event, element in elementtree.iterparse(data):
            element.tag = element.tag.lower()  # normalize
    except elementtree.ParseError:
        return

    for ref in element.findall('entry/ref[@href]'):
        yield fix_asf_uri(ref.get('href', '').strip())

    for entry in element.findall('entry[@href]'):
        yield fix_asf_uri(entry.get('href', '').strip())

def parse_asx(url):
    urls = []
    data = Helper.downloadString(url)

    m = re.search(r'\<[Rr]ef.*\"(http.*)\"', data, re.I)
    url = m.group(1)
    print("Found asx url: " + url)
    urls.append(url)

    return urls

    if b'asx' in data.getvalue()[0:50].lower():
        url = parse_new_asx(data)
        urls.append(url)
    else:
        url = parse_old_asx(data)
        urls.append(url)

    return urls

'''
def find_playlist_parser(extension, content_type):
    extension_map = {'.asx': parse_asx,
                     '.m3u': parse_m3u,
                     '.pls': parse_pls}
    content_type_map = {'video/x-ms-asf': parse_asx,
                        'application/x-mpegurl': parse_m3u,
                        'audio/x-scpls': parse_pls}

    parser = extension_map.get(extension, None)
    if not parser and content_type:
        # Annoying case where the url gave us no hints so try and work it out
        # from the header's content-type instead.
        # This might turn out to be server-specific...
        parser = content_type_map.get(content_type.lower(), None)
    return parser
'''

def _call(url, param=None):
    # print('call radio with path=%s, param=%s', path, param)
    if param:
        param['key'] = yt_key
        url += '?' + urlencode(param)
    print("call tunein with url: " + url)
    response = Helper.downloadString(url)
    json_data = json.loads(response)
    return json_data

class TuneIn(object):
    """Wrapper for the TuneIn API."""

    def __init__(self, timeout):
        self._base_uri = 'http://opml.radiotime.com/%s'
        self._timeout = timeout / 1000.0
        self._stations = {}

    def reload(self):
        self._stations.clear()
        self._tunein.clear()
        self._get_playlist.clear()

    def _flatten(self, data):
        results = []
        for item in data:
            if 'children' in item:
                results.extend(item['children'])
            else:
                results.append(item)
        return results

    def _filter_results(self, data, section_name=None, map_func=None):
        results = []

        def grab_item(item):
            if 'guide_id' not in item:
                return
            if map_func:
                station = map_func(item)
            elif item.get('type', 'link') == 'link':
                results.append(item)
                return
            else:
                station = item
            self._stations[station['guide_id']] = station
            results.append(station)

        for item in data:
            if section_name is not None:
                section_key = item.get('key', '').lower()
                if section_key.startswith(section_name.lower()):
                    for child in item['children']:
                        grab_item(child)
            else:
                grab_item(item)
        return results

    def categories(self, category=''):
        if category == 'location':
            args = '&id=r0'  # Annoying special case
        elif category == 'language':
            args = '&c=lang'
            return []  # TuneIn's API is a mess here, cba
        else:
            args = '&c=' + category

        # Take a copy so we don't modify the cached data
        results = list(self._tunein('Browse.ashx', args))
        if category in ('podcast', 'local'):
            # Flatten the results!
            results = self._filter_results(self._flatten(results))
        elif category == 'trending':
            results = self._filter_results(results)
            results = _make_stream_models(results)
        elif category == '':
            trending = {'text': 'Trending',
                        'key': 'trending',
                        'type': 'link',
                        'URL': self._base_uri % 'Browse.ashx?c=trending'}
            # Filter out the language root category for now
            results = [x for x in results if x['key'] != 'language']
            results.append(trending)
        else:
            results = self._filter_results(results)
        return results

    def locations(self, location):
        args = '&id=' + location
        results = self._tunein('Browse.ashx', args)
        # TODO: Support filters here
        return [x for x in results if x.get('type', '') == 'link']

    def _browse(self, section_name, guide_id):
        args = '&id=' + guide_id
        results = self._tunein('Browse.ashx', args)
        return self._filter_results(results, section_name)

    def featured(self, guide_id):
        return self._browse('Featured', guide_id)

    def local(self, guide_id):
        return self._browse('Local', guide_id)

    def stations(self, guide_id):
        return self._browse('Station', guide_id)

    def related(self, guide_id):
        return self._browse('Related', guide_id)

    def shows(self, guide_id):
        return self._browse('Show', guide_id)

    def episodes(self, guide_id):
        args = '&c=pbrowse&id=' + guide_id
        results = self._tunein('Tune.ashx', args)
        return self._filter_results(results, 'Topic')

    def _map_listing(self, listing):
        # We've already checked 'guide_id' exists
        url_args = 'Tune.ashx?id=%s' % listing['guide_id']
        return {'text': listing.get('name', '???'),
                'guide_id': listing['guide_id'],
                'type': 'audio',
                'subtext': listing.get('slogan', ''),
                'URL': self._base_uri % url_args}

    def _station_info(self, station_id):
        args = '&c=composite&detail=listing&id=' + station_id
        results = self._tunein('Describe.ashx', args)
        listings = self._filter_results(results, 'Listing', self._map_listing)
        if listings:
            return listings[0]

    def parse_stream_url(self, url):
        extension = urlparse.urlparse(url).path[-4:]
        print('Using TuneIn extension parsing: %s , extension %s', \
            url, extension)
        if extension in ['.mp3', '.wma']:
            return [url]  # Catch these easy ones
        elif extension == '.pls':
            return parse_pls(url)
        elif extension == '.m3u':
            return parse_m3u(url)
        #elif extension == '.asx':
        #    return parse_asx(url)
        else:
            return [url]

        # old code from mopidy tunein
        results = []
        playlist, content_type = self._get_playlist(url)
        if playlist:
            parser = find_playlist_parser(extension, content_type)
            if parser:
                print("parse_stream_url " + playlist)
                #playlist_data = StringIO.StringIO(playlist)
                #results = [u for u in parser(playlist_data) if u is not None]

        if not results:
            print(
                'Parsing failure, possibly malformed playlist: %s' % playlist)
        else:
            print('TuneIn found URI: %s', results[0])
        return results

    def tune(self, station_id, parse_url=True):
        print('Tuning station id %s' % station_id)
        args = '&id=' + station_id
        for stream in self._tunein('Tune.ashx', args):
            if 'url' in stream:
                # TODO Cache these playable stream urls?
                if parse_url:
                    return self.parse_stream_url(stream['url'])
                else:
                    return [stream['url']]

        print('Failed to tune station id %s' % station_id)
        return []

    def station(self, station_id):
        if station_id in self._stations:
            station = self._stations[station_id]
        else:
            station = self._station_info(station_id)
            self._stations['station_id'] = station
        return station

    def search(self, query):
        # "Search.ashx?query=" + query + filterVal
        if not query:
            print('Empty search query')
            return []
        print('Searching for "%s"' % query)
        #args = '&query=' + query
        args ={
            "query": query
        }
        args = "&" + urlencode(args)
        search_results = self._tunein('Search.ashx', args)
        results = []
        for item in self._flatten(search_results):
            # use only audio and filter out wma
            # wma is not playable from gstreamer
            if item.get('type', '') == 'audio' \
            and not 'wma' in item.get('formats',''):
                # Only return stations
                self._stations[item['guide_id']] = item
                results.append(item)

        return results

    def _tunein(self, variant, args):
        uri = (self._base_uri % variant) + '?render=json' + args
        print('TuneIn request: %s', uri)
        try:
            data = _call(uri)
            if (data['head']['status'] != '200'):
                print("Error requesting tunein")
            return data['body']
        except Exception as e:
            print('TuneIn request failed: %s', e)
        return {}

    def _get_playlist(self, uri):
        try:
            # Defer downloading the body until know it's not a stream
            response = _call(uri)
            response.raise_for_status()
            content_type = response.headers.get('content-type', 'audio/mpeg')
            content_type = content_type.split(';')[0]
            print('Content type: %s', content_type)
            if content_type == 'audio/mpeg':
                print('Found streaming audio at %s' % uri)
                data = None
            else:
                data = response.content.decode('utf-8', errors='ignore')
            response.close()
            return (data, content_type)
        except Exception as e:
            print('TuneIn playlist request failed: %s', e)
        return (None, None)

singelton = TuneIn(3000)


# @author: Gautam Patel
# Problem Description URL: https://www.hackerrank.com/challenges/super-reduced-string/problem
#!/bin/python3

import math
import os
import random
import re
import sys
from itertools import groupby

# Complete the superReducedString function below.
def superReducedString(s):
    old_s = s
    new_s = ''
    while True:
        new_s = ''
        for k, g in groupby(old_s):
            if len(list(g))%2 == 1:
                new_s += k
        if new_s == old_s:
            break
        old_s = new_s
    return (new_s if len(new_s) > 0 else 'Empty String')

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    s = input()

    result = superReducedString(s)

    fptr.write(result + '\n')

    fptr.close()


import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import pickle

def loadData(filepath, lowerbound, upperbound):

    with open(filepath, 'rb') as file:
        data = pickle.load(file)
        positions = data['positions']
        times = data['time']

        new_positions = np.nan * np.ones((len(positions), 50, 2))
        for i, c in enumerate(positions):
            for j, cc in enumerate(c):
                if cc[1] < lowerbound or cc[1] > upperbound:
                    new_positions[i, j, :] = [np.nan, np.nan]
                else:
                    new_positions[i, j, :] = cc

    return times, new_positions

prepend = 'E:/rec/15cm/1x/'
lowerbound, upperbound = 0, 12.8
a100times, a100pos = loadData('{}1_before_pressure_increase_15cm_1x_light.pickle'.format(prepend), lowerbound, upperbound)
b100times, b100pos = loadData('{}2_pressure_increase_15cm_1x_light.pickle'.format(prepend), lowerbound, upperbound)
c100times, c100pos = loadData('{}3_after_pressure_increase_15cm_1x_light.pickle'.format(prepend), lowerbound, upperbound)
d100times, d100pos = loadData('{}4_pressure_decrease_15cm_1x_light.pickle'.format(prepend),lowerbound, upperbound)
e100times, e100pos = loadData('{}5_after_pressure_decrease_15cm_1x_light.pickle'.format(prepend),lowerbound, upperbound)

def normParams(values):
    return np.nanmean(values), np.nanstd(values)

a100_mean, a100_std = normParams(a100pos[5:3000, :5, 1])
b100_mean, b100_std = normParams(b100pos[5:3000, 1:3, 1])
c100_mean, c100_std = normParams(c100pos[5:3000, :5, 1])
d100_mean, d100_std = normParams(d100pos[5:3000, :3, 1])
e100_mean, e100_std = normParams(e100pos[5:3000, :4, 1])

Recordings = ['before PI', 'during PI', 'after PI', 'during PD', 'after PD']
count = np.arange(len(Recordings))
means = [a100_mean, b100_mean, c100_mean, d100_mean, e100_mean]
std = [a100_std, b100_std, c100_std, d100_std, e100_std]

fig, ax = plt.subplots(figsize=(10,6))

ax.bar(count, means, width=0.5, edgecolor='black', yerr=std, alpha=0.5, ecolor='black', capsize=5)
ax.set_xticks(count)
ax.set_xticklabels(Recordings)
ax.yaxis.grid(True)
ax.set_ylabel('Distance from Water surface', fontsize=12)
ax.set_xlabel('Phase')
ax.set_title('Mean Swimmingheights during Pressure Change at 15cm Waterlevel')

ax.legend()

plt.tight_layout()
plt.show()


import logging
import os

from optparse import make_option

from django.core.management.base import CommandError

from multiprocessing import Pool, cpu_count

from softwarecollections.management.commands import LoggingBaseCommand
from softwarecollections.scls.models import OtherRepo

logger = logging.getLogger(__name__)

def centos_sync(args):
    repo, timeout = args
    logger.info('Syncing {}'.format(repo))
    try:
        repo.sync(timeout)
    except Exception as e:
        logger.error('Failed to sync {}: {}'.format(repo, e))
        return 1
    return 0

class Command(LoggingBaseCommand):
    option_list = LoggingBaseCommand.option_list + (
        make_option(
            '-P', '--max-procs', action='store', dest='max_procs', default=cpu_count(),
            help='Run up to MAX_PROCS processes at a time (default {})'.format(cpu_count())
        ),
        make_option(
            '-t', '--timeout', action='store', dest='timeout', default=None,
            help='Timeout in seconds for each step of centos (repocentos, rpmbuild, createrepo, dump_provides)',
        ),
    )

    help = 'Refresh other repos information.'

    def handle(self, *args, **options):
        self.configure_logging(options['verbosity'])
        timeout = options['timeout'] and int(options['timeout'])
        with Pool(processes=int(options['max_procs'])) as pool:
            errors = sum(pool.map(
                centos_sync,
                [(repo, timeout) for repo in OtherRepo.objects.all()],
            ))
            if errors > 0:
                raise CommandError('Failed to sync other repos: {} error(s)'.format(errors))


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 23 17:28:54 2014

@author: cesar zamora
"""

import numpy as np
from scipy.spatial import Delaunay

n = 100

points = np.zeros((n,2))

for i in range(0,n): #genera n puntos aleatorios dentro del cir de radio 1
    point = np.array([1.,1.])
    while np.linalg.norm(point) >= 1:
        g_point = 2 * np.random.random_sample((1, 2)) - 1
        point = g_point
    points[i] = point

#print points

#points = np.random.rand(n, 2) # 30 points in 2-d
tri = Delaunay(points)

# Make a list of line segments:
# edge_points = [ ((x1_1, y1_1), (x2_1, y2_1)),
#                 ((x1_2, y1_2), (x2_2, y2_2)),5
#                 ... ]
edge_points = []
edges = set()

def add_edge(i, j):
    """Add a line between the i-th and j-th points, if not in the list already"""
    if (i, j) in edges or (j, i) in edges:
        # already added
        return
    edges.add( (i, j) )
    edge_points.append(points[ [i, j] ])

# loop over triangles:
# ia, ib, ic = indices of corner points of the triangle
for ia, ib, ic in tri.vertices:
    add_edge(ia, ib)
    add_edge(ib, ic)
    add_edge(ic, ia)

# plot it: the LineCollection is just a (maybe) faster way to plot lots of
# lines at once
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection

lines = LineCollection(edge_points)
plt.figure()
plt.title('Delaunay triangulation')
plt.gca().add_collection(lines)
plt.plot(points[:,0], points[:,1], 'o', hold=1)
#plt.xlim(-2, 2)
#plt.ylim(-2, 2)

# -- the same stuff for the convex hull

plt.show()


# -*- coding: utf-8 -*-
"""
parrot plugin: Sends out messages to MUC or JID.

"""

__author__ = "Petr Morávek (xificurk@gmail.com)"
__copyright__ = ["Copyright (C) 2007 Kevin Smith",
                 "Copyright (C) 2009-2011 Petr Morávek"]
__license__ = "GPL 3.0"

__version__ = "0.5.0"

import logging

log = logging.getLogger(__name__)
__ = lambda x: x # Fake gettext function

class parrot:
    def __init__(self, bot, config):
        self.send_message = bot.send_message
        self.get_our_nick = bot.get_our_nick
        self.gettext = bot.gettext
        self.ngettext = bot.ngettext

        bot.add_command("say", self.say, __("Send message into MUC"), __("Bot sends the groupchat message into the specified MUC."), __("room@server text"))
        bot.add_command("tell", self.tell, __("Send message to JID"), __("Bot sends the message to the specified JID."), __("user@server text"))

    def say(self, command, args, msg, uc):
        if args.count(" ") >= 1:
            room, text = args.split(" ", 1)
        else:
            return self.gettext("You've forgot to specify the MUC.", uc.lang)
        if self.get_our_nick(room) is None:
            return self.gettext("I'm not in the room {}.", uc.lang).format(room)
        self.send_message(room, text, mtype="groupchat")
        return self.gettext("Sent", uc.lang)

    def tell(self, command, args, msg, uc):
        if args.count(" ") >= 1:
            jid, text = args.split(" ", 1)
        else:
            return self.gettext("You've forgot to specify the JID.", uc.lang)
        self.send_message(jid, text, mtype="chat")
        return self.gettext("Sent", uc.lang)


import json
import os
import requests
import time

from kafka import KafkaProducer

def fetch() -> dict:
    endpoint = 'https://api.coindesk.com/v1/bpi/currentprice.json'
    res = requests.get(endpoint)
    res_json = json.loads(res.text)
    return res_json

def clean_record(raw: dict) -> dict:
    return {
        'time': raw['time']['updated'],
        'currenty': raw['bpi']['USD']['code'],
        'rate': raw['bpi']['USD']['rate_float']
    }

def publish_record(producer: KafkaProducer) -> None:
    res = fetch()
    cleaned_response = clean_record(res)

    producer.send('bpi-raw-stream', cleaned_response)

if __name__ == '__main__':
    KAFKA_BROKER_URL = os.environ.get('KAFKA_BROKER_URL')

    producer = KafkaProducer(
        bootstrap_servers=KAFKA_BROKER_URL,
        value_serializer=lambda x: json.dumps(x).encode('utf-8')
        )

    while True:
        publish_record(producer)
        producer.flush()
        time.sleep(1)


# Verdi GUI is a small tool to interface with a Coherent Verdi laser through Serial
# Verdi GUI is distributed under the MIT Expat License

from PyQt4 import QtCore, QtGui
import os, serial, sys, time
from win32event import CreateMutex
from win32api import CloseHandle, GetLastError
from winerror import ERROR_ALREADY_EXISTS

sys.path.insert(0, './forms')   # tell Python where to look for the next imports

# import the code for all the ui window classes
import icon_qrc
from verdiconnect import Ui_Form as Connect_Form
from verdidoc import Ui_Form as Doc_Form
from verdifault import Ui_Form as Fault_Form
from verdiname import Ui_Form as Name_Form
from verdiwindow import Ui_Form as Window_Form

# Verdi class contains information and methods that directly deal with the laser:
#    1. All methods that handle serial communication (establishing connection, sending, receiving)
#    2. It also contains information obtained from laser status updates
#    3. Finally, it contains static dictionaries of fault error codes and serial commands

class Verdi(object):

    # Fault dictionary, contains laser fault codes as keys and the corresponding fault message as values
    FAULTS = {
        "1": "Laser Head Interlock Fault",
        "2": "External Interlock Fault",
        "3": "PS Cover Interlock Fault",
        "4": "LBO Temperature Fault",
        "5": "LBO Not Locked at Set Temp",
        "6": "Vanadate Temp. Fault",
        "7": "Etalon Temp. Fault",
        "8": "Diode 1 Temp. Fault",
        "10": "Baseplate Temp. Fault",
        "11": "Heatsink 1 Temp. Fault",
        "16": "Diode 1 Over Current Fault",
        "18": "Over Current Fault",
        "19": "Diode 1 Under Volt Fault",
        "21": "Diode 1 Over Volt Fault",
        "25": "Diode 1 EEPROM Fault",
        "27": "Laser Head EEPROM Fault",
        "28": "Power Supply EEPROM Fault",
        "29": "PS-Head Mismatch Fault",
        "31": "Shutter State Mismatch",
        "40": "Head-Diode Mismatch Fault",
        "47": "Vanadate2 Temp Fault"
    }

    # STATS and HOURS tuples contain queries to get the desire laser health stats and operating hours

    STATS = ("C", "D1C", "D2C", "D1HST", "D2HST", "ET", "VT", "BT", "LBOT")

    HOURS = ("HH", "D1H", "D2H")

    def __init__(self):
        object.__init__(self)

        # populate laser initial values. These will change later.
        self.connected = False
        self.baud = 19200
        self.powerValue = 0
        self.fault = 0
        self.shutter = "0"

        # iterate over STATS and HOURS and populate initial values for each laser health stat
        for i in self.STATS:
            exec("".join(["self.", i, "= 'N/A'"]))

        for h in self.HOURS:
            exec("".join(["self.", h, "= 'N/A'"]))

        self.serSetup()        # call method to set up the serial port

    def serSetup(self):
        # Windows ports are named "COM1," "COM2", "COM3, ...
        # iterate through all of these to find a connection
        # once a connection is found, test to see if it is the laser
        # if it is the laser, flag the connection and escape the loop
        for i in range(0, 10):
            self.port = "COM" + str(i)
            try:
                self.ser = serial.Serial(self.port, baudrate=self.baud, timeout=0.2)
            except serial.serialutil.SerialException:
                continue
            else:
                try:
                    # make sure that this is the laser's port
                    replies = ("PRINT SHUTTER", "Verdi>", "1", "0")
                    self.send("PRINT SHUTTER")
                    reply = self.read()
                    if reply in replies:
                        self.connected = True
                        break
                    else:
                        continue
                except:
                    continue

        if self.connected:
            # make sure Verdi is not in ECHO or PROMPT mode as that will mess with reading results
            self.send("ECHO = 0")
            self.send("PROMPT = 0")
            self.ser.flush()            # flush the buffer so there is no old data left

            # now that connection is ready, do the first round of updates
            self.updateStats()
            self.checkPower()
            self.checkShutter()

    def read(self):
        # read 1 line from serial buffer, stripping off termination characters automatically
        try:
            line = self.ser.readline().strip()
            return line
        except serial.serialutil.SerialTimeoutException:
            window.pred("ERROR: Readline timed out.")
        except:
            return "No response received."

    def send(self, cmd):
        # automatically add termination chars if none are given, then send command
        if cmd[-2:] != "\r\n":
            cmd += "\r\n"
        try:
            self.ser.write(cmd)
        except serial.serialutil.SerialTimeoutException:
            window.pred("ERROR: Send command timed out.")
        except serial.serialutil.SerialException as e:
            if "WriteFile failed" in str(e):
                self.connected = False
                window.serialDisconnected()
            else:
                raise

    def updateSer(self):
        # this function is called when serial port, baudrate or timeout are updated
        # program will close pre-existing serial port and open a new one
        try:
            self.ser.close()
        except:
            pass
        finally:
            try:
                self.ser = serial.Serial(self.port, baudrate=self.baud, timeout=0.2)
            except serial.serialutil.SerialException:
                window.pred("Could not connect with these values.")
                self.connected = False
            else:
                self.connected = True

    def updateStats(self):
        # get updated values from the laser for currents and temperatures
        # but first make sure there are no faults
        if self.connected:
            self.checkFault()
            if not self.fault:
                for stat in self.STATS:
                    query = "?" + stat
                    self.send(query)
                    reply = self.read()
                    if reply == "No response received.":
                        reply = "N/A"
                    exec ("".join(["self.", stat, " = reply"]))
                self.checkPower()
            else:
                fault = self.fault
                window.triggerFault(fault, self.FAULTS[fault])

    def getHours(self):
        # get values for laser head, diode 1, diode 2 operating hours
        if self.connected:
            for item in self.HOURS:
                query = "?" + item
                self.send(query)
                reply = self.read()
                if reply == "No response received.":
                    reply = "???"
                exec("".join(["self.", item, " = reply + ' h'"]))

    def checkShutter(self):
        # check if shutter is open or closed
        if self.connected:
            self.send("?S")
            self.shutter = self.read()

    def checkFault(self):
        # query the laser for active faults
        # laser will return "SYSTEM OK" if no faults are active
        self.send("?F")
        fault = self.read()
        if fault == "SYSTEM OK":
            self.fault = 0
        elif fault == "No response received.":
            pass
        else:
            self.fault = fault

    def setPower(self, power):
        # set the laser's power to a new value
        if self.connected:
            power = round(power, 2)
            pstring = str(power)
            cmd = "P = " + pstring
            self.send(cmd)
            self.read()     # discard a line
            verdi.powerValue = power
            # window.pblue(" ".join(["Power has been set to", str(power), "watts."]))

    def checkPower(self):
        # get the laser's set power value
        if self.connected:
            self.send("?SP")
            power = self.read()
            if power != '':
                self.powerValue = float(power)

    def toggleShutter(self, open):
        # open or close the shutter
        if self.connected:
            if open:
                self.send("S = 1")
            else:
                self.send("S = 0")
        self.checkShutter()

# Create the connection pop-up window class

class ConnectWindow(QtGui.QWidget, Connect_Form):
    def __init__(self, parent=None):
        super(ConnectWindow, self).__init__(parent)
        # try to tell Windows to keep this window on top
        QtGui.QMainWindow.__init__(self, None, QtCore.Qt.WindowStaysOnTopHint)
        self.setupUi(self)
        self.resize(200,100)
        self.setWindowIcon(QtGui.QIcon(':/icon/verdigui.ico'))
        # assign handlers for the two buttons
        self.retry_btn.clicked.connect(self.retryConnection)
        self.cancel_btn.clicked.connect(self.cancelConnection)

    def retryConnection(self):
        verdi.serSetup()
        if verdi.connected:
            window.pblue("Verdi GUI is now connected to port: " + verdi.port)
            window.updateAll()
            window.checkShutter()
            window.formatPower()
            window.timer.start()
            self.hide()
        else:
            self.label.setText("Still no connection.")
            self.label_2.setText("Are you sure your computer is connected to the laser?")

    def cancelConnection(self):
        window.warned = True
        window.pblue("Verdi GUI did not establish a connection to a laser.\n"
                 "To try again at a later time, use the command \"-scan\".")
        self.hide()

# Create the documentation pop-up window

class DocWindow(QtGui.QWidget, Doc_Form):
    def __init__(self, parent=None):
        super(DocWindow, self).__init__(parent)
        self.setupUi(self)
        self.setWindowIcon(QtGui.QIcon(':/icon/verdigui.ico'))

# Create the fault pop-up window

class FaultWindow(QtGui.QWidget, Fault_Form):
    def __init__(self, parent=None):
        super(FaultWindow, self).__init__(parent)
        QtGui.QMainWindow.__init__(self, None, QtCore.Qt.WindowStaysOnTopHint)
        self.setupUi(self)
        self.setWindowIcon(QtGui.QIcon(':/icon/verdigui.ico'))
        self.pushButton.clicked.connect(self.clearFault)

    def clearFault(self):
        # tell the laser that the fault is (probably) cleared
        verdi.send("L = 1")
        verdi.read()
        window.timer.start(window.interval)
        window.checkShutter()
        self.hide()

# Create the export pop-up window
# This window request the user's name, then logs laser stats

class NameWindow(QtGui.QWidget, Name_Form):

    def __init__(self, parent=None):
        super(NameWindow, self).__init__(parent)
        self.setupUi(self)
        self.setWindowIcon(QtGui.QIcon(':/icon/verdigui.ico'))

        # assign handlers to the buttons
        self.okay_btn.clicked.connect(self.exportData)
        self.name_box.returnPressed.connect(self.exportData)
        self.cancel_btn.clicked.connect(self.hide)

    def exportData(self):
        # record user's name and log that name, along with all the stats, to the log file
        name = str(self.name_box.text()).strip()
        comment = str(self.comment_box.text()).strip()
        if len(name) == 0:      # catch empty strings
            return
        else:
            verdi.getHours()
            try:
                # try to open the file
                log = open("verdilog.csv")
            except IOError:
                # an IOError would mean that the file does not exist, so create one
                log = open("verdilog.csv", "a")
                cols = ("TIME,LOGGER,POWER,AVG CURRENT,D1 CURRENT,D2 CURRENT,D1 HEATSINK,D2 HEATSINK,"
                        "BASEPLATE,ETALON,VANADATE,LBO,HEAD HRS,D1 HRS, D2 HRS, COMMENT\n")
                log.write(cols)
            else:
                # if file already exists, switch to "append" mode
                log = open("verdilog.csv", "a")
            finally:
                # now that the file definitely exists and is open, format stats and append them to the file
                datetime = time.strftime("%Y-%m-%d %H:%M:%S")
                stats = (datetime, name, str(verdi.powerValue), verdi.C, verdi.D1C, verdi.D2C, verdi.D1HST, verdi.D2HST,
                         verdi.BT, verdi.ET, verdi.VT, verdi.LBOT, verdi.HH, verdi.D1H, verdi.D2H, comment)
                writestring = ",".join(stats)
                try:
                    log.write(writestring + "\n")
                except IOError as e:
                    # an IOError here would mean the program doesn't have permission to write
                    if str(e) == "File not open for writing":
                        window.pred("Verdi GUI doesn't have permission to write to the log.\n"
                                 "Make sure the log file is closed and you have permission to write "
                                 "in the folder in which Verdi GUI is installed.")
                        self.hide()
                        return
                log.close()
                # get Verdi GUI's folder's path
                path = os.path.realpath(sys.argv[0])
                if not os.path.isdir(path):
                    path = os.path.dirname(path)
                window.pblue("".join(["Current stats have been logged to ", path, "\\verdilog.csv"]))
            self.hide()

# Main UI window

class MainWindow(QtGui.QWidget, Window_Form):

    COLORS = {
        "black": "#000000",
        "green": "#508a59",
        "red": "#a83636",
        "blue": "#586890",
    }

    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)

        # resize the window based on user's screen resolution
        screen = app.desktop().screenGeometry()
        width = int(screen.width()*0.22)
        height = int(screen.height()*0.52)
        self.resize(width, height)

        # set a custom icon
        self.setWindowIcon(QtGui.QIcon(':/icon/verdigui.ico'))

        # initialize the Verdi class when the GUI is created
        global verdi
        verdi = Verdi()

        self.assignHandlers()
        self.createTooltips()
        self.updateAll()
        self.checkShutter()
        self.formatPower()

        # initialize the auto-update timer with default interval of 5 seconds
        self.interval = 2000
        self.timer = QtCore.QTimer()
        self.timer.timeout.connect(self.updateAll)
        self.timer.start(self.interval)

        # flag to remember whether the user has already been warned about connection issues
        # so that they are not repeatedly spammed with pop-up warning windows
        self.warned = False

        # say hello to the user
        self.pgreen("Welcome to Verdi GUI! ")
        if verdi.connected:
            self.pgreen("Verdi GUI is connected via Windows port: " + verdi.port)
        else:
            self.serialDisconnected()

    def assignHandlers(self):
        # connect buttons to handler functions so that they actually do something when clicked
        self.shutter_btn.clicked.connect(self.toggleShutter)
        self.export_btn.clicked.connect(self.exportButtonPushed)
        self.openLog_btn.clicked.connect(self.openLog)
        self.doc_btn.clicked.connect(self.openDocumentation)
        self.input_box.returnPressed.connect(self.inputEntered)
        self.power_up.clicked.connect(self.powerUp)
        self.power_down.clicked.connect(self.powerDown)

    def createTooltips(self):
        # add helpful tooltips to everything
        self.power_up.setToolTip("Increase the laser power by 10 mW.")
        self.power_down.setToolTip("Decrease the laser power by 10 mW.")
        self.shutter_btn.setToolTip("Shutter is currently CLOSED. Click to OPEN.")
        self.export_btn.setToolTip("Record currents, temperatures, and operating hours to log file.")
        self.C_box.setToolTip("Average measured current, in amperes, of the two diodes.")
        self.D1C_box.setToolTip("Measured current, in amperes, of Diode 1.")
        self.D2C_box.setToolTip("Measured current, in amperes, of Diode 2.")
        self.D1HST_box.setToolTip("Measured temperature, in degrees Celsius, of Diode 1's heatsink.")
        self.D2HST_box.setToolTip("Measured temperature, in degrees Celsius, of Diode 2's heatsink.")
        self.BT_box.setToolTip("Measured temperature, in degrees Celsius, of laser head baseplate.")
        self.ET_box.setToolTip("Measured temperature, in degrees Celsius, of the etalon.")
        self.VT_box.setToolTip("Measured temperature, in degrees Celsius, of the vanadate element.")
        self.LBOT_box.setToolTip("Measured temperature, in degrees Celsius, of the LBO crystal.")
        self.doc_btn.setToolTip("Open the full list of commands.")

    # print colorful messages for better visibility
    def pblack(self, msg):
        msg = "".join(["", msg, ""])
        self.output_box.append(msg)
        self.output_box.moveCursor(QtGui.QTextCursor.End)

    def pgreen(self, msg):
        msg = "".join(["", msg, ""])
        self.output_box.append(msg)
        self.output_box.moveCursor(QtGui.QTextCursor.End)

    def pblue(self, msg):
        msg = "".join(["", msg, ""])
        self.output_box.append(msg)
        self.output_box.moveCursor(QtGui.QTextCursor.End)

    def pred(self, msg):
        msg = "".join(["", msg, ""])
        self.output_box.append(msg)
        self.output_box.moveCursor(QtGui.QTextCursor.End)

    def updateAll(self):
        # query the laser for the latest health stats, then update the display boxes to match
        verdi.updateStats()
        self.checkShutter()
        self.C_box.setText(verdi.C)
        self.D1C_box.setText(verdi.D1C)
        self.D2C_box.setText(verdi.D2C)
        self.D1HST_box.setText(verdi.D1HST)
        self.D2HST_box.setText(verdi.D2HST)
        self.ET_box.setText(verdi.ET)
        self.VT_box.setText(verdi.VT)
        self.BT_box.setText(verdi.BT)
        self.LBOT_box.setText(verdi.LBOT)

    def formatPower(self):
        power = "{:.2f}".format(verdi.powerValue)
        self.power_lcd.display(power)

    def powerUp(self):
        # increase the laser's power by 0.01 W, unless power is already too high
        power = verdi.powerValue + 0.01
        if power <= 5.001:
            verdi.setPower(power)
            self.formatPower()
        else:
            window.pred("Power cannot be higher than 5.00 W.")

    def powerDown(self):
        # decrease the laser's power by 0.01 W, unless power is already too low
        power = verdi.powerValue - 0.01
        if power >= 0.009:
            verdi.setPower(power)
            self.formatPower()
        else:
            window.pred("Power cannot be lower than 0.01 W.")

    def toggleShutter(self):
        # toggle laser's shutter when button is pushed
        if verdi.shutter == "1":
            verdi.toggleShutter(False)
        else:
            verdi.toggleShutter(True)
        self.checkShutter()

    def checkShutter(self):
        # make sure GUI's shutter status button matches laser's real shutter status
        btn = self.shutter_btn
        verdi.checkShutter()
        if verdi.shutter == "1":
            btn.setChecked(True)
            btn.setText("OPEN")
            btn.setToolTip("Shutter is currently OPEN. Click to CLOSE.")
        else:
            btn.setChecked(False)
            btn.setText("CLOSED")
            btn.setToolTip("Shutter is currently CLOSED. Click to OPEN.")

    def exportButtonPushed(self):
        # when the export button is pushed, launch the export window
        self.name_popup = NameWindow()
        self.name_popup.show()

    def openLog(self):
        try:
            os.startfile("verdilog.csv")
        except WindowsError:
            self.pred("No log found. Click \"Export\" to start a log.")

    def openDocumentation(self):
        # when documentation button is pushed, launch the documentation window
        self.doc_popup = DocWindow()
        self.doc_popup.show()

    def inputEntered(self):
        # take whatever the user has typed and parse it for commands
        # give the user help, if they are asking for help
        # if user types a GUI command that beings with a dash (-), figure out what it is
        # all other commands are sent straight to the laser

        cmd = str(self.input_box.text()).strip()
        HELP = ["help", "-", "-help", "HELP", "-HELP", "-h"]

        if len(cmd) == 0:       # catch accidental enter press
            return

        self.pblack("» " + cmd)
        if cmd in HELP:
            self.showHelp()
        elif cmd[0] == "-":
            self.dashCommand(cmd[1:].split(" ", 1))
        else:
            if verdi.connected:
                verdi.send(cmd)
                reply = verdi.read()
                self.pblue("Reply received: " + reply)
        self.input_box.setText("")

    def dashCommand(self, cmd):
        # find out what the user wants and do it
        CMDS = ("port", "baud", "update")
        if cmd[0] == "scan":
            verdi.serSetup()
            if verdi.connected:
                self.pblue("Verdi GUI is now connected to port: " + verdi.port)
                self.warned = False
            else:
                self.pblue("Verdi GUI could not establish a connection to a laser.")
        elif cmd[0] not in CMDS:
            self.pred("Not a value command.")
        else:
            try:
                val = int(cmd[1])       # try to convert the second word to an integer
            except ValueError:
                # if we get a ValueError, that means there is a second word, but it's not a number
                # so we can't do anything with it. In this case, give the user an appropriate error message.
                self.pred("ERROR: New setting must be an integer")
            except IndexError:
                # if we get an IndexError, that means there was only one word in the command, which is fine
                # we can execute that one word command
                if cmd[0] == "port":
                    self.pblue("Current port is: " + str(verdi.port))
                elif cmd[0] == "baud":
                    self.pblue("Current baudrate is: " + str(verdi.baud))
                elif cmd[0] == "update":
                    if verdi.connected:
                        self.updateAll()
                        self.pblue("Updated laser stats.")
                    else:
                        self.pred("Verdi GUI cannot update stats while laser is disconnected.")
            else:
                # if no errors are raised, that means the user issued a valid 2-word command
                # figure out what it is and execute it
                if cmd[0] == "port":
                    verdi.port = "COM" + cmd[1]
                    self.pblue("Switching to port: " + cmd[1])
                elif cmd[0] == "baud":
                    verdi.baud = val
                    self.pblue("Baudrate changed to: " + cmd[1])
                elif cmd[0] == "update":
                    if val > 0:
                        self.interval = val*1000
                        self.timer.setInterval(self.interval)
                        self.pblue(" ".join(["Update frequency changed to", cmd[1], "seconds"]))
                    else:
                        self.pred("Update interval must be an integer greater than 0.")

                if verdi.connected:
                    verdi.updateSer()

        # clear the input box
        self.input_box.setText("")

    def showHelp(self):
        # mini-documentation, accessed in the program by typing 'help', '-h', etc
        helpstring = ("VERDI GUI QUICK COMMANDS REFERENCE:
"
                      "    -scan: sweep COM ports to try and find the laser
"
                      "    -port n: switch to using COM port n
"
                      "    -baud n: switch to baudrate n
"
                      "    -update: force Verdi GUI to fetch new stat values
"
                      "    -update n: change update frequency to every n seconds
"
                      "Commands that do not being with a dash (-) are sent directly to the laser.")
        self.pblue(helpstring)

    def serialDisconnected(self):
        # check warning flag to see if user has already been warned about the lack of connection
        # if not, make the warning pop up window
        self.timer.stop()
        if not self.warned:
            self.connection_popup = ConnectWindow()
            self.connection_popup.show()

    def triggerFault(self, fault, faultstring):
        # oh no there is a fault
        # better make a new window
        self.fault_popup = FaultWindow()
        self.fault_popup.errorCode_label.setText("Error code: " + fault)
        self.fault_popup.errorMsg_label.setText(faultstring)
        self.fault_popup.show()
        self.timer.stop()   # temporarily suspend the update timer

# prevent duplicate windows by registered a mutex
class Singleton:

    def __init__(self):
        self.mutex = CreateMutex(None, False, "Verdi_GUI_Windows_Registered_Mutex")
        self.lasterror = GetLastError()

    def isRunning(self):
        return (self.lasterror == ERROR_ALREADY_EXISTS)

    def __del__(self):
        if self.mutex:
            CloseHandle(self.mutex)

if __name__ == "__main__":
    # check for pre-existing window immediately; kill new window if it is a duplicate
    instance = Singleton()
    if instance.isRunning():
        sys.exit(0)
    else:
        app = QtGui.QApplication(sys.argv)
        window = MainWindow()
        window.show()
        sys.exit(app.exec_())


#!/usr/bin/env python3
from Crypto.Cipher import AES
import requests
import time
import string

def encrypt(payload):
    url = "http://aes.cryptohack.org/ecb_oracle/encrypt/"
    r = requests.get(url + payload + '/')
    return r.json()['ciphertext']

def print_blk(hex_blks, sz):
   for i in range(0, len(hex_blks), sz):
       print(hex_blks[i:i+sz], ' ', end='')
   print()

def bruteforce():
    flag = ''
    total = 32 - 1
    alphabet = '_'+'@'+'}'+string.digits+string.ascii_lowercase+string.ascii_uppercase

    while True:
        payload = '1' * (total-len(flag))
        expected = encrypt(payload.encode().hex())
        print('E', '', end='')
        print_blk(expected, 32)

        for c in alphabet:
            res = encrypt(bytes.hex((payload + flag + c).encode()))
            print(c, '', end='')
            print_blk(res, 32)
            if res[32:64] == expected[32:64]:
                flag += c
                print(flag)
                break
            time.sleep(1)

        if flag.endswith('}'): break

    print(flag)

bruteforce()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

import os
import sys
sys.path.append('../../../..')

import argparse
import math
import cv2
import numpy as np
import csv
import glob
import yaml
import pickle
from multiprocessing import Pool
import shutil
import pickle
import copy
from sklearn.cluster import KMeans

from mysettings import FileManager, DLConfig
import align2d_merge
import align2d_config

import logging
from logging import basicConfig, Formatter, FileHandler, getLogger, DEBUG
# log_format = '[%(asctime)s %(levelname)s] %(name)s: %(message)s'
log_format = '[%(asctime)s %(levelname)s] %(message)s'
formatter = Formatter(log_format)
basicConfig(level=DEBUG, format=log_format, datefmt="%Y-%m-%d %H:%M:%S")
logger = getLogger(__name__)
fh = FileHandler('./log.txt')
fh.setFormatter(formatter)
logger.addHandler(fh)

def on_floor_height(z, floor_height, height):
    return z - floor_height >= 0 and z - floor_height < height

def fetch_on_floor_shots(tra_dict, center, height):
    shot_list = []
    for k in tra_dict:
        if on_floor_height(tra_dict[k][2], center - height/2, height):
            shot_list.append(k)
    return shot_list

def find_floors(tra_dict, height):
    z_list = []
    for k in tra_dict:
        z_list.append(tra_dict[k][2])

    # prepare for clustering
    z_list = np.array(z_list)
    z_list = np.c_[z_list, np.zeros(z_list.shape[0])]    # expand axis

    for clu_num in range(1, 6):
        km = KMeans(n_clusters=clu_num)
        km.fit(z_list)
        centers = km.cluster_centers_[:, 0]
        centers.sort()

        floor_heights = [centers[i + 1] - centers[i] for i in range(clu_num-1)]
        shot_nums = [len(fetch_on_floor_shots(tra_dict, centers[i], height)) for i in range(clu_num)]
        if clu_num == 1:
            c = centers[0]
            floor_candidates = centers
        if clu_num != 1 and all([height > 15 for height in floor_heights]) and all([n > 100 for n in shot_nums]) :
            floor_candidates = []
            floor_candidates = centers
            # print(clu_num, centers, shot_nums)

    floor_candidates.sort()
    floor_candidates = [math.floor(h - height/2 + 0.5) for h in floor_candidates]
    return floor_candidates

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='create score map')
    parser.add_argument('test_num_dir', help='path to data dir to be processed')
    parser.add_argument('-d', '--data', nargs='?', type=str, default='data.yaml', help='load data yaml file(default=data.yaml)')
    parser.add_argument('-f', '--target_floors', nargs='*', type=str, help='target floor names')
    parser.add_argument('-t', '--targets', nargs='*', type=str, help='target data names')
    parser.add_argument('-c', '--config', nargs='?', type=str, default='alignment2d_config.yaml', help='load config yaml file(defalut=alignment2d_config.yaml)')
    # parser.add_argument('-a', '--plot_all', default=False, action='store_true', help='create for all floor(default=False)')
    parser.add_argument('-j', '--process_num', nargs='?', type=int, default='4', help='process number(default=4)')
    # parser.add_argument('-m', '--mask_dir', nargs='?', type=str, default=None, help='path to walkable mask dir')
    # parser.add_argument('--load_align_result', default=False, action='store_true', help='load align result(default=False)')
    # parser.add_argument('--sort_by_z', default=False, action='store_true', help='sort by z(default=False)')
    # parser.add_argument('--merge', default=False, action='store_true', help='merge results(default=False)')
    parser.add_argument('--merge', default=False, action='store_true', help='run_merge(default=False)')
    # parser.add_argument('--eval_voxel', default=False, action='store_true', help='calc score by each voxel. non floor shot cannot be evaluated(20180521). (default=False).')
    args = parser.parse_args()

    run_merge = args.merge

    # set args
    config = align2d_config.Align2dConfig(args)
    # get target pairs
    target_pairs = config.get_target_pairs()
    for dn in target_pairs:
        logger.info('target pairs: {}'.format(dn))
        for floor in target_pairs[dn]:
            logger.info('    {}'.format(floor))

    for data_name in target_pairs:
        distribution_plot_data = {}

        # load trajectory
        tra_fn = config.fm.trajectory_fp(data_name)
        tra_dict = {}
        with open(tra_fn, 'r') as f:
            reader = csv.reader(f)
            for row in reader:
                k = row[0]
                x = int(row[1])
                y = int(row[2])
                z = float(row[4]) * config.pix_per_meter    # z will come in meter
                tra_dict[k] = [x, y, z]
        logger.info('load {} trajectory points from {}'.format(len(tra_dict), tra_fn))

        # calc trajectory size
        minx, miny, minz = 1000000000, 1000000000, 1000000000
        maxx, maxy, maxz = 0, 0, 0
        for shot in tra_dict:
            x, y, z = tra_dict[shot]
            minx, miny, minz = min(minx, x), min(miny, y), min(minz, z)
            maxx, maxy, maxz = max(maxx, x), max(maxy, y), max(maxz, z)
        logger.info('trajectory size = ({}, {}, {})'.format(maxx-minx, maxy-miny, int(math.ceil(maxz-minz))))

        # find floor heights
        floor_heights = find_floors(tra_dict, config.align_voxel_size[2])
        for h in floor_heights:
            logger.info('floor heights: {}'.format(h))

        actual_result = {}

        # calc soore for each floor
        for floor in target_pairs[data_name]:
            target_floor_dir = config.fm.t_floor_dir(data_name, floor)
            if not os.path.exists(target_floor_dir):
                logger.info('target floor not found: {}'.format(target_floor_dir))
                continue
            logger.info('target floor: {}'.format(floor))

            fp_fn = config.fm.floorplan_fp(floor)
            fp_img = cv2.imread(fp_fn)

            # load score
            score_fn_list = config.fm.score_fp_list(data_name, floor)
            score_fn_list.sort()
            score_fn_list = score_fn_list[::config.decimate]
            logger.info('load {} scores'.format(len(score_fn_list)))

            logger.info('create align result...')
            target_tra_dict = {}
            range_x, range_y = None, None
            score_yxz_dict = {}
            floormask_yxz_dict = {}
            score_map_count = None

            # create range_x and range_y
            floor_minx, floor_miny = 1000000000, 1000000000
            floor_maxx, floor_maxy = 0, 0
            ##  get floor mask size
            if config.load_wmask:
                if config.create_mask_from_score:    # create  load mask from score
                    score_fn = score_fn_list[0]
                    with open(score_fn, 'r') as f:
                        reader = csv.reader(f)
                        for row in reader:
                            x = int(row[0])
                            y = int(row[1])
                            floor_minx, floor_miny = min(floor_minx, x), min(floor_miny, y)
                            floor_maxx, floor_maxy = max(floor_maxx, x), max(floor_maxy, y)
                else:    # create mask from mask image
                    mask_fn = config.fm.wmask_fp(floor)
                    mask_img = cv2.imread(mask_fn)
                    mask_img = cv2.cvtColor(mask_img, cv2.COLOR_BGR2GRAY)
                    # mask_img = cv2.dilate(mask_img, np.ones((5, 5), np.uint8), iterations=1)
                    # mask_img = cv2.dilate(mask_img, np.ones((5, 5), np.uint8), iterations=1)

                    mask_map = np.zeros((fp_img.shape[0]//config.align_voxel_size[0], fp_img.shape[1]//config.align_voxel_size[1]), dtype=bool)
                    mask_map_voxel = np.zeros((fp_img.shape[0], fp_img.shape[1]), dtype=bool)
                    for y in range(config.crop_voxel_ratio_y//2, mask_map.shape[0], config.crop_voxel_ratio_y):
                        for x in range(config.crop_voxel_ratio_y//2, mask_map.shape[1], config.crop_voxel_ratio_x):
                            if np.sum(mask_img[y*config.align_voxel_size[1]-config.crop_step//2: y*config.align_voxel_size[1]+config.crop_step//2,
                                    x*config.align_voxel_size[0]-config.crop_step//2: x*config.align_voxel_size[0]+config.crop_step//2] > 0) > 0:
                                mask_map[y-config.crop_voxel_ratio_y//2:y+config.crop_voxel_ratio_y//2, x-config.crop_voxel_ratio_x//2:x+config.crop_voxel_ratio_x//2] = True
                                # mask_map_voxel[y*align_voxel_size[1]-crop_step//2: y*align_voxel_size[1]+crop_step//2, x*align_voxel_size[0]-crop_step//2: x*align_voxel_size[0]+crop_step//2] = True

                    # create range_x and range_y
                    ##  get walkable area mask size
                    mask_i = np.where(mask_map == True)
                    floor_minx, floor_miny = min(mask_i[1]) * config.align_voxel_size[1], min(mask_i[0]) * config.align_voxel_size[0]
                    floor_maxx, floor_maxy = max(mask_i[1]) * config.align_voxel_size[1], max(mask_i[0]) * config.align_voxel_size[0]
                    # create not floor map
                    not_floor = mask_map == False

                    # cv2.imwrite('/mnt/host/share/tmp/{}_mask_map.png'.format(floor), mask_map*255)
                    # cv2.imwrite('/mnt/host/share/tmp/{}_mask_ori.png'.format(floor), mask_img)
                    # cv2.imwrite('/mnt/host/share/tmp/{}_mask.png'.format(floor), mask_map_voxel*255)
                    # cv2.imwrite('/mnt/host/share/tmp/{}_mask.png'.format(floor), cv2.addWeighted(fp_img, 0.5, cv2.cvtColor((mask_map_voxel*255).astype(np.uint8), cv2.COLOR_GRAY2BGR), 0.5, 0.0))
                    # cv2.imwrite('/mnt/host/share/tmp/{}_mask_ori.png'.format(floor), cv2.addWeighted(fp_img, 0.5, cv2.cvtColor(mask_img.astype(np.uint8), cv2.COLOR_GRAY2BGR), 0.5, 0.0))
                    # cv2.imwrite('/mnt/host/share/tmp/{}.png'.format(floor), fp_img)
            else:    # not load wmask
                pass

            floor_minx -= int(config.crop_step/2)
            floor_miny -= int(config.crop_step/2)
            floor_maxx += int(config.crop_step/2)
            floor_maxy += int(config.crop_step/2)
            range_x = list(range(-(maxx-floor_minx), floor_maxx-minx, config.align_step[0]))
            range_y = list(range(-(maxy-floor_miny), floor_maxy-miny, config.align_step[1]))

            logger.info('floor mask size = ({}, {})'.format(floor_maxx - floor_minx, floor_maxy - floor_miny))
            logger.info('range x: {} to {}'.format(range_x[0], range_x[-1]))
            logger.info('range y: {} to {}'.format(range_y[0], range_y[-1]))
            # print(floor_heights)
            logger.info('total num: {}x{}x{}={}'.format(len(range_x), len(range_y), len(floor_heights), len(range_x)*len(range_y)*len(floor_heights)))

            ## load score map from csv
            logger.info('loading score map from csv...')
            score_map_dict = {}
            for score_fn in score_fn_list:
                shot_name = os.path.splitext(os.path.basename(score_fn))[0]+'.png'
                # score_map = np.zeros((fp_img.shape[0], fp_img.shape[1], 1), dtype=float)
                score_map = np.zeros((fp_img.shape[0]//config.align_voxel_size[1], fp_img.shape[1]//config.align_voxel_size[0], 1), dtype=float)

                score_map_voxel = np.zeros((fp_img.shape[0], fp_img.shape[1]), dtype=float)    # for debug
                if config.load_wmask and config.create_mask_from_score:
                    score_map += -1    # for out-range of floormask
                with open(score_fn, 'r') as f:
                    reader = csv.reader(f)
                    for row in reader:
                        x = (int(row[0]) - config.crop_step//2)//config.align_voxel_size[0]    # center to topleft
                        y = (int(row[1]) - config.crop_step//2)//config.align_voxel_size[1]    # center to topleft

                        if config.load_wmask and not config.create_mask_from_score and mask_map[y][x] == False:
                            # score_map[int(y-crop_step/2)//align_voxel_size[1]:int(y+crop_step/2)//align_voxel_size[1],
                            #           int(x-crop_step/2)//align_voxel_size[0]:int(x+crop_step/2)//align_voxel_size[0]] = -1
                            score_map[y: y + config.crop_voxel_ratio_y,
                                    x: x + config.crop_voxel_ratio_x] = -1
                            continue

                        score = float(row[2])
                        # score_map[int(y-crop_step/2):int(y+crop_step/2), int(x-crop_step/2):int(x+crop_step/2)] = score
                        # score_map[int(y-crop_step/2)//align_voxel_size[1]:int(y+crop_step/2)//align_voxel_size[1],
                        #           int(x-crop_step/2)//align_voxel_size[0]:int(x+crop_step/2)//align_voxel_size[0]] = score
                        score_map[y: y + config.crop_voxel_ratio_y,
                                x: x + config.crop_voxel_ratio_x] = score    # center to topleft
                        score_map_voxel[y*config.align_voxel_size[1]:y*config.align_voxel_size[1]+config.crop_step,
                                        x*config.align_voxel_size[0]:x*config.align_voxel_size[0]+config.crop_step] = score

                if config.load_wmask and not config.create_mask_from_score:
                    score_map[not_floor] = -1
                score_map_dict[shot_name] = score_map
                # cv2.imwrite('/mnt/host/share/tmp/{}_score_index.png'.format(shot_name), score_map*255)
                # cv2.imwrite('/mnt/host/share/tmp/{}_score_single.png'.format(shot_name), cv2.addWeighted(fp_img, 0.5, cv2.cvtColor((score_map_voxel*255).astype(np.uint8), cv2.COLOR_GRAY2BGR), 0.5, 0.0))
                # cv2.imwrite('/mnt/host/share/tmp/{}_score.png'.format(shot_name), score_map_voxel*255)

            # create score map of each translation(len(range_y)*len(range_x))
            logger.info('start to create score map of each translation')
            results = [[] for z in range(len(floor_heights))]

            def calc_score(arg):
                tra_y = arg[0]
                tra_x = arg[1]
                result = [None for z in range(len(floor_heights))]

                # count number of shots which hit floor
                hit_shot_list = []
                not_hit_shot_list = []
                for score_fn in score_fn_list:
                    shot_name = os.path.splitext(os.path.basename(score_fn))[0]+'.png'
                    if shot_name not in tra_dict:
                        continue
                    xyz = tra_dict[shot_name]
                    y = (xyz[1] + tra_y) // config.align_voxel_size[1]
                    x = (xyz[0] + tra_x) // config.align_voxel_size[0]
                    if y < 0 or y >= score_map_dict[shot_name].shape[0] or \
                                    x < 0 or x >= score_map_dict[shot_name].shape[1] or \
                                    score_map_dict[shot_name][y][x] < 0:    # out of range
                        not_hit_shot_list.append(shot_name)
                    else:
                        hit_shot_list.append(shot_name)
                if len(hit_shot_list) < config.hit_shot_count_threshold:
                    return result
                # count number of shots which hit each voxel
                # count_yxz = np.zeros((fp_img.shape[0], fp_img.shape[1], len(range_z)), dtype=int)
                count_yxz = np.zeros((fp_img.shape[0]//config.align_voxel_size[1], fp_img.shape[1]//config.align_voxel_size[0], len(floor_heights)), dtype=int)
                hit_voxel_count = [0 for z in range(len(floor_heights))]
                hit_shot_z_list = [[] for z in range(len(floor_heights))]
                for shot_name in hit_shot_list:
                    xyz = tra_dict[shot_name]
                    # y = ((xyz[1] + tra_y)//crop_step) * crop_step
                    # x = ((xyz[0] + tra_x)//crop_step) * crop_step
                    y = (xyz[1] + tra_y)//config.align_voxel_size[1]
                    x = (xyz[0] + tra_x)//config.align_voxel_size[0]

                    for z, floor_height in enumerate(floor_heights):
                        if on_floor_height(xyz[2], floor_height, config.align_voxel_size[2]):
                            # print(xyz[2], floor_height, shot_name)
                            hit_shot_z_list[z].append(shot_name)
                            if count_yxz[y, x, z] == 0:
                                hit_voxel_count[z] += 1
                            count_yxz[y, x, z] += 1

                # sum scores for each voxel
                for z in range(len(floor_heights)):
                    if hit_voxel_count[z] < config.floor_voxel_count_threshold:
                        continue

                    # print('z: ', z, ', len: ', len(hit_shot_z_list[z]), ', hit_shot_len: ', len(hit_shot_list), 'not_hit_shot_len', len(not_hit_shot_list))
                    score = 0.0
                    in_floor = hit_shot_z_list[z]
                    fire = []
                    c = 0
                    for shot_name in hit_shot_z_list[z]:
                        xyz = tra_dict[shot_name]
                        y = (xyz[1] + tra_y) // config.align_voxel_size[1]
                        x = (xyz[0] + tra_x) // config.align_voxel_size[0]
                        s = score_map_dict[shot_name][y][x]
                        if s > config.fire_threshold:
                            fire.append(shot_name)
                        # score += s/float(count_yxz[y][x][z]) if config.eval_voxel else s
                        score += s

                    # add non floor shot score
                    score += config.non_floor_score * len(not_hit_shot_list)

                    # average
                    # score /= float(floor_voxel_count[z]) if config.eval_voxel else float(len(score_fn_list))
                    score /= float(len(score_fn_list))

                    # print(score)

                    if score < config.good_consistency_threshold:
                        continue
                    sub_result = {}
                    sub_result['x'] = int(tra_x)
                    sub_result['y'] = int(tra_y)
                    sub_result['z'] = - int(floor_heights[z])
                    sub_result['score'] = float(score)
                    sub_result['in_floor'] = in_floor
                    sub_result['fire'] = fire
                    result[z] = sub_result
                return result

            # calc score for each

            for tra_y in range_y:
                arg = []
                for tra_x in range_x:
                    arg.append([tra_y, tra_x])
                p = Pool(config.process_num)
                result = p.map_async(calc_score, arg).get(9999999)
                for r in result:
                    for z in range(len(floor_heights)):
                        if r[z] is not None:
                            results[z].append(r[z])
                p.close()

            logger.info('finish to count number of shots 2d!!')

            logger.info('creating result dict to save')
            align_result = []
            for z, floor_height in enumerate(floor_heights):
                results[z].sort(key=lambda x: -x['score'])
                if len(results[z]) == 0:
                    continue
                logger.info('max score in z={:3d} is {:.8f}({}, {})(in {} candidates)'.format(-floor_height, results[z][0]['score'], results[z][0]['x'], results[z][0]['y'], len(results[z])))
                for i, result in enumerate(results[z]):
                    # if i > max_save_znum - 1:
                    #     break
                    align_result.append(result)

            if config.calc_correct:
                # get tra_x, tra_y, tra_z
                tra_floor_fn = config.fm.trajectory_floor_fp(data_name, floor)
                tra_fn = config.fm.trajectory_fp(data_name)    # for getting tra_z
                if not os.path.exists(tra_floor_fn) or not os.path.exists(tra_fn):
                    logger.info('{} does not belong to {}'.format(data_name, floor))
                else:
                    tra_floor_dict = {}
                    c = 0
                    tra_x, tra_y, tra_z = 0, 0, 0
                    #  calc tra_x and tra_y and tra_z
                    with open(tra_floor_fn, 'r') as f:
                        reader = csv.reader(f)
                        for row in reader:
                            k = row[0]
                            tra_x += int(row[1]) - tra_dict[k][0]
                            tra_y += int(row[2]) - tra_dict[k][1]
                            tra_z += float(row[4]) * config.pix_per_meter - tra_dict[k][2]
                            c += 1
                    tra_x /= c
                    tra_y /= c
                    tra_z /= c
                    tra_z += config.align_voxel_size[2]/2
                    tra_x, tra_y, tra_z = int(tra_x), int(tra_y), int(tra_z)
                    # print('actual tranclation ', tra_x, tra_y, tra_z)
                    # tra_z = - range_z[int((- tra_z - range_z[0])//config.align_step[2])]

                    actual = calc_score([tra_y, tra_x])
                    # actual = [tmp for tmp in actual if tmp is not None]
                    for r in actual:
                        if r is not None:
                            logger.debug('actual result cand: {} ({}, {}, {})'.format(r['score'], r['x'], r['y'], r['z']))

                    # get actual z
                    min_diff = 1000000000000000
                    act_z = None
                    for z, floor_height in enumerate(floor_heights):
                        diff = abs(tra_z + floor_height)
                        if diff < min_diff:
                            min_diff = diff
                            act_z = z
                        # print(z, floor_height, diff)
                    if actual[act_z] is None:
                        logger.warning('actual score not found: {} - {}'.format(data_name, floor))
                    else:
                        logger.info('actual score on {}: {:.8f} ({}, {}, {})'.format(floor, actual[act_z]['score'], actual[act_z]['x'], actual[act_z]['y'], actual[act_z]['z']))
                        actual = [actual[act_z]]
                        actual_result[floor] = actual

            data = {'align_result': align_result, 'actual_result': actual_result, 'range_x': range_x, 'range_y': range_y, 'range_z': None, 'floor_heights': floor_heights}
            config.fm.save_align_result(data_name, floor, data)
            logger.info('save align result to {}'.format(config.fm.align_result_fp(data_name, floor)))

        if run_merge:
            logger.info('select and plot candidates: {}'.format(config.fm.align_result_fp(data_name, floor)))
            config_tmp = copy.copy(config)
            config_tmp.targets = [data_name]
            config_tmp.target_floors = target_pairs[data_name]
            align2d_merge.run(config_tmp)


"""Action Handler Module."""
from collections import deque

from ..context import context
from ._action import Action, HandlerAction, ReversibleAction

class ActionHandler:
    """Action Handler Class.

    Implements action execution and Undo/Redo functionality.
    """

    DEFAULT_MAX_UNDO = 100

    def __init__(self, target, max_undo: int = DEFAULT_MAX_UNDO) -> None:
        """Initialize the action handler."""
        self.target = target
        self.max_undo = max_undo
        self.undo_history: deque[ReversibleAction] = deque(maxlen=max_undo)
        self.redo_history: deque[ReversibleAction] = deque(maxlen=max_undo)

    def do(self, action: Action) -> None:  # pylint: disable=invalid-name
        """Process and perform action."""
        action.target = self.target
        action.do()
        if isinstance(action, HandlerAction):
            return
        if isinstance(action, ReversibleAction):
            self.undo_history.append(action)
        context.previous_action = action
        self.redo_history.clear()

    def redo(self) -> None:
        """Redo action."""
        if len(self.redo_history) == 0:
            return
        last_action = self.redo_history.pop()
        last_action.redo()
        self.undo_history.append(last_action)

    def undo(self) -> None:
        """Undo action."""
        if len(self.undo_history) == 0:
            return
        last_action = self.undo_history.pop()
        last_action.undo()
        self.redo_history.append(last_action)


#!/usr/bin/python3

# COMP3311 22T3 Assignment 2
# Print a list of character roles played by an actor/actress

import sys
import psycopg2

### Globals

db = None
usage = f"Usage: {sys.argv[0]} FullName"

### Command-line args

if len(sys.argv) < 2:
   print(usage)
   exit(1)

# process the command-line args ...
name  = sys.argv[1]

### Queries
qry1 = """
   select p.id
   from People p
   where p.name = %s
   order by p.id
   ;
"""

qry2 = """
   select pl.role, m.title, m.year
   from People p
   join Principals pr on pr.person = p.id
   join Movies m on m.id = pr.movie
   join Playsrole pl on pl.inMovie = pr.id
   where (p.id = %s) and (pr.job = 'self' or pr.job = 'actor' or pr.job = 'actress')
   order by m.year, m.title, pl.role
   ;
"""

### Manipulating database
db = psycopg2.connect("dbname=ass2")
cur = db.cursor()

try:
   # your code goes here
   cur.execute(qry1, [name])
   person = cur.fetchall()

   countPerson = 0
   if not person:
      print("No such person")
      exit(0)
   else:
      countPerson = len(person)

   for p in person:
      cur.execute(qry2,person[0])

   roles = cur.fetchall()
   numRoles = len(roles)

   if countPerson == 1 :
      if numRoles == 0:
         print("No acting roles")
      else:
         for r in roles:
            print(f"{r[0]} in {r[1]} ({r[2]})")

   else :
      counter = 1
      for p in person:
         cur.execute(qry2,[p[0]])
         roles = cur.fetchall()
         numRoles = len(roles)
         print(f"{name} #{counter}")
         counter +=1
         if numRoles == 0:
            print(f"No acting roles")
         else:
            for r in roles:
               print(f"{r[0]} in {r[1]} ({(r[2])})")

except Exception as err:
   print("DB error: ", err)
finally:
   if db:
      db.close()


# KunShan Assignment 6
# q1a
def natureNumbers(n: int) -> None:
    def natureNumber(n: int) -> None:
        if n <= 1:
            print('1', end=' ')
        else:
            natureNumber(n - 1)
            print(n, end=' ')
    natureNumber(n)
    print()

# q2a
def alt(s: str, t: str) -> str:
    if len(s) == 0 or len(s) != len(t):
        return ''
    else:
        return s[0]+t[0]+alt(s[1:], t[1:])

# q2
def factorial(n: int) -> int:
    res = 1
    for i in range(1, n+1):
        res *= i
    return res

def factorialRecur(n: int) -> int:
    if n == 0 or n == 1:
        return 1
    else:
        return n * factorialRecur(n - 1)

def checkTime(func, arg):
    t1 = time.time()
    func(arg)
    t2 = time.time()
    return t2 - t1

if __name__ == '__main__':
    # q1a
    natureNumbers(10)
    natureNumbers(100)
    # q1b
    print(alt('hello', 'world'))
    print(alt('hello,', 'world!'))
    print(alt('hello', 'world!'))
    # q2
    import matplotlib.pyplot as plt
    import time
    k, step = 2_000, 1
    x = [n for n in range(0, k, step)]
    lst1 = [checkTime(factorial, n) for n in x]
    lst2 = [checkTime(factorialRecur, n) for n in x]
    plt.plot(x, lst1, label='Iter')
    plt.plot(x, lst2, label='Recu')
    plt.legend()
    plt.show()


from pathlib import Path
from unittest.mock import mock_open, patch

from gen_unit_tables import write_unit_table

def test_write_unit_table(snapshot):
    with patch.object(Path, "open", mock_open()) as mock_open_file:
        write_unit_table(filename=Path())

    lines = [
        args[0] for args in mock_open_file.return_value.write.call_args_list
    ]
    # Exclude the first line, because it contains "Generated by /home/username/..."
    assert lines[1:] == snapshot


import pprint
import pandas as pd
from sklearn.metrics import roc_auc_score, average_precision_score

def score_counter(val, y_true, y_pred):
    scores = {
            "TP": 0,
            "FP": 0,
            "TN": 0,
            "FN": 0
        }

    for i in range(len(y_pred)):
        if y_pred[i] == val:
            # positives
            if y_true[i] == val:
                scores["TP"] += 1
            else:
                scores["FP"] += 1
        else:
            # negatives
            if y_true[i] == val:
                scores["FN"] += 1
            else:
                scores["TN"] += 1

    for key in scores.keys():
        scores[key] = float(scores[key])

    return scores

def accuracy(scores):
    correct = scores["TP"] + scores["TN"]
    incorrect = scores["FP"] + scores["FN"]
    return correct / (correct + incorrect)

def specificity(scores):
    # TP / TP + FN
    if scores["TP"] == 0: return 0
    return scores["TP"] / (scores["TP"] + scores["FN"])

def sensitivity(scores):
    # TN / FP + TN
    if scores["TN"] == 0: return 0
    return scores["TN"] / (scores["FP"] + scores["TN"])

def positive_predictive_value(scores):
    # TP / TP + FP
    if scores["TP"] == 0: return 0
    return scores["TP"] / (scores["TP"] + scores["FP"])

def negative_predictive_value(scores):
    # TN / TN + FN
    if scores["TN"] == 0: return 0
    return scores["TN"] / (scores["TN"] + scores["FN"])

def to_binary_labels(val, y):
    bin_copy = y.copy()
    for i in range(len(bin_copy)):
        if bin_copy[i] == val:
            bin_copy[i] = 1
        else:
            bin_copy[i] = 0
    return bin_copy

def classification_report(y_true, y_pred, labelEncoder):
    # Returns a dataframe representing a range of scores for each label
    classes = labelEncoder.classes_
    class_report = {
            "Class": [],
            "Accuracy": [],
            "Specificity": [],
            "Sensitivity": [],
            "PPV": [],
            "NPV": [],
            "ROC_AUC": [],
            "Average Precision Score": []
        }
    report_df = pd.DataFrame(data=class_report)
    for i in range(len(classes)):
        scores = score_counter(i, y_true, y_pred)
        print(scores)
        print(y_true, y_pred)
        row = {
                "Class": classes[i],
                "Accuracy": accuracy(scores),
                "Specificity": specificity(scores),
                "Sensitivity": sensitivity(scores),
                "PPV": positive_predictive_value(scores),
                "NPV": negative_predictive_value(scores)
            }
        # ROC AUC is undefined if all values are the same. So verify difference fist
        bin_y_true = to_binary_labels(i, y_true)
        bin_y_pred = to_binary_labels(i, y_pred)
        if len(set(bin_y_true)) > 1 and len(set(bin_y_pred)) > 1:
            row["ROC_AUC"] = roc_auc_score(bin_y_true, bin_y_pred)
            row["Average Precision Score"] = average_precision_score(bin_y_true, bin_y_pred)
        else:
            row["ROC_AUC"] = "undefined"
            row["Average Precision Score"] = "undefined"

        report_df = report_df.append(row, ignore_index=True)

    return report_df


import coloredlogs, logging
coloredlogs.install(fmt='%(levelname)7s %(asctime)s %(hostname)s %(message)s')

import numpy as np
import matplotlib.pyplot as plt

# a combination of different randoms

Bn = 256
Sn = 80

A = np.concatenate((np.random.randn(Bn) * 0.2 + 8, np.random.randn(Bn) * 3 - 10, np.random.randn(Bn), np.random.rand(Bn) + 2, np.random.pareto(3.0, size=Bn)))

# plt.hist(A, 100)
# plt.show()

means = []
stds = []

for k in range(100):
    sample_idx = np.random.choice(A.shape[0], size=Sn * A.shape[0] // 100, replace=False)
    h = A[sample_idx]
    means.append(h.mean())
    stds.append(h.std())

means = np.array(means)
stds = np.array(stds)

print(means.mean(), means.std())
print(stds.mean(), stds.std())


import os
import argparse
import json
import numpy as np
import torch

from utils.util import get_mask_mnr, get_mask_bm, get_mask_rm
from utils.util import find_max_epoch, print_size, sampling, calc_diffusion_hyperparams

from imputers.DiffWaveImputer import DiffWaveImputer
from imputers.SSSDSAImputer import SSSDSAImputer
from imputers.SSSDS4Imputer import SSSDS4Imputer

from sklearn.metrics import mean_squared_error
from statistics import mean

def generate(output_directory,
             num_samples,
             ckpt_path,
             data_path,
             ckpt_iter,
             use_model,
             masking,
             missing_k,
             only_generate_missing):

    """
    Generate data based on ground truth

    Parameters:
    output_directory (str):           save generated speeches to this path
    num_samples (int):                number of samples to generate, default is 4
    ckpt_path (str):                  checkpoint path
    ckpt_iter (int or 'max'):         the pretrained checkpoint to be loaded;
                                      automitically selects the maximum iteration if 'max' is selected
    data_path (str):                  path to dataset, numpy array.
    use_model (int):                  0:DiffWave. 1:SSSDSA. 2:SSSDS4.
    masking (str):                    'mnr': missing not at random, 'bm': black-out, 'rm': random missing
    only_generate_missing (int):      0:all sample diffusion.  1:only apply diffusion to missing portions of the signal
    missing_k (int)                   k missing time points for each channel across the length.
    """

    # generate experiment (local) path
    local_path = "T{}_beta0{}_betaT{}".format(diffusion_config["T"],
                                              diffusion_config["beta_0"],
                                              diffusion_config["beta_T"])

    # Get shared output_directory ready
    output_directory = os.path.join(output_directory, local_path)
    if not os.path.isdir(output_directory):
        os.makedirs(output_directory)
        os.chmod(output_directory, 0o775)
    print("output directory", output_directory, flush=True)

    # map diffusion hyperparameters to gpu
    for key in diffusion_hyperparams:
        if key != "T":
            diffusion_hyperparams[key] = diffusion_hyperparams[key].cuda()

    # predefine model
    if use_model == 0:
        net = DiffWaveImputer(**model_config).cuda()
    elif use_model == 1:
        net = SSSDSAImputer(**model_config).cuda()
    elif use_model == 2:
        net = SSSDS4Imputer(**model_config).cuda()
    else:
        print('Model chosen not available.')
    print_size(net)

    # load checkpoint
    ckpt_path = os.path.join(ckpt_path, local_path)
    if ckpt_iter == 'max':
        ckpt_iter = find_max_epoch(ckpt_path)
    model_path = os.path.join(ckpt_path, '{}.pkl'.format(ckpt_iter))
    try:
        checkpoint = torch.load(model_path, map_location='cpu')
        net.load_state_dict(checkpoint['model_state_dict'])
        print('Successfully loaded model at iteration {}'.format(ckpt_iter))
    except:
        raise Exception('No valid model found')

    ### Custom data loading and reshaping ###

    testing_data = np.load(trainset_config['test_data_path'])
    testing_data = np.split(testing_data, 4, 0)
    testing_data = np.array(testing_data)
    testing_data = torch.from_numpy(testing_data).float().cuda()
    print('Data loaded')

    all_mse = []

    for i, batch in enumerate(testing_data):

        if masking == 'mnr':
            mask_T = get_mask_mnr(batch[0], missing_k)
            mask = mask_T.permute(1, 0)
            mask = mask.repeat(batch.size()[0], 1, 1)
            mask = mask.type(torch.float).cuda()

        elif masking == 'bm':
            mask_T = get_mask_bm(batch[0], missing_k)
            mask = mask_T.permute(1, 0)
            mask = mask.repeat(batch.size()[0], 1, 1)
            mask = mask.type(torch.float).cuda()

        elif masking == 'rm':
            mask_T = get_mask_rm(batch[0], missing_k)
            mask = mask_T.permute(1, 0)
            mask = mask.repeat(batch.size()[0], 1, 1).float().cuda()

        batch = batch.permute(0,2,1)

        start = torch.cuda.Event(enable_timing=True)
        end = torch.cuda.Event(enable_timing=True)
        start.record()

        sample_length = batch.size(2)
        sample_channels = batch.size(1)
        generated_audio = sampling(net, (num_samples, sample_channels, sample_length),
                                   diffusion_hyperparams,
                                   cond=batch,
                                   mask=mask,
                                   only_generate_missing=only_generate_missing)

        end.record()
        torch.cuda.synchronize()

        print('generated {} utterances of random_digit at iteration {} in {} seconds'.format(num_samples,
                                                                                ckpt_iter,
                                                                                int(start.elapsed_time(
                                                                                end) / 1000)))

        generated_audio = generated_audio.detach().cpu().numpy()
        batch = batch.detach().cpu().numpy()
        mask = mask.detach().cpu().numpy()

        outfile = f'imputation{i}.npy'
        new_out = os.path.join(ckpt_path, outfile)
        np.save(new_out, generated_audio)

        outfile = f'original{i}.npy'
        new_out = os.path.join(ckpt_path, outfile)
        np.save(new_out, batch)

        outfile = f'mask{i}.npy'
        new_out = os.path.join(ckpt_path, outfile)
        np.save(new_out, mask)

        print('saved generated samples at iteration %s' % ckpt_iter)

        mse = mean_squared_error(generated_audio[~mask.astype(bool)], batch[~mask.astype(bool)])
        all_mse.append(mse)

    print('Total MSE:', mean(all_mse))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, default='config.json',
                        help='JSON file for configuration')
    parser.add_argument('-ckpt_iter', '--ckpt_iter', default='max',
                        help='Which checkpoint to use; assign a number or "max"')
    parser.add_argument('-n', '--num_samples', type=int, default=500,
                        help='Number of utterances to be generated')
    args = parser.parse_args()

    # Parse configs. Globals nicer in this case
    with open(args.config) as f:
        data = f.read()
    config = json.loads(data)
    print(config)

    gen_config = config['gen_config']

    train_config = config["train_config"]  # training parameters

    global trainset_config
    trainset_config = config["trainset_config"]  # to load trainset

    global diffusion_config
    diffusion_config = config["diffusion_config"]  # basic hyperparameters

    global diffusion_hyperparams
    diffusion_hyperparams = calc_diffusion_hyperparams(
        **diffusion_config)  # dictionary of all diffusion hyperparameters

    global model_config
    if train_config['use_model'] == 0:
        model_config = config['wavenet_config']
    elif train_config['use_model'] == 1:
        model_config = config['sashimi_config']
    elif train_config['use_model'] == 2:
        model_config = config['wavenet_config']

    generate(**gen_config,
             ckpt_iter=args.ckpt_iter,
             num_samples=args.num_samples,
             use_model=train_config["use_model"],
             data_path=trainset_config["test_data_path"],
             masking=train_config["masking"],
             missing_k=train_config["missing_k"],
             only_generate_missing=train_config["only_generate_missing"])


import os
import re
import itertools
import logging
import pandas as pd
from slugify import slugify

from labonneboite.conf import settings
from labonneboite.common import mapping as mapping_util
from labonneboite.common import hiring_type_util
from labonneboite.common import geocoding
from labonneboite.common.search import HiddenMarketFetcher

logging.basicConfig(level=logging.INFO)

INPUT_FILENAME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mailing_data/users_sample.csv')
INPUT_FILE_CSV_DELIMITER = ','
OUTPUT_FILENAME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mailing_data/users_sample_prepared.csv')

COMMUNE_COLUMN = 'Code INSEE commune'
ROME_1_COLUMN = 'ROME'
ROME_2_COLUMN = 'ROME 2'

COLUMNS_FOR_EACH_OFFICE = ['siret', 'name', 'commune_id', 'url']
OFFICES_PER_USER = 5
ADDITIONAL_COLUMNS = ['search_url', 'used_distance', 'used_rome_id'] + \
    [
        'office%s_%s' % (1 + office_index, column_name)
    for office_index in range(OFFICES_PER_USER)
    for column_name in COLUMNS_FOR_EACH_OFFICE
] + \
    ['INFO1']  # stands for 'main_text'

DISTANCE_ATTEMPTS = [10, 20, 30, 50, 100, 200]

# is this mailing wip or for real?
IS_THIS_MAILING_WORK_IN_PROGRESS = False

# Google Analytics tracking
GA_TRACKING_MEDIUM = 'mailing'
GA_TRACKING_SOURCE = 'mailing_20180209_chaumont'  # FIXME carefully update date for every mailing

if IS_THIS_MAILING_WORK_IN_PROGRESS:
    GA_TRACKING_MEDIUM = "%s_wip" % GA_TRACKING_MEDIUM
    GA_TRACKING_SOURCE = "%s_wip" % GA_TRACKING_SOURCE

GA_TRACKING_SEARCH = 'utm_medium=%s&utm_source=%s&utm_campaign=%s_search' % (
    GA_TRACKING_MEDIUM, GA_TRACKING_SOURCE, GA_TRACKING_SOURCE)
GA_TRACKING_DETAIL = 'utm_medium=%s&utm_source=%s&utm_campaign=%s_detail' % (
    GA_TRACKING_MEDIUM, GA_TRACKING_SOURCE, GA_TRACKING_SOURCE)

SEND_ALL_EMAILS_TO_DEBUG_EMAIL = IS_THIS_MAILING_WORK_IN_PROGRESS

ENABLE_FAKE_METZ_INSEE_CODE_FOR_ALL = False  # sometimes convenient while working on local dev
METZ_INSEE_CODE = '57463'

def get_results(commune_id, rome_1_id, rome_2_id):
    """
    Try to find enough offices matching this commune_id,
    for either of the two provided rome_ids,
    and within as small of a distance as possible.

    The result is a dictionary:
    - offices found
    - distance which was eventually used for the result
    - rome_id which was eventually used for the result
    """
    if ENABLE_FAKE_METZ_INSEE_CODE_FOR_ALL:
        commune_id = METZ_INSEE_CODE

    city = geocoding.get_city_by_commune_id(commune_id)

    if city is None:
        return {'offices': [], 'distance': settings.DISTANCE_FILTER_DEFAULT, 'rome_id': rome_1_id}

    latitude = city['coords']['lat']
    longitude = city['coords']['lon']

    offices = []

    # the only case where those two stay None is if there is no result at all
    used_distance = None
    used_rome_id = None

    # itertools.product iterates over all combinations of values in multiple lists, see:
    # https://stackoverflow.com/questions/16384109/iterate-over-all-combinations-of-values-in-multiple-lists-in-python
    for distance, rome_id in itertools.product(DISTANCE_ATTEMPTS, [rome_1_id, rome_2_id]):
        if mapping_util.rome_is_valid(rome_id):
            naf_code_list = mapping_util.map_romes_to_nafs([rome_id])
            # FIXME randomize per user to avoid spamming companies
            fetcher = HiddenMarketFetcher(
                naf_codes=naf_code_list,
                romes=[rome_id],
                latitude=latitude,
                longitude=longitude,
                distance=distance,
                from_number=1,
                to_number=OFFICES_PER_USER,
                hiring_type=hiring_type_util.DPAE,
            )
            offices, _ = fetcher.get_offices()
            if len(offices) >= 1:
                used_rome_id = rome_id
                used_distance = distance
            if len(offices) >= OFFICES_PER_USER:
                break

    return {'offices': offices, 'distance': used_distance, 'rome_id': used_rome_id}

def get_search_url(commune_id, distance, rome_id):
    """
    Build search URL for given commune_id, distance and rome_id
    """
    if mapping_util.rome_is_valid(rome_id):
        rome_description = settings.ROME_DESCRIPTIONS[rome_id.upper()]
        slugified_rome_description = slugify(rome_description)
        city = geocoding.get_city_by_commune_id(commune_id)
        if city:
            # Here we hardcode the URL since this script typically runs in staging
            # for data confidentiality reasons and not in production.
            # However we need the resulting URLs to be pointing to production,
            # this is why we do not use `url_for`.
            search_url = 'https://labonneboite.pole-emploi.fr/entreprises/%s-%s/%s?d=%s&%s' % (
                city['slug'],
                city['zipcode'],
                slugified_rome_description,
                distance,
                GA_TRACKING_SEARCH,
            )
            return search_url
        else:
            logging.info("WARNING city not found : %s", commune_id)
    else:
        logging.info("WARNING invalid rome_code : %s", rome_id)

    return None

def compute_additional_data(commune_id, rome_1_id, rome_2_id):
    """
    Compute value of all additional CSV columns for a given commune_id and rome_id.
    """
    results = get_results(commune_id, rome_1_id, rome_2_id)
    offices = results['offices']
    distance = results['distance']
    rome_id = results['rome_id']

    search_url = get_search_url(commune_id, distance, rome_id)
    values = [search_url, distance, rome_id]

    def url_from_siret(siret):
        return 'https://labonneboite.pole-emploi.fr/%s/details?%s' % (
            siret,
            GA_TRACKING_DETAIL,
        )

    for i in range(OFFICES_PER_USER):
        # about try except else
        # see https://stackoverflow.com/questions/16138232/is-it-a-good-practice-to-use-try-except-else-in-python
        try:
            office = offices[i]
        except IndexError:
            values += [''] * len(COLUMNS_FOR_EACH_OFFICE)
        else:
            office_url = url_from_siret(office.siret)
            # names of these columns are in COLUMNS_FOR_EACH_OFFICE
            values += [office.siret, office.name, office.city_code, office_url]

    list_text = ''.join(
        ["
%s
" % (url_from_siret(o.siret), o.name) for o in offices]
    )

    # For the record GMS does not directly support utf8 characters with accent (e.g. 'é').
    # It also does not support html encoding (e.g. 'é') as it breaks at the `;`.
    main_text = "
%s
" % list_text

    # remove new lines
    main_text = main_text.strip().replace('\n', '').replace('\r', '')
    # remove duplicate spaces
    main_text = re.sub(' +', ' ', main_text)
    values += [main_text]

    return values

def prepare_mailing_data():
    """
        Enrich CSV user data in order to prepare mailing.
    """
    df = pd.read_table(INPUT_FILENAME, sep=INPUT_FILE_CSV_DELIMITER)

    # compute additional columns and add them to initial dataframe
    # inspired by
    # https://stackoverflow.com/questions/16236684/apply-pandas-function-to-column-to-create-multiple-new-columns
    # and https://stackoverflow.com/questions/13331698/how-to-apply-a-function-to-two-columns-of-pandas-dataframe
    temp = list(zip(*df[[COMMUNE_COLUMN, ROME_1_COLUMN, ROME_2_COLUMN]
                        ].apply(lambda x: compute_additional_data(*x), axis=1)))
    for index, column in enumerate(ADDITIONAL_COLUMNS):
        df[column] = temp[index]

    raw_total_rows = len(df)

    # delete rows with empty or buggy results
    # df = df[condition] will remove from df all rows which do not match condition
    df = df[df.search_url.isnull() == False]  # noqa
    df = df[df.office1_url.isnull() == False]  # noqa

    net_total_rows = len(df)
    empty_rows = raw_total_rows - net_total_rows
    logging.info("deleted %s empty rows out of %s", empty_rows, raw_total_rows)

    # custom manipulation of email column
    if 'Courriel' not in df.columns:
        raise Exception('required column Courriel is missing')
    df.rename(columns={'Courriel': 'EMAIL1'}, inplace=True)
    if SEND_ALL_EMAILS_TO_DEBUG_EMAIL:
        df.EMAIL1 = "labonneboite.pe@gmail.com"

    # index=False avoids writing leading index column
    # encoding='ascii' is the default, see
    # https://stackoverflow.com/questions/31331358/unicode-encode-error-when-writing-pandas-df-to-csv
    df.to_csv(OUTPUT_FILENAME, sep=',', index=False, encoding='utf-8')

    logging.info("please consult result in file %s", OUTPUT_FILENAME)

if __name__ == '__main__':
    prepare_mailing_data()


from typing import List
class Solution:
    def permute(self, nums: List[int]) -> List[List[int]]:
        self.result = []
        self.list = []
        self.backtrack(nums)
        return self.result
    def backtrack(self, nums):
        if len(nums) == len(self.list):
            self.result.append(self.list[:])

        for i in range(len(nums)):
            if nums[i] in self.list:
                continue
            self.list.append(nums[i])
            self.backtrack(nums)
            self.list.pop()