"""
The aim of this program is to convert a .dat file which is used for the ARTERE program.
This program reads and grabs the data from the .dat file and creates a Python file in order
to make a network with pandapower.
"""
import math
import shlex

busList = []
init_vm_pu_list = []
init_va_degree_list = []
load_list = []
generatorList = []
transformersList = []
linesList = []
switchList = []
shuntsList = []
busDict = {}
bus_voltage_dict = {}
genDict = {}
recordList = []
controllersList = []

def checkStatus(element_status):
    if element_status == 1:
        return True
    return False

def from_artere(inputFile):
    records = ""
    global frequency
    frequency = 50

    with open(inputFile) as artereFile:
        lines = artereFile.readlines()
        for record in lines:
            if record[0][0] == "#" or record[0][0] == "!":
                pass
            else:
                records += record.rstrip()

    for line in records.split(";"):
        if line:
            recordList.append(shlex.split(line))

    for record in recordList:
        if record[0] == 'SLACK':
            slackBus = record[1]

        if record[0] == 'FNOM':
            frequency = float(record[1])

    for record in recordList:
        if record[0] == 'BUS':
            bus_name = record[1]
            vn_kv = float(record[2])
            Pload = float(record[3])
            Qload = float(record[4])
            Bshunt = float(record[5])
            Qshunt = float(record[6])

            # List of bus names
            try:
                bus_name = bus_name.replace(" ", "_")
            except:
                pass

            # Create a bus
            busDict[bus_name] = {}
            busDict[bus_name]["name"] = bus_name
            busDict[bus_name]["voltage"] = vn_kv

            # Create a string that is going to be stored in the Python file.
            created_bus = f"{bus_name} = pp.create_bus(net, vn_kv={vn_kv}, name=\"{bus_name}\", in_service=True)\n"
            busList.append(created_bus)

            # If the bus has a load, create a load connected to the bus.
            if Pload != 0 or Qload != 0:
                created_load = f"pp.create_load(net, bus={bus_name}, p_mw={Pload}, q_mvar={Qload}, name=\"{bus_name}\", in_service=True)\n"
                load_list.append(created_load)

            # if the bus has a shunt, create a shunt to the bus
            # Qshunt-> Create load, Bshunt-> Create shunt
            if Qshunt != 0:
                created_load = f"pp.create_load(net, bus={bus_name}, name=\"{bus_name}_Shunt\", p_mw=0, q_mvar={Qshunt}, in_service=True) \n"
                load_list.append(created_load)
            if Bshunt != 0:
                created_shunt = f"pp.create_shunt(net, bus={bus_name}, q_mvar={Bshunt}, in_service=True)\n"
                shuntsList.append(created_shunt)

        # Transmittion lines
        if record[0] == 'LINE':
            line_name = record[1]
            from_bus = record[2]
            to_bus = record[3]
            R_Ohm = float(record[4])
            X_Ohm = float(record[5])
            half_shunt_suceptance = float(record[6]) / 10 ** 6  # It is given in μS
            Snom = float(record[7])
            status = checkStatus(int(record[8]))

            if Snom == 0:
                Snom = 9999

            # Calculate the maximum curent
            line_voltage = busDict[from_bus]["voltage"]
            Imax = (Snom * 10 ** 6) / (math.sqrt(3) * line_voltage * 10 ** 3)
            Imax = Imax / 1000
            Imax = format(Imax, '.6f')

            # Calculate the capacitance in nF.  ωC/2
            Capacitance = (half_shunt_suceptance * 2) / (2 * math.pi * frequency)
            Capacitance = Capacitance * 10 ** 9  # nF
            Capacitance = format(Capacitance, '.6f')

            created_line = f'pp.create_line_from_parameters(net, from_bus={busDict[from_bus]["name"]}, to_bus={busDict[to_bus]["name"]}, name="{line_name}"' \
                           f', length_km=1, r_ohm_per_km={R_Ohm}, x_ohm_per_km={X_Ohm}, max_i_ka={Imax}, c_nf_per_km={Capacitance}, in_service={status})\n'
            linesList.append(created_line)

        # LTC-V
        if record[0] == 'LTC-V':
            print('LTC-V is not supported. ')
            raise Warning('LTC-V is not supported.')

        # Transformers for TRANSFO record
        if record[0] == 'TRANSFO':
            transfo_name = record[1]
            from_bus = record[2]
            to_bus = record[3]
            rk = float(record[4]) # Resistance Ohm
            xk = float(record[5]) # Reactance
            SusceptanceB1 = float(record[6])
            SusceptanceB2 = float(record[7])
            N_ratio = float(record[8])
            ym = (SusceptanceB1 + SusceptanceB2) / 2.0
            Snom = float(record[10])
            status = int(record[11])
            PHI = float(record[9])

            status = checkStatus(status)

            # vk, vkr, i0_percent

            vk_percent = math.sqrt(pow(rk, 2) + pow(xk, 2))  # zk
            vkr_percent = rk
            i0_percent = ym

            # Check which bus is High level and which is low.
            if busDict[from_bus]['voltage'] > busDict[to_bus]['voltage']:
                high_lvl_bus = from_bus
                low_lvl_bus = to_bus
            else:
                high_lvl_bus = to_bus
                low_lvl_bus = from_bus

            if N_ratio > 100:
                tap_neutral = 0
                tap_min = N_ratio - 100
                tap_max = N_ratio - 100
                tap_pos = N_ratio - 100
                tap_step_percent = 1
                tap_side = "\"hv\""

            elif N_ratio < 100:
                tap_neutral = 0
                tap_min = -100 + N_ratio
                tap_max = -100 + N_ratio
                tap_pos = -100 + N_ratio
                tap_step_percent = 1
                tap_side = "\"hv\""

            # Create a transformer
            # if the ratio is not 100, we should take into account the taps.
            if N_ratio == 100:
                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, ' \
                                      f'name=\"{transfo_name}\", shift_degree={PHI}, vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent}, pfe_kw=0, i0_percent={i0_percent}, in_service={status})\n'
            else:
                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, name={transfo_name}, ' \
                                      f'shift_degree={PHI}, vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent},' \
                                      f' pfe_kw=0, i0_percent={i0_percent}, tap_min={tap_min}, tap_max={tap_max}, tap_step_percent={tap_step_percent}, tap_pos={tap_pos},tap_neutral={tap_neutral}, tap_side={tap_side}, in_service={status})\n'

            transformersList.append(created_transformer)

        # Transformer for TRFO record
        if record[0] == 'TRFO':
            transfo_name = record[1]
            from_bus = record[2]
            to_bus = record[3]
            CON_BUS = record[4]
            Ohm_Resistance = (record[5])

            if CON_BUS == '\'' and Ohm_Resistance == '\'':
                CON_BUS = None
                Ohm_Resistance = float(record[6])
                Xreactance = float(record[6 + 1])
                Susceptance = float(record[7 + 1])
                N_ratio = float(record[8 + 1])
                Snom = float(record[9 + 1])
                N_First = float(record[10 + 1]) * 100
                N_Last = float(record[11 + 1]) * 100
                NBPOS = int(float(record[12 + 1]))
                TOLV = record[13 + 1]
                VDES = record[14 + 1]
                On_off = int(record[15 + 1])
            else:
                Ohm_Resistance = float(record[5])
                Xreactance = float(record[6])
                Susceptance = float(record[7])
                N_ratio = float(record[8])
                Snom = float(record[9])
                N_First = float(record[10])
                N_Last = float(record[11])
                NBPOS = int(float(record[12]))
                TOLV = record[13]
                VDES = record[14]
                On_off = int(record[15])

            status = checkStatus(On_off)

            # if TOLV != "0" and VDES != "0":
            #     CreateController = f"ContinuousTapControl(net, tid, vm_set_pu={VDES}, tol={TOLV}, side='{tap_side}', trafotype='2W', in_service=True) \n"
            #     controllersList.append(CreateController)

            # vkr, vk, i0 percent
            Ym = Susceptance
            rk = Ohm_Resistance
            xk = Xreactance
            zk = math.sqrt(pow(rk, 2) + pow(xk, 2))
            vk_percent = zk
            vkr_percent = rk
            i0_percent = Ym

            # Check which bus is High level and which is low.
            if busDict[from_bus]["voltage"] > busDict[to_bus]["voltage"]:
                high_lvl_bus = from_bus
                low_lvl_bus = to_bus
            else:
                high_lvl_bus = to_bus
                low_lvl_bus = from_bus

            # TAPS
            try:
                if CON_BUS.isdigit():
                    CON_BUS = "b" + CON_BUS
            except:
                pass

            try:
                if busDict[CON_BUS]['voltage'] == busDict[high_lvl_bus]['voltage']:
                    tap_side = "\"hv\""
                elif busDict[CON_BUS]['voltage'] == busDict[low_lvl_bus]['voltage']:
                    tap_side = "\"lv\""
                else:
                    tap_side = None
            except:
                tap_side = None

            # tap_max = NBPOS, tap_min=0
            # check if the N ratio is not 100, while NBPOS, N_first and N_Last are 0.
            if N_ratio != 100 and NBPOS == 0 and N_First == 0 and N_Last == 0:
                if N_ratio > 100:
                    tap_neutral = 0
                    tap_min = N_ratio - 100
                    tap_max = N_ratio - 100
                    tap_pos = N_ratio - 100
                    tap_step_percent = 1
                    tap_side = "\"hv\""

                elif N_ratio < 100:
                    tap_neutral = 0
                    tap_min = -100 + N_ratio
                    tap_max = -100 + N_ratio
                    tap_pos = -100 + N_ratio
                    tap_step_percent = 1
                    tap_side = "\"lv\""

                # Create a string
                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, name="{transfo_name}", ' \
                                      f'vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent}, pfe_kw=0, i0_percent={i0_percent}, ' \
                                      f'tap_pos={tap_pos}, tap_neutral={tap_neutral}, tap_step_percent={tap_step_percent}, tap_side={tap_side}, tap_min={tap_min}, tap_max={tap_max}, in_service={status})\n'

                transformersList.append(created_transformer)

            # if N_ratio , N_First, N_last are equal:
            elif N_ratio != 100 and N_First == N_Last and N_Last == N_ratio:
                if N_ratio > 100:
                    tap_neutral = 0
                    tap_min = N_ratio - 100
                    tap_max = N_ratio - 100
                    tap_pos = N_ratio - 100
                    tap_step_percent = 1
                    # tap_side = "\"hv\""

                elif N_ratio < 100:
                    tap_neutral = 0
                    tap_min = -100 + N_ratio
                    tap_max = -100 + N_ratio
                    tap_pos = -100 + N_ratio
                    tap_step_percent = 1
                    # tap_side = "\"lv\""

                # Create a string
                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, name="{transfo_name}", ' \
                                      f'vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent}, pfe_kw=0, i0_percent={i0_percent}, ' \
                                      f'tap_pos={tap_pos}, tap_neutral={tap_neutral}, tap_step_percent={tap_step_percent}, tap_side={tap_side}, tap_min={tap_min}, tap_max={tap_max}, in_service={status})\n'

                # Store the string to a list
                transformersList.append(created_transformer)

            elif N_ratio == 100 and N_First == 0 and N_Last == 0:
                # Create a string
                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, name="{transfo_name}", ' \
                                      f'vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent}, pfe_kw=0, ' \
                                      f'i0_percent={i0_percent}, in_service={status})\n'

                # Store the string to a list
                transformersList.append(created_transformer)

            elif N_ratio > 0 and N_First > 0 and N_Last > 0 and NBPOS > 0:

                tap_step_percent = (N_Last - N_First) / (NBPOS - 1)

                tap_neutral = (100 - float(N_First)) / tap_step_percent
                tap_neutral = round(tap_neutral)

                tap_pos = (float(N_ratio) - N_First) / tap_step_percent
                tap_pos = round(tap_pos)

                created_transformer = f'pp.create_transformer_from_parameters(net, hv_bus={busDict[high_lvl_bus]["name"]}, lv_bus={busDict[low_lvl_bus]["name"]}, sn_mva={Snom}, name="{transfo_name}", ' \
                                      f'vn_hv_kv={busDict[high_lvl_bus]["voltage"]}, vn_lv_kv={busDict[low_lvl_bus]["voltage"]}, vkr_percent={vkr_percent}, vk_percent={vk_percent}, pfe_kw=0, i0_percent={i0_percent}, ' \
                                      f'tap_pos={tap_pos}, tap_neutral={tap_neutral}, tap_step_percent={tap_step_percent}, tap_side={tap_side}, tap_min=0, tap_max={NBPOS - 1}, in_service={status})\n'

                transformersList.append(created_transformer)

        # Generators
        if record[0] == 'GENER':
            gener_name = record[1]
            bus_connected = record[2]
            Pproduction = float(record[4])
            Qproduction = float(record[5])
            Snom = record[7]
            VIMP = record[6]
            Qmin = float(record[8])
            Qmax = float(record[9])
            On_off = int(record[10])

            # Check if it is in service
            status = checkStatus(On_off)

            # if VIMP is zero, the generator is treated as a PQ bus (we create sgen)
            # Otherwise it is treated as a PV bus and the Q is ignored
            if VIMP == "0":
                created_sgen = f'pp.create_sgen(net, bus={busDict[bus_connected]["name"]}, p_mw={Pproduction}, q_mvar={Qproduction}, sn_mva={Snom}, name="{gener_name}", max_q_mvar={Qmax}, min_q_mvar={Qmin}, in_service={status})\n'
                generatorList.append(created_sgen)

            else:
                if gener_name == slackBus:
                    created_gen = f'pp.create_gen(net, p_mw={Pproduction}, max_q_mvar={Qmax}, min_q_mvar={Qmin}, sn_mva={Snom}, bus={busDict[bus_connected]["name"]}, vm_pu={VIMP}, name="{gener_name}", slack={True}, in_service={status})\n'
                    generatorList.append(created_gen)

                elif gener_name == "Ext_Grid":
                    created_gen = f'pp.create_gen(net, p_mw={Pproduction}, max_q_mvar={Qmax}, min_q_mvar={Qmin}, sn_mva={Snom}, bus={busDict[bus_connected]["name"]}, vm_pu={VIMP}, name="{gener_name}", slack={True}, in_service={status})\n'
                    generatorList.append(created_gen)

                else:
                    created_gen = f'pp.create_gen(net, p_mw={Pproduction}, max_q_mvar={Qmax}, min_q_mvar={Qmin}, sn_mva={Snom}, bus={busDict[bus_connected]["name"]}, vm_pu={VIMP}, name="{gener_name}", slack={False}, in_service={status})\n'
                    generatorList.append(created_gen)

        # Switches
        if record[0] == 'SWITCH':
            switch_name = record[1]
            first_bus = record[2]
            second_bus = record[3]
            On_off = int(record[4])

            status = checkStatus(On_off)

            created_switch = f'pp.create_switch(net, bus={busDict[first_bus]["name"]}, name="{switch_name}" ,element={second_bus}, et="b" ,closed={status})\n'
            switchList.append(created_switch)

        # LFRESV specifies the voltage magnitude and phase at a bus.
        if record[0] == 'LFRESV':
            init_vm_pu = float(record[2])  # Voltage magnitude in pu  # MODULE
            init_va_degree = float(record[3])  # Phase angle in radians   # PHASE

            init_vm_pu_list.append(init_vm_pu)
            init_va_degree_list.append(init_va_degree)

    createNewFile(inputFile)

def createNewFile(file):
    # Those lines get the removes the .dat extension and puts .py
    newPythonFile = file.split('.')[0] + ".py"

    with open(newPythonFile, 'w') as pythonFile:
        pythonFile.write(f"""import pandapower as pp
from pandapower.control.controller.trafo.ContinuousTapControl import ContinuousTapControl

net = pp.create_empty_network(f_hz={frequency}, sn_mva={100})

    """)
        pythonFile.write('\n# list of Buses:\n')
        for bus in busList:
            pythonFile.write(bus)

        pythonFile.write('\n\n# list of Loads:\n')
        for load in load_list:
            pythonFile.write(load)

        pythonFile.write('\n\n# List of Shunts:\n')
        for shunt in shuntsList:
            pythonFile.write(shunt)

        pythonFile.write('\n\n# list of Lines:\n')
        for line in linesList:
            pythonFile.write(line)

        pythonFile.write('\n\n# list of Transformers:\n')
        for transformer in transformersList:
            pythonFile.write(transformer)

        pythonFile.write('\n\n# list of Generators:\n')
        for generator in generatorList:
            pythonFile.write(generator)

        pythonFile.write('\n\n# List of Switches:\n')
        for switch in switchList:
            pythonFile.write(switch)

        pythonFile.write('\n\n# List of controllers:\n')
        for controller in controllersList:
            pythonFile.write(controller)

        pythonFile.write(f"""
init_va_degree_list = {init_va_degree_list}
init_vm_pu_list = {init_vm_pu_list}

# Run power flow
if len(init_va_degree_list) > 0 and len(init_vm_pu_list) > 0:
    pp.runpp(net, init_va_degree=init_va_degree_list, init_vm_pu=init_vm_pu_list)
else:
    pp.runpp(net)

print(net.res_bus)          # bus results
""")


#!/usr/bin/env python

from bravado.client import SwaggerClient
from bravado.requests_client import RequestsClient
from BitMEXAPIKeyAuthenticator import APIKeyAuthenticator

HOST = "https://testnet.bitmex.com"
SPEC_URI = HOST + "/api/explorer/swagger.json"

# See full config options at http://bravado.readthedocs.io/en/latest/configuration.html
config = {
  # Don't use models (Python classes) instead of dicts for #/definitions/{models}
  'use_models': False,
  # This library has some issues with nullable fields
  'validate_responses': False,
}

bitMEX = SwaggerClient.from_url(
  SPEC_URI,
  config=config)

# You can get a feel for what is available by printing these objects.
# See also https://testnet.bitmex.com/api/explorer
print(dir(bitMEX))
print(dir(bitMEX.Trade))

# Basic unauthenticated call
res = bitMEX.Trade.Trade_get(symbol='XBTUSD', count=40).result()
print(res)

#
# Authenticated calls
#
# To do authentication, you must generate an API key.
# Do so at https:https://testnet.bitmex.com/app/apiKeys

API_KEY = '<API_KEY_HERE>'
API_SECRET = '<API_SECRET_HERE>'

request_client = RequestsClient()
request_client.authenticator = APIKeyAuthenticator(HOST, API_KEY, API_SECRET)

bitMEXAuthenticated = SwaggerClient.from_url(
  SPEC_URI,
  config=config,
  http_client=request_client)

print(dir(bitMEXAuthenticated.Position))

# Basic unauthenticated call
res = bitMEXAuthenticated.Position.Position_get().result()
print(res)


import wikipedia
import folders
from pydub import AudioSegment
from pydub import effects

import sys
sys.path.append('/path/to/ffmpeg')

def write_file(file,data):
	# f = open(file, "w")
	# f.write(data)
	# f.close()
	with open(file, 'w') as f:
		f.write(data)

def getwiki(path_wiki,title):
	wikipedia.set_lang("hu")
	mycontent = wikipedia.page(title).content
	mycontent = mycontent.split('\n')
	mycontent = [x for x in mycontent if len(x)>0]
	mycontent = '\n'.join(mycontent)

	filename = title.replace("-","_").replace(":","_").replace("!","_").replace("\\","_")
	write_file(path_wiki+'/'+filename+'.txt',mycontent)

def mp3_speed(root):
	# root = r'original/abc.mp3'
	velocidad_X = 1.5 # No puede estar por debajo de 1.0

	sound = AudioSegment.from_mp3(root)
	so = sound.speedup(velocidad_X, 150, 25)
	so.export(root[:-4] + '_out.mp3', format = 'mp3')

def main(path_wiki,path_out):

	# print(mp3files)
	# folders.create_folder(path_wiki)

	# with open('list.txt','r') as f:
	# 	data = f.read()

	# titles = data.split('\n')
	# print(titles)
	# for title in titles:
	# 	getwiki(path_wiki,title)

	# folders.folder_in_out(path_wiki,path_out)
	mp3files = folders.list_files2('output', 'mp3')
	# mp3_mod = [mp3_speed(x) for x in mp3files]
	for mp3file in mp3files:
		print(mp3file)
		mp3_speed(mp3file)

if __name__ == '__main__':
	path_wiki = 'output/wiki'
	path_out = 'output/mp3/'
	main(path_wiki,path_out)

	# with open(file,'w') as f:
		# f.write(data)

# mycontent = ny.content

# file.
# u'New York is a state in the Northeastern region of the United States. New York is the 27th-most exten'...
# ny.links[0]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr  3 23:31:06 2020

@author: Ricky Garza-Giron
University of California, Santa Cruz
Earth and Planetary Sciences
Seismology Laboratory
"""

import os
import numpy as np
import datetime
from obspy.core import read, UTCDateTime, Stream, Trace
from obspy.core import Stats
import datetime

import pickle

os.mkdir('RESULTS')

class EVENT(object):
    """
    Events catalog with arrival times to each station

    :type event_id: str
    :param event_id: Event ID

    :type origin_time: datetime.datetime
    :param origin_time: Origin time of earthquake

    :type latitude: float
    :param latitude: Latitude of hypocenter

    :type longitude: float
    :param longitude: Longitude of hypocenter

    :type depth: float
    :param depth: Depth of hypocenter

    :type magnitude: float
    :param magnitude: Magnitude of earthquake

    :type stations: list
    :param stations: List of stations for event

    :type arrival_times: list of datetime.datetime
    :param arrival_times: List of arrival times of P or S to each station

    :type travel_times: list of datetime.datetime
    :param travel_times: List of travel times of P or S to each station

    :type phase_types: list
    :param phase_types: List of type of arrival (P or S)

    """
    def __init__(self, event_id, origin_time,
                 latitude, longitude,
                 depth, magnitude_catalog=None,magnitude_inv=None,net_sta_chan=None,arrival_times=None,
                 travel_times=None,phase_types=None):
        self.event_id = event_id
        self.origin_time = origin_time
        self.latitude = latitude
        self.longitude = longitude
        self.depth = depth
        self.magnitude_catalog = magnitude_catalog
        self.magnitude_inv = magnitude_inv
        self.net_sta_chan = net_sta_chan
        self.arrival_times = arrival_times
        self.travel_times = travel_times
        self.phase_types = phase_types

    def __repr__(self):
        print_str = ' '.join(['event_id=', str(self.event_id), '\n',
                              'origin_time=', str(self.origin_time), '\n',
                              'latitude=', str(self.latitude), '\n',
                              'longitude=', str(self.longitude), '\n',
                              'depth=', str(self.depth), '\n',
                              'magnitude_catalog=', str(self.magnitude_catalog), '\n',
                              'magnitude_inv=', str(self.magnitude_inv), '\n',
                              'net_sta_chan=', str(self.net_sta_chan), '\n',
                              'arrival_times=', '[...]', '\n',
                              'travel_times=', '[...]', '\n',
                              'phase_types=', '[...]', '\n'])
        return "EVENT(" + print_str + ")"

filename='INPUT/CATALOG.txt'
def read_catalog(fname,header=True):
    '''
    Function to make events of :class: EVENT from a catalog with format:
        DATE(yr/m/d),TIME(H:M:S.microsec),LAT,LON,DEPTH,MAG,ID

    Parameters
    ----------
    fname : TYPE: str
        DESCRIPTION: Filename of events catalog
    header : TYPE, optional
        DESCRIPTION. The default is True.

    Returns
    -------
    events : TYPE: list
        DESCRIPTION: list of events with :class:EVENT

    '''
    events=[]
    if header==False:
        F=open(fname,'r')
        for f in F.readlines():
            events.append(EVENT(event_id=f.rstrip('\n').split(',')[6],
            origin_time=datetime.datetime.strptime(f.rstrip('\n').split(',')[0]+
            ' '+f.rstrip('\n').split(',')[1],'%Y/%m/%d %H:%M:%S.%f'),
            latitude=float(f.rstrip('\n').split(',')[2]),
            longitude=float(f.rstrip('\n').split(',')[3]),
            depth=float(f.rstrip('\n').split(',')[4]),
            magnitude_catalog=float(f.rstrip('\n').split(',')[5])))
    else:
        F=open(fname,'r')
        for f in F.readlines()[1:]:
            events.append(EVENT(event_id=f.rstrip('\n').split(',')[6],
            origin_time=datetime.datetime.strptime(f.rstrip('\n').split(',')[0]+
            ' '+f.rstrip('\n').split(',')[1],'%Y/%m/%d %H:%M:%S.%f'),
            latitude=float(f.rstrip('\n').split(',')[2]),
            longitude=float(f.rstrip('\n').split(',')[3]),
            depth=float(f.rstrip('\n').split(',')[4]),
            magnitude_catalog=float(f.rstrip('\n').split(',')[5])))
    return events

events=read_catalog(filename)

print('###################################################################')
print('###################################################################')
print('###############        READING EVENTS          ###################')
print('###################################################################')
print('###################################################################')

filename="INPUT/SC_HYPOINVERSE2000_clean.txt"

def picks_from_hypoinverse2000(filename,events):
    '''
    Function to add phase pick information to events from Hypoinverse
    2000 Phase file.

    For an example check out:
        https://service.scedc.caltech.edu/eq-catalogs/date_mag_loc.php
    Select "Hypoinverse 2000 Phase" from the Output Format drop-down menu

    NOTE: The Hypoinverse 2000 Phase format files sometimes append rows with
    the number of events and phases at the end. YOU MUST REMOVE THESE LINES
    FOR THIS FUNCTION TO WORK.

    Parameters
    ----------
    filename : TYPE: str
        DESCRIPTION: Filename of phase catalog

    events : TYPE: list
        DESCRIPTION: list of events with :class:EVENT

    Returns
    -------
    events : TYPE: list
        DESCRIPTION: list of events with :class:EVENT

    '''
    f=open(filename)
    F=f.readlines()
    BREAKS=[i for i,FF in enumerate(F) if len(FF)>160]
    BREAKS.append(len(F))

    for j in np.arange(0,len(BREAKS)-1):
        eID=F[BREAKS[j]][138:146] #event ID
        [e for e in events if e.event_id==eID][0].net_sta_chan=[]
        [e for e in events if e.event_id==eID][0].arrival_times=[]
        [e for e in events if e.event_id==eID][0].travel_times=[]
        [e for e in events if e.event_id==eID][0].phase_types=[]

        for k in np.arange(BREAKS[j]+1,BREAKS[j+1]):
            st=F[k][0:4]

            if ' ' in st:
                st=st.replace(' ','') #Get station name
            ch=F[k][9:12] #Get channel
            net=F[k][5:7]
            #If empty get p of next one (this was needed for N and E
            #components, should not matter for only vertical components)
            if F[k][14]=='P':
                if F[k][30:34]!='    ':
                    ph_type='P'
                    if (int(F[k][30:34])/100)==60:
                        if int(F[k][27:29])==59:
                            pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+str(int(F[k][25:27])+1)+':'+'00'+':'+'00.00'
                        else:
                            pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+str(int(F[k][27:29])+1)+':'+'00.00'
                    else:
                        pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+F[k][27:29]+':'+str(int(F[k][30:34])/100)
                else:
                    ph_type='P'
                    if (int(F[k][42:46])/100)==60:
                        if int(F[k][27:29])==59:
                            pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+str(int(F[k][25:27])+1)+':'+'00'+':'+'00.00'
                        else:
                            pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+str(int(F[k][27:29])+1)+':'+'00.00'
                    else:
                        pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+F[k][27:29]+':'+str(int(F[k][42:46])/100)
            else:
                ph_type='S'
                if (int(F[k][42:46])/100)==60:
                    if int(F[k][27:29])==59:
                        pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+str(int(F[k][25:27])+1)+':'+'00'+':'+'00.00'
                    else:
                        pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+str(int(F[k][27:29])+1)+':'+'00.00'
                else:
                    pick=F[k][17:21]+'-'+F[k][21:23]+'-'+F[k][23:25]+' '+F[k][25:27]+':'+F[k][27:29]+':'+str(int(F[k][42:46])/100)

            [e for e in events if e.event_id==eID][0].net_sta_chan.append((net,st,ch))
            [e for e in events if e.event_id==eID][0].arrival_times.append(UTCDateTime(pick).datetime)
            [e for e in events if e.event_id==eID][0].travel_times.append(UTCDateTime(pick)-UTCDateTime([e.origin_time for e in events if e.event_id==eID][0]))
            [e for e in events if e.event_id==eID][0].phase_types.append(ph_type)

    return events

events=picks_from_hypoinverse2000(filename,events)

class NETWORK(object):
    """
    Network information

    :type network: str
    :param network: Network that station belongs to

    :type station: str
    :param station: Station

    :type channel: str
    :param channel: Channel

    :type latitude: float
    :param latitude: Latitude of station in decimal degrees

    :type longitude: float
    :param longitude: Longitude of station in decimal degrees

    :type elevation: float
    :param elevation: Elevation of station in meters

    :type correction: float
    :param correction: Station correction for the attenuation relationship
    (calculated in inversion)

    NOTE: Station elevation is usually provided in meters, be careful not to
        give it in km!

    """
    def __init__(self, network, station,channel,
                 latitude, longitude,
                 elevation, correction=None):
        self.network = network
        self.station = station
        self.channel = channel
        self.latitude = latitude
        self.longitude = longitude
        self.elevation = elevation
        self.correction = correction

    def __repr__(self):
        print_str = ' '.join(['network=', str(self.network), '\n',
                              'station=', str(self.station), '\n',
                              'channel=', str(self.channel), '\n',
                              'latitude=', str(self.latitude), '\n',
                              'longitude=', str(self.longitude), '\n',
                              'elevation=', str(self.elevation), '\n',
                              'correction=', str(self.correction), '\n'])
        return "NETWORK(" + print_str + ")"

print('###################################################################')
print('###################################################################')
print('###############        READING NETWORK          ###################')
print('###################################################################')
print('###################################################################')

filename='INPUT/NETWORK.txt'
def read_network(fname,header=True):
    '''
    Funtion to read network from file with the format:
            NETWORK,STATION,CHANNEL,LATITUDE,LONGITUDE,ELEVATION

    Parameters
    ----------
    fname : TYPE: str
        DESCRIPTION: Filename of network

    header : TYPE, optional
        DESCRIPTION. The default is True.

    Returns
    -------
    network : TYPE: list
        DESCRIPTION: list of events with :class:NETWORK

    '''
    network=[]
    if header==False:
        F=open(fname,'r')
        for f in F.readlines():
            network.append(NETWORK(network=f.rstrip('\n').split(',')[0],
                                   station=f.rstrip('\n').split(',')[1],
            channel=f.rstrip('\n').split(',')[2],
            latitude=float(f.rstrip('\n').split(',')[3]),
            longitude=float(f.rstrip('\n').split(',')[4]),
            elevation=float(f.rstrip('\n').split(',')[5])))
    else:
        F=open(fname,'r')
        for f in F.readlines()[1:]:
            network.append(NETWORK(network=f.rstrip('\n').split(',')[0],
                                   station=f.rstrip('\n').split(',')[1],
            channel=f.rstrip('\n').split(',')[2],
            latitude=float(f.rstrip('\n').split(',')[3]),
            longitude=float(f.rstrip('\n').split(',')[4]),
            elevation=float(f.rstrip('\n').split(',')[5])))

    return network

network=read_network(filename)

def great_circle(lat1, lon1, lat2, lon2):
    '''
    Function to calculate great circle distances between two points on
    Earth

    Parameters
    ----------
    lat1 : TYPE: float
        DESCRIPTION: Latitude of point 1 (in decimal degrees)

    lon1 : TYPE: float
        DESCRIPTION: Longitude of point 1 (in decimal degrees)

    lat2 : TYPE:float
        DESCRIPTION: Latitude of point 2 (in decimal degrees)

    lon2 : TYPE:float
        DESCRIPTION: Longitude of point 2 (in decimal degrees)

    Returns
    -------
    dist_km : TYPE: float
        DESCRIPTION: great circle distance in km

    '''
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])

    # great_circle formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    # Radius of earth in kilometers is 6371
    dist_km = 6371* c
    return dist_km

def peak2peak(tr):
    '''
    Function to calculate peak-to-peak amplitudes of a Wood-Anderson-converted
    waveform

    This function attemps to find the maximum absolute amplitude in the trace
    and then find if the peak preceding or following it is larger so it has the
    complete pulse from which we get the amplitude and period

    Parameters
    ----------
    tr : TYPE: obspy.core.trace.Trace
        DESCRIPTION: Wood-Anderson converted trace

    Returns
    -------
    amplitude : TYPE: float
        DESCRIPTION: peak-to-peak amplitude in mm
    timespan : TYPE: float
        DESCRIPTION: period of pulse in seconds

    '''
    #Get peak-to-peak amplitude and period
    #FOR POSITIVE
    if abs(max(tr.data))/abs(min(tr.data))>=1:
        A_max=max(tr.data)
        time_A_max=tr.times()[np.argmax(tr.data)]
        #Look for the zero crossing before and after pulse
            #Find the samples between zero and interpolate to find zero-crossing (make a line and find value at zero)
                #Find lowest sample point and interpolate with sample above
        for jj in np.arange(np.argmax(tr.data),0,-1):
            if tr.data[jj]<0:
                low_cross_point=jj
                break
        y1=tr.data[low_cross_point] #Lowest sample
        y2=tr.data[low_cross_point+1] #sample above
        x1=tr.times()[low_cross_point]
        x2=tr.times()[low_cross_point+1]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_1=-b/m
        for jj in np.arange(low_cross_point,0,-1):
            if tr.data[jj]>0:
                lowest_cross_point=jj
                break
        y1=tr.data[lowest_cross_point] #Lowest sample
        y2=tr.data[lowest_cross_point+1] #sample above
        x1=tr.times()[lowest_cross_point]
        x2=tr.times()[lowest_cross_point+1]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_0=-b/m
        #Look for next crossing (mid-step towards finding the full pulse)
        for jj in np.arange(np.argmax(tr.data),len(tr.data),1):
            if tr.data[jj]<0:
                mid_cross_point=jj
                break
        y1=tr.data[mid_cross_point-1]
        y2=tr.data[mid_cross_point]
        x1=tr.times()[mid_cross_point-1]
        x2=tr.times()[mid_cross_point]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_2=-b/m
        #look for final zero crossing (FULL PULSE)
        for jj in np.arange(mid_cross_point,len(tr.data),1):
            if tr.data[jj]>0:
                high_cross_point=jj
                break
        y1=tr.data[high_cross_point-1]
        y2=tr.data[high_cross_point]
        x1=tr.times()[high_cross_point-1]
        x2=tr.times()[high_cross_point]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_3=-b/m #for plotting

        if min(tr.data[lowest_cross_point:low_cross_point])<min(tr.data[mid_cross_point:high_cross_point]):
            A_min=min(tr.data[lowest_cross_point:low_cross_point])
            time_A_min=tr.times()[lowest_cross_point+np.argmin(tr.data[lowest_cross_point:low_cross_point])]
        else:
            A_min=min(tr.data[mid_cross_point:high_cross_point])
            time_A_min=tr.times()[mid_cross_point+np.argmin(tr.data[mid_cross_point:high_cross_point])]
        #Get peak-to-peak amplitude
        amplitude=A_max+abs(A_min)
        #Get period of peak-to-peak amplitude pulse
        timespan=abs(time_A_max-time_A_min)

    #FOR NEGATIVE
    else:
        A_min=min(tr.data)
        time_A_min=tr.times()[np.argmin(tr.data)]
        #Look for the zero crossing before and after pulse
        #Find the samples between zero and interpolate to find zero-crossing (make a line and find value at zero)
            #Find lowest sample point and interpolate with sample above
        for jj in np.arange(np.argmin(tr.data),0,-1):
            if tr.data[jj]>0:
                low_cross_point=jj
                break
        y1=tr.data[low_cross_point]
        y2=tr.data[low_cross_point+1]
        x1=tr.times()[low_cross_point]
        x2=tr.times()[low_cross_point+1]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_1=-b/m
        for jj in np.arange(low_cross_point,0,-1):
            if tr.data[jj]<0:
                lowest_cross_point=jj
                break
        y1=tr.data[lowest_cross_point] #Lowest sample
        y2=tr.data[lowest_cross_point+1] #sample above
        x1=tr.times()[lowest_cross_point]
        x2=tr.times()[lowest_cross_point+1]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_0=-b/m
        #Look for next crossing (mid-step towards finding the full pulse)
        for jj in np.arange(np.argmin(tr.data),len(tr.data),1):
            if tr.data[jj]>0:
                mid_cross_point=jj
                break
        y1=tr.data[mid_cross_point-1]
        y2=tr.data[mid_cross_point]
        x1=tr.times()[mid_cross_point-1]
        x2=tr.times()[mid_cross_point]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_2=-b/m
        #Look for final zero crossing (FULL PULSE)
        for jj in np.arange(mid_cross_point,len(tr.data),1):
            if tr.data[jj]<0:
                high_cross_point=jj
                break
        y1=tr.data[high_cross_point-1]
        y2=tr.data[high_cross_point]
        x1=tr.times()[high_cross_point-1]
        x2=tr.times()[high_cross_point]
        m=(y1-y2)/(x1-x2)
        b=(x1*y2 - x2*y1)/(x1-x2)
        zero_crossing_3=-b/m #for plotting

        if max(tr.data[lowest_cross_point:low_cross_point])>max(tr.data[mid_cross_point:high_cross_point]):
            A_max=max(tr.data[lowest_cross_point:low_cross_point])
            time_A_max=tr.times()[lowest_cross_point+np.argmax(tr.data[lowest_cross_point:low_cross_point])]
        else:
            A_max=max(tr.data[mid_cross_point:high_cross_point])
            time_A_max=tr.times()[mid_cross_point+np.argmax(tr.data[mid_cross_point:high_cross_point])]
        #Get peak-to-peak amplitude in mm
        amplitude=(A_max+abs(A_min))*1e-6
        #Get period of peak-to-peak amplitude pulse
        timespan=abs(time_A_max-time_A_min)
    # if plot=True:
    #     plt.close('all')
    #     plt.figure('Peak-to-peak amplitude')
    #     plt.plot(t_vec,tr.data,'r')
    #     plt.plot(t_vec,[0]*len(t_vec),'k',linewidth=0.5)
    #     plt.axvline(x=(tr.stats.starttime+time_A_max).datetime,color='black',linewidth=2,linestyle='--')
    #     plt.axvline(x=(tr.stats.starttime+time_A_min).datetime,color='black',linewidth=2,linestyle='--')

    #     plt.axvline(x=(tr.stats.starttime+zero_crossing_0).datetime,color='black',linewidth=2,linestyle='--')
    #     plt.axvline(x=(tr.stats.starttime+zero_crossing_1).datetime,color='black',linewidth=2,linestyle='--')
    #     plt.axvline(x=(tr.stats.starttime+zero_crossing_2).datetime,color='black',linewidth=2,linestyle='--')
    #     plt.axvline(x=(tr.stats.starttime+zero_crossing_3).datetime,color='black',linewidth=2,linestyle='--')

    return amplitude,timespan

def SNR(tr,p_phase,W_SNR,type='rms'):
    '''
    Function to calculate the signal to noise ratio.

    For type rms:

        Noise is defined as the root-mean-square of the data W_SNR seconds
        before the earthquake

        Signal is defined as the root-mean-square of the data W_SNR seconds
        after the P phase arrival of the earthquake.

    For type max:

        Noise is defined as the root-mean-square of the data W_SNR seconds
        before the earthquake

        Signal is defined as the maximum amplitude of the data W_SNR seconds
        after the P phase arrival of the earthquake.

    Parameters
    ----------
    tr: TYPE: obspy.core.trace.Trace
        DESCRIPTION: Wood-Anderson converted trace

    p_phase: TYPE: datetime.datetime
        DESCRIPTION: P phase arrival to the corresponding station

    W_SNR: TYPE: int or float
        DESCRIPTION: Window (in seconds) to be used for both signal and noise

    type: TYPE: str
        DESCRIPTION: The type of SNR to be calculated, defaults to rms

    Returns
    -------
    SIGNAL/NOISE : TYPE: float
        DESCRIPTION: signal-to-noise ratio
    '''
    p_phase=UTCDateTime(p_phase)
    #NOISE
    dbp=tr.slice(starttime=p_phase-W_SNR,endtime=p_phase).data
    NOISE=np.sqrt(np.mean(dbp**2)) #Root mean square of noise part
    #SIGNAL
    if type=='rms':
        ds=tr.slice(starttime=p_phase,endtime=p_phase+W_SNR).data
        SIGNAL=np.sqrt(np.mean(ds**2)) #Root mean square of signal part
    elif type=='max':
        ds=tr.slice(starttime=p_phase,endtime=p_phase+W_SNR).data
        SIGNAL=np.max(ds) #Root mean square of signal part

    return SIGNAL/NOISE

print('###################################################################')
print('###################################################################')
print('###############        READING WAVEFORMS          ###################')
print('###################################################################')
print('###################################################################')
'''
GET WOOD-ANDERSON/FROM SAC CONVERTED TRACES
-RTR
-TAPER WIDTH 0.5 (BECAUSE WE HAVE LONG RECORD WE CAN DO A LONGER TAPER)
-TRANSFER FROM EVALRESP TO WA

'''

DIR_WA='INPUT/WAVEFORMS/WOODANDERSON/'
ids_waveforms=list(np.unique([x.split('.')[0] for x in os.listdir(DIR_WA)]))

events=[e for e in events if e.event_id in ids_waveforms]

print('Loading waveforms')
tags=[]
WAVEFORMS_WA=[]

[(WAVEFORMS_WA.append(read(DIR_WA+e.event_id+'*.sac')),tags.append(e.event_id)) for e in events]

print('Done loading waveforms')

RESP_FILES=os.listdir('INPUT/WAVEFORMS/SC_RESP/')
for i in range(len(WAVEFORMS_WA)):
    for tr in WAVEFORMS_WA[i]:
        if 'RESP.'+tr.stats.network+'.'+tr.stats.station+'..'+tr.stats.channel not in RESP_FILES:
            WAVEFORMS_WA[i].remove(tr)

'''CHECK FOR EMPTY TRACES'''
for i in range(len(WAVEFORMS_WA)):
    for tr in WAVEFORMS_WA[i]:
        if not any(tr.data):
            WAVEFORMS_WA[i].remove(tr)

'''CHECK FOR NAN IN TRACES'''
for i in range(len(WAVEFORMS_WA)):
    for tr in WAVEFORMS_WA[i]:
        if any(np.isnan(tr.data)):
            WAVEFORMS_WA[i].remove(tr)

'''SNR'''
W_SNR=10 #10 seconds window
for i in range(len(WAVEFORMS_WA)):
    e=[e for e in events if e.event_id==tags[i]][0]
    for j,tr in enumerate(WAVEFORMS_WA[i]):
        if SNR(tr,e.arrival_times[j],W_SNR,type='rms')<5:
            WAVEFORMS_WA[i].remove(tr)

'''REMOVE STREAMS WITH LESS THAN NT TRACES'''
NT=3
empty_index=[i for i,x in enumerate(WAVEFORMS_WA) if len(x)<3]
if len(empty_index)>0:
    for index in empty_index[::-1]:
        del WAVEFORMS_WA[index]
        del tags[index]

events=[e for e in events if e.event_id in tags]

'''
LEAST-SQUARES INVERSION
'''

def make_kernel(tags,equation='hutton_boore'):

    print('Making G')
    A=[]
    hypo_dist=[]
    G=[]

    for i,tag in enumerate(tags):
        print(i)
        e=[e for e in events if e.event_id==tag][0]

        #Event magnitude vector init
        M_vec=[0]*len(tags)
        #Turn on magnitude for event
        M_vec[i]=1

        for tr in WAVEFORMS_WA[tags.index(e.event_id)]:

            ROW=[]

            AMP,period=peak2peak(tr)

            A.append(AMP/2)

            #get index of network,station,channel
            nsc_index=[i for i,n in enumerate(network) if (n.network,n.station,n.channel)==(tr.stats.network,tr.stats.station,tr.stats.channel)][0]

            #Station correction vector init
            dMl_vec=[0]*len(network)
            #Turn on station correction for channel
            dMl_vec[nsc_index]=-1

            #Epicentral distance
            epicenter=great_circle(e.latitude,e.longitude,network[nsc_index].latitude,network[nsc_index].longitude)

            depth=e.depth+network[nsc_index].elevation/1000

            #Hypocentral distance
            R=np.sqrt(depth**2+epicenter**2)

            hypo_dist.append(R)

            if equation=='hutton_boore':
                ROW.extend([np.log10(R/17)])
                ROW.extend([R-17])
            elif equation=='richter':
                ROW.extend([np.log10(R/100)])
                ROW.extend([R-100])

            ROW.extend(M_vec)
            ROW.extend(dMl_vec)

            G.append(ROW)

    return(A,G,hypo_dist)

A,G,hypo_dist=make_kernel(tags)

print('Done...')

G=np.asmatrix(G)

A_fname='RESULTS/G'
with open(A_fname, 'wb') as f:
    pickle.dump(G, f)

REFERENCE_VECTOR=['ALPHA','K']
REFERENCE_VECTOR.extend(events)
REFERENCE_VECTOR.extend(network)

#If no event or channel is represented in matrix its an empty column, we have
#to take it out
NO_EVENT_OR_CHANNEL=[i for i in range(G.shape[1]) if not any(G[:,i])]

#List of missing events or channels
MISSING=[REFERENCE_VECTOR[x] for x in NO_EVENT_OR_CHANNEL]
print('Missing '+str(len(MISSING))+' channels of '+str(len(network)))

EVENTS=[x for x in events if x not in MISSING]
CHANNELS=[x for x in network if x not in MISSING]

for INDEX in NO_EVENT_OR_CHANNEL[::-1]:
    G = np.delete(G, INDEX, 1)

#This last row guarantees that the sum of all station corrections = zero
ROW_CORECTIONS=[0]*(len(EVENTS)+2)
ROW_CORECTIONS.extend([1]*len(CHANNELS))

G=np.vstack([G,ROW_CORECTIONS])

print('Inversion')

###The +2 factor is used instead of the +3 defined by Richter to be consistent with the definition
#proposed by Hutton and Boore (1987) (10mm @ 17km instead of 1mm @ 100km)
logA=(np.log10(np.asarray(A))+2).reshape(len(A),1)

d=logA
d=np.append(d,0)
d=np.reshape(d,(d.shape[0],1))

c=(np.transpose(G)@G).I@(np.transpose(G)@(d))

alpha=c.item(0)
K=c.item(1)

print('Value for K is = '+str(K))
print('Value for alpha is = '+str(alpha))

A_fname='RESULTS/alpha'
with open(A_fname, 'wb') as f:
    pickle.dump(alpha, f)

A_fname='RESULTS/K'
with open(A_fname, 'wb') as f:
    pickle.dump(K, f)

Magnitudes=[]
for k in np.arange(2,len(EVENTS)+2):
    Magnitudes.append(np.round(c.item(k),1))

A_fname='RESULTS/magnitudes'
with open(A_fname, 'wb') as f:
    pickle.dump(Magnitudes, f)

dMl=[]
for k in np.arange(len(EVENTS)+2,c.shape[0]):
    dMl.append(c.item(k))

A_fname='RESULTS/dMl'
with open(A_fname, 'wb') as f:
    pickle.dump(dMl, f)


import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
from sklearn.externals.six import StringIO
from sklearn.cluster import KMeans
import pickle as pk
import pydotplus
import pensieve
import pensiedt
import robustmpc
import robustmdt
import robustlin
import argparse
import load_trace
import fixed_env as env
from multiprocessing.dummy import Pool as ThreadPool
import limes
import pensilin
import fixed_env_hotdash as env_hotdash
import hotdash
import hotdadt
import hotdlin
#from actor_wrapper import Actor_LIME
#NN_MODEL = './models/pretrain_linear_reward.ckpt'

S_INFO = 6
S_INFO_P = 6  # bit_rate, buffer_size, next_chunk_size, bandwidth_measurement(throughput and time), chunk_til_video_end
S_INFO_R = 5  # bit_rate, buffer_size, rebuffering_time, bandwidth_measurement, chunk_til_video_end
S_LEN = 8  # take how many frames in the past
A_DIM = 6
ACTOR_LR_RATE = 0.0001
CRITIC_LR_RATE = 0.001
VIDEO_BIT_RATE = [300,750,1200,1850,2850,4300]  # Kbps
BITRATE_REWARD = [1, 2, 3, 12, 15, 20]
BUFFER_NORM_FACTOR = 10.0
CHUNK_TIL_VIDEO_END_CAP = 48.0
M_IN_K = 1000.0
REBUF_PENALTY = 4.3  # 1 sec rebuffering -> 3 Mbps
SMOOTH_PENALTY = 1
DEFAULT_QUALITY = 1  # default video quality without agent
RANDOM_SEED = 42
RAND_RANGE = 1000
SUMMARY_DIR = './results'
LOG_FILE = './results/log_pensieve'
# log in format of time_stamp bit_rate buffer_size rebuffer_time chunk_size download_time reward
NN_MODEL = './models/pretrain_linear_reward.ckpt'
THREAD = 16

def split_train_test(obss, acts, train_frac):
    n_train = int(train_frac * len(obss))
    idx = np.arange(len(obss))
    np.random.shuffle(idx)
    obss_train = obss[idx[:n_train]]
    acts_train = acts[idx[:n_train]]
    obss_test = obss[idx[n_train:]]
    acts_test = acts[idx[n_train:]]
    return obss_train, acts_train, obss_test, acts_test

def get_rollouts(env, policy, args, n_batch_rollouts, lin=None):
    rollouts = []
    if lin is None:
        for i in range(n_batch_rollouts):
            rollouts.extend(policy.main(args, env, flag =1))
    else:
        for i in range(n_batch_rollouts):
            rollouts.extend(policy.main(args, env, lime = lin,flag =1))
    return rollouts

def resample(states, actions, serials, max_pts):
    idx = np.random.choice(len(states), size=max_pts)
    return states[idx], actions[idx], serials[idx]

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--abr', metavar='ABR', choices=['pensieve', 'robustmpc', 'hotdash'])
    #parser.add_argument('-n', '--leaf-nodes', type=int)
    parser.add_argument('-q', '--qoe-metric', choices=['lin', 'log', 'hd'])
    parser.add_argument('-l', '--log', action='store_true')
    parser.add_argument('-i', '--lin', action='store_true')
    parser.add_argument('-m', '--iters', type=int)
    parser.add_argument('-t', '--traces', choices=['norway', 'fcc', 'oboe'])

    args = parser.parse_args()
    n_batch_rollouts = 10
    max_iters = args.iters
    max_pts = 200000
    train_frac = 0.8
    np.random.seed(RANDOM_SEED)
    states, actions, serials = [], [], []
    precision = []
    #trees = []
    all_cooked_time, all_cooked_bw, all_file_names = load_trace.load_trace(args.traces)
    if args.abr == 'hotdash':
        net_env = env_hotdash.Environment(all_cooked_time=all_cooked_time, all_cooked_bw=all_cooked_bw,
                              all_file_names=all_file_names)
    else:
        net_env = env.Environment(all_cooked_time=all_cooked_time, all_cooked_bw=all_cooked_bw,
                              all_file_names=all_file_names)

    if args.abr == 'pensieve':
        teacher = pensieve.Pensieve()
        student = pensilin.Pensilin()
        #test = pensieve.Pensieve()
    elif args.abr == 'robustmpc':
        teacher = robustmpc.RobustMPC()
        student = robustlin.Robustlin()
    elif args.abr == 'hotdash':
        teacher = hotdash.Hotdash()
        student = hotdlin.Hotdlin()
    else:
        raise NotImplementedError

    # Step 1: Initialization for the first iteration
    trace = get_rollouts(env=net_env, policy=teacher, args=args, n_batch_rollouts=n_batch_rollouts)
    states.extend((state for state, _, _ in trace))
    actions.extend((action for _, action, _ in trace))
    serials.extend(serial for _, _, serial in trace)

    def predict_fn2(data):
        if args.abr == 'pensieve':
            input_data = np.zeros((data.shape[0], S_INFO, S_LEN))
            input_data[:, 0, -1] = data[:, 0]
            input_data[:, 1, -1] = data[:, 1]
            input_data[:, 2, :] = data[:, 2:10]
            input_data[:, 3, :] = data[:, 10:18]
            input_data[:, 4, :A_DIM] = data[:, 18:24]
            input_data[:, 5, -1] = data[:, 24]
            return teacher.actor.predict(input_data)
        elif args.abr == 'hotdash':
            input_data = np.zeros((data.shape[0], S_INFO, S_LEN))
            input_data[:, 0, -1] = data[:, 0]
            input_data[:, 1, -1] = data[:, 1]
            input_data[:, 2, :] = data[:, 2:10]
            input_data[:, 3, :] = data[:, 10:18]
            input_data[:, 4, :A_DIM] = data[:, 18:24]
            input_data[:, 5, -1] = data[:, 24]
            return teacher.actor_bitr.predict(input_data)
        else:
            pass
    #test_data=[20,21,22,23,24]
    #receive_data=[]
    #def he_test(j):
    grade=np.zeros((100,3))
    for j in range(27,30):
        for i in range(max_iters):
            # Step 2:
            print('cluster number:{},Iteration {}/{}'.format(j, i, max_iters))
            cur_states, cur_actions, cur_serials = resample(np.array(states), np.array(actions), np.array(serials), max_pts)
            print('Training student with {} points'.format(len(cur_serials)))
            serials_train, actions_train, serials_val, actions_val = split_train_test(cur_serials, cur_actions, train_frac)
            #print(serials_train.shape[0])
            #def predict_fn1(state):
                #print(state.shape)
                #print(state.shape[0])
                #result = np.zeros((state.shape[0],6))
                #for i in range (state.shape[0]):
                    #for j in range(6):
                        #if(actions_train[i] ==j ) :
                            #result[i,j] = 0.5
                        #else:
                            #result[i,j] = 0.1
                #print(result)
                #return result

            lin = limes.LIMESimpleModel(cluster_num = j, cluster_method = KMeans, random_state = None)
            lin.fit(serials_train, actions_train, predict_fn = predict_fn2, labels_num = 6)
            #print('Train accuracy: {}'.format(np.mean(actions_train == lin.predict(serials_train))))
            #print('Val accuracy: {}'.format(np.mean(actions_val == lin.predict(serials_val))))
            precision.append(np.mean(lin.predict(serials_val) == actions_val))
            print('unpruned precision', precision[-1])
            grade[i][j-27] =precision[-1]
            student_trace = get_rollouts(env=net_env, policy=student, args=args, n_batch_rollouts=n_batch_rollouts,lin= lin)
            student_states = [state for state, _, _ in student_trace]
            student_actions = [action for _, action, _ in student_trace]
            student_serials = [serial for _, _, serial in student_trace]

            if args.abr == 'pensieve':
                teacher_actions = map(teacher.predict, student_states)
            elif args.abr == 'hotdash':
                teacher_actions = map(teacher.get_abr_rl_bitrate, student_states)
            else:
                pool = ThreadPool(THREAD)
                teacher_actions = pool.map(teacher.predict, student_states)
                pool.close()
                pool.join()
                print (teacher_actions)

        # teacher_actions = []
        # for student_state in student_states:
        #     teacher_actions.append(teacher.predict(student_state))

            states.extend(student_states)
            actions.extend(teacher_actions)
            serials.extend(student_serials)
        #print('cluster number ={}, average curancy: {}',format(j,sum(grade)/max_iters))
        #trees.append(lin)
        #return(grade)
    #pool = ThreadPool(THREAD)
    #receive_data=pool.map(he_test, test_data)
    #pool.close()
    #pool.join()
    np.savetxt('dd.txt',grade,fmt='%.5f')
    #best_lin = trees[-1]
    # save decision tree to file
    #with open('lime/' + args.abr + '.pk3', 'wb') as f:
        #pk.dump(lin, f)

    #dot_data = StringIO()
    #export_graphviz(lin, out_file=dot_data, filled=True)
    #out_graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
    #out_graph.write_svg('lime/' + args.abr + '.svg')


#!/usr/bin/python
# -*- coding: utf-8 -*-

import random

class GameError(Exception):
    pass

class StateError(GameError):
    pass

class NotAliveError(GameError):
    pass

class WrongPlayerNumberError(GameError):
    pass

class UnrecognizedRole(GameError):
    pass

class MoreThanOneWorP(GameError):
    pass

class Role:
    def __init__(self, name, good, special):
        self.name = name
        self.good = good
        self.special = special

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.__str__()

contadino = Role("Contadino", True, False)
lupo = Role("Lupo", False, False)
veggente = Role("Veggente", True, True)
protettore = Role("Protettore", True, True)
figlio_del_lupo = Role("Figlio del Lupo", True, True)

def ch2role(c):
    """Role translation"""
    if c == "c":
        return contadino
    elif c == "l":
        return lupo
    elif c == "v":
        return veggente
    elif c == "p":
        return protettore
    elif c == "f":
        return figlio_del_lupo
    else:
        raise UnrecognizedRole

NIGHT = 0
NIGHT_END = 0.5
DAY = 1
DAY_END = 3
FINISH = -1
PRE = -2
RUOLI_ASSEGNATI = -3

W_TIE = 0
W_GOOD = 1
W_BAD = 2

def stateName(state):
    if state == NIGHT:
        return "NOTTE"
    elif state == NIGHT_END:
        return "NOTTE CONCLUSA"
    elif state == DAY:
        return "GIORNO"
    elif state == DAY_END:
        return "GIORNO CONCLUSA"
    elif state == PRE:
        return "PRE"
    elif state == FINISH:
        return "PARTITA CONCLUSA"
    elif state == RUOLI_ASSEGNATI:
        return "conto alla rovescia per la notte..."
    else:
        return "ERRORE"

def sideName(side):
    if side == W_TIE:
        return "PARITÀ ⚖"
    elif side == W_GOOD:
        return "I BUONI vincono! 👩🏻🌾 👨🏻🌾"
    elif side == W_BAD:
        return "I LUPI vincono! 🐺"
    else:
        return "ERRORE"

class Game:
    """Game class"""
    def __init__(self, rolestring):
        self.rolelist = []
        self.check_roles(rolestring)
        for c in rolestring:
            r = ch2role(c)
            self.rolelist.append(r)

        self.turn = 0
        self.state = PRE
        self.stato_lupi = False
        self.stato_veggente = False if "v" in rolestring else True
        self.stato_protettore = False if "p" in rolestring else True
        self.figlio_del_lupo = False
        self.players = []

    def check_roles(self, rolestring):
        if len(rolestring.split("v")) > 2 or len(rolestring.split("p")) > 2:
            raise MoreThanOneWorP

    def setPlayers(self, players):
        """Give roles to players"""
        if len(players) != len(self.rolelist):
            raise WrongPlayerNumberError

        random.shuffle(self.rolelist)

        for i, p in enumerate(players):
            p.role = self.rolelist[i]

        self.state = RUOLI_ASSEGNATI

    def alivePlayers(self):
        return [p for p in self.players if p.alive]

    def goodPlayers(self):
        return [p for p in self.players if (p.alive and p.role.good)]

    def badPlayers(self):
        return [p for p in self.players if (p.alive and (not p.role.good))]

    def wolves(self):
        return [p for p in self.players if (p.alive and p.role == lupo)]

    def watcher(self):
        return [p for p in self.players if (p.alive and p.role == veggente)]

    def protector(self):
        return [p for p in self.players if (p.alive and p.role == protettore)]

    def checkEnd(self):
        """Check if the game end"""
        if len(self.alivePlayers()) == 0:
            self.state = FINISH
            self.win = W_TIE
            return
        if len(self.goodPlayers()) == 0:
            self.state = FINISH
            self.win = W_BAD
            return
        if len(self.badPlayers()) == 0:
            self.state = FINISH
            self.win = W_GOOD

    def euthanise(self, toe):
        """Delete one player"""
        self.players[toe].alive = False
        self.players[toe].choice = None
        self.checkEnd()

    def inputNight(self, results):
        """Night step"""
        if self.state != NIGHT_END:
            raise StateError

        killed_now = []
        out = {}

        tp = None
        if 'toprotect' in results:
            tp = results['toprotect']

            if not self.players[tp].alive:
                raise NotAliveError

        if 'tomurder' in results:
            tm = results['tomurder'][0]
            if tm is not None:
                if not self.players[tm].alive:
                    raise NotAliveError
                elif tp != tm and self.players[tm].role != figlio_del_lupo:
                    killed_now.append(tm)
                elif self.players[tm].role == figlio_del_lupo:
                    self.figlio_del_lupo = True
                    self.players[tm].role = lupo

            out['killed_now'] = killed_now

        if 'toview' in results:
            tv = results['toview']
            out['viewed'] = self.players[tv].role.good

            if not self.players[tv].alive:
                raise NotAliveError

        for tokill in killed_now:
            self.players[tokill].alive = False

        self.state = DAY

        self.checkEnd()

        return out

    def inputDay(self, voted, n_veggenti, n_protettori):
        """Day syep"""
        if not self.players[voted].alive:
            raise NotAliveError

        self.players[voted].alive = False

        self.state = NIGHT
        self.stato_veggente = False if n_veggenti >= 1 else True
        self.stato_protettore = False if n_protettori >= 1 else True

        self.checkEnd()

    def recompute_player_index(self):
        """Recompute players index"""
        for i, p in enumerate(self.players):
            p.index = i


# -*- coding: utf-8 -*-

"""
Test the main module

SPDX-FileCopyrightText: 2016-2021 Uwe Krien <krien@uni-bremen.de>

SPDX-License-Identifier: MIT
"""
# import deflex.tools

__copyright__ = "Uwe Krien <krien@uni-bremen.de>"
__license__ = "MIT"

from oemof.solph import Bus, GenericStorage, Sink, Source, Transformer

from deflex import (
    DeflexGraph,
    calculate_key_values,
    fetch_attributes_of_commodity_sources,
    fetch_test_files,
    get_combined_bus_balance,
    get_converter_balance,
    get_time_index,
    nodes2table,
    postprocessing,
    restore_results,
    scenario_tools,
)
from deflex.postprocessing.analyses import (
    _calculate_marginal_costs,
    _fetch_electricity_flows,
    fetch_converter_parameters,
)
from deflex.postprocessing.basic import _get_all_nodes_from_results

def test_string_label():
    label = scenario_tools.nodes.Label(
        cat="my*category", tag="my+tag", subtag="my-subtag", region="my-region"
    )
    assert label.tag == "my+tag"
    assert postprocessing.analyses.label2str(label) == (
        "my*category_my+tag_my-subtag_my-region"
    )

class TestAnalysis:
    @classmethod
    def setup_class(cls):
        cls.results = restore_results(fetch_test_files("de02_no-heat.dflx"))

    def test_all_nodes_from_results(self):
        nodes = _get_all_nodes_from_results(self.results)
        assert len(nodes) == 141
        # 15 + 26 + 87 + 1 + 12 = 141
        assert len([n for n in nodes if isinstance(n, Sink)]) == 15
        assert len([n for n in nodes if isinstance(n, Source)]) == 26
        assert len([n for n in nodes if isinstance(n, Transformer)]) == 87
        assert len([n for n in nodes if isinstance(n, GenericStorage)]) == 1
        assert len([n for n in nodes if isinstance(n, Bus)]) == 12
        print(set([type(n) for n in nodes]))

    def test_graph_and_nodes(self):
        graph = DeflexGraph(self.results)
        nodes = postprocessing.basic._get_all_nodes_from_results(self.results)
        n = 0
        for k, v in graph.group_nodes_by_type().items():
            n += len(v)
            assert len([n for n in nodes if isinstance(n, k)]) == len(v)
        assert n == len(nodes)
        print(n)

    def test_nodes_to_table(self):
        """
        The sum of all inputs must be the sum of all outputs.
        In the de02-example the sum of the outflows of the electricity Bus must
        equal the input flows of the storage and the demand Sink.
        """
        all_nodes = nodes2table(self.results)
        assert int(all_nodes.sum()["out"]) == int(all_nodes.sum()["in"])
        assert int(
            all_nodes.loc[("Bus", "electricity", "all", "all", "DE01"), "out"]
        ) == int(
            all_nodes.loc["Sink", "electricity demand"]["in"].sum()
            + all_nodes.loc["GenericStorage"]["in"].sum()
        )

    def test_resource_parameter(self):
        nodes = postprocessing.basic._get_all_nodes_from_results(self.results)
        cdf = fetch_attributes_of_commodity_sources(self.results)
        buses = [
            n
            for n in nodes
            if isinstance(n, Bus) and n.label.cat == "commodity"
        ]
        all_costs = 0
        emission = 0
        number = 0
        for bus in buses:
            number += 1
            all_costs += float(cdf.loc[cdf.to_node == bus]["variable_costs"])
            emission += float(cdf.loc[cdf.to_node == bus]["emission"])
        assert round(all_costs / number, 2) == 19.19
        assert round(emission / number, 4) == 0.1804

    def test_converter_parameters(self):
        nodes = postprocessing.basic._get_all_nodes_from_results(self.results)
        transformer = [
            n
            for n in nodes
            if isinstance(n, Transformer) and n.label.subtag == "lignite"
        ]
        cp = postprocessing.analyses.fetch_converter_parameters(
            self.results, transformer
        )
        assert cp["fuel"].unique() == ["lignite, DE"]
        assert round(cp["efficiency, electricity"].mean(), 3) == 0.361

    def test_time_index(self):
        time_index = get_time_index(self.results)
        assert len(time_index) == 48
        assert time_index[5].year == 2014
        assert time_index[20].hour == 20

    def test_parameter_of_commodity_sources(self):
        fetch_attributes_of_commodity_sources(self.results)

    def test_marginal_costs(self):
        nodes = postprocessing.basic._get_all_nodes_from_results(self.results)
        transformer = [
            n
            for n in nodes
            if isinstance(n, Transformer) and n.label.subtag == "hard coal"
        ]
        cp = fetch_converter_parameters(self.results, transformer)
        mc = _calculate_marginal_costs(cp)
        assert round(mc["marginal costs"].mean(), 2) == 47.51

    def test_electricity_flows(self):
        _fetch_electricity_flows(self.results)

    def test_calculate_key_values(self):
        calculate_key_values(self.results)

    def test_combined_bus_balance(self):
        get_combined_bus_balance(self.results)

    def test_converter_balance(self):
        get_converter_balance(self.results)


#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock

import nova.conf
from nova import context
from nova import objects
from nova.objects import base as obj_base
from nova.scheduler.client import report
from nova import test
from nova.tests import uuidsentinel as uuids

CONF = nova.conf.CONF

class SchedulerReportClientTestCase(test.NoDBTestCase):

    def setUp(self):
        super(SchedulerReportClientTestCase, self).setUp()
        self.context = context.get_admin_context()
        self.ks_sess_mock = mock.Mock()

        with test.nested(
                mock.patch('keystoneauth1.session.Session',
                           return_value=self.ks_sess_mock),
                mock.patch('keystoneauth1.loading.load_auth_from_conf_options')
        ) as (_auth_mock, _sess_mock):
            self.client = report.SchedulerReportClient()

    @mock.patch('keystoneauth1.session.Session')
    @mock.patch('keystoneauth1.loading.load_auth_from_conf_options')
    def test_constructor(self, load_auth_mock, ks_sess_mock):
        report.SchedulerReportClient()

        load_auth_mock.assert_called_once_with(CONF, 'placement')
        ks_sess_mock.assert_called_once_with(auth=load_auth_mock.return_value)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_exists_in_cache(self, get_rp_mock,
            create_rp_mock):
        # Override the client object's cache to contain a resource provider
        # object for the compute host and check that
        # _ensure_resource_provider() doesn't call _get_resource_provider() or
        # _create_resource_provider()
        self.client._resource_providers = {
            uuids.compute_node: mock.sentinel.rp
        }

        self.client._ensure_resource_provider(uuids.compute_node)
        self.assertFalse(get_rp_mock.called)
        self.assertFalse(create_rp_mock.called)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_get(self, get_rp_mock, create_rp_mock):
        # No resource provider exists in the client's cache, so validate that
        # if we get the resource provider from the placement API that we don't
        # try to create the resource provider.
        get_rp_mock.return_value = mock.sentinel.rp

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        self.assertEqual({uuids.compute_node: mock.sentinel.rp},
                          self.client._resource_providers)
        self.assertFalse(create_rp_mock.called)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_create_none(self, get_rp_mock,
            create_rp_mock):
        # No resource provider exists in the client's cache, and
        # _create_provider returns None, indicating there was an error with the
        # create call. Ensure we don't populate the resource provider cache
        # with a None value.
        get_rp_mock.return_value = None
        create_rp_mock.return_value = None

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        create_rp_mock.assert_called_once_with(uuids.compute_node,
                                               uuids.compute_node)
        self.assertEqual({}, self.client._resource_providers)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_create(self, get_rp_mock,
            create_rp_mock):
        # No resource provider exists in the client's cache and no resource
        # provider was returned from the placement API, so verify that in this
        # case we try to create the resource provider via the placement API.
        get_rp_mock.return_value = None
        create_rp_mock.return_value = mock.sentinel.rp

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        create_rp_mock.assert_called_once_with(
                uuids.compute_node,
                uuids.compute_node,  # name param defaults to UUID if None
        )
        self.assertEqual({uuids.compute_node: mock.sentinel.rp},
                          self.client._resource_providers)

        create_rp_mock.reset_mock()
        self.client._resource_providers = {}

        self.client._ensure_resource_provider(uuids.compute_node,
                                              mock.sentinel.name)

        create_rp_mock.assert_called_once_with(
                uuids.compute_node,
                mock.sentinel.name,
        )

    def test_get_resource_provider_found(self):
        # Ensure _get_resource_provider() returns a ResourceProvider object if
        # it finds a resource provider record from the placement API
        uuid = uuids.compute_node
        resp_mock = mock.Mock(status_code=200)
        json_data = {
            'uuid': uuid,
            'name': uuid,
            'generation': 42,
        }
        resp_mock.json.return_value = json_data
        self.ks_sess_mock.get.return_value = resp_mock

        result = self.client._get_resource_provider(uuid)

        expected_provider = objects.ResourceProvider(
                uuid=uuid,
                name=uuid,
                generation=42,
        )
        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        self.assertTrue(obj_base.obj_equal_prims(expected_provider,
                                                 result))

    def test_get_resource_provider_not_found(self):
        # Ensure _get_resource_provider() just returns None when the placement
        # API doesn't find a resource provider matching a UUID
        resp_mock = mock.Mock(status_code=404)
        self.ks_sess_mock.get.return_value = resp_mock

        uuid = uuids.compute_node
        result = self.client._get_resource_provider(uuid)

        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        self.assertIsNone(result)

    @mock.patch.object(report.LOG, 'error')
    def test_get_resource_provider_error(self, logging_mock):
        # Ensure _get_resource_provider() sets the error flag when trying to
        # communicate with the placement API and not getting an error we can
        # deal with
        resp_mock = mock.Mock(status_code=503)
        self.ks_sess_mock.get.return_value = resp_mock

        uuid = uuids.compute_node
        result = self.client._get_resource_provider(uuid)

        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        # A 503 Service Unavailable should trigger an error logged and
        # return None from _get_resource_provider()
        self.assertTrue(logging_mock.called)
        self.assertIsNone(result)

    def test_create_resource_provider(self):
        # Ensure _create_resource_provider() returns a ResourceProvider object
        # constructed after creating a resource provider record in the
        # placement API
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=201)
        self.ks_sess_mock.post.return_value = resp_mock

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_provider = objects.ResourceProvider(
            uuid=uuid,
            name=name,
            generation=1,
        )
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        self.assertTrue(obj_base.obj_equal_prims(expected_provider,
                                                 result))

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_create_resource_provider_concurrent_create(self, get_rp_mock):
        # Ensure _create_resource_provider() returns a ResourceProvider object
        # gotten from _get_resource_provider() if the call to create the
        # resource provider in the placement API returned a 409 Conflict,
        # indicating another thread concurrently created the resource provider
        # record.
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=409)
        self.ks_sess_mock.post.return_value = resp_mock

        get_rp_mock.return_value = mock.sentinel.get_rp

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        self.assertEqual(mock.sentinel.get_rp, result)

    @mock.patch.object(report.LOG, 'error')
    def test_create_resource_provider_error(self, logging_mock):
        # Ensure _create_resource_provider() sets the error flag when trying to
        # communicate with the placement API and not getting an error we can
        # deal with
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=503)
        self.ks_sess_mock.post.return_value = resp_mock

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        # A 503 Service Unavailable should log an error and
        # _create_resource_provider() should return None
        self.assertTrue(logging_mock.called)
        self.assertIsNone(result)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_ensure_resource_provider')
    @mock.patch.object(objects.ComputeNode, 'save')
    def test_update_resource_stats_saves(self, mock_save, mock_ensure):
        cn = objects.ComputeNode(context=self.context,
                                 uuid=uuids.compute_node,
                                 hypervisor_hostname='host1')
        self.client.update_resource_stats(cn)
        mock_save.assert_called_once_with()
        mock_ensure.assert_called_once_with(uuids.compute_node, 'host1')


import logging
from optparse import make_option

from django.core.management.base import BaseCommand

from amo.utils import chunked

from mkt.developers.tasks import refresh_iarc_ratings

log = logging.getLogger('z.task')

class Command(BaseCommand):
    """
    Refresh old or corrupt IARC ratings by re-fetching the certificate.
    """
    option_list = BaseCommand.option_list + (
        make_option('--apps',
                    help='Webapp ids to process. Use commas to separate '
                         'multiple ids.'),
    )
    help = __doc__

    def handle(self, *args, **kw):
        from mkt.webapps.models import Webapp

        # Get apps.
        apps = Webapp.objects.filter(iarc_info__isnull=False)
        ids = kw.get('apps')
        if ids:
            apps = apps.filter(
                id__in=(int(id.strip()) for id in ids.split(',')))

        for chunk in chunked(apps.values_list('id', flat=True), 100):
            refresh_iarc_ratings.delay(chunk)


"""
  Time complexity: O(m * nlogn)
	m: m strings
	n: the longest string in strs
"""
# https://leetcode.com/problems/group-anagrams/description/
class Solution(object):
    def groupAnagrams(self, strs):
        """
        :type strs: List[str]
        :rtype: List[List[str]]
        """
        res = {}

        for word in strs:
            key = tuple(sorted(word))
            if key not in res:
                res[key] = [word]
            else:
                res[key].append(word)

        return res.values()


from fastapi.encoders import jsonable_encoder
from fastapi.exception_handlers import request_validation_exception_handler
from fastapi.exceptions import RequestValidationError
from starlette.requests import Request
from starlette.responses import JSONResponse
from .responses import fhir_rest_response
from fhirpath.utils import lookup_fhir_class

HTTP_400_FHIR_VALIDATION = 400
FHIR = False

async def fhir_request_validation_exception_handler(
    request: Request, exc: RequestValidationError
) -> JSONResponse:
    # do your custom
    # identify as FHIR service
    # create FHIR outcome
    # response
    if "FHIR_REQUEST_ID" not in request.scope:
        # no dealings with no FHIR service
        return await request_validation_exception_handler(request, exc)

    # create operation outcome
    outcome = make_outcome(request, exc)

    return fhir_rest_response(
        request,
        outcome,
        status_code=request.scope.get("http_error_code", HTTP_400_FHIR_VALIDATION),
    )

def make_outcome(request: Request, exc: RequestValidationError):
    """
    https://terminology.hl7.org/2.0.0/CodeSystem-operation-outcome.html
    :param exc:
    :param status_code:
    :return:
    """
    klass = lookup_fhir_class(
        "OperationOutcome", fhir_release=request.scope["FHIR_VERSION"]
    )
    issues = list()
    for error in exc.errors():
        issue = {
            "severity": "error",
            "code": exc.code,
            "details": {
                "coding": [
                    {
                        "system": "http://terminology.hl7.org/CodeSystem/operation-outcome",
                        "code": exc.system_code,
                        "display": exc.body,
                    }
                ]
            },
            "diagnostics": f"loc: {error['loc']}, message: {error['msg']}",
        }
        issues.append(issue)

    outcome = klass(**{"id": str(request.scope["FHIR_REQUEST_ID"]), "issue": issues})
    return outcome


url = input("请输入图片路径：")
a = open(file=url,mode="rb")
import random
z = random.randint(0,9999999999)
c = open(file="E:\Python\day01\\" + str(z) +".jpg",mode="wb")

b = a.read()
c.write(b)

c.close()
a.close()
print("上传成功！！！")


##################################################################################
# MIT License
#
# Copyright (c) 2018 新的天 Xin de Tian
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
##################################################################################
# https://github.com/Xin-tian/npKeras
##################################################################################

import numpy as np
import pickle as pkl

from keras import Model
from keras.models import Sequential
from keras.layers import Input, Conv2D, MaxPooling2D
from keras.layers import Flatten, Dense
####################################################
from keras.datasets import mnist
####################################################
from keras_weights_exporter import keras_weights

def myModel():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), activation = 'relu', padding='same',
                      input_shape=(28, 28, 1), name='conv2d_1'))
    model.add(Conv2D(32, (3, 3), activation='relu', padding='same',
                      name='conv2d_2'))
    model.add(MaxPooling2D((2, 2)))
    model.add(Conv2D(32, (3, 3), activation='relu', padding='same',
                      name='conv2d_3'))
    model.add(Conv2D(32, (3, 3), activation='relu', padding='same',
                      name='conv2d_4'))
    model.add(MaxPooling2D((2, 2)))
    model.add(Conv2D(64, (3, 3), activation='relu', padding='same',
                      name='conv2d_5'))
    model.add(Conv2D(64, (3, 3), activation='relu', padding='same',
                      name='conv2d_6'))
    model.add(Flatten())
    model.add(Dense(64, activation='relu',   name='dense_1'))
    model.add(Dense(10, activation='softmax', name='dense_2'))
    return model

if __name__ == "__main__":

    MNIST_data = mnist.load_data()
    (X_train, y_train), (X_test, y_test) = MNIST_data

    # Save data for inference
    pkl.dump( MNIST_data, open('MNIST_data.pkl', 'wb'))

    X_train = np.reshape(X_train,(-1, 28, 28, 1))
    X_test  = np.reshape(X_test,(-1, 28, 28, 1))

    model = myModel()
    model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    model.fit(X_train, y_train, batch_size=32, epochs=3,
              validation_split=0.2, verbose=1)

    y_preds = np.argmax(model.predict(X_test, verbose=1), axis=1)
    print('Accuracy = ' + str(np.mean(y_test == y_preds)))

    ######################################################
    # Export Keras model weights
    keras_weights(model, 'MNIST_weights.pkl', verbose=1)
    ######################################################


def prob_199_6():
  lines = open("dataset_199_6.txt").read().splitlines()

  k = int(lines[0])
  string = lines[1]

  fout = open("out.txt", "w")
  for key, value in DeBruijnGraph(string, k).items():
    if value != []:
      fout.write(key + " -> " + ",".join(value) + "\n")
  fout.close()

def DeBruijnGraph(string, k):
  graph = {}
  comp = Composition(string, k)

  for kmer in comp:
    prefix = kmer[:-1]
    suffix = kmer[1:]
    if prefix not in graph:
      graph[prefix] = [suffix]
    else:
      graph[prefix] += [suffix]

  return graph

def Composition(text, k):
  kmers = []
  for i in range(len(text)-k+1):
    kmer = text[i:i+k]
    kmers.append(kmer)
  return kmers

if __name__ == '__main__':
  prob_199_6()


"""
Common narrative logging functions.

Other biokbase modules can use the logging like this:

    from biokbase.narrative.common.kblogging import get_logger
    _log = get_logger(__name__)

Log messages can be free-form, *but* the desired format is:

    Event_name;Key1=value1 Key2=Value2 (..etc..)

In this format the "Event_name" is taken as the canonical name of what event in the
system this is logging and the Key/value pairs are parsed into a mapping.
Any non-key/value text will be combined into a single text field.

Logging to MongoDB will be enabled, via proxy, if the proxy is
running on the pre-configured host/port
"""
__author__ = 'Dan Gunter <dkgunter@lbl.gov>'
__date__ = '2014-07-31'

import collections
import logging
from logging import handlers
import os
import re
import threading
import time
# Local
from .util import kbase_env
from . import log_proxy
from .log_common import format_event

## Constants

KBASE_TMP_DIR = "/tmp"
KBASE_TMP_LOGFILE = os.path.join(KBASE_TMP_DIR, "kbase-narrative.log")

# env var with location of proxy config file
KBASE_PROXY_ENV = 'KBASE_PROXY_CONFIG'

## Internal logging

_log = logging.getLogger("kblogging")
_h = logging.StreamHandler()
_h.setFormatter(logging.Formatter("[%(levelname)s] %(asctime)s %(name)s: %(message)s"))
_log.addHandler(_h)
lvl = logging.DEBUG if os.environ.get('KBASE_DEBUG', False) else logging.WARN
_log.setLevel(lvl)

## Functions

def get_logger(name="", init=False):
    """Get a given KBase log obj.

    :param name: name (a.b.c) of the logging namespace, which may be
                 relative or absolute (starting with 'biokbase.'), or
                 empty in which case the 'biokbase' logger is returned
    :param init: If true, re-initialize the file/socket log handlers
    :return: Log object
    :rtype: LogAdapter
    """
    if init:
        reset_handlers()
    log = logging.getLogger(_kbase_log_name(name))
    return LogAdapter(log, _get_meta())

def _kbase_log_name(name):
    # no name => root
    if not name:
        return "biokbase"
    # absolute name
    if name.startswith("biokbase."):
        return name
    # relative name
    return "biokbase." + name

def log_event(log, event, mapping):
    """Log an event and a mapping."""
    log.info(format_event(event, mapping))

class LogAdapter(logging.LoggerAdapter):
    """Add some extra methods to the stock LoggerAdapter."""
    def __init__(self, log, extra):
        logging.LoggerAdapter.__init__(self, log, extra)
        self.handlers = log.handlers
        self.addHandler = log.addHandler
        self.removeHandler = log.removeHandler
        self.setLevel = log.setLevel
        self.isEnabledFor = log.isEnabledFor
        self.file_handler, self.socket_handler = None, None

    def addHandler(self, h):
        """Track known handlers for efficient access later."""
        if isinstance(h, logging.FileHandler):
            self.file_handler = h
        elif isinstance(h, handlers.SocketHandler):
            self.socket_handler = h
        self.logger.addHandler(h)

    def removeHandler(self, h):
        """Track known handlers for efficient access later."""
        if isinstance(h, logging.FileHandler):
            self.file_handler = None
        elif isinstance(h, handlers.SocketHandler):
            self.socket_handler = None
        self.logger.removeHandler(h)

    def shutdown(self):
        """Close and remove all handlers."""
        map(self.removeHandler, self.handlers)

def _get_meta():
    meta = {}

    # Auth values
    token = kbase_env.auth_token

    if token:
        # User
        m = re.search('un=([^|]+)', token)
        if m is not None:
            meta['user'] = m.group(1)

    # Session id
    sess = kbase_env.session
    if sess:
        meta['session_id'] = sess

    # Notebook name
    if kbase_env.narrative:
        meta['narr'] = kbase_env.narrative

    # Client IP
    meta['client_ip'] = kbase_env.client_ip or "0.0.0.0"

    return meta

class MetaFormatter(logging.Formatter):
    def __init__(self):
        """Format with metadata in the mix."""
        logging.Formatter.__init__(
            self,
            "%(levelname)s %(asctime)s %(name)s %(message)s")

    def format(self, record):
        s = logging.Formatter.format(self, record)
        return s
        # XXX: This version adds env crap
        # return "{} {}".format(s, ' '.join(["env.{}={}".format(k, v)
        #                                    for k, v in os.environ.items()
        #                                    if k.startswith('KB_')]))

class BufferedSocketHandler(handlers.SocketHandler):
    """Proxy for another handler that always returns immediately
    and queues up messages to send.
    """
    def __init__(self, *args):
        handlers.SocketHandler.__init__(self, *args)
        _log.debug("Created SocketHandler with args = {}".format(args))
        self.buf = collections.deque([], 100)
        self.buf_lock = threading.Lock()
        # start thread to send data from buffer
        self.thr = threading.Thread(target=self.emitter)
        self.thr.daemon = True
        self._stop = False
        self.thr.start()

    def close(self):
        if self.thr:
            self._stop = True
            self.thr.join()
            self.thr = None
        handlers.SocketHandler.close(self)

    def emitter(self):
        while not self._stop:
            try:
                self.buf_lock.acquire()
                item = self.buf.popleft()
                if not self._emit(item):
                    self.buf.appendleft(item)
                    self.buf_lock.release()
                    time.sleep(0.1)
                else:
                    self.buf_lock.release()
            except IndexError:
                self.buf_lock.release()
                time.sleep(0.1)

    def emit(self, record):
        if _log.isEnabledFor(logging.DEBUG):
            _log.debug("Emit 1 record")
        self.buf_lock.acquire()
        try:
            self.buf.append(record)
        finally:
            self.buf_lock.release()

    def _emit(self, record):
        """Re-implement to return a success code."""
        success = False
        try:
            s = self.makePickle(record)
            self.send(s)
            success = True
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as err:
            _log.debug("Emit record to socket failed: {}".format(err))
            self.handleError(record)
        if success and _log.isEnabledFor(logging.DEBUG):
            _log.debug("Record sent to socket")
        return success

def init_handlers():
    """Initialize and add the log handlers."""
    # Turn on debugging by setting environment variable KBASE_DEBUG.
    if os.environ.get("KBASE_DEBUG", None):
        g_log.setLevel(logging.DEBUG)
    else:
        g_log.setLevel(logging.INFO)

    if not g_log.file_handler:
        hndlr = logging.FileHandler(KBASE_TMP_LOGFILE)
        hndlr.setFormatter(MetaFormatter())
        g_log.addHandler(hndlr)

    if not g_log.socket_handler:
        cfg = get_proxy_config()
        g_log.debug("Opening socket to proxy at {}:{}".format(
            cfg.host, cfg.port))
        sock_handler = BufferedSocketHandler(cfg.host, cfg.port)
        g_log.addHandler(sock_handler)

def get_proxy_config():
    config_file = os.environ.get(KBASE_PROXY_ENV, None)
    if config_file:
        _log.info("Configuring KBase logging from file '{}'".format(config_file))
    else:
        _log.warn("Configuring KBase logging from defaults ({} is empty, or not found)"
                  .format(KBASE_PROXY_ENV))
#    return log_proxy.ProxyConfiguration(config_file)
    return log_proxy.ProxyConfigurationWrapper(config_file)

def reset_handlers():
    """Remove & re-add all handlers."""
    while g_log.handlers:
        g_log.removeHandler(g_log.handlers.pop())
    init_handlers()

## Run the rest of this on import

# Get root log obj.
g_log = get_logger()

# If no handlers, initialize them
if not g_log.handlers:
    init_handlers()