from typing import List, Union, Type
from numpy import random
import torch
from pytorch_lightning import LightningDataModule
from torch.utils.data import Dataset, DataLoader, Subset, WeightedRandomSampler
from partglot.datamodules.datasets.partglot_dataset import PartglotDataset
from partglot.utils.simple_utils import unpickle_data
from partglot.datamodules.data_utils import split_indices_with_unseen_target_geo_in_test
from partglot.datamodules.data_utils import part_names
import os.path as osp
import os
import numpy as np
import h5py

class PartglotDataModule(LightningDataModule):
    def __init__(
        self,
        batch_size: int,
        only_correct: bool,
        only_easy_context: bool,
        max_seq_len: int,
        only_one_part_name: bool,
        seed: int,
        split_sizes: List[float],
        balance: bool,
        data_dir: str,
        **kwargs
    ):
        super().__init__()
        self.save_hyperparameters(logger=False)

        self.data_train = self.data_val = self.data_test = None
        self.prepare_data()

    def prepare_data(self):
        (
            self.game_data,
            self.word2int,
            self.int2word,
            self.int2sn,
            self.sn2int,
            self.sorted_sn,
        ) = unpickle_data(osp.join(self.hparams.data_dir, "game_data.pkl"))
        self.h5_data = h5py.File(
            osp.join(self.hparams.data_dir, "cic_bsp.h5"), "r"
        )

    def setup(self, stage=None):
        if not self.data_train and not self.data_val and not self.data_test:
            dataset = PartglotDataset(
                self.hparams, self.game_data, self.h5_data, self.word2int
            )
            split_ids = split_indices_with_unseen_target_geo_in_test(
                self.hparams.split_sizes, dataset.geo_ids, dataset.labels, seed = 2004
            )

            for i, s in enumerate(["train", "val", "test"]):
                unq = dataset.geo_ids[split_ids[s]]
                unq = np.unique(unq.flatten())
                print(f"{s} unique shape:", len(unq))

            self.data_train = Subset(dataset, split_ids["train"])
            self.data_val = Subset(dataset, split_ids["val"])
            self.data_test = Subset(dataset, split_ids["test"])

    def _build_dataloader(self, mode: str):
        self.setup()
        ds = getattr(self, "data_" + mode)

        def seed_worker(worker_id):
            worker_seed = torch.initial_seed() % 2 ** 32
            np.random.seed(worker_seed)
            random.seed(worker_seed)

        g = torch.Generator()
        g.manual_seed(0)

        if self.hparams.balance and mode == "train":
            sampler = self.get_balance_sampler(ds)

            return DataLoader(
                ds,
                batch_size=self.hparams.batch_size,
                shuffle=False,
                num_workers=4,
                pin_memory=False,
                drop_last=True,
                sampler=sampler,
                worker_init_fn=seed_worker,
                generator=g,
            )
        else:
            return DataLoader(
                ds,
                batch_size=self.hparams.batch_size,
                shuffle=mode == "train",
                num_workers=4,
                pin_memory=False,
                drop_last=mode == "train",
                worker_init_fn=seed_worker,
                generator=g,
            )
    def train_dataloader(self):
        return self._build_dataloader("train")

    def val_dataloader(self):
        return self._build_dataloader("val")

    def test_dataloader(self):
        return self._build_dataloader("test")

    def get_balance_sampler(self, ds: Union[Type[Dataset], Subset]):
        """
        Weights of Weighted Sampler are inversely proportional to the number of each part utterances.
        """
        # TODO: check Type[Dataset]

        total_ds_len = len(ds)
        weights = np.zeros(total_ds_len)

        if isinstance(ds, Subset):
            part_ind = ds.dataset.part_indicator[ds.indices]
        else:
            part_ind = ds.part_indicator

        num_example_per_part = np.sum(part_ind, axis=0)
        # fmt = f"total: {num_example_per_part.sum()}"
        # for i in range(4):
            # fmt += f"{part_names[i]}: {num_example_per_part[i]} "
        # print(fmt)
        part_ind = np.argmax(part_ind, axis=1)

        assert len(part_ind) == total_ds_len and len(num_example_per_part) == len(
            part_names
        )

        for i in range(len(part_names)):
            indices_per_part = (part_ind == i).nonzero()
            weights[indices_per_part] = 1 / num_example_per_part[i]

        g = torch.Generator()
        g.manual_seed(0)

        return WeightedRandomSampler(list(weights), len(ds), replacement=True, generator=g)


# -*- coding: utf-8 -*-
import sys
from PyQt5.QtWidgets import QApplication, QTextEdit, QWidget, QVBoxLayout, QPushButton

class MainWin(QWidget):
    def __init__(self, parent=None):
        super(MainWin, self).__init__(parent)
        self.textEdit = QTextEdit()
        self.btnPress1 = QPushButton('顯示文字')
        self.btnPress2 = QPushButton('顯示HTML')
        layout = QVBoxLayout()
        layout.addWidget(self.textEdit)
        layout.addWidget(self.btnPress1)
        layout.addWidget(self.btnPress2)
        self.setLayout(layout)
        self.btnPress1.clicked.connect(self.Press1)
        self.btnPress2.clicked.connect(self.Press2)

        self.resize(350, 270)
        self.setWindowTitle('QTextEdit範例')

    def Press1(self):
        self.textEdit.setPlainText('Hello PyQt5!\n點擊按鈕')

    def Press2(self):
        self.textEdit.setHtml(
            "<red>Hello PyQt5!\n點擊按鈕")

if __name__ == '__main__':
    app = QApplication(sys.argv)
    win = MainWin()
    win.show()
    sys.exit(app.exec_())


import os
import sys
import numpy as np
import matplotlib.pyplot as plt

def plot_solution (data):
    plt.grid()
    plt.plot(data[:,0],data[:,1],label="aprox",c="blue",linestyle='--')
    plt.plot(data[:,0],data[:,3],label="analit",c="red")
    plt.xlabel("t",fontsize=15)
    plt.ylabel("y",fontsize=15)
    plt.title("Analitical x Aproximation",fontsize=14)
    plt.legend(loc=0,fontsize=14)
    plt.show()

def main():

    if (len(sys.argv) != 2):
        print("==========================================================")
        print("Usage:> %s <input_file>" % (sys.argv[0]))
        print("==========================================================")
        sys.exit(1)
    else:
        input_file = sys.argv[1]

        data = np.genfromtxt(input_file)

        plot_solution(data)

if __name__ == "__main__":
    main()


import numpy as np

# Assigns velocities with Maxwell-Boltzmann (Gaussian) distribution and positions with uniform random distribution
def initial(seed_int,N,Lv,pos,vel,T_desired):

    np.random.seed(seed=seed_int)

    sig = np.sqrt(T_desired/3)      # standard deviation in terms of temperature in reduced units

    #Box-Muller transform to generate Gaussian random numbers from uniform random numbers
    u1 = np.random.random(N)
    u2 = np.random.random(N)
    u3 = np.random.random(N)
    u4 = np.random.random(N)

    vel[:,0] = sig*np.sqrt(-2*np.log(u1))*np.cos(2*np.pi*u2) #distribution of vx
    vel[:,1] = sig*np.sqrt(-2*np.log(u1))*np.sin(2*np.pi*u2) #distribution of vy
    vel[:,2] = sig*np.sqrt(-2*np.log(u3))*np.cos(2*np.pi*u4) #distribution of vz

    #computing the mean of each velocity component to impose mean value of the velocity components to be zero
    vx_mean = np.mean(vel[:,0])
    vy_mean = np.mean(vel[:,1])
    vz_mean = np.mean(vel[:,2])

    #mean value of the velocity components to be zero
    vel[:,0] = vel[:,0] - vx_mean
    vel[:,1] = vel[:,1] - vy_mean
    vel[:,2] = vel[:,2] - vz_mean

    # initializing with random positions
    pos[:,0] = Lv[0]*np.random.random(N)
    pos[:,1] = Lv[1]*np.random.random(N)
    pos[:,2] = Lv[2]*np.random.random(N)

    return pos, vel


# 16 - Faça um programa para uma loja de tintas. O programa deverá pedir o tamanho em metros quadrados da área
#  a ser pintada. Considere que a cobertura da tinta é de 1 litro para cada 3 metros quadrados e que a tinta
#  é vendida em latas de 18 litros, que custam R$ 80,00. Informe ao usuário a quantidades de latas de tinta
#  a serem compradas e o preço total.

areaPintura = float(input("Informe a área total a ser pintada (em metros quadrados): "))

totalLitros = areaPintura / 3
totalLatas = totalLitros / 18

if totalLatas % 18 != 0:
    totalLatas += 1

precoTotal = totalLatas * 80

print("Você irá precisar de %d latas de tinta" % totalLatas)
print("O valor total é de R$", precoTotal)


import glob
import sys

from pathlib import Path

from format_glob import format_glob

def processPaths(name, filter=None, file_name=None):
    results = []
    meta_data = []
    # if the function is called with a list or tuple, iterate over those and join the results
    if isinstance(name, (tuple, list)):
        for n in name:
            r, m = processPaths(n, filter=filter, file_name=file_name)
            results.extend(r)
            meta_data.extend(m)
    else:
        # add the file_name pattern if there is one and if it is not already there
        if file_name is not None and Path(name).suffix != Path(file_name).suffix:
            name = Path(name) / "**" / file_name
        # if it is a glob pattern, add all matching elements
        if "{" in str(name):
            for file, data in format_glob(name):
                results.append(file)
                meta_data.append(data)
        elif "*" in str(name):
            results = glob.glob(str(name), recursive=True)
            meta_data = [{}]*len(results)
        # or add the name directly
        elif Path(name).exists():
            results = [name]
            meta_data = [{}]

        # filter results if a filter is provided
        if filter is not None:
            meta_data2 = []
            results2 = []
            for n, m in zip(results, meta_data):
                print("filter", n, filter(n))
                if filter(n):
                    meta_data2.append(m)
                    results2.append(n)
            results = results2
            meta_data = meta_data2

        # if nothing was found, try to give a meaningful error message
        if len(results) == 0:
            # get a list of all parent folders
            name = Path(name).absolute()
            hierarchy = []
            while name.parent != name:
                hierarchy.append(name)
                name = name.parent
            # iterate over the parent folders, starting from the lowest
            for path in hierarchy[::-1]:
                # check if the path exists (or is a glob pattern with matches)
                if "*" in str(path):
                    exists = len(glob.glob(str(path)))
                else:
                    exists = path.exists()
                # if it does not exist, we have found our problem
                if not exists:
                    target = f"No file/folder \"{path.name}\""
                    if "*" in str(path.name):
                        target = f"Pattern \"{path.name}\" not found"
                    source = f"in folder \"{path.parent}\""
                    if "*" in str(path.parent):
                        source = f"in any folder matching the pattern \"{path.parent}\""
                    print(f"WARNING: {target} {source}", file=sys.stderr)
                    break

    return zip(results, meta_data)

if __name__ == "__main__":
    from mock_dir import MockDir
    # to test we create an artificial folder structure
    file_structure = {
        "tmp": {
            "run-1": ["run_nodes3_name-Alice.txt", "run_nodes4_name-Bob.txt", "run_nodes7_name-Foo.txt", "run_nodes7.8_name-Bar.txt"],
            "run-2": ["run_nodes3_name-Alice.txt", "run_nodes4_name-Bob.txt", "run_nodes7_name-Foo.txt", "run_nodes7.8_name-Bar.txt"],
        }
    }
    with MockDir(file_structure):

        # or get the result as a pandas dataframe
        for results, meta_data in processPaths("tmp/**/run_nodes{n:f}_name-{name}x", file_name=".txt"):
            print(results)
            print(meta_data)


#!/usr/bin/env python
import rospy
from sensor_msgs.msg import Image
import cv2
from cv_bridge import CvBridge
import numpy as np
from numpy.linalg import *
import signal

# SOURCE: LAB 4, IMAGE_PROCESS.PY
# Create a CvBridge to convert ROS messages to OpenCV images
bridge = CvBridge()

NUM_CUPS = 3
EPSILON = 5
BALL = -1

centroids = []

# Converts a ROS Image message to a NumPy array to be displayed by OpenCV
def ros_to_np_img(ros_img_msg):
  return np.array(bridge.imgmsg_to_cv2(ros_img_msg,'bgr8'))

def np_img_to_ros(cv2_array):
	return bridge.cv2_to_imgmsg(cv2_array, encoding='passthrough')

def do_kmeans(data, n_clusters):
    """Uses opencv to perform k-means clustering on the data given. Clusters it into
       n_clusters clusters.

       Args:
         data: ndarray of shape (n_datapoints, dim)
         n_clusters: int, number of clusters to divide into.

       Returns:
         clusters: integer array of length n_datapoints. clusters[i] is
         a number in range(n_clusters) specifying which cluster data[i]
         was assigned to.
    """
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.2)
    _, clusters, centers = kmeans = cv2.kmeans(data.astype(np.float32), n_clusters, bestLabels=None, criteria=criteria, attempts=1, flags=cv2.KMEANS_RANDOM_CENTERS)

    return clusters

def euclidian(p1, p2):
	return np.sqrt((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2)

def callback(img):
	# Guassian Blur
	img_array = ros_to_np_img(img)
	img_d = cv2.resize(img_array, dsize=(int(img_array.shape[1]/4), int(img_array.shape[0]/4)), interpolation=cv2.INTER_NEAREST)
	img_blur = cv2.GaussianBlur(img_d, (21,21), 0)

	# Select out RED items
	img_mask = cv2.inRange(img_blur, np.array([0, 0, 70]), np.array([63, 63, 150]))
	img_u = cv2.resize(src=img_mask, dsize=(img_array.shape[1], img_array.shape[0]), interpolation=cv2.INTER_NEAREST)
	img_message = np_img_to_ros(img_u.astype(np.uint8))

	# publish mask
	try:
		pub_mask.publish(img_message)
	except rospy.ROSInterruptException: pass

	# Find centers of mass
	# https://learnopencv.com/find-center-of-blob-centroid-using-opencv-cpp-python/
	contours, hierarchy = cv2.findContours(img_mask,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
	new_centroids = []

	for c in contours:
		M = cv2.moments(c)
		if M["m00"] != 0:
			cX = int(M["m10"] / M["m00"])
			cY = int(M["m01"] / M["m00"])

			# Rule out centroids which are too close to others
			far = True
			for centroid in new_centroids:
				if euclidian([cX, cY], centroid) < EPSILON:
					far = False

			if far:
				new_centroids.append([cX, cY])

			# put text and highlight the center
			cv2.circle(img_mask, (cX, cY), 5, (0, 0, 0), -1)

	global centroids
	if len(centroids) == 0:
		new_centroids.sort(key = lambda c: c[0])
		centroids = new_centroids
		print("First iteration: " + str(centroids))
		return

	a_img_u = cv2.resize(src=img_mask, dsize=(img_array.shape[1], img_array.shape[0]), interpolation=cv2.INTER_NEAREST)
	a_img_message = np_img_to_ros(a_img_u.astype(np.uint8))

	# publish centroids
	try:
		pub_annotated.publish(a_img_message)
	except rospy.ROSInterruptException: pass

	dist = np.full((len(new_centroids), NUM_CUPS), float('inf'))

	for i in range(len(new_centroids)):
		for j in range(NUM_CUPS):
			dist[i][j] = euclidian(new_centroids[i], centroids[j])

	for n in range(NUM_CUPS):
		min_index = np.argmin(dist)
		min_i, min_j = min_index // NUM_CUPS, min_index % NUM_CUPS
		centroids[min_j] = new_centroids[min_i]
		dist[:, min_j] = np.full((1, len(new_centroids)), float('inf'))
		dist[min_i, :] = np.full(NUM_CUPS, float('inf'))

	assert len(centroids) == NUM_CUPS
	assert max(max(centroids)) < float('inf')

	print(centroids)

def observe():
	rospy.Subscriber("/usb_cam/image_raw", Image, callback)
	rospy.spin()

def handler(signum, frame):
	x_centroids = [c[0] for c in centroids]
	order = np.argsort(x_centroids)
	ball_pos = np.where(order == BALL)[0][0]
	print("The ball is at position " + str(ball_pos) + " from the left.")
	print("The ball is at " + str(centroids[BALL]))
	exit(1)

signal.signal(signal.SIGINT, handler)

if __name__ == '__main__':
	rospy.init_node('guesser', anonymous = 'True')
	pub_mask = rospy.Publisher("/guesser/image_mask", Image, queue_size=10)
	pub_bin = rospy.Publisher("guesser/image_bin", Image, queue_size=10)
	pub_annotated = rospy.Publisher("guesser/image_centroid", Image, queue_size=10)

	print("How many cups are in the game?")
	NUM_CUPS = int(input())
	assert NUM_CUPS > 0

	print("Which cup is the ping pong ball under?")
	BALL = int(input())
	assert BALL < NUM_CUPS

	observe()


# -*- coding: utf-8 -*-
"""
Created on Sun Jun  3 10:22:52 2018

@author: sande
"""
#This part of code is used to plot the residuals and find outliers in training dataset and
#find the best threshold which gives minimum number of false positives
import numpy as np
from sklearn.decomposition import PCA
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn import decomposition
import matplotlib.pyplot as plt
import sys
sys.path.append('../')
from Data.datareader import *
plt.style.use('ggplot')

features = ['L_T1', 'L_T2', 'L_T3', 'L_T4', 'L_T5', 'L_T6', 'L_T7',
     'F_PU1', 'S_PU1', 'F_PU2', 'S_PU2', 'F_PU3', 'S_PU3', 'F_PU4', 'S_PU4',
     'F_PU5', 'S_PU5', 'F_PU6', 'S_PU6', 'F_PU7', 'S_PU7', 'F_PU8', 'S_PU8',
     'F_PU9', 'S_PU9', 'F_PU10', 'S_PU10', 'F_PU11', 'S_PU11', 'F_V2', 'S_V2',
     'P_J280', 'P_J269', 'P_J300', 'P_J256', 'P_J289', 'P_J415', 'P_J302',
     'P_J306', 'P_J307', 'P_J317', 'P_J14', 'P_J422']

df3 = read_3_df()
df3 = df3[features]

df_without_outliers = df3
scaler = StandardScaler()
normalized_training = scaler.fit_transform(df3)
spe = np.zeros((normalized_training.shape[0]))

pca = decomposition.PCA(n_components=25)
pca.fit(normalized_training)
pca_model = pca.transform(normalized_training)
eigenvectors = pca.components_
P = np.transpose(eigenvectors[:-10])
P_T = np.transpose(P)
C = np.dot(P, P_T)

# Identity Matrix with the number of columns as given by training data  43 * 43
I = np.identity(43)
y_residual = np.zeros((normalized_training.shape))
for i in range(normalized_training.shape[0]):
    # Convert row to column vector
    y = np.transpose(normalized_training[i])
    y_residual[i] = np.dot(I - C, y)

#for threshold in range(270,560,30):
for threshold in range(200,500,20):
# flagged will be set to 1 if the spe is greater than the threshold
    count = 0
    flagged = np.zeros((normalized_training.shape[0]))
    for i in range(normalized_training.shape[0]):
        spe[i] = np.square(np.sum(np.subtract(y_residual[i], normalized_training[i])))
        if(spe[i] > threshold):
            flagged[i] = 1
            count = count + 1
            #hardcoding the best threshold value to avoid creating one more redundant loop just to create new dataframe
            if (threshold == 380):
                #A dataframe which contains training data without outliers and this will be used in pca_all.py
                df_without_outliers = df_without_outliers.drop(df_without_outliers.index[i])
                #Save the df in a file TrainingDfwithoutOutliers
                df_without_outliers.to_pickle("TrainingDfwithoutOutliers")

    df3 = df3.assign(ResidualVector=spe)
    df3['ResidualVector'].plot(figsize=(15,5),label="Residuals in normal data ")
    plt.axhline(y=400, c='darkgrey', linestyle='--', linewidth=1, label="400 threshold")
    plt.xlabel('Date')
    plt.ylabel('Residuals')
    plt.savefig("normal_residuals.svg", bbox_inches='tight')
    print ("number of outliers or false positives ", count, "for threshold ", threshold)
    #print ("total ", normalized_training.shape[0])

print (df3.shape)
print (df_without_outliers.shape)


import os
from src.Datos.Precipitacion import Precipitacion
from src.Datos.Nubes import Nubes
from src.Datos.Viento import Viento
from src.Datos.Temperatura import Temperatura

def main():

    dir_path = os.path.dirname(os.path.abspath(__file__))
    path = dir_path + "/../textos/csv/crear/"

    fecha = "2019-01-08"
    nubes = Nubes(path,fecha)
    precipitacion = Precipitacion(path,fecha)
    temperatura = Temperatura(path,fecha)

    viento = Viento(path,fecha)

    informe =  ""
    indice = 0

    result = escribirinfore(nubes.frase,indice,informe)
    informe = result[0]
    indice = result[1]

    result = escribirinfore(precipitacion.frase,indice,informe)
    informe = result[0]
    indice = result[1]

    result = escribirinfore(temperatura.frase,indice,informe)
    informe = result[0]
    indice = result[1]
    print(temperatura.frase)
    result = escribirinfore(viento.frase,indice,informe)
    informe = result[0]
    indice = result[1]

    informe +=  "\n\n\n"
    informe +=  temperatura.fraseTablaTemperatura

    print(informe)

def escribirinfore(frase,indice,informe):

    for c in frase:

        if indice >= 57:

            if c == " ":
                informe +="\n"
                indice = 0
            else: informe +=c
        else: informe +=c

        indice +=1

    if frase != "":
        informe += " "
    return informe,indice

main()


import pandas as pd
import os
import click

#input = "/groups/gerlich/members/ZsuzsannaTakacs/DataScience/PythonBookClub/Tidy_data_Homework/Localinput/"
#outdir = "/groups/gerlich/members/ZsuzsannaTakacs/DataScience/PythonBookClub/Tidy_data_Homework/Localinput/test/"
#suffix = "csv"

@click.command()
@click.option('--input', '-i', help='Path of input files')
@click.option('--outdir', '-o', help='Path of output directory')
@click.option('--suffix', '-s', help='File extension')

def tidy_data_local(input, outdir, suffix):
    for filename in os.listdir(input):
        if filename.endswith('txt'):
            with open(os.path.join(input, filename)) as f:
                df = pd.read_csv(f, sep = '\t')
                df = pd.melt(df, id_vars = "Absolute time", var_name='trace')
                df['trace'] = df['trace'].str.replace('Trace', '').astype('float')
                df.to_csv(os.path.join(outdir, f'{filename}_tidy.{suffix}'), index = False)

if __name__ == '__main__':
    tidy_data_local()


N = int(input())

getinput = input().split()
arr = [int(i) for i in getinput]
arr.sort()

nowTime = 0
for i in range(N):
    nowTime += int(arr[i])*(N-i)

print (nowTime)


import ssl
import gzip
import socket
import selectors
from typing import Any, cast

import pytest

from pytest_mock import MockerFixture

from proxy.http import HttpProtocolHandler, HttpClientConnection, httpMethods
from proxy.http.proxy import HttpProxyPlugin
from proxy.common.flag import FlagParser
from proxy.http.parser import HttpParser, httpParserTypes
from proxy.common.utils import bytes_, build_http_request
from proxy.http.responses import (
    PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, okResponse,
)
from proxy.core.connection import TcpServerConnection
from .utils import get_plugin_by_test_name
from ..test_assertions import Assertions

class TestHttpProxyPluginExamplesWithTlsInterception(Assertions):

    @pytest.fixture(autouse=True)   # type: ignore[misc]
    def _setUp(self, request: Any, mocker: MockerFixture) -> None:
        self.mock_socket_dup = mocker.patch('socket.dup')
        self.mock_selector = mocker.patch('selectors.DefaultSelector')
        self.mock_sign_csr = mocker.patch('proxy.http.proxy.server.sign_csr')
        self.mock_gen_csr = mocker.patch('proxy.http.proxy.server.gen_csr')
        self.mock_gen_public_key = mocker.patch(
            'proxy.http.proxy.server.gen_public_key',
        )
        self.mock_server_conn = mocker.patch(
            'proxy.http.proxy.server.TcpServerConnection',
        )
        self.mock_ssl_context = mocker.patch('ssl.create_default_context')
        self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket')

        self.mock_sign_csr.return_value = True
        self.mock_gen_csr.return_value = True
        self.mock_gen_public_key.return_value = True

        self.fileno = 10
        self.mock_socket_dup.side_effect = lambda fd: fd

        self._addr = ('127.0.0.1', 54382)
        self.flags = FlagParser.initialize(
            ca_cert_file='ca-cert.pem',
            ca_key_file='ca-key.pem',
            ca_signing_key_file='ca-signing-key.pem',
            threaded=True,
        )
        self.plugin = mocker.MagicMock()

        plugin = get_plugin_by_test_name(request.param)

        self.flags.plugins = {
            b'HttpProtocolHandlerPlugin': [HttpProxyPlugin],
            b'HttpProxyBasePlugin': [plugin],
        }
        self._conn = mocker.MagicMock(spec=socket.socket)

        self.protocol_handler = HttpProtocolHandler(
            HttpClientConnection(self._conn, self._addr), flags=self.flags,
        )
        self.protocol_handler.initialize()

        self.server = self.mock_server_conn.return_value

        self.server_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection
        self.client_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_wrap.return_value = self.client_ssl_connection

        def has_buffer() -> bool:
            return cast(bool, self.server.queue.called)

        def closed() -> bool:
            return not self.server.connect.called

        def mock_connection() -> Any:
            if self.mock_ssl_context.return_value.wrap_socket.called:
                return self.server_ssl_connection
            return self._conn

        # Do not mock the original wrap method
        self.server.wrap.side_effect = \
            lambda x, y, as_non_blocking: TcpServerConnection.wrap(
                self.server, x, y, as_non_blocking=as_non_blocking,
            )

        self.server.has_buffer.side_effect = has_buffer
        type(self.server).closed = mocker.PropertyMock(side_effect=closed)
        type(
            self.server,
        ).connection = mocker.PropertyMock(
            side_effect=mock_connection,
        )

        self.mock_selector.return_value.select.side_effect = [
            [(
                selectors.SelectorKey(
                    fileobj=self._conn.fileno(),
                    fd=self._conn.fileno(),
                    events=selectors.EVENT_READ,
                    data=None,
                ),
                selectors.EVENT_READ,
            )],
            [(
                selectors.SelectorKey(
                    fileobj=self.client_ssl_connection.fileno(),
                    fd=self.client_ssl_connection.fileno(),
                    events=selectors.EVENT_READ,
                    data=None,
                ),
                selectors.EVENT_READ,
            )],
            [(
                selectors.SelectorKey(
                    fileobj=self.server_ssl_connection.fileno(),
                    fd=self.server_ssl_connection.fileno(),
                    events=selectors.EVENT_WRITE,
                    data=None,
                ),
                selectors.EVENT_WRITE,
            )],
            [(
                selectors.SelectorKey(
                    fileobj=self.server_ssl_connection.fileno(),
                    fd=self.server_ssl_connection.fileno(),
                    events=selectors.EVENT_READ,
                    data=None,
                ),
                selectors.EVENT_READ,
            )],
        ]

        # Connect
        def send(raw: bytes) -> int:
            return len(raw)

        self._conn.send.side_effect = send
        self._conn.recv.return_value = build_http_request(
            httpMethods.CONNECT,
            b'uni.corn:443',
            no_ua=True,
        )

    @pytest.mark.asyncio    # type: ignore[misc]
    @pytest.mark.parametrize(
        '_setUp',
        (
            ('test_modify_post_data_plugin'),
        ),
        indirect=True,
    )   # type: ignore[misc]
    async def test_modify_post_data_plugin(self) -> None:
        await self.protocol_handler._run_once()

        self.assertEqual(self.mock_sign_csr.call_count, 1)
        self.assertEqual(self.mock_gen_csr.call_count, 1)
        self.assertEqual(self.mock_gen_public_key.call_count, 1)

        self.mock_server_conn.assert_called_once_with('uni.corn', 443)
        self.server.connect.assert_called()
        self.assertEqual(
            self.protocol_handler.work.connection,
            self.client_ssl_connection,
        )
        self.assertEqual(self.server.connection, self.server_ssl_connection)
        self._conn.send.assert_called_with(
            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
        )
        self.assertFalse(self.protocol_handler.work.has_buffer())

        #
        original = b'{"key": "value"}'
        modified = b'{"key": "modified"}'
        self.client_ssl_connection.recv.return_value = build_http_request(
            b'POST', b'/',
            headers={
                b'Host': b'uni.corn',
                b'Content-Length': bytes_(len(original)),
                b'Content-Type': b'application/x-www-form-urlencoded',
            },
            body=original,
            no_ua=True,
        )
        await self.protocol_handler._run_once()
        self.server.queue.assert_called_once()
        # pkt = build_http_request(
        #     b'POST', b'/',
        #     headers={
        #         b'Host': b'uni.corn',
        #         b'Content-Length': bytes_(len(modified)),
        #         b'Content-Type': b'application/json',
        #     },
        #     body=modified,
        # )
        response = HttpParser.response(
            self.server.queue.call_args_list[0][0][0].tobytes(),
        )
        self.assertEqual(response.body, modified)

    @pytest.mark.asyncio    # type: ignore[misc]
    @pytest.mark.parametrize(
        '_setUp',
        (
            ('test_man_in_the_middle_plugin'),
        ),
        indirect=True,
    )   # type: ignore[misc]
    async def test_man_in_the_middle_plugin(self) -> None:
        await self.protocol_handler._run_once()

        self.assertEqual(self.mock_sign_csr.call_count, 1)
        self.assertEqual(self.mock_gen_csr.call_count, 1)
        self.assertEqual(self.mock_gen_public_key.call_count, 1)

        self.mock_server_conn.assert_called_once_with('uni.corn', 443)
        self.server.connect.assert_called()
        self.assertEqual(
            self.protocol_handler.work.connection,
            self.client_ssl_connection,
        )
        self.assertEqual(self.server.connection, self.server_ssl_connection)
        self._conn.send.assert_called_with(
            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
        )
        self.assertFalse(self.protocol_handler.work.has_buffer())
        #
        request = build_http_request(
            b'GET', b'/',
            headers={
                b'Host': b'uni.corn',
            },
            no_ua=True,
        )
        self.client_ssl_connection.recv.return_value = request

        # Client read
        await self.protocol_handler._run_once()
        self.server.queue.assert_called_once_with(request)

        # Server write
        await self.protocol_handler._run_once()
        self.server.flush.assert_called_once()

        # Server read
        self.server.recv.return_value = okResponse(
            content=b'Original Response From Upstream',
        )
        await self.protocol_handler._run_once()
        response = HttpParser(httpParserTypes.RESPONSE_PARSER)
        response.parse(self.protocol_handler.work.buffer[0])
        assert response.body
        self.assertEqual(
            gzip.decompress(response.body),
            b'Hello from man in the middle',
        )


# 程序创建@李宏宇
# 创建时间 ：2023/3/10

"""
1.操作文件和目录：CSDN  https://blog.csdn.net/m0_63794226/article/details/126435297?ops_request_misc=&request_id=&biz_id=102&utm_term=python%E6%93%8D%E4%BD%9C%E6%96%87%E4%BB%B6%E5%92%8C%E7%9B%AE%E5%BD%95&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-0-126435297.142^v73^insert_down3,201^v4^add_ask,239^v2^insert_chatgpt&spm=1018.2226.3001.4187
2.序列化：把对象转化为可传输的二进制流数据  反序列化：把二进制流数据转化为对象
    2.1.

"""
import json
import keyword
import pickle

# f = open('E:/IO编程/1.txt', 'r', encoding = 'utf-8')

# print(f.read())

import pickle

d = dict(name = 'Bob', age = 20, score = 88)
print(pickle.dumps(d))

f = open('E:/IO编程/1.txt', 'rb')
d = pickle.load(f)
f.close()
print(d)

obj = dict(name = '小明', age = 20)
s = json.dumps(obj, ensure_ascii = True)
print(s)
s = json.dumps(obj, ensure_ascii = False)
print(s)

print(keyword.kwlist)

py基础语法/IO编程/操作文件和目录、序列化.py

操作文件和目录、序列化.py


"""
These are the implementations of classes related to gameplay and graphics.
"""

# an instance of GameStats has attributes for values that need to be recorded
# or kept track of, and that will be reset each round
class GameStats:
    def __init__(self):
        self.in_round = False   # False if betting, True after dealing
        self.bet = 0    # stores bet of the round

        self.user_cards = 0     # number of cards in user's hand
        self.dealer_cards = 0   # number of cards in dealer's hand

        self.user_score = 0     # value of user's hand
        self.dealer_score = 0   # value of dealer's hand

        self.user_aces = 0      # number of aces in user's hand
        self.dealer_aces = 0    # number of aces in dealer's hand

        # stores the dealer's second card so that it can be reprinted
        # (uncovered) after the user's turn
        self.dealer_hidden = None

# an instance of this class has attributes for constants related to the size
# and position of graphics to be displayed
class Pixels:
    def __init__(self):
        self.valtext_x = 350    # x position of coffer/bet text
        self.valtext_y = 450    # y position of coffer/bet text

        self.card_h = 145       # heigh of card
        self.card_w = 100       # width of card
        self.card_x = 20        # x position of first card in each hand
        self.usercard_y = 285   # y position of cards in user hand
        self.dealercard_y = 20  # y position of cards in dealer hand

        self.midtext_x = 50     # x position of text that says who won
        self.midtext_y = 210    # y position of text that says who won

        self.move_x = 560       # x position of deal/hit button
        self.move_y = 200       # y position of deal/hit, stand, double buttons

        self.bet_x = 50         # x position of first bet button
        self.bet_y = 450        # y position of bet buttons

        self.button_w = 51      # button width
        self.button_h = 49      # button height


coffee = 10
while True:
    money = int(input("돈을 넣어주세요 :"))
    if money == 300:
        print("커피를준다")
        coffee = coffee - 1
    elif money > 300 :
        print("남은돈 %d원을 주며 커피를 준다." % (money - 300))
        coffee = coffee -1
    else :
        print("돈없으면 커피 없어요")
        print("남은 커피는 %d잔 입니다." % coffee)
    if coffee == 0:
        print("SOLD OUT CLOSE THE HERE")
        break


#!/usr/bin/env python
# -*- coding: GB2312 -*-
# Last modified:

"""docstring
"""

__revision__ = '0.1'
fins = []
#fins.append([open("aveenv/merge.csv"), 1.1])
fins.append([open("xgboost/merge.csv"), 1])
fins.append([open("subenv/merge.csv"), 1])
fins.append([open("v1subenv/merge.csv"), 1.2])
N = sum([k[1] for k in fins])
fout = open("merge.csv","w")
while fins[0]:
    ps = []
    k = 1
    for i in range(len(fins)):
        line = fins[i][0].next()
        id,pred = line.strip().split(",")
        ps.append(float(pred) * fins[i][1])
        k = float(pred) * k
    #fout.write("%s,%f\n" % (id, sum(ps)/N))
    fout.write("%s,%f\n" % (id, k))


"""
#Python Programm zur Umrechnung von beliebligen römischen Zahlen in Dezimalzahlen
"""
#variablen

romZif = {
    'I': 1,
    'V': 5,
    'X': 10,
    'L': 50,
    'C': 100,
    'D': 500,
    'M': 1000
}

wert1 = 0
dezimal = 0

#input

romZahl = input("Bitte römische Zahl eingeben: ")

#methoden

#für jeden character aus der Eingabe wird folgendes durchgeführt
for char in romZahl:
    #wenn der character in dem Dictionary vorhanden ist
    if char in romZif:
        #soll er mit dem vorherigen wert1 verglichen werden

            #falls er größer ist als wert1, wird der wert1 von dezimal subtrahiert
        if romZif[char] > wert1:
            dezimal -= wert1
            #ansonsten wird dezimal und wert addiert

        else:
            dezimal += wert1

    # wert 1 soll romZif aus dem eingegebenen input werden
    wert1 = romZif[char]
#dezimal plus wert1 sei dezimal
dezimal += wert1

#das ergebnis ist ein String aus dezimal
print("Ergebnis: " + str(dezimal))


#!/usr/bin/env python
import os
import argparse
import numpy as np
from astropy.table import Table
from astropy.coordinates import SkyCoord
import astropy.units as u

from Ska.Shell import bash
from Chandra.Time import DateTime

task = 'dark_cal_att_gen'
SHARE = os.path.join(os.environ['SKA_SHARE'], task)
DATA = os.path.join(os.environ['SKA_DATA'], task)

def get_options():
    parser = argparse.ArgumentParser(
        description="""Make a filtered version of dark_attitudes.dat using the zodiacal brightness
                       data from the attitude.pl CGI""")
    parser.add_argument("--date",
                        help="Requested date of expected dark current calibration. Defaults to now + 2 weeks")
    opt = parser.parse_args()
    return opt

def filter_atts(date):
    """
    Run the attitude.pl Perl CGI with a date and return a filtered version to remake
    dark_attitudes.dat (since the FOT is already parsing that file, I want to return a
    file that is formatted the same way).

    :param date: Chandra.Time compatible date
    :returns: astropy table of dark attitudes filtered by zodiacal brightness limits
    """
    date = DateTime(date).date

    Z_LOW = 0
    Z_UP = 250
    zodi_annotated_html = bash(
        'perl {}/attitude.pl date="{date}" type="ATT List" z_low="{z_low}" z_up="{z_up}"'.format(
            SHARE, date=date[0:8], z_low=Z_LOW, z_up=Z_UP))
    zodi_list = Table.read(zodi_annotated_html, format='ascii.html')
    ok = (zodi_list['Zodi'] >= Z_LOW) & (zodi_list['Zodi'] <= Z_UP)
    zodi_ok_coords = SkyCoord(zodi_list[ok]['RA'], zodi_list[ok]['DEC'], unit='deg')

    all_atts = Table.read(os.path.join(DATA, 'full_dark_attitudes.dat'),
                          format='ascii.fixed_width_two_line')
    ok = []
    for att in all_atts:
        coord = SkyCoord(att['eq_ra'], att['eq_dec'], unit='deg')
        ok.append(np.any(coord.separation(zodi_ok_coords) < (1 * u.arcsec)))
    ok = np.array(ok)
    cut_list = all_atts[ok]
    return cut_list

if __name__ == '__main__':
    opt = get_options()
    if opt.date is not None:
        date = DateTime(opt.date)
    else:
        date = DateTime() + 14
    cut_list = filter_atts(date)
    print("Writing out dark_attitudes.dat filtered for {}".format(date.date))
    cut_list.write(os.path.join(DATA, "dark_attitudes.dat"),
                   format='ascii.fixed_width_two_line')


# let suppose a dictionary named students contains the names and marks respectivly
students = {
    "Aaditay": 78,
    "Abhay": 90,
    "Sachin": 67,
    "Faizal": 78,
    "Fiza": 96,
    "Vishal": 34,
    "Smriti": 39,
    "Tanishk": 41,
}

lst = filter(lambda n: n[1] >= 40, students.items())

print(list(lst))


ls = ['Shashi','Ashish','Kanha','Rahul','Badhe','Prasad']

for name in ls:
    if('s' in name):
        continue
    else:
        print(name)

print()

for name in ls:
    flag=0
    for i in name:
        if(i == 's'):
            flag=1
            break
    print(name) if(flag==0) else 1


import argparse
from config import conf

parser = argparse.ArgumentParser(description='PyTorch Training')

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def str2str(v):
    if isinstance(v, str):
        if v == 'None':
            return ''
        else:
            return v
    else:
        raise ValueError()

def str2tuple(v):
    if isinstance(v, (tuple,list)):
        return v
    elif isinstance(v, str):
        v=[int(k) for k in v.split('_')]
        return v
    else:
        raise argparse.ArgumentTypeError()

for k, v in conf.__dict__.items():
    if isinstance(v, bool):
        parser.add_argument(f"--{k}", type=str2bool, nargs='?',
                            const=True, default=v)
    elif isinstance(v, (float, int, )):  # note bool is int
        parser.add_argument(f'--{k}', default=v, type=type(v))
    elif isinstance(v, (str,)):
        parser.add_argument(f'--{k}', default=v, type=str2str, nargs='?', const=True, )
    elif isinstance(v, (tuple, list)):
        parser.add_argument(f'--{k}', default=v, type=str2tuple, nargs='?', const=True, )

parser.add_argument('--local_rank', default=None, type=int, )
parser.add_argument('--work_path', default=None, type=str, )
parser.add_argument('--data_url', default=None, type=str)
parser.add_argument('--init_method', default=None, type=str)
parser.add_argument('--train_url', default=None, type=str)

if __name__ == '__main__':
    args = parser.parse_args()
    print(args)


import os
import pathlib
import string

async def check_cancel(message):
    """
    Проверка ввода комментария пользователем.
    @param message: сообщение пользователя.
    @return: флаг.
    """
    if message.text == '-':
        return True
    else:
        return None

def clear_whitespace(value):
    """
    Получение строки без пробелов.
    @param value: строка.
    @return: строка без пробелов.
    """
    return value.translate({ord(c): None for c in string.whitespace})

async def check_double(message, state, value):
    """
    Валидация сообщения с десятичным числом.
    @param message: объект для отправки сообщения.
    @param state: состояние бота.
    @param value: сообщение пользователя.
    @return: флаг.
    """
    try:
        if float(value) < 0:
            await message.answer("Некорректный ввод, повторите ввод:")
            return None
        return float(value)
    except:
        await message.answer("Некорректный ввод, повторите ввод:")
        await state.set()
        return None

async def check_int(message, state, value):
    """
        Валидация сообщения с целым числом.
        @param message: объект для отправки сообщения.
        @param state: состояние бота.
        @param value: сообщение пользователя.
        @return: флаг.
    """
    try:
        if int(value) < 0:
            await message.answer("Некорректный ввод, повторите ввод:")
            return None
        return int(value)
    except:
        await message.answer("Некорректный ввод, повторите ввод:")
        await state.set()
        return None

def check_folders(main_dir):
    """
    проверка на наличие папок прокта.
    @param main_dir: главная дирректория проекта.
    """
    if not os.path.exists(str(pathlib.Path(main_dir, 'images'))):
        os.mkdir(pathlib.Path(main_dir, 'images'))
        os.mkdir(str(pathlib.Path(main_dir, 'images', 'logo')))
        os.mkdir(str(pathlib.Path(main_dir, 'images', 'products_photo')))


#!/usr/bin/env python3
import json
import os
import re
import signal
import sys
import time
import traceback
from pathlib import Path
from urllib import request

import cv2
import numpy as np
import onnxruntime

from daemon import Daemon

__version__ = "0.8.12"

class AIDaemon(Daemon):
    def load_metadata(self, meta_file: Path) -> dict:
        if not meta_file.is_file():
            return {}
        with open(meta_file, "r") as fp:
            return json.load(fp)

    def update_metadata(self, meta_file: Path, data: dict) -> None:
        metadata = self.load_metadata(meta_file)
        if "update_time" not in metadata:
            metadata["update_time"] = time.time()
        metadata.update(data)
        with open(meta_file, "w") as fp:
            json.dump(metadata, fp)

    def ai(
        self, source_file: Path, prepared_file: Path, meta_file: Path
    ) -> None:
        pid = os.fork()
        if pid != 0:
            return

        try:
            ref_size = 512

            im = cv2.imread(str(source_file))
            out_im = cv2.cvtColor(im, cv2.COLOR_BGR2BGRA)
            im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)

            if len(im.shape) == 2:
                im = im[:, :, None]
            if im.shape[2] == 1:
                im = np.repeat(im, 3, axis=2)
            elif im.shape[2] == 4:
                im = im[:, :, 0:3]

            im = (im - 127.5) / 127.5
            im_h, im_w, im_c = im.shape

            def get_scale_factor(im_h, im_w, ref_size):
                if max(im_h, im_w) < ref_size or min(im_h, im_w) > ref_size:
                    if im_w >= im_h:
                        im_rh = ref_size
                        im_rw = int(im_w / im_h * ref_size)
                    elif im_w < im_h:
                        im_rw = ref_size
                        im_rh = int(im_h / im_w * ref_size)
                else:
                    im_rh = im_h
                    im_rw = im_w

                im_rw = im_rw - im_rw % 32
                im_rh = im_rh - im_rh % 32

                x_scale_factor = im_rw / im_w
                y_scale_factor = im_rh / im_h

                return x_scale_factor, y_scale_factor

            x, y = get_scale_factor(im_h, im_w, ref_size)

            im = cv2.resize(im, None, fx=x, fy=y, interpolation=cv2.INTER_AREA)
            im = np.transpose(im)
            im = np.swapaxes(im, 1, 2)
            im = np.expand_dims(im, axis=0).astype("float32")

            MODEL_PATH = os.environ.get(
                "MODEL_PATH",
                "/opt/app/modnet_photographic_portrait_matting.onnx",
            )
            session = onnxruntime.InferenceSession(MODEL_PATH, None)
            input_name = session.get_inputs()[0].name
            output_name = session.get_outputs()[0].name
            result = session.run([output_name], {input_name: im})

            matte = (np.squeeze(result[0]) * 255).astype("uint8")
            matte = cv2.resize(
                matte, dsize=(im_w, im_h), interpolation=cv2.INTER_AREA
            )
            out_im[:, :, 3] = matte
            out_im = out_im.astype(float)

            # Load metadata
            metadata = self.load_metadata(meta_file)

            background = metadata.get("background", "").strip(" #")
            if len(background) == 6:
                background = background.lower() + "ff"
            color_re = re.compile(r"(^[A-Za-z0-9]{6}$)|(^[A-Za-z0-9]{8}$)")
            background_alpha = 0
            background_im = None
            if background.startswith("http://") or background.startswith(
                "https://"
            ):
                try:
                    req = request.urlopen(background)
                    arr = np.asarray(bytearray(req.read()), dtype=np.uint8)
                    background_im = cv2.imdecode(arr, cv2.IMREAD_UNCHANGED)
                    background_im = cv2.resize(
                        background_im,
                        dsize=(im_w, im_h),
                        interpolation=cv2.INTER_AREA,
                    )
                    if background_im.shape[2] == 4:
                        background_im = background_im.astype(float)
                        background_im = background_im[:, :, :3] * np.repeat(
                            background_im[:, :, 3].reshape(
                                out_im.shape[:2] + (1,)
                            )
                            / 255.0,
                            3,
                            axis=2,
                        )
                    background_alpha = 1.0
                except:
                    background_im = None
            elif color_re.match(background):
                red = int(background[0:2], 16)
                green = int(background[2:4], 16)
                blue = int(background[4:6], 16)
                if len(background) == 8:
                    background_alpha = int(background[6:8], 16) / 255.0
                else:
                    background_alpha = 1.0
                background_im = np.full(
                    (im_h, im_w, 3), [blue, green, red], dtype=np.float32
                )

            alpha_mask = np.repeat(
                out_im[:, :, 3].reshape(out_im.shape[:2] + (1,)) / 255.0,
                3,
                axis=2,
            )
            out_im[:, :, :3] = out_im[:, :, :3] * alpha_mask

            if (
                background_im is None
                and metadata.get("type", "") != "image/png"
            ):
                background_im = np.full(
                    (im_h, im_w, 3), [255.0, 255.0, 255.0], dtype=np.float32
                )
                background_alpha = 1.0

            if background_im is not None:
                out_im[:, :, :3] = out_im[:, :, :3] + background_im[
                    :, :, :3
                ] * background_alpha * (1.0 - alpha_mask)
                out_im[:, :, 3] = 255.0

            cv2.imwrite(str(prepared_file), out_im.astype("uint8"))
            self.update_metadata(
                meta_file,
                {
                    "processed": "true",
                },
            )
        except Exception as e:
            if os.environ.get("DEBUG", "false").lower() in ("true", "1", "on"):
                print(traceback.format_exc())
            self.update_metadata(
                meta_file,
                {
                    "processed": "error",
                },
            )

        source_file.unlink()
        sys.exit()

    def queue(self) -> None:
        STAGED_PATH = os.environ.get("STAGED_PATH", "/tmp/ai/staged")
        SOURCE_PATH = os.environ.get("SOURCE_PATH", "/tmp/ai/source")
        PREPARED_PATH = os.environ.get("PREPARED_PATH", "/tmp/ai/prepared")
        MAX_FORK = int(os.environ.get("MAX_FORK", 8))
        CHUNK_SIZE = int(os.environ.get("CHUNK_SIZE", 4096))

        staged_files = sorted(
            [
                f
                for f in Path(STAGED_PATH).glob("*")
                if f.is_file() and f.suffix != ".json"
            ],
            key=lambda f: f.stat().st_mtime,
        )
        source_files = [f for f in Path(SOURCE_PATH).glob("*") if f.is_file()]
        source_files_count = len(source_files)

        while source_files_count < MAX_FORK and staged_files:
            source_files_count += 1
            staged_file = staged_files.pop(0)

            meta_file = staged_file.with_suffix(".json")
            source_file = Path(SOURCE_PATH) / staged_file.name
            prepared_file = Path(PREPARED_PATH) / (
                staged_file.stem + staged_file.suffix
            )

            with staged_file.open("rb") as src_fp, source_file.open(
                "wb"
            ) as dst_fp:
                while True:
                    chunk = src_fp.read(CHUNK_SIZE)
                    if not chunk:
                        break
                    dst_fp.write(chunk)

            staged_file.unlink()
            self.ai(source_file, prepared_file, meta_file)

    def run(self) -> None:
        signal.signal(signal.SIGCHLD, signal.SIG_IGN)
        while True:
            self.queue()
            time.sleep(float(os.environ.get("QUEUE_LATENCY", 1.0)))

if __name__ == "__main__":
    CHROOT_PATH = os.environ.get("CHROOT_PATH", "/opt/app")
    PIDFILE_PATH = os.environ.get("PIDFILE_PATH", "/opt/app/run/ai.pid")

    AIDaemon(pidfile=PIDFILE_PATH, chroot=CHROOT_PATH).start()


from typing import Optional, Tuple, Union

import torch
from torch import nn
from transformers.models.gptj.modeling_gptj import get_embed_positions, apply_rotary_pos_emb, create_sinusoidal_positions
from transformers.models.gpt_neox.modeling_gpt_neox import apply_rotary_pos_emb as apply_rotary_pos_emb_neox
from transformers.utils import is_torch_fx_proxy

class OverrideGPTJAttention(nn.Module):
    def __init__(self,  attention, attn_override, attn_override_mask):
        super(OverrideGPTJAttention, self).__init__()

        max_positions = 2048
        self.register_buffer(
            "bias",
            torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(
                1, 1, max_positions, max_positions
            ),
        )
        self.register_buffer("masked_bias", torch.tensor(-1e9), persistent=False)

        self.attn_dropout = attention.attn_dropout
        self.resid_dropout = attention.resid_dropout

        self.embed_dim = attention.embed_dim
        self.num_attention_heads = attention.num_attention_heads
        self.head_dim = self.embed_dim // self.num_attention_heads
        if self.head_dim * self.num_attention_heads != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and `num_attention_heads`: {self.num_attention_heads})."
            )
        self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())

        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
        self.rotary_dim = attention.rotary_dim
        pos_embd_dim = self.rotary_dim or self.embed_dim
        self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim)
        # override
        self.attn_override = attn_override
        self.attn_override_mask = attn_override_mask

    def _split_heads(self, tensor, num_attention_heads, attn_head_size, rotary):
        """
        Splits hidden dim into attn_head_size and num_attention_heads
        """
        new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size)
        tensor = tensor.view(new_shape)
        if rotary:
            return tensor
        if len(tensor.shape) == 5:
            return tensor.permute(0, 1, 3, 2, 4)  # (batch, blocks, head, block_length, head_features)
        elif len(tensor.shape) == 4:
            return tensor.permute(0, 2, 1, 3)  # (batch, head, seq_length, head_features)
        else:
            raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")

    def _merge_heads(self, tensor, num_attention_heads, attn_head_size):
        """
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        """
        if len(tensor.shape) == 5:
            tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()
        elif len(tensor.shape) == 4:
            tensor = tensor.permute(0, 2, 1, 3).contiguous()
        else:
            raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")
        new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)
        return tensor.view(new_shape)

    def _attn(
        self,
        query,
        key,
        value,
        attention_mask=None,
        head_mask=None,
    ):
        # compute causal mask from causal mask buffer
        query_length, key_length = query.size(-2), key.size(-2)
        causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length]

        # Keep the attention weights computation in fp32 to avoid overflow issues
        query = query.to(torch.float32)
        key = key.to(torch.float32)

        attn_weights = torch.matmul(query, key.transpose(-1, -2))

        mask_value = torch.finfo(attn_weights.dtype).min
        # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
        # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
        mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
        attn_weights = torch.where(causal_mask, attn_weights, mask_value)

        attn_weights = attn_weights / self.scale_attn

        if attention_mask is not None:
            # Apply the attention mask
            attn_weights = attn_weights + attention_mask

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
        attn_weights = attn_weights.to(value.dtype)
        attn_weights = self.attn_dropout(attn_weights)

        # Mask heads if we want to
        if head_mask is not None:
            attn_weights = attn_weights * head_mask

        # override attention weights
        override_seq_len = self.attn_override_mask.shape[-1]
        attn_weights[:, :, :override_seq_len, :override_seq_len] = torch.where(
            self.attn_override_mask,
            self.attn_override,
            attn_weights[:, :, :override_seq_len, :override_seq_len],
        )

        attn_output = torch.matmul(attn_weights, value)

        return attn_output, attn_weights

    def _get_embed_positions(self, position_ids):
        embed_positions = self.embed_positions
        if embed_positions.device != position_ids.device:
            embed_positions = embed_positions.to(position_ids.device)
            self.embed_positions = embed_positions
        return embed_positions.repeat(position_ids.shape[0], 1, 1)

    def forward(
        self,
        hidden_states: torch.FloatTensor,
        layer_past: Optional[Tuple[torch.Tensor]] = None,
        attention_mask: Optional[torch.FloatTensor] = None,
        position_ids: Optional[torch.LongTensor] = None,
        head_mask: Optional[torch.FloatTensor] = None,
        use_cache: Optional[bool] = False,
        output_attentions: Optional[bool] = False,
    ) -> Union[
        Tuple[torch.Tensor, Tuple[torch.Tensor]],
        Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],
    ]:
        query = self.q_proj(hidden_states)
        key = self.k_proj(hidden_states)
        value = self.v_proj(hidden_states)

        query = self._split_heads(query, self.num_attention_heads, self.head_dim, True)
        key = self._split_heads(key, self.num_attention_heads, self.head_dim, True)
        value = self._split_heads(value, self.num_attention_heads, self.head_dim, False)

        if is_torch_fx_proxy(position_ids):
            # The logic to conditionally copy to GPU could not be traced, so we do this
            # every time in the torch.fx case
            embed_positions = get_embed_positions(self.embed_positions, position_ids)
        else:
            embed_positions = self._get_embed_positions(position_ids)

        repeated_position_ids = position_ids.unsqueeze(-1).repeat(1, 1, embed_positions.shape[-1])
        sincos = torch.gather(embed_positions, 1, repeated_position_ids)
        sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1)

        if self.rotary_dim is not None:
            k_rot = key[:, :, :, : self.rotary_dim]
            k_pass = key[:, :, :, self.rotary_dim :]

            q_rot = query[:, :, :, : self.rotary_dim]
            q_pass = query[:, :, :, self.rotary_dim :]

            k_rot = apply_rotary_pos_emb(k_rot, sin, cos)
            q_rot = apply_rotary_pos_emb(q_rot, sin, cos)

            key = torch.cat([k_rot, k_pass], dim=-1)
            query = torch.cat([q_rot, q_pass], dim=-1)
        else:
            key = apply_rotary_pos_emb(key, sin, cos)
            query = apply_rotary_pos_emb(query, sin, cos)

        key = key.permute(0, 2, 1, 3)
        query = query.permute(0, 2, 1, 3)

        if layer_past is not None:
            past_key = layer_past[0]
            past_value = layer_past[1]
            key = torch.cat((past_key, key), dim=-2)
            value = torch.cat((past_value, value), dim=-2)

        if use_cache is True:
            present = (key, value)
        else:
            present = None

        # compute self-attention: V x Softmax(QK^T)
        attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)

        attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)
        attn_output = self.out_proj(attn_output)
        attn_output = self.resid_dropout(attn_output)

        outputs = (attn_output, present)
        if output_attentions:
            outputs += (attn_weights,)

        # shape of attn_output is (batch_size, seq_len, hidden_size)
        return outputs  # a, present, (attentions)

class OverrideGPTNeoXAttention(nn.Module):

    def __init__(self, attention, attn_override, attn_override_mask):
        super(OverrideGPTNeoXAttention, self).__init__()

        self.num_attention_heads = attention.num_attention_heads
        self.hidden_size = attention.hidden_size
        self.head_size = self.hidden_size // self.num_attention_heads
        self.rotary_ndims = attention.rotary_ndims
        max_positions = 2048
        self.register_buffer(
            "bias",
            torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view(
                1, 1, max_positions, max_positions
            ),
        )
        self.register_buffer("masked_bias", torch.tensor(-1e9))
        self.rotary_emb = attention.rotary_emb
        self.norm_factor = torch.sqrt(torch.tensor(self.head_size, dtype=torch.float32)).to(
            torch.get_default_dtype())
        self.query_key_value = nn.Linear(self.hidden_size, 3 * self.hidden_size)
        self.dense = nn.Linear(self.hidden_size, self.hidden_size)

        # override
        self.attn_override = attn_override
        self.attn_override_mask = attn_override_mask

    def forward(
            self,
            hidden_states,
            attention_mask,
            head_mask=None,
            layer_past=None,
            use_cache=False,
            output_attentions=False,
    ):
        has_layer_past = layer_past is not None

        # Compute QKV
        # Attention heads [batch, seq_len, hidden_size]
        #   --> [batch, seq_len, (np * 3 * head_size)]
        qkv = self.query_key_value(hidden_states)

        # [batch, seq_len, (num_heads * 3 * head_size)]
        #   --> [batch, seq_len, num_heads, 3 * head_size]
        new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size)
        qkv = qkv.view(*new_qkv_shape)

        # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size]
        query = qkv[..., : self.head_size].permute(0, 2, 1, 3)
        key = qkv[..., self.head_size: 2 * self.head_size].permute(0, 2, 1, 3)
        value = qkv[..., 2 * self.head_size:].permute(0, 2, 1, 3)

        # Compute rotary embeddings on rotary_ndims
        query_rot = query[..., : self.rotary_ndims]
        query_pass = query[..., self.rotary_ndims:]
        key_rot = key[..., : self.rotary_ndims]
        key_pass = key[..., self.rotary_ndims:]

        # Compute token offset for rotary embeddings (when decoding)
        seq_len = key.shape[-2]
        offset = 0
        if has_layer_past:
            offset = layer_past[0].shape[-2]
            seq_len += offset
        cos, sin = self.rotary_emb(value, seq_len=seq_len)
        query, key = apply_rotary_pos_emb_neox(query_rot, key_rot, cos, sin, offset=offset)
        query = torch.cat((query, query_pass), dim=-1)
        key = torch.cat((key, key_pass), dim=-1)

        # Cache QKV values
        if has_layer_past:
            past_key = layer_past[0]
            past_value = layer_past[1]
            key = torch.cat((past_key, key), dim=-2)
            value = torch.cat((past_value, value), dim=-2)
        present = (key, value) if use_cache else None

        # Compute attention
        attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)

        # Reshape outputs
        attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_size)
        attn_output = self.dense(attn_output)

        outputs = (attn_output, present)
        if output_attentions:
            outputs += (attn_weights,)

        return outputs

    @classmethod
    def _split_heads(cls, tensor, num_attention_heads, attn_head_size):
        """
        Splits hidden dim into attn_head_size and num_attention_heads
        """
        # tensor: [bs, seq_len, hidden_size]
        new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size)
        # -> [bs, seq_len, num_attention_heads, attn_head_size]
        tensor = tensor.view(new_shape)
        # -> [bs, num_attention_heads, seq_len, attn_head_size]
        tensor = tensor.permute(0, 2, 1, 3)
        return tensor

    @classmethod
    def _merge_heads(cls, tensor, num_attention_heads, attn_head_size):
        """
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        """
        # tensor [bs, num_attention_heads, seq_len, attn_head_size]
        tensor = tensor.permute(0, 2, 1, 3).contiguous()
        # -> [bs, seq_len, num_attention_heads, attn_head_size]
        tensor = tensor.view(tensor.size(0), tensor.size(1), num_attention_heads * attn_head_size)
        # -> [bs, seq_len, hidden_size]
        return tensor

    def _attn(self, query, key, value, attention_mask=None, head_mask=None):
        # q, k, v: [bs, num_attention_heads, seq_len, attn_head_size]
        # compute causal mask from causal mask buffer
        batch_size, num_attention_heads, query_length, attn_head_size = query.size()
        key_length = key.size(-2)

        causal_mask = self.bias[:, :, key_length - query_length: key_length, :key_length].bool()

        query = query.view(batch_size * num_attention_heads, query_length, attn_head_size)
        key = key.view(batch_size * num_attention_heads, key_length, attn_head_size)
        attn_scores = torch.zeros(
            batch_size * num_attention_heads,
            query_length,
            key_length,
            dtype=query.dtype,
            device=key.device,
        )
        attn_scores = torch.baddbmm(
            attn_scores,
            query,
            key.transpose(1, 2),
            beta=1.0,
            alpha=(torch.tensor(1.0, dtype=self.norm_factor.dtype,
                                device=self.norm_factor.device) / self.norm_factor),
        )
        attn_scores = attn_scores.view(batch_size, num_attention_heads, query_length, key_length)

        mask_value = torch.finfo(attn_scores.dtype).min
        # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
        # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
        mask_value = torch.tensor(mask_value, dtype=attn_scores.dtype).to(attn_scores.device)
        attn_scores = torch.where(causal_mask, attn_scores, mask_value)

        if attention_mask is not None:
            # Apply the attention mask
            attn_scores = attn_scores + attention_mask

        attn_weights = nn.functional.softmax(attn_scores, dim=-1)
        attn_weights = attn_weights.to(value.dtype)

        # Mask heads if we want to
        if head_mask is not None:
            attn_weights = attn_weights * head_mask

        # override attention weights
        override_seq_len = self.attn_override_mask.shape[-1]
        attn_weights[:, :, :override_seq_len, :override_seq_len] = torch.where(
            self.attn_override_mask,
            self.attn_override,
            attn_weights[:, :, :override_seq_len, :override_seq_len],
        )

        attn_output = torch.matmul(attn_weights, value)
        return attn_output, attn_weights

    def attention_mask_func(attention_scores, ltor_mask):
        attention_scores.masked_fill_(~ltor_mask, torch.finfo(attention_scores.dtype).min)
        return attention_scores


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jun  2 22:43:58 2020

@author: Disco
"""

import random
import pandas as pd
import time
import pickle

conSchool = False #

hogares =[]
personas = []
trabajo = []
edu_sup = []
edu_basica = []
l_hogares = []
usuariosTP = {}

regionSelected = 13 #Region Metropolitana
regionSelected = 11 #Es Aysen
Comunas = []

Actualizar = False
GranInicial = time.perf_counter()

#%%
aPath = ""

###Carga de datos pesados
inicial = time.perf_counter()
aViviendas = aPath + "Microdato_Censo2017-Viviendas.csv"
viviendas = pd.read_csv(aViviendas, sep=";")
viviendas = viviendas[['REGION', 'PROVINCIA','COMUNA','DC','AREA','ID_ZONA_LOC','NVIV']]
final = time.perf_counter()
print("Carga de Datos Viviendas {0:f}".format(final-inicial))

inicial = time.perf_counter()
if Actualizar==True:
    aHogares = aPath + "Microdato_Censo2017-Hogares.csv"
    hogares = pd.read_csv(aHogares, sep=";")
    hogares = hogares[['REGION', 'PROVINCIA','COMUNA','DC','AREA','ID_ZONA_LOC','NVIV','NHOGAR']]
else:
    hogares=pd.read_csv("Hogares_Creado0707.csv")
final = time.perf_counter()
print("Carga de Datos Hogares {0:f}".format(final-inicial))

inicial = time.perf_counter()
if Actualizar==True:
    aPersonas = aPath + "Microdato_Censo2017-Personas.csv"
    personas = pd.read_csv(aPersonas, sep=";")
    personas = personas[['REGION', 'PROVINCIA','COMUNA','DC','AREA','ID_ZONA_LOC','NVIV'
                  ,'NHOGAR','PERSONAN','P08','P09','P13','P15','P17']]
else:
    personas=pd.read_csv('Personas_Creado0707.csv')
final = time.perf_counter()
print("Carga de Datos Personas {0:f}".format(final-inicial))

inicial = time.perf_counter()
aEmpresas = aPath + "empresas2017.csv"
Empresas = pd.read_csv(aEmpresas, sep=";")
Empresas.rename(columns={"Cod Region":"REGION", "Cod Comuna":"COMUNA"},inplace=True)
final = time.perf_counter()
##########Reasignacion de empresas entre comunas para corregir cupos
rPer = personas[personas['P17']==1].groupby('COMUNA').agg({'P17':'count','REGION':'min'}) #Personas por Comuna que trabajan
rFir =  Empresas.groupby('COMUNA').sum()['Trabajadores'] #Oferta de trabajo de las empresas
rFir.rename("Puestos", inplace=True)

capacidades = pd.concat([rPer,rFir],axis=1, sort=False)
capacidades['indice'] = capacidades['P17'] / capacidades['Puestos']

relLaboral = capacidades.groupby('REGION').sum()
relLaboral['ratio'] = relLaboral['P17']/relLaboral['Puestos']

Superavit = relLaboral[relLaboral['ratio']<=1] #La Unica Región es la 13

comunasSuperavit = capacidades[(capacidades['REGION']==13) & (capacidades['indice']<1)]
cDeficit = capacidades[(capacidades['indice']>1)]

cS = comunasSuperavit.sort_values(by="indice", ascending=True).copy(deep=True)

for idxdc, comd in cDeficit.iterrows(): #trab > cupos
    trad = comd['P17']
    pud = comd['Puestos']
    for idxsc, coms in cS.iterrows(): #trab < cupos
        tras = coms['P17']
        pus = coms['Puestos']
        if (tras/pus) < 0.9: #Tomando el deficit general 7/8=0.875
            for idxf, comf in Empresas[Empresas['COMUNA']==idxsc].iterrows():
                Empresas.at[idxf,'COMUNA'] = idxdc
                Empresas.at[idxf,'REGION'] = comd['REGION']
                pus = pus - comf['Trabajadores'] #Menos superavitaria
                cS.at[idxsc,'Puestos'] = pus
                pud = pud + comf['Trabajadores'] #se reduce el deficit
                cDeficit.at[idxdc,'Puestos'] = pud
                if (tras/pus) > 0.9:
                    break
                if pud==0:
                    continue
                if (trad/pud) < 1: #Todos van a quedar deficitaria
                    break
            if (trad/pud) < 1:
                break
print("Carga de Datos Empresas {0:f}".format(final-inicial))

inicial = time.perf_counter()
aUniversidades = aPath + "Matricula_U_2017.csv"
Universidades = pd.read_csv(aUniversidades, sep=";")
Universidades.rename(columns={"Region":"REGION","Comuna":"COMUNA"},inplace=True)
Univer = Universidades.groupby(["COMUNA","Jornada"]).agg(
{"Total":["sum"],
  "REGION":["max"]})
Univer = Univer.reset_index(level="Jornada")
final = time.perf_counter()
print("Carga de Datos Universidades {0:f}".format(final-inicial))

inicial = time.perf_counter()
aEducacion = aPath + "Matricula_SE_2017.csv"
Educacion = pd.read_csv(aEducacion, sep="\t")
Educacion.rename(columns={"COD_REG_RBD":"REGION","COD_COM_RBD":"COMUNA"},inplace=True)
final = time.perf_counter()
print("Carga de Datos Educacion {0:f}".format(final-inicial))

aPermisos = aPath + 'permisos-de-circulacion2017.csv'
inicial = time.perf_counter()
Vehiculos = pd.read_csv(aPermisos, sep=";")
final = time.perf_counter()

Vehiculos['Total'] = Vehiculos['Catalitico'] + Vehiculos['NoCatalitico']
Vehic = Vehiculos[Vehiculos["Destino"]==1].groupby('COMUNA').agg({'Total':'sum','REGION':'min'})

#%%

#Primero los datos llegan filtrados hasta aqui

personas["T_asignado"] = 0
personas["U_asignado"] = 0
personas["E_asignado"] = 0

per = personas[personas['REGION']==regionSelected]
#viv = viviendas[viviendas['REGION']==regionSelected]
#hog = hogares[hogares['REGION']==regionSelected]
res = Univer['REGION']==regionSelected
univ = Univer[res["max"]]
edu = Educacion[Educacion['REGION']==regionSelected]
firms = Empresas[Empresas['REGION']==regionSelected]

#%% Se asignan trabajadores a sus puestos de trabajo
#Se asignan los trabajadores de una comuna a los puestos de trabajos disponibles

inicial = time.perf_counter()
rPer = per[per['P17']==1].groupby('COMUNA').count()['P17'] #Personas por Comuna que trabajan
rPer.rename("Trabaj", inplace=True)
rFir =  firms.groupby('COMUNA').sum()['Trabajadores'] #Oferta de trabajo de las empresas
rFir.rename("Puestos", inplace=True)

capacidades = pd.concat([rPer,rFir],axis=1, sort=False)
capacidades["usadas"] = 0

#ATENCION:
# ****** Revisar

#personas["T_asignado"] = 0
Empresas["asignados"] = 0
trabajo = pd.DataFrame([],columns=['id_p','id_f', 'pos'])
#trabajod = dict()
for idxc, datos in capacidades.iterrows():
    lista = []
    #Se construyen las capacidades
    for idx, firma in Empresas[Empresas["COMUNA"]==idxc].iterrows():
        for i in range(int(firma["Trabajadores"])):
            lista.append([0, idx, i])
    if len(lista)==0:
        continue
    muestra = random.sample(range(len(lista)),len(lista))

    i = 0
    for idx, persona in personas[(personas["COMUNA"]==idxc) & (personas['P17']==1) & (personas['T_asignado']==0)].iterrows():
        # if persona['P17']==1 and persona["T_asignado"]==0:
        personas.at[idx,"T_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
        lista[muestra[i]][0] = idx
        i = i + 1
        if i == len(lista):
            break

    trabajo = pd.concat([trabajo, pd.DataFrame(lista,
                                columns=['id_p','id_f', 'pos'])],axis=0,sort=False)

    #Se asignan los que quedaron fuera
    # cuposlibres = trabajo[trabajo["id_p"]==0].index.tolist()

    # if len(cuposlibres)==0:
    #     continue #Se asignaron todos los puestos de trabajo disponibles

    # i = 0
    # for idx, persona in personas[(personas["REGION"]==regionSelected) & (personas['P17']==1) & (personas['T_asignado']==0)].iterrows(): ####MUY INEFICIENTE
    #     # if persona['P17']==1 and persona["T_asignado"]==0:
    #     trabajo.at[cuposlibres[i],"id_p"] = idx
    #     personas.at[idx,"T_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
    #     i = i + 1
    #     if i == len(cuposlibres):
    #         break

final = time.perf_counter()
print("Tiempo para terminar el proceso de trabajo: {0:f} ".format(final-inicial))

#%% Sistema de Educacion Superior

inicial = time.perf_counter()
#Personas por Comuna que estudian en educacion universitaria
#P13, es si asiste a la educacion formal
rPer = per[per['P13']==1 & per['P15'].between(11,14)].groupby('COMUNA').count()['P13']
rPer.rename("Estudiantes", inplace=True)
rUniv = univ.groupby('COMUNA').sum()['Total']["sum"] #Para obtener una serie
rUniv.rename("Matricula", inplace=True)
capacidades = pd.concat([rPer,rUniv],axis=1, sort=False).fillna(0)
capacidades["usadas"] = 0

#ATENCION:
# ****** Funciona bien para cuando las Matriculas son mayores que los Estudiantes

#personas["U_asignado"] = 0
Universidades["asignados"] = 0
edu_sup = pd.DataFrame([],columns=['id_p','id_u', 'pos'])
for idxc, datos in capacidades.iterrows():
    lista = []
    #Se construyen las capacidades
    for idx, carrera in Universidades[Universidades["COMUNA"]==idxc].iterrows():
        for i in range(int(carrera["Total"])):
            lista.append([0, idx, i])
    if len(lista)==0:
        continue
    muestra = random.sample(range(len(lista)),len(lista))

    i = 0
    for idx, persona in personas[(personas["COMUNA"]==idxc) & (personas['P13']==1) &
                                 (personas['P15'] >= 11) & (personas['P15'] <= 14) & (personas["U_asignado"]==0)].iterrows():
        # if persona['P13']==1 and 11 <= persona['P15'] <= 14 and persona["U_asignado"]==0:
        personas.at[idx,"U_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
        lista[muestra[i]][0] = idx
        i = i + 1
        if i == len(lista):
            break

    edu_sup = pd.concat([edu_sup, pd.DataFrame(lista,
                                columns=['id_p','id_u', 'pos'])],axis=0,sort=False)

    #Se asignan los que quedaron fuera
    cuposlibres = edu_sup[edu_sup["id_p"]==0].index.tolist()

    if len(cuposlibres)==0:
        continue #Se asignaron todos

    i = 0
    for idx, persona in personas[(personas["REGION"]==regionSelected) & (personas['P13']==1) &
                                 (personas['P15'] >= 11) & (personas['P15'] <= 14) & (personas["U_asignado"]==0)].iterrows(): ####MUY INEFICIENTE
        # if persona['P13']==1 and 11 <= persona['P15'] <= 14 and persona["U_asignado"]==0:
        edu_sup.at[cuposlibres[i],"id_p"] = idx
        personas.at[idx,"U_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
        i = i + 1
        if i == len(cuposlibres):
            break

final = time.perf_counter()
print("Tiempo para terminar el proceso de edu superior: {0:f} ".format(final-inicial))

#%%
#Sistema de educación primara y secundaria
inicial = time.perf_counter()

rPer = per[per['P13']==1 & per['P15'].between(1,10)].groupby('COMUNA').count()['P13']
rPer.rename("Estudiantes", inplace=True)
rEdu = edu.groupby('COMUNA').sum()['MAT_TOTAL'] #Para obtener una serie
rEdu.rename("Matricula", inplace=True)
capacidades = pd.concat([rPer,rEdu],axis=1, sort=False).fillna(0)
capacidades["usadas"] = 0

#personas["E_asignado"] = 0
Educacion["asignados"] = 0
edu_basica = pd.DataFrame([],columns=['id_p','id_e', 'pos'])
for idxc, datos in capacidades.iterrows():
    lista = []
    #Se construyen las capacidades
    for idx, dependencia in Educacion[Educacion["COMUNA"]==idxc].iterrows():
        for i in range(int(dependencia["MAT_TOTAL"])):
            lista.append([0, idx, i])
    if len(lista)==0:
        continue
    muestra = random.sample(range(len(lista)),len(lista))

    i = 0
    for idx, persona in personas[(personas["COMUNA"]==idxc) & (personas['P13']==1) &
                                 (personas['P15']>=1) & (personas['P15'] <= 10) & (personas["E_asignado"]==0)].iterrows():
    # for idx, persona in per[per["COMUNA"]==idxc].iterrows():
    #     if persona['P13']==1 and 1 <= persona['P15'] <= 10 and persona["E_asignado"]==0:
        personas.at[idx,"E_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
        lista[muestra[i]][0] = idx
        i = i + 1
        if i == len(lista):
            break

    edu_basica = pd.concat([edu_basica, pd.DataFrame(lista,
                                columns=['id_p','id_e', 'pos'])],axis=0,sort=False)

    #Se asignan los que quedaron fuera
    cuposlibres = edu_basica[edu_basica["id_p"]==0].index.tolist()
    if len(cuposlibres)==0:
        continue #Esto quiere decir que no hay cupos libres, y quedaron mas estudiantes que cupos

    i = 0
    for idx, persona in personas[(personas["REGION"]==regionSelected) & (personas['P13']==1) &
                                 (personas['P15'] >=1 ) & (personas['P15'] <= 10) & (personas["E_asignado"]==0)].iterrows(): ####MUY INEFICIENTE: ####MUY INEFICIENTE
        # if persona['P13']==1 and 1 <= persona['P15'] <= 10 and persona["E_asignado"]==0:
        edu_basica.at[cuposlibres[i],"id_p"] = idx
        personas.at[idx,"E_asignado"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME
        i = i + 1
        if i == len(cuposlibres):
            break

final = time.perf_counter()
print('Tiempo para terminar el proceso de edu basica: {0:2f} '.format(final-inicial))

#%%
#Ahora se construyen diccionarios de todas las estructuras de datos
#trabajo = pd.DataFrame([],columns=['id_p','id_f', 'pos'])
#edu_sup = pd.DataFrame([],columns=['id_p','id_u', 'pos'])
#edu_basica = pd.DataFrame([],columns=['id_p','id_e', 'pos'])
lista = []
edu_basicad = dict()
for index, relacion in edu_basica.iterrows():
    if relacion['pos']==0:
        edu_basicad[relacion['id_e']] = lista
        lista = []
    lista.append(relacion['id_p'])

#Se hace el último grupo
edu_basicad[relacion['id_e']] = lista

##Educacion superior
lista = []
edu_supd = dict()
for index, relacion in edu_sup.iterrows():
    if relacion['pos']==0:
        if len(lista)>0:
            edu_supd[relacion['id_u']] = lista
            lista = []
    lista.append(relacion['id_p'])

#Se hace el último grupo
edu_supd[relacion['id_u']] = lista

##trabajo
lista = []
trabajod = dict()
for index, relacion in trabajo.iterrows():
    if relacion['pos']==0:
        if len(lista)>0:
            trabajod[relacion['id_f']] = lista
            lista = []
    lista.append(relacion['id_p'])

#Se hace el último grupo
trabajod[relacion['id_f']] = lista

#%% Distribucion de Transporte entre los hogares

inicial = time.perf_counter()

rHogar = hogares[hogares['REGION']==regionSelected].groupby('COMUNA').count()['NHOGAR']
rHogar.rename('Hogares', inplace=True)
rVehic = Vehic[Vehic['REGION']==regionSelected].groupby('COMUNA').sum()['Total'] #Para obtener una serie
rVehic.rename("Autos", inplace=True)
capacidades = pd.concat([rHogar,rVehic],axis=1, sort=False).fillna(0)

hogares['conAuto'] = 0
for idxc, datos in capacidades.iterrows():
    hogaresconauto = int(min((datos['Autos'] * 0.9),datos['Hogares']))

    muestra = random.sample(range(hogaresconauto),hogaresconauto)

    indices = hogares[hogares['COMUNA']==idxc]
    for i in range(hogaresconauto):
        idx = indices.index[i]
        hogares.at[idx,"conAuto"] = 1 ##NOTAR QUE SE ASIGNA SOBRE EL GRAN DATAFRAME

final = time.perf_counter()
print('Tiempo para terminar la asignación de autos: {0:2f} '.format(final-inicial))

#%%

################################################################
#def preparaEstructurasComplementarias(conSchool):

if Actualizar==True:
    hogares['ID'] = (hogares['ID_ZONA_LOC']+100000).astype(str)+(hogares['NVIV']+1000).astype(str)+hogares['NHOGAR'].astype(str)
    personas['ID'] = (personas['ID_ZONA_LOC']+100000).astype(str)+(personas['NVIV']+1000).astype(str)+personas['NHOGAR'].astype(str)
hogares.set_index('ID')
personas.set_index('ID')
Comunas = hogares.groupby('COMUNA').agg({'REGION':'min'})
print('Indices Establecidos en Personas y Hogares')
# #hogares['ID'] = hogares[['ID_ZONA_LOC', 'NVIV', 'NHOGAR']].agg('-'.join, axis=1)

#%% Estructura de Probabilidad

p = 0.01 #La probabilida de progresión de un bebé
oddratio = p / (1-p)

personas['Pprogreso'] = oddratio * (1.08)**personas['P09']/(1 + oddratio * (1.08)**personas['P09'])

#%%
print('Creando la estructura de pasajeros y de educacion')
if conSchool == True:
    Sistema = "Con Escuelas"
    l_hogares = []
    usuariosTP = {}
    for idxc, comuna in Comunas[Comunas['REGION']==regionSelected].iterrows():
        perss = personas[personas['COMUNA']==idxc]
        pasajeros = []
        for idxh, hogar in hogares[hogares['COMUNA']==idxc].iterrows():
            idHogar = hogar['ID']
            grupo = perss[perss['ID']==idHogar].index
            l_hogares.append(grupo) #Esta tiene la lista de personas en cada hogar
            if hogar['conAuto'] == 0: #Si no tienen auto
                pasajeros.extend(grupo.tolist())
                usuariosTP[idxc] = pasajeros
else:

    Sistema = "Sin Escuelas"
    l_hogares = []
    usuariosTP = {}
    for idxc, comuna in Comunas[Comunas['REGION']==regionSelected].iterrows():
        perss = personas[personas['COMUNA']==idxc]
        pasajeros = []
        for idxh, hogar in hogares[hogares['COMUNA']==idxc].iterrows():
            idHogar = hogar['ID']
            grupo = perss[perss['ID']==idHogar].index
            l_hogares.append(grupo) #Esta tiene la lista de personas en cada hogar
            if hogar['conAuto'] == 0: #Si no tienen auto
                for idpersona in grupo:
                    if perss.loc[idpersona,'P13']!=1 or perss.loc[idpersona,'P17']!=5:
                        pasajeros.append(idpersona)
        usuariosTP[idxc] = pasajeros

####################################################
#def grabaDatos():
losdatos = {'personas': personas, "hogares":hogares, "edu_basica":edu_basicad, "edu_sup":edu_supd,
    "trabajo":trabajod, "regionSelected":regionSelected, "Comunas":Comunas,
    "usuariosTP":usuariosTP,"l_hogares":l_hogares}
if conSchool == True:
    archivo = "datosinicialesconSchool.pickle"
else:
    archivo = "datosinicialessinSchool.pickle"

with open(archivo, 'wb') as handle:
    pickle.dump(losdatos, handle, protocol=pickle.HIGHEST_PROTOCOL)

if Actualizar==True:
    hogares.to_csv(r'Hogares_Creado0707.csv')
    personas.to_csv(r'Personas_Creado0707.csv')

final = time.perf_counter()
print('Tiempo para terminar TODO {0:f}'.format(final-GranInicial))


import torch
import pygame
import random
import numpy as np

from collections import deque

import Actions
import Rewards
import BoardState

from Game import Game
from PriorityBuffer import PriorityBuffer
from PygameRenderer import PygameRenderer

class ExperienceBufferBuilder(Game):

    # Only allow the model to observe state and take an action every n frames
    # Note that there will be many repeated frames in the game while animations occur etc.
    ActionFrameSkip = 1

    DiscountFactor = 0.99

    def __init__(self):
        super().__init__()

        pygame.init()

        self.image_dimensions = (50, 100)

        self.model_renderer = PygameRenderer(self, bgcolour=(255,255,255))
        self.model_renderer.resize(*self.image_dimensions)

        self.device = 'cuda'
        self.experience_buffer = PriorityBuffer(10000)

        self.board.game = self
        self.difficulty = 29
        self.dynamic_difficulty = False

    def start(self):
        super().start()

    #-------------------------------------------------------------------------
    # Reward Function Contributors

    def notify_score(self, points_scored):
        print( "Line cleared" )
        self.reward += Rewards.line_cleared(points_scored)
        self.priority_update = True

    def notify_piece_landed(self):
        self.reward += Rewards.piece_landed(self.board)

    #-------------------------------------------------------------------------

    def read_state(self):
        render_target = pygame.Surface( self.image_dimensions )

        self.model_renderer.render_board(render_target)

        pixels = pygame.surfarray.array3d(render_target).astype(np.float32)

        greyscale = pixels.dot([0.298, 0.587, 0.114])
        greyscale = greyscale / 255.0
        greyscale = torch.tensor(greyscale, dtype=torch.float).to(self.device)
        greyscale = greyscale.unsqueeze(0)
        greyscale = greyscale.unsqueeze(0)

        return greyscale

    def update(self, action):
        self.reward += Rewards.action_taken(action)

        super().update( [action] )

        if self.board.game_over:
            self.reward += Rewards.game_over()

        self.state = self.read_state()

    #-------------------------------------------------------------------------------------------

    def prepare_run(self):
        output_dimensions = (200, 400)

        self.display = pygame.display.set_mode( output_dimensions )
        pygame.display.set_caption('Tetris')
        self.clock = pygame.time.Clock()

        self.test_renderer = PygameRenderer(self)
        self.test_renderer.resize( *output_dimensions )

    def run(self, path):
        self.start()
        self.state = self.read_state()
        total_reward = 0

        while not self.game_complete:
            self.reward = 0
            before_state = self.state
            self.priority_update = False

            action = Actions.Nothing
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.complete = True
                elif event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_LEFT:
                        action = Actions.Left
                    elif event.key == pygame.K_RIGHT:
                        action = Actions.Right
#                    elif event.key == pygame.K_DOWN:
#                        action = Actions.FastFallOn
                    elif event.key == pygame.K_x:
                        action = Actions.RotateLeft
                    elif event.key == pygame.K_z:
                        action = Actions.RotateRight
                elif event.type == pygame.KEYUP:
                    pass
#                    if event.key == pygame.K_DOWN:
#                        action = Actions.FastFallOff

            if (self.frame_count % self.ActionFrameSkip) == 0 and self.board.state not in (BoardState.Clearing, BoardState.Falling):
                self.update(action)
                after_state = self.state
                action_index = torch.tensor( [self.ActionSet.index(action)] )

                if self.priority_update:
                    self.experience_buffer.append_priority (
                        (before_state, after_state, action_index, self.reward, self.game_complete)
                    )
                else:
                    self.experience_buffer.append (
                        (before_state, after_state, action_index, self.reward, self.game_complete)
                    )
            else:
                self.update(None)

            total_reward += self.reward
            self.test_renderer.render_board(self.display)
            pygame.display.update()

            # Lower than usual frame rate - will slow down disappearing line animations unnaturally,
            # but should allow a human to follow what the computer's doing in real time!
            self.clock.tick(10)

        print( "Final reward: %.1f"%total_reward )
        print( "Saving to file %s ..."%path )
        self.save(path)

    #-------------------------------------------------------------------------------------------

    def save(self, path):
        self.experience_buffer.save(path, True)

    def load(self,path):
        self.experience_buffer.load(path)

if __name__ == '__main__':
    game = ExperienceBufferBuilder()

    game.prepare_run()
    game.run('pgbuffer1.bf')


from core.models import Balance, OrderProduct, Product

def get_total_cash():
    cashed_in_balance_objects = Balance.objects.filter(balance='cashed')
    current_cash = 0
    for cash in cashed_in_balance_objects:
        current_cash = current_cash + int(cash.bill)
    return current_cash

def equalize(bill, expense):
    cashed_objects = Balance.objects.filter(balance='cashed')
    current_cash = bill - expense
    equal_amount = current_cash / cashed_objects.count()
    for cash in cashed_objects:
        balance = Balance.objects.get(id=cash.id)
        balance.bill = str(int(equal_amount))
        balance.save()

def generate_balances(sale, purchase):
    current_cash = get_total_cash()
    sale = sale
    purchase = purchase
    profit = sale - purchase
    expense = purchase
    equalize(current_cash, expense)
    generate_sale(sale)
    generate_expense(expense)
    generate_profit(profit)

def generate_sale(bill):
    Balance.objects.create(
        title="Auto Generated Sale Balance",
        description="generated_automatically_for_sales",
        bill=str(bill),
        balance="sale",
    ).save()

def generate_expense(bill):
    Balance.objects.create(
        title="Auto Generated Expense",
        description="generated_automatically_for_calculating_expense",
        bill=str(bill),
        balance="expense",
    ).save()

def generate_profit(bill):
    Balance.objects.create(
        title="Auto Generated Profit",
        description="generated_automatically_for_calculating_profit",
        bill=str(bill),
        balance="profit",
    ).save()

def generate_from_order_product(product_id, quantity):
    cash_in_hand = get_total_cash()
    product = Product.objects.get(id=product_id)
    product.stock = str(int(product.stock) - int(quantity))
    product.save()
    sale_bill = int(product.sale_price) * int(quantity)
    purchase_bill = int(product.purchase_price) * int(quantity)
    order_product = OrderProduct.objects.create(
        product=product,
        quantity=quantity,
        sale_bill=str(sale_bill),
        purchase_bill=str(purchase_bill)
    )
    generate_sale(sale_bill)
    generate_profit(sale_bill-purchase_bill)
    equalize(cash_in_hand, purchase_bill)
    order_product.save()
    return order_product


from splinter import Browser
from bs4 import BeautifulSoup as bs
from webdriver_manager.chrome import ChromeDriverManager
import pandas as pd
import time

def scrape():

    executable_path = {"executable_path": ChromeDriverManager().install()}
    browser = Browser("chrome", **executable_path, headless=False)

    mars_dict = {}

    # Scrape Mars News Articles
    news_url = "https://redplanetscience.com/"
    browser.visit(news_url)

    time.sleep(1)

    news_html = browser.html
    soup_news = bs(news_html, 'html.parser')

    news_title = soup_news.find_all('div', class_="content_title")[0].text
    news_p = soup_news.find_all('div', class_="article_teaser_body")[0].text

    # Mars Space Images
    image_url = "https://spaceimages-mars.com/"
    browser.visit(image_url)

    time.sleep(1)

    image_html = browser.html
    image_soup = bs(image_html, 'html.parser')

    result = image_soup.find_all("img", class_="headerimage fade-in")

    for r in result:
        src = r.get("src")
        featured_image_url = f"{image_url}{src}"

    # Mars Facts
    facts_url = "https://galaxyfacts-mars.com/"
    tables = pd.read_html(facts_url)
    facts_df = tables[1]
    facts_df.columns = [" ", "Facts"]
    facts_df.set_index(" ", inplace=True)
    table_html = facts_html = facts_df.to_html()
    facts_html = facts_html.replace('\n', '')

    # Mars Hemisphere
    url_hemi = "https://marshemispheres.com/"
    browser.visit(url_hemi)

    time.sleep(1)

    html_hemi = browser.html
    soup_hemi = bs(html_hemi, 'html.parser')

    results_hemi = soup_hemi.find_all('h3')
    results_tag = soup_hemi.find_all('div', class_="description")

    hemisphere_image_urls = []

    for x in range(4):

        # Title
        img_title = results_hemi[x].text

        # Images
        browser.visit(f"{url_hemi}{results_tag[x].a['href']}")
        html_x = browser.html
        soup_tag = bs(html_x, "html.parser")
        tag = soup_tag.find('img', class_="wide-image").get("src")
        img_url = f"{url_hemi}{tag}"

        # Build the dictionary
        post = {}
        post['title'] = img_title
        post['img_url'] = img_url
        hemisphere_image_urls.append(post)

    # Build a dictionary for MongoDB
    mars_dict["news_title"] = news_title
    mars_dict["news_p"] = news_p
    mars_dict["featured_image_url"] = featured_image_url
    mars_dict["fact_table"] = str(facts_html)
    mars_dict["hemisphere_images"] = hemisphere_image_urls

    browser.quit()

    return mars_dict


import json
import logging
import http
import uuid
import re

import queries
from board_membership_status import BoardMembershipStatus
from websocket_service import WebsocketService

logger = logging.getLogger()
logger.setLevel(logging.INFO)

websocket_service = WebsocketService()

def handle(event, context):
	'''Expected body:
	{
		photoKey: string, (a uuid)
		s3BucketName: string,
		s3ObjectKey: string
	}
	'''
	response = {
		'headers': {
			'Access-Control-Allow-Headers': '*',
			'Access-Control-Allow-Origin': '*',
			'Access-Control-Allow-Methods': 'OPTIONS,POST,GET,PUT'
		},
		'statusCode': http.HTTPStatus.OK
	}

	try:
		cognito_username = event['requestContext']['authorizer']['claims']['cognito:username']
	except:
		cognito_username = None

	try:
		photo_key, s3_bucket_name, s3_object_key, join_code = _parse_args(event)
	except:
		logging.exception('Invalid arguments.')
		response['statusCode'] = http.HTTPStatus.BAD_REQUEST
		return response

	response = _create_board_photo_map(cognito_username,
								   photo_key,
								   s3_bucket_name,
								   s3_object_key,
								   join_code,
								   response)

	if response['statusCode'] == http.HTTPStatus.OK:
		websocket_service.push_photo_to_clients(join_code, photo_key)

	return response

def _parse_args(event):
	body = json.loads(event['body'])
	photo_key = body['photoKey']
	s3_bucket_name = body['s3BucketName']
	s3_object_key = body['s3ObjectKey']

	uuid_regex = re.compile('.*/boards/([^/]*)/photos') # matches '/boards/<uuid>/photos', capturing <uuid>
	path = event['path']
	uuid_path_component = uuid_regex.match(path).group(1)
	join_code = str(uuid.UUID(uuid_path_component))

	return photo_key, s3_bucket_name, s3_object_key, join_code

def _create_board_photo_map(cognito_username,
							photo_key,
							s3_bucket_name,
							s3_object_key,
							join_code,
							response):
	try:
		queries.create_photo_for_board(cognito_username,
									   photo_key,
									   s3_bucket_name,
									   s3_object_key,
									   join_code,
									   BoardMembershipStatus.ACTIVE)
	except:
		logging.exception('Could not create board photo map.')
		response['statusCode'] = http.HTTPStatus.INTERNAL_SERVER_ERROR
	finally:
		return response


#!/bin/env python3
import itertools
import re
import sys
from os import path, walk

def main(argv):
    testnet_dir = path.dirname(path.abspath(argv[0]))
    env_dir = path.join(testnet_dir, "env")

    res = list(itertools.chain(*map(triples_from_file, host_files(env_dir))))

    testnets = sorted(set(m[0] for m in res))
    hosts = sorted(set(m[3] for m in res))

    table = {h: {tn: [] for tn in testnets} for h in hosts}

    for tn, typ, idx, host in res:
        table[host][tn].append("{}.{}".format(typ, idx))

    print("host;{}".format(";".join(sorted(testnets))))
    for k, v in sorted(table.items()):
        line = ";".join([", ".join(v[tn]) for tn in testnets])
        print("{};{}".format(k, line))

#    print(sorted(testnets))
#    print(len(list(testnets)))
#    print(sorted(hosts))
#    print(len(list(hosts)))
#    print(table)

def host_files(p):
    for (root, dirs, files) in walk(p):
        for f in filter(lambda f: f.endswith(".ini"), files):
            yield path.join(root, f)

NODE_DEF_RE = re.compile(r'^\s*([-a-zA-Z.0-9]+)\s+.*ic_host="([a-zA-Z-0-9.]+)"')
NODE_DEF_RE = re.compile(r'^\s*([-a-zA-Z0-9]+)\.([a-zA-Z0-9]+)\.([a-zA-Z0-9]+)\s+.*ic_host="([a-zA-Z-0-9.]+)"')

def triples_from_file(h):
    with open(h, "r") as f:
        matches = (NODE_DEF_RE.match(line) for line in f)
        matches = (m for m in matches if m is not None)
        return [(m[1], m[2], m[3], m[4]) for m in matches]

if __name__ == "__main__":
    main(sys.argv)