import matplotlib.pyplot as plt
import seaborn as sns

import matplotlib as mpl
import numpy as np
from six import string_types

def contour_plot(x, y, z, shade=False, legend=True, shade_lowest=True,
                 cbar=False, cbar_ax=None, cbar_kws=None, ax=None, **kwargs):

    """Plot a contour.

    Parameters
    ----------
    x : 1d array-like
        Input data.
    y: 1d array-like,
        Second input data.
    shade : bool, optional
        If True, shade in the area between contours
    legend : bool, optional
        If True, add a legend or label the axes when possible.
    shade_lowest : bool, optional
        If True, shade the lowest contour. Setting this to ``False`` can be
        useful when you want multiple contours on the same Axes.
    cbar : bool, optional
        If True and drawing a bivariate KDE plot, add a colorbar.
    cbar_ax : matplotlib axes, optional
        Existing axes to draw the colorbar onto, otherwise space is taken
        from the main axes.
    cbar_kws : dict, optional
        Keyword arguments for ``fig.colorbar()``.
    ax : matplotlib axes, optional
        Axes to plot on, otherwise uses current axes.
    kwargs : key, value pairings
        Other keyword arguments are passed to ``plt.plot()`` or
        ``plt.contour{f}``

    Returns
    -------
    ax : matplotlib Axes
        Axes with plot.

    Examples
    --------

    Plot a basic contour:

    .. plot::
        :context: close-figs

        >>> import numpy as np; np.random.seed(10)
        >>> import seaborn as sns; sns.set(color_codes=True)
        >>> import vis
        >>> x = np.arange(1, 10)
        >>> y = x.reshape(-1, 1)
        >>> z = x * y
        >>> ax = vis.contour_plot(x, y, z)

    Use an existing axis:

    .. plot::
        :context: close-figs

        >>> sns.set(style="darkgrid")
        >>> f, ax = plt.subplots(figsize=(8, 8))
        >>> ax.set_aspect("equal")
        >>> start, stop, n_values = -8, 8, 800
        >>> x_vals = np.linspace(start, stop, n_values)
        >>> y_vals = np.linspace(start, stop, n_values)
        >>> x, y = np.meshgrid(x_vals, y_vals)
        >>> z = np.sqrt(x**2 + y**2)
        >>> vis.contour_plot(x, y, z, ax=ax)

    Shade under the contour and use a different color:

    .. plot::
        :context: close-figs

        >>> ax = vis.contour_plot(x, shade=True, color="r")

    Use more contour levels and a different color palette:

    .. plot::
        :context: close-figs

        >>> ax = vis.contour_plot(x, y, z, n_levels=30, cmap="Purples_d")

    Specify contour level with a different color:

    .. plot::
        :context: close-figs

        >>> ax = vis.contour_plot(x, y, z, n_levels=[0.1, 0.2], color="r")

    Add a colorbar for the contours:

    .. plot::
        :context: close-figs

        >>> ax = vis.contour_plot(x, y, cbar=True)

    TODO extend functionality to plot text on contours:
    #https://stackoverflow.com/questions/33169093/how-to-label-a-seaborn-contour-plot

    """
    if ax is None:
        ax = plt.gca()

    if isinstance(x, list):
        x = np.asarray(x)
    if len(x) == 0:
        return ax
    x = x.astype(np.float64)

    if isinstance(y, list):
        y = np.asarray(y)
    y = y.astype(np.float64)
    if isinstance(z, list):
        z = np.asarray(z)
    z = z.astype(np.float64)

    ax = _contour_plot(
        x, y, z, shade, shade_lowest, legend, cbar, cbar_ax, cbar_kws, ax,
        **kwargs)

    return ax

def _contour_plot(x, y, z, filled, fill_lowest,
                       axlabel, cbar, cbar_ax, cbar_kws, ax, **kwargs):
    """Plot a contour."""
    # Plot the contours
    n_levels = kwargs.pop("n_levels", 10)

    scout, = ax.plot([], [])
    default_color = scout.get_color()
    scout.remove()

    colors = kwargs.pop("colors", None)
    if colors is None:
        color = kwargs.pop("color", default_color)
        cmap = kwargs.pop("cmap", None)
        if cmap is None:
            if filled:
                cmap = light_palette(color, as_cmap=True)
            else:
                cmap = dark_palette(color, as_cmap=True)
        if isinstance(cmap, string_types):
            if cmap.endswith("_d"):
                pal = ["#333333"]
                pal.extend(color_palette(cmap.replace("_d", "_r"), 2))
                cmap = blend_palette(pal, as_cmap=True)
            else:
                cmap = mpl.cm.get_cmap(cmap)
        kwargs["cmap"] = cmap
    else:
        kwargs["colors"] = colors

    label = kwargs.pop("label", None)

    contour_func = ax.contourf if filled else ax.contour
    cset = contour_func(x, y, z, n_levels, **kwargs)
    if filled and not fill_lowest:
        cset.collections[0].set_alpha(0)
    kwargs["n_levels"] = n_levels

    if cbar:
        cbar_kws = {} if cbar_kws is None else cbar_kws
        ax.figure.colorbar(cset, cbar_ax, ax, **cbar_kws)

    # Label the axes
    if hasattr(x, "name") and axlabel:
        ax.set_xlabel(x.name)
    if hasattr(y, "name") and axlabel:
        ax.set_ylabel(y.name)

    if label is not None:
        legend_color = cmap(.95) if color is None else color
        if filled:
            ax.fill_between([], [], color=legend_color, label=label)
        else:
            ax.plot([], [], color=legend_color, label=label)

    return ax


import msal
import requests
import pandas as pd
from pathlib import Path

def get_access_token():

    app_id = '*********************************************'
    pbi_tenant_id = '*********************************************'
    app_secret = '*********************************************'

    authority_url = f'https://login.microsoftonline.com/{pbi_tenant_id}'
    scopes = [r'https://analysis.windows.net/powerbi/api/.default']

    client = msal.ConfidentialClientApplication(app_id, authority=authority_url, client_credential=app_secret)

    response = client.acquire_token_for_client(scopes)
    token = response.get('access_token')

    return token

token = get_access_token()
headers = {
    'Content-Type': 'application/json',
    'Authorization': f'Bearer {token}'
}

def get_workspaces():

    workspace_endpoint = 'https://api.powerbi.com/v1.0/myorg/groups?$filter=(isOnDedicatedCapacity eq true)'
    response_request = requests.get(workspace_endpoint, headers=headers)
    result = response_request.json()

    workspace_id = [workspace['id'] for workspace in result['value']]
    workspace_name = [workspace['name'] for workspace in result['value']]

    return zip(workspace_id, workspace_name)

def get_datasets(workspace_id = None):

    if workspace_id is None:
        all_workspaces = get_workspaces()
    else:
        all_workspaces = [workspace_id]

    dataset_ids = []
    dataset_names = []
    workspace_ids = []

    for workspace_id, workspace_name in all_workspaces:

        dataset_endpoint = fr"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}/datasets"
        response_request = requests.get(dataset_endpoint, headers=headers)
        result = response_request.json()

        for item in result['value']:
            dataset_ids.append(item['id'])
            dataset_names.append(item['name'])
            workspace_ids.append(workspace_id)

    return zip(dataset_ids, dataset_names, workspace_ids)

def get_refresh_history():

    dataset_ids, dataset_names, workspace_ids = zip(*list(get_datasets()))
    dataset_workspace = zip(dataset_ids, workspace_ids)
    result = []

    for dataset_id, workspace_id in dataset_workspace:
        refresh_endpoint = fr"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes"
        response_request = requests.get(refresh_endpoint, headers=headers)

        if response_request.status_code == 200:
            response_text = response_request.json()['value']
            for entry in response_text:
                entry['DatasetID'] = dataset_id
                entry['WorkspaceID'] = workspace_id

            result.append(response_text)

    flatlist = []

    for sublist in result:
        for element in sublist:
            flatlist.append(element)

    return flatlist

workspace_df = pd.DataFrame.from_records(get_workspaces(), columns = ['WorkspaceID', 'WorkspaceName'])
dataset_df = pd.DataFrame.from_records(get_datasets(), columns = ['DatasetID', 'DatasetName', 'WorkspaceID'])

refresh_df = pd.DataFrame(get_refresh_history())
columns_to_keep = ['refreshType', 'startTime', 'endTime', 'status', 'DatasetID', 'WorkspaceID', 'serviceExceptionJson']
capitalize_names = [name[0].capitalize() + name[1:] for name in columns_to_keep]
refresh_df = refresh_df[columns_to_keep]
refresh_df.columns = capitalize_names
refresh_df = refresh_df.astype({'StartTime': 'datetime64[ns]', 'EndTime': 'datetime64[ns]'})
refresh_df['StartTime'] = refresh_df['StartTime'].dt.date
refresh_df['EndTime'] = refresh_df['EndTime'].dt.date

final_df = pd.merge(workspace_df, dataset_df, how='inner', left_on = 'WorkspaceID', right_on = 'WorkspaceID').merge(refresh_df, how = 'inner', on = 'WorkspaceID')
columns_to_keep = ['WorkspaceID', 'WorkspaceName', 'DatasetID_x', 'DatasetName', 'RefreshType', 'StartTime', 'EndTime', 'ServiceExceptionJson']
final_df = final_df[columns_to_keep]
final_df.rename(columns = {'DatasetID_x':'DatasetID'}, inplace = True)

home = str(Path.home())
final_df.to_excel(home + r"\Desktop\Refresh History.xlsx", index= False)


import random
import pygame
import words

def main():
    width, height = 435, 570
    pygame.init()
    surface = pygame.display.set_mode((width, height))
    pygame.display.set_caption('Wordle Clone')
    game = Game(surface)
    game.play()
    pygame.quit()

class Game():
    def __init__(self, surface):
        self.surface = surface
        self.bg_color = (18, 18, 18)
        self.font = pygame.font.SysFont('Ariel', 70)

        self.FPS = 60
        self.game_Clock = pygame.time.Clock()
        self.run = True

        self.secret_word = words.WORDS[random.randint(0, len(words.WORDS))].upper()

        self.board = [
            [' ', ' ', ' ', ' ', ' '],
            [' ', ' ', ' ', ' ', ' '],
            [' ', ' ', ' ', ' ', ' '],
            [' ', ' ', ' ', ' ', ' '],
            [' ', ' ', ' ', ' ', ' '],
            [' ', ' ', ' ', ' ', ' ']]

        self.attempt = 0
        self.letter = 0
        self.active_attempt = True
        self.game_over = False

    def play(self):
        while self.run:
            self.handle_events()
            self.draw()
            self.game_Clock.tick(self.FPS)

    def draw(self):
        self.surface.fill(self.bg_color)
        for row in range(len(self.board)):
            for col in range(len(self.board[0])):
                pygame.draw.rect(self.surface, (60, 60, 60), [col * 85 + 10, row * 85 + 10, 75, 75], 2)

                if self.board[row][col] == self.secret_word[col] and row < self.attempt:
                    pygame.draw.rect(self.surface, (80, 140, 80), [col * 85 + 10, row * 85 + 10, 75, 75])
                elif self.board[row][col] in self.secret_word and row < self.attempt:
                    pygame.draw.rect(self.surface, (180, 160, 60), [col * 85 + 10, row * 85 + 10, 75, 75])
                elif row < self.attempt:
                    pygame.draw.rect(self.surface, (60, 60, 60), [col * 85 + 10, row * 85 + 10, 75, 75])

                text = self.font.render(self.board[row][col], True, (255, 255, 255))
                self.surface.blit(text, (col * 85 + 30, row * 85 + 25))

        pygame.display.update()

    def check_win(self):
        if ''.join(self.board[self.attempt - 1]) == self.secret_word:
            print('win')
            self.game_over = True
        elif self.attempt > 5:
            print('lose')
            self.game_over = True

    def handle_events(self):
        events = pygame.event.get()
        for event in events:
            if event.type == pygame.QUIT:
                self.run = False

            if event.type == pygame.KEYDOWN and not self.game_over:

                if event.key == pygame.K_RETURN and self.letter == 5:
                    self.attempt += 1
                    self.letter = 0
                    self.active_attempt = True
                    self.check_win()

                elif event.key == pygame.K_BACKSPACE and self.letter > 0:
                    self.board[self.attempt][self.letter - 1] = ' '
                    self.letter -= 1

            if event.type == pygame.TEXTINPUT and self.active_attempt and not self.game_over:
                key = event.__getattribute__('text')
                if key.isalpha():
                    self.board[self.attempt][self.letter] = key.upper()
                    self.letter += 1

            if self.letter == 5:
                self.active_attempt = False
            if self.letter < 5:
                self.active_attempt = True

if __name__ == "__main__":
    main()


import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
from time import sleep
from selenium import webdriver

url = 'https://www.usquidditch.org/teams'

#open selenium - use your own info for PATH
PATH = (r"C:\\Users\\Gustavo\\Desktop\\scoutfy\\Team USA\\chromedriver.exe")

driver = webdriver.Chrome(PATH)
driver.get(url)
driver.implicitly_wait(3)

#wait for page loading
sleep(2)
#html got with selenium
selenium_get = driver.page_source

soup = bs(selenium_get, 'html.parser')

link=[]

#close selenium
driver.close()

for a in soup.findAll('a'):
    link.append(a['href'])

#get every team website
link=link[115:346]
#get every team's info from the main page
info = soup.find('tbody').findAll('tr')

def find_acronym(s):
    #return only letters between ()
    return s[s.find("(")+1:s.find(")")]

def find_class(soup):
    list =[]
    for el in range(230):
        element = (soup.find("table", attrs = {"class" : "teams-display"}))
        list.append(element.find("tr", attrs ={"id" : "t"+str(el)}).find_all("td")[2].get_text())
# find_class(soup)

def find_socialmedia(soup):
    refs = []
    listsocial = []
    acuts = (soup.find_all("a"))
    for el in acuts:
        refs.append(el.get("href"))
    listsocial.append(refs)
    return listsocial
# find_socialmedia(soup)

def main():
    # this list will store every dictionary of each club
    my_list = []

    for i in range(len(info)):
       # This dict will store the information of each club
        my_dict = {}

        my_dict['Name'] = info[i].findAll('td')[0].text
        my_dict['City'] = info[i].findAll('td')[1].text.split(',')[0].strip()
        my_dict['State'] = info[i].findAll('td')[1].text.split(',')[1].strip()

        #here it's oppened every team's new page with another info

        url = 'https://www.usquidditch.org' + str(link[i])

        agent = {"User-Agent":'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}
        req = requests.get(url, headers=agent)
        soup = bs(req.content, features="lxml")

        my_dict['Acronym'] = find_acronym(soup.find('h2').text)
        my_dict['social_media'] = find_socialmedia(soup.find("ul", attrs ={"class" : "team-social-media-links"}))

        #this are all the constants that must be added
        my_dict['Category'] = 'Non Recognized'
        my_dict['Sport'] = 'Quidditch'
        my_dict['Country'] = 'United States of America'
        my_dict['Affilated'] = 'US Quidditch'
        my_dict['Recorded By'] = 'Gustavo Florindo Oliveira'
        my_dict['Recorded Date'] = '27/02/2022'

        my_list.append(my_dict)

    return my_list

res = main()

df = pd.DataFrame(data=res)
df.to_excel('US_Quidditch.xlsx', index=False)


#!/usr/bin/env python3
"""
SfM point filtering
"""
import sys
import time
import numpy
import glob
from termcolor import colored
import pylab as plt

from PointFiltering import LazData, VegetationIndices, Filter, Grid, PlotTools
from PointFiltering.Helpers import saveimg, savergb

def run(filter, **kwargs):
    """ Run point filter
    Args:
        filter (string): options are 'isl', 'vi', 'isl_vi'
        **kwargs (dict): optional keyword arguments
    Raises:
        ValueError: when invalid filter or VI is provided
    """
    print('Run {} filter'.format(filter))

    FOLDER = kwargs.get('folder', 'example_data/')
    RES = kwargs.get('res', 0.25)
    VI = kwargs.get('vi', 'exg')
    PLOT = kwargs.get('plot', False)
    wkt = kwargs.get('wkt', '')

    if VI == 'exg':
        THRESHOLD = kwargs.get('threshold', 0.01)
    elif VI == 'vvi':
        THRESHOLD = kwargs.get('threshold', 5e-5)
    else:
        raise ValueError('invalid VI')

    OK = "green"

    if not filter or filter not in Filter.FILTERS:
        raise ValueError('invalid filter')

    nr_points = 0
    nr_ground_points = 0

    i = 0
    LISTDIR = glob.glob(FOLDER + "*.las")
    LISTDIR.sort()
    if len(LISTDIR) == 0:
        raise RuntimeError('No las files found in folder')
    print("Processing tiles")

    # iterate over tiles
    for LASFILE in LISTDIR:
        print(LASFILE+"..."),
        points = LazData.createlaz(LASFILE)
        nr_points += len(points.point_x)
        if len(points.ground) == 0:
            print ("no points in tile, skipping"),
        else:
            try:
                # In the first iteration all points are considered ground points.
                # The ground point labels are stored as an array in points.ground as bool
                # each filter sets the label to False when points are considered not-ground

                print ("total: %6d" % len(points.ground)),
                # first filter based on vegetation index
                if VI == "vvi":
                    points.vegindex = VegetationIndices.vvi(points)
                    if filter in ['vi', 'isl_vi']:
                        Filter.filter_by_vvi(points, THRESHOLD)
                if VI == "exg":
                    points.vegindex = VegetationIndices.exg(points)
                    if filter in ['vi', 'isl_vi']:
                        Filter.filter_by_exg(points, THRESHOLD)
                # then filter by ISL
                if filter in ['isl', 'isl_vi']:
                    Filter.filter_by_dtm(points)
                print ("ground: %d" % sum(points.ground)),
                nr_ground_points += sum(points.ground)

            except Exception as err:
                print (err)
            finally:
                if "GRID" not in locals():
                    GRID = Grid.Grid(points, res=RES)
                    GRID.filter_method = points.filter_method
                else:
                    GRID_PART = Grid.Grid(points, res=RES)
                    GRID.merge_grids(GRID_PART)
        i += 1
        print(colored("done", OK)),

    print('total nr points: {}; nr ground points: {}'.format(
        nr_points, nr_ground_points))
    # create raster files
    t0 = time.time()
    print("rasterizing %d points into a (%d, %d) grid..." %
           (len(GRID.point_x), GRID.grid_x.shape[0], GRID.grid_x.shape[1])),
    GRID.create_dtm()
    GRID.create_dsm()
    GRID.create_rgb()
    GRID.create_vi()
    print(colored("done", OK)),

    chm = GRID.dsm - GRID.dtm
    chm[chm < 0] = 0

    print(colored("done", OK))

    if PLOT:
        print ("plotting..."),
        # points vi
        values = numpy.isnan(GRID.point_vi) == 0
        data = GRID.point_vi[values]
        clim = PlotTools.stretch(data)
        filename = "out/plots/points_vi.png"
        PlotTools.scatter(
            GRID.point_x[values],
            GRID.point_y[values],
            data,
            clim,
            filename,
            title='VI points'
        )
        # points gvi
        values = numpy.isnan(GRID.point_dtm) == 0
        data = GRID.point_gvi[values]
        # use clim from vi
        filename = "out/plots/points_vi-ground" + GRID.filter_method + ".png"
        PlotTools.scatter(
            GRID.point_x[values],
            GRID.point_y[values],
            data,
            clim,
            filename,
            title='VI (ground only)'
        )
        # points dsm -----------------------------
        values = numpy.isnan(GRID.point_dsm) == 0
        data = GRID.point_dsm[values]
        clim = PlotTools.stretch(data)
        fn = "out/plots/points_dsm" + GRID.filter_method + ".png"
        PlotTools.scatter(
            GRID.point_x[values],
            GRID.point_y[values],
            data,
            clim,
            fn,
            title='DSM points'
        )

        # ground points z ------------------------
        values = numpy.isnan(GRID.point_dtm) == 0
        PlotTools.scatter(
            GRID.point_x[values],
            GRID.point_y[values],
            GRID.point_dtm[values],
            clim,
            "out/plots/points_dtm" + GRID.filter_method + ".png",
            title='DTM points'
        )
        # dsm
        PlotTools.image(
            GRID.dsm,
            "out/plots/grid_dsm.png",
            do_shade=True,
            res=RES,
            title='DSM grid'
        )
        # dtm
        PlotTools.image(
            GRID.dtm,
            "out/plots/grid_dtm" + GRID.filter_method + ".png",
            do_shade=True,
            res=RES,
            title='DTM grid'
        )
        # chm
        PlotTools.image(
            chm,
            "out/plots/grid_chm" + GRID.filter_method + ".png",
            title='CHM grid'
        )
        print(colored("done", OK))

    # save images
    saveimg(GRID, GRID.dtm, 'out/dtm.tif', wkt=wkt)
    saveimg(GRID, GRID.dsm, 'out/dsm.tif', wkt=wkt)
    saveimg(GRID, GRID.dsm-GRID.dtm, 'out/chm.tif', wkt=wkt)
    saveimg(GRID, GRID.raster_vi, 'out/vi.tif', wkt=wkt)
    savergb(GRID, GRID.rgb, 'out/rgb.tif', wkt=wkt)

if __name__ == "__main__":
    args = sys.argv

    kwargs = {}
    # get filter from command line argument
    if len(args) > 1:
        for arg in args:
            if '=' not in arg:
                continue
            key = arg.split('=')[0]
            value = arg.split('=')[1]
            if 'filter' in arg:
                filter = value.lower()
            else:
                kwargs[key] = value
    # use default if not provided
    if 'filter' not in locals():
        filter = 'isl_vi'
    wkt = 'PROJCS["WGS 84 / UTM zone 32N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],UNIT["metre",1,AUTHORITY["EPSG","9001"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",9],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],AUTHORITY["EPSG","32632"],AXIS["Easting",EAST],AXIS["Northing",NORTH]]'
    run(filter=filter, plot=True, wkt=wkt, **kwargs)


import requests
from bs4 import BeautifulSoup
import pandas as pd

url = 'https://exame.com/brasil/as-50-melhores-escolas-de-cada-estado-do-brasil-no-enem/'
response = requests.get(url)

soup = BeautifulSoup(response.content, 'html.parser')

table = soup.find('table')

data = []
for row in table.find_all('tr')[1:6]:
    cells = row.find_all('td')
    posicao = cells[0].get_text().strip()
    nome_escola = cells[1].get_text().strip()
    nota_media = cells[2].get_text().strip()
    uf = cells[3].get_text().strip()
    data.append([posicao, nome_escola, nota_media, uf])

df = pd.DataFrame(data, columns=['Posicao', 'Nome da escola', 'Nota media', 'UF'])

print(df)

Raspagem das 5 melhores escolas do país.py


import math
import tensorflow as tf
import tensorflow_datasets as tfds
import matplotlib.pyplot as plt

def plot_image_examples(images : tf.Tensor, n=9):
    rows = int(math.sqrt(n))
    fig, axs = plt.subplots(rows, math.ceil(n / rows))
    for i in range(n):
        axs[i // rows, i % rows].imshow(images[i], cmap="gray")
    fig.show()
    plt.show()

def plot_changed_images(images : tf.Tensor, noised_image : tf.Tensor, changed_images : tf.Tensor, n=8):
    fig, axs = plt.subplots(3, n)
    for i in range(n):
        axs[0, i].imshow(images[i], cmap="gray")
        axs[1, i].imshow(noised_image[i], cmap="gray")
        axs[2, i].imshow(changed_images[i], cmap="gray")
    fig.show()
    plt.show()


#---------------------------------------------------------------
#
#   This script contains the functions to
#   receive a new pattern and modify
#   according the existent rule base.
#
#   To avoid unnecessary processing,
#   the function " intersected_connected_sets "
#   find which of the original connected sets
#   are intersected by the new pattern.
#
#  Those connected sets together with the new pattern
#  form a " new (connected) set" which will be cut
#  following the maximum volume heuristic
#
#  The resulting rule base is build with the partition
#  of the new connected_set + the partitions comming from
#  the not_intersected connected sets.
#
#----------------------------------------------------------------

import json

from splitR_version1 import *  #  CURRENT VERSION  #
from optimum_partition_for_Q import *
from rulex_2 import *                   #    if necessary . . .

# Read a json file ---------------------
def read(file_name):
    with open(file_name) as json_data:
        file_content = json.load(json_data)
        return file_content
#---------------------------------------------------
# Read files with all_connected_sets and their indexes
all_connected_sets = read('all_connected_sets.json')
#indexes_all_connected_sets = read('indexes_connected_sets.json')
#for i in all_connected_sets: print(i)

# Find wich of the original connected_sets are intersected by the new instance
# get its indexes
def intersected_connected_sets( new_pattern, all_connected_sets ):
    indexes_of_intersected_sets = [ ]
    intersected_sets = []
    index_counter = -1
    for connected_set in all_connected_sets:
        index_counter += 1
        include_set = False
        for rule in connected_set:
            print(new_pattern, rule)
            #intersects = intersection(new_pattern, rule) # INTERSECTIONS OF ALREADY RULEXED INSTANCES
            # Function that considers intersection or possible rule formation with Rulex
            intersects = intersection_or_possible_rule_formation( new_pattern, rule, 1 )
            print('Intersection with this rule: ', intersects)
        if intersects == True:
            include_set = True
        if include_set == True:
            indexes_of_intersected_sets.append(index_counter)
            intersected_sets.append(connected_set)
    return [ intersected_sets,  indexes_of_intersected_sets ]
#   NEW INSTANCE
#pattern = ( 2, 5, 'A' )

#[ intersected_sets,  indexes_of_intersected_sets ] = intersected_connected_sets( pattern, all_connected_sets)

#Create new set with the new pattern + its intersections
def pattern_plus_intersections(pattern, intersected_sets):
        new_set = []
        for intersected_set in intersected_sets:
            for rule in intersected_set:
                new_set.append(rule)
        new_set.append(pattern)
        return new_set

#print('This is the set to cut : ' )
#new_set = pattern_plus_intersections(pattern, intersected_sets)
#print(new_set)

#print('This is the set without inconsistencies among the rules: ')
#new_set = optimum_partition( new_set )
#print(new_set)

#------------------------------------------------------------------------------------
#                           Exclude from optimim partitions
#                           the indexes_of_intersected_sets
#                           keep the rest of the partitions
#------------------------------------------------------------------------------------
optimum_partitions = read('optimum_partitions.json')
optimum_partitions_indexes = read('connected_rules_indexes.json')  #  They share indexes
lonly_rules = read('lonly_rules.json')
lonly_rules_indexes = read('lonly_rules_indexes.json')

#print('------------------')
#print('indexes of intersected sets :  ' , indexes_of_intersected_sets)
#print('optimum_partitions_indexes',optimum_partitions_indexes)
#print('lonly_rules_indexes', lonly_rules_indexes)

#  compare the indexes of optimum partitions (partitions for the connected rules) and lonly rules
#  with the indexes of the intersected sets
#  if they match, eliminate the entrance either in the optimim_partitions of in the lonly_rules
#  return as final set {lonly_rules, optimum_partitions}\{elements_at_indexes_of_intersected_rules}
def remaining_partitions(optimum_partitions, optimum_partitions_indexes, lonly_rules, lonly_rules_indexes, indexes_of_intersected_sets):
    kept_partitions = []
    kept_lonly = []

    for i in range( len(optimum_partitions_indexes) ):
        index = optimum_partitions_indexes[i]
        flag = index in indexes_of_intersected_sets
        if flag == False:
            kept_partitions = kept_partitions + optimum_partitions[i]

    for i in range( len(lonly_rules_indexes) ):
        index = lonly_rules_indexes[i]
        flag = index in indexes_of_intersected_sets
        if flag == False:
            kept_lonly.append(lonly_rules[i])
    return [ kept_partitions, kept_lonly ]

#print( 'REMAINING PARTITIONS : ')

#not_intersected = remaining_partitions(optimum_partitions, optimum_partitions_indexes, lonly_rules, lonly_rules_indexes, indexes_of_intersected_sets)
#print(not_intersected)

#---------------------------------------------------------------
#    The resulting rule base incorporating the new parrern
#    is the set { remaining_partitions U new_set }
#
#---------------------------------------------------------------
def not_intersected_union_new_set(not_intersected, new_set):
    union = []
    for i in not_intersected:
        union = union + i
    for i in new_set:
        union.append(i)
    return union
#print('result: ')
#print( not_intersected_union_new_set(not_intersected, new_set) )

#-----------------------------------------------------------------
#
#           PROCESS  ---   NEW  ---  PATTERN
#
#-----------------------------------------------------------------
#    Write json data
#--------------------------------------
def write(file_name,data):
    with open (file_name, 'w') as f:
        json.dump(data,f)

def as_tuple(iterable):
    _list = []
    for item in iterable:
        if type(item) == list:
            item = tuple(item)
            _list.append(item)
        else:
            _list.append(item)
            the_tuple = tuple(_list)
    return the_tuple

def format_new_set(new_set):
    formatted = []
    for i in new_set:
        i = as_tuple(i)
        formatted.append(i)
    return formatted

def as_list(new_set):
    _list = []
    for i in new_set:
        i = list(i)
        _list.append(i)
    return _list

def remove_risk(new_set):
    _list = []
    for rule in new_set:
        rule.pop()
        _list.append(rule)
    return _list

#--------------------------------------
def process_new_pattern(pattern):
    all_connected_sets = read('all_connected_sets.json')
    [ intersected_sets, indexes_of_intersected_sets ] = intersected_connected_sets( pattern,  all_connected_sets )
    new_set = pattern_plus_intersections(pattern, intersected_sets)
    # print('new set = pattern + intersected_Sets', new_set)

    # Add risk parameter to the new (connected) set for Rules
    for rule in new_set: rule.append(1)
    #print('New connected set without RULEX', new_set)
    #FORMART INTO TUPLES
    new_set = format_new_set(new_set)
    new_set = rulex(new_set)
    print('New set after RULEX', new_set)

    new_set = as_list(new_set)
    new_set = remove_risk(new_set)  ###################
    print( 'New set for cut', new_set )
    new_set = optimum_partition( new_set )
    print('New set optimum partition', new_set)

    optimum_partitions = read('optimum_partitions.json')
    optimum_partitions_indexes = read('connected_rules_indexes.json')
    lonly_rules = read('lonly_rules.json')
    lonly_rules_indexes = read('lonly_rules_indexes.json')

    not_intersected = remaining_partitions(optimum_partitions, optimum_partitions_indexes, lonly_rules, lonly_rules_indexes,indexes_of_intersected_sets)
    print('Not intersected connected sets : ', not_intersected)
   # for j in not_intersected:
    #    print('j',j)
     #   for k in j: print(k)
    if new_set != False:
        new_rule_base = not_intersected_union_new_set(not_intersected,new_set)
    else:
        print('The new pattern do not intersect any connected set')
        del pattern[-1]
        not_intersected.append( [pattern] )
        new_rule_base = not_intersected
    return new_rule_base

#pattern = [ 2,  5,  'A' ]
#print('new rule base: ', process_new_pattern(pattern))


# Name: Gerardo Palacios
# Date: 11/26/2020
# Honor Statement: I have not given or received any unauthorized assistance on this assignment
# YouTube Link: https://youtu.be/DXaabepMEvw

import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

class SixSidedDie:
    """A simulation of a six-sided dice.

    Attributes
    ----------
    face_value : int
        Current face value of the simulated dice

    num_sides : int
        Number of sides of the simulated dice"""

    face_value = int()
    num_sides = 6

    def roll(self):
        """Simulate rolling the dice and return face value."""

        self.face_value = random.randint(1, self.num_sides)
        return self.face_value

    def getFaceValue(self):
        """Read the current face value of the dice."""

        return self.face_value

    def __repr__(self):
        """Canonical string representation of class name and current face value"""

        repr_msg = '{}({})'.format(self.__class__.__name__, self.face_value)
        return repr_msg

class TenSidedDie(SixSidedDie):
    """A simulation of a ten-sided dice.

    Attributes
    ----------
    face_value : int
        Current face value of the simulated dice

    num_sides : int
        Number of sides of the simulated dice"""

    num_sides = 10

class TwentySidedDie(SixSidedDie):
    """A simulation of a twenty-sided dice.

    Attributes
    ----------
    face_value : int
        Current face value of the simulated dice

    num_sides : int
        Number of sides of the simulated dice"""

    num_sides = 20

class SimulatedData:
    """Simulation data handler."""

    def __init__(self):
        # initialize empty attributes

        self.parameters = {}
        self.data = pd.DataFrame({
            'actors': [],
            'jailed': [],
            'wealth': [],
            'thieves': [],
            'lieutenants': [],
            'jailed_thieves': [],
            'jailed_lieutenants': [],
            'collected': [],
            'recovered': [],
            'bribes': []
        })
        self.heists = pd.DataFrame({
            'heist_value': [],
            'solve_probability': [],
            'detective_assigned': [],
            'recovered_wealth': [],
        })
        self.weekly_collection = 0
        self.weekly_thieves = 0
        self.weekly_lieutenants = 0
        self.weekly_jailed_thieves = 0
        self.weekly_jailed_lieutenants = 0
        self.weekly_recovered_assets = 0

    def load_parameters(self, filename):
        """Load parameters from work directory csv file."""
        self.parameters = pd.read_csv(filename).to_dict('records')[0]

    def add_new_row(self, df, new_row):
        """Adds new row data frame.

        Parameters
        ----------
        df : str
            Name of data frame to add new row
        new_row
            Data for new row
        """

        if df == 'data':
            self.data = self.data.append(new_row, ignore_index=True)

        elif df == 'heists':
            self.heists = self.heists.append(new_row, ignore_index=True)

    def replace_row(self, df, row_index, new_row, columns=None):
        """Replaces elements for entire row or row and column.

        Parameters
        ----------
        df : str
            Name of data frame

        row_index : int
            Row index for data frame to look at.

        new_row : int or list of ints
            Data to replace row

        columns : str optional(default=None)
            Name of column to replace in row

        """

        if df == 'data':
            if columns is None:
                self.data.loc[row_index] = new_row
            else:
                self.data.loc[row_index:row_index, columns] = new_row

        elif df == 'heists':
            if columns is None:
                self.heists.loc[row_index] = new_row
            else:
                self.heists.loc[row_index:row_index, columns] = new_row

    def get_row(self, df, row_index):
        """Returns a pandas data frame for the given row index.

        Parameters
        ----------
        df : str
            Name of data frame to add new row

        row_index : int
            Row index for data frame to look at.
        """

        if df == 'data':
            return self.data.loc[row_index]

        elif df == 'heists':
            return self.heists.loc[row_index]

    def create_potential_heists(self):
        """Creates a randomly generated data frame for potential heists."""

        # calculate number of potential heists
        num_heists = int(self.data['thieves'].sum() - self.data['jailed_thieves'].sum())

        # Create random data frame
        self.heists = pd.DataFrame({
            'heist_value': [self.parameters['heist_coefficient']*(twenty_sided.roll()**2) for i in range(num_heists)],
            'solve_probability': np.random.rand(num_heists).tolist(),  # Create a list of n random numbers
            'detective_assigned': [-1] * num_heists,  # start out with no detectives assigned to heist
            'recovered_wealth': [0] * num_heists,  # start out with no recovered wealth
        })

    def reset_stats(self):
        """Resets weekly collected stats."""

        self.weekly_collection = 0
        self.weekly_thieves = 0
        self.weekly_lieutenants = 0
        self.weekly_jailed_thieves = 0
        self.weekly_jailed_lieutenants = 0
        self.weekly_recovered_assets = 0

    def save_results(self, filename, columns=('actors', 'jailed', 'wealth', 'bribes')):
        """Save results to csv file

        Parameters
        ----------

        filename : str
            File name including file extension '.csv'
        columns : list of str optional(default=['actors', 'jailed', 'wealth', 'bribes'])
            Columns to be saved to file. Default are actors, jailed, wealth, and bribes

        """
        df = self.data.loc[:, columns]
        df.to_csv(filename)

    def plot_time_series(self, title, stat):
        """Pleasantly displays current board"""

        plt.title(title)
        if isinstance(stat, list):
            for var in stat:
                plt.plot(self.data.index, self.data[var], linestyle='solid')
        else:
            plt.plot(self.data.index, self.data[stat], linestyle='solid')

        plt.yticks(rotation=45)
        plt.xticks(rotation=45)
        plt.show()

class Thief:
    """Low ranking actor in Mr. Biggs crime syndicate

    Attributes
    ----------

    wallet : int
        Collected profits for thief.

    jail : bool
        Boolean if thief has been apprehended
    """

    def __init__(self):
        self.wallet = 0
        self.jail = False
        self.new_recruit()

    def new_recruit(self):
        """Add a new recruit to the weekly stats."""

        simulated_data.weekly_thieves += 1

    def collect_heist(self):
        """Collect bounty from random heist."""

        select = None
        # initialize the total number of heists available
        number_of_heists = int(simulated_data.data['thieves'].sum() - simulated_data.data['jailed_thieves'].sum())

        # Assign a random heist that has not been assigned yet
        while select in crymland.syndicate.heists_assigned or select is None:
            select = random.randint(0, number_of_heists - 1)

        # Record assigned heist
        crymland.syndicate.heists_assigned.update({select: True})
        # Get heist data
        heist_data = simulated_data.get_row('heists', select)
        kickback = self.check_collection(heist_data, select)
        return kickback

    def check_collection(self, heist_data, row_index):
        """Evaluates if thief gets away or gets arrested. Returns kickback to boss.

        Parameters
        ----------
        heist_data : pandas data frame
            Assigned heist data

        row_index : int
            Assigned heist data frame row index
        """

        if heist_data['heist_value'] == -1:  # Detective solved this crime
            taken_away = heist_data['recovered_wealth'] + self.wallet  # Bounty AND wealth is recovered
            simulated_data.replace_row('heists', row_index, taken_away, 'recovered_wealth')  # Saved recovered to df
            simulated_data.weekly_jailed_thieves += 1  # add 1 to weekly stats
            simulated_data.weekly_recovered_assets += self.wallet
            self.wallet = 0  # Zero out wallet
            self.jail = True  # Set jail to true
            return -1

        else:  # Thief got away!
            simulated_data.weekly_collection += heist_data['heist_value']  # Save total collected
            profit = heist_data['heist_value'] * 0.50  # Keep half, kickback half
            self.wallet += profit
            return profit

class Lieutenant(Thief):
    """Mid ranking actor in Mr. Biggs crime syndicate.

    Attributes
    ----------

    wallet : int
        Collected profits for thief.

    jail : bool
        Boolean if thief has been apprehended

    thieves : dict of Thief class
        Dictionary of Thieves being managed.

    lieutenants : dict of Lieutenant class
        Dictionary of Lieutenants being managed.

    in_jail : int
        Number of actors on team that are in jail.

    """

    def __init__(self):
        super().__init__()
        self.thieves = {i: Thief() for i in range(simulated_data.parameters['num_thieves'])}
        self.lieutenants = {}
        self.in_jail = 0
        self.new_recruit()

    def new_recruit(self):
        """Add a new recruit to the weekly lieutenant stats."""

        simulated_data.weekly_lieutenants += 1

    def collect_heist(self):
        """Collect kickbacks from team."""

        collection = self.check_collection(self.check_thieves() + self.check_lieutenants())
        self.promote_thieves()

        return collection

    def check_thieves(self):
        """Collect kickbacks from low ranking actors. Returns total thief kickbacks"""

        collection = 0  # initialize empty collection variable
        remove_thieves = []  # initialize empty list
        for thief in self.thieves:  # iterate through thief list
            value = self.thieves[thief].collect_heist()

            if value > 0:  # Thief gave you proper kickback
                collection += value * 0.50
            else:  # He got caught
                self.in_jail += 1  # Record jailed actor
                remove_thieves.append(thief)  # Add actor in jail to list

        for thief in remove_thieves:  # Remove jailed actors from team
            self.thieves.pop(thief)

        return collection

    def check_lieutenants(self):
        """Collect kickbacks from mid ranking actors. Returns total lieutenant kickbacks"""

        collection = 0  # initialize empty collection variable
        remove_lieutenants = []  # initialize empty list
        for lieutenant in self.lieutenants:  # iterate through lieutenant list
            value = self.lieutenants[lieutenant].collect_heist()  # Check kickbacks

            if value > 0:  # Lieutenant gives proper kickback
                collection += value * 0.50
            else:  # Lieutenant in jail
                self.in_jail += 1
                remove_lieutenants.append(lieutenant)

        for lieutenant in remove_lieutenants:  # Remove jailed lieutenants from team
            self.lieutenants.pop(lieutenant)

        return collection

    def promote_thieves(self):
        """Promote a low-ranking thief to a mid-ranking lieutenant."""
        remove = []  # initialize empty list
        if not self.in_jail:  # Can't promote from jail
            for thief in self.thieves:  # iterate through thieves

                # if thieves have achieved a personal wealth greater than set parameter... promote
                if self.thieves[thief].wallet >= simulated_data.parameters['promotion_wealth']:
                    self.lieutenants.update({thief: Lieutenant()})  # create a new lieutenant
                    self.lieutenants[thief].wallet = self.thieves[thief].wallet  # transfer wealth
                    remove.append(thief)

            for remove_thief in remove:  # remove promoted thief from thief team. No longer commit heists.
                self.thieves.pop(remove_thief)

    def check_collection(self, collection, **kwargs):
        """Checks if collected actors talked in jail. Returns False or kickbacks."""

        # if thieves/LTs have grouped more than given parameter. go to jail.

        if self.in_jail >= simulated_data.parameters['jailed_thieves']:  # Too many members talked
            simulated_data.weekly_jailed_lieutenants += 1
            simulated_data.weekly_recovered_assets += self.wallet + collection
            self.jail = True
            self.wallet = 0
            return -1
        else:
            self.wallet += collection  # Add to personal wealth
            return collection

class Syndicate:
    """Mr. Biggs crime syndicate.

    Attributes
    ----------

    bigg : object
        Object of class Lieutenant. Mr. Bigg.

    heists_assigned : dict
        dictionary of already assigned heists.

    successful_bribes : int
        Number of successful bribes

    """

    def __init__(self):
        self.bigg = Lieutenant()
        self.heists_assigned = {}
        self.successful_bribes = 0

    def reset_stats(self):
        """Reset heists assigned."""

        self.heists_assigned = {}

    def initiate_heists(self):
        """Inform syndicate to being heists."""

        self.bigg.collect_heist()

    def bribes(self):
        """Attempt to bribe detectives that have seized large portion of assets."""

        for detective in crymland.academy.detectives:  # Iterate through detectives
            # Get parameters
            detective = crymland.academy.detectives[detective]
            assets = detective.get_recovered_assets()
            first, after = simulated_data.parameters['seizes_first'], simulated_data.parameters['seizes_thereafter']
            bribe_rate, weekly_collection = simulated_data.parameters['initial_bribe'], simulated_data.weekly_collection

            # Whenever a detective first seizes a total of $1,000,000 from thieves or lieutenants, and every
            # $1,000,000 thereafter, Mr. Bigg attempts to bribe the detective

            if assets >= first and assets >= first + after * self.successful_bribes:
                if detective.evaluate_bribe(bribe_rate * weekly_collection):
                    self.successful_bribes += 1

class Academy:
    """Detective Agency.

    Attributes
    ----------

    detectives : dict
        Dictionary of dedicated detectives.

    """

    def __init__(self):
        self.detectives = {i: Detective() for i in range(simulated_data.parameters['num_detectives'])}
        self.detectives_assigned = {}

    def reset_stats(self):
        """Resets stats."""

        self.detectives_assigned = {}
        for detective in self.detectives:
            self.detectives[detective].bribe_assets = 0

    def assign_heists(self):
        """Assigned detectives on team a heist to investigate."""

        for detective in self.detectives:  # Iterate through each detective
            select = None  # Initialize empty variable
            number_of_heists = int(  # Calculate total number of current heists.
                simulated_data.data['thieves'].sum() - simulated_data.data['jailed_thieves'].sum())

            if detective < number_of_heists:  # As long as there heists left to assign
                while select in self.detectives_assigned or select is None:  # assign a random heist
                    select = random.randint(0, number_of_heists - 1)

                # Record assigned heist
                self.detectives_assigned.update({select: detective})
                # Assign heist
                simulated_data.replace_row('heists', select, detective, 'detective_assigned')

                # Order detective to investigate heist
                self.detectives[detective].investigate(select)

    def review_recovered_assets(self):
        """Review and tally recovered assets from detectives."""

        for detective in self.detectives:  # iterate through detectives
            recovered = simulated_data.heists.loc[  # get data assigned from heist
                simulated_data.heists['detective_assigned'] == detective]

            # Tally the recovered assets
            if recovered['recovered_wealth'].to_list():
                self.detectives[detective].new_recovered_assets(int(recovered['recovered_wealth']))

    def fight_corruption(self):
        """Replace corrupt detectives with an uncorrupted detective."""

        for detective in self.detectives:  # iterate through detectives
            if self.detectives[detective].bribed:  # if corrupted
                search_chance = random.random()

                if search_chance <= self.detectives[detective].discovery_risk:  # attempt to catch
                    self.detectives.update({detective: Detective()})
                else:
                    self.detectives[detective].discovery_risk += twenty_sided.roll() / 100
                    # increase chances of discovery

class Detective:
    """Dedicated detective.

    Attributes
    ----------

    solve_rate : float
        probability detective solves heist.

    solve_cap : float
        max probability detective can solve a heist.

    bribed : bool
        Boolean if detective is bribed.

    bribe_assets : float
        Value of accepted bribe.

    recovered_assets : float
        Value of total recovered assets from investigations.

    discovery_risk : float
        Probability of being caught as a corrupted detective.

    """

    def __init__(self):
        self.solve_rate = simulated_data.parameters['solve_init']
        self.solve_cap = simulated_data.parameters['solve_cap']
        self.bribed = False
        self.bribe_assets = 0
        self.recovered_assets = 0
        self.discovery_risk = 0

    def get_recovered_assets(self):
        """Return the total recovered assets from detective"""

        return self.recovered_assets

    def gain_experience(self):
        """Improve investigation experience."""

        self.solve_rate += ten_sided.roll() / 100  # increase experience

        if self.solve_rate >= self.solve_cap:  # Cap experience
            self.solve_rate = self.solve_cap

    def investigate(self, heist_index):
        """Investigate assigned heist"""

        heist = simulated_data.get_row('heists', heist_index)  # Get heist data
        if not self.bribed and heist['solve_probability'] <= self.solve_rate:  # If not corrupted. Attempt to solve
            saved = heist['heist_value']  # Collect heist_value
            simulated_data.replace_row('heists', heist_index, saved, ['recovered_wealth'])  # Add to recovered wealth
            simulated_data.replace_row('heists', heist_index, -1, ['heist_value'])  # negate heist value
            self.gain_experience()  # Gain experience due to successful investigation

    def new_recovered_assets(self, new):
        """Add to current recovered assets."""

        self.recovered_assets += new
        return self.recovered_assets

    def evaluate_bribe(self, bribe):
        """Evaluate bribe from Mr.Biggs. Returns True if bribe is accepted """

        # Evaluate bribe
        if bribe <= 10000:
            probability = .05

        elif bribe <= 100000:
            probability = .1

        elif bribe <= 1000000:
            probability = .25

        else:
            probability = .5
        # calculate probability
        random_prob = random.random()
        if random_prob <= probability:

            # Accept bribe and become a corrupted detective
            self.bribed = True
            self.bribe_assets += bribe
            self.discovery_risk = simulated_data.parameters['initial_risk']

            return True
        else:
            return False

class Crymland:
    """Land of Crymland.

    Attributes
    ----------

    syndicate : object
        Object of class Syndicate.
    academy : object
        Object of class Academy

    """

    def __init__(self):
        self.syndicate = Syndicate()
        self.academy = Academy()

    def run_sim(self, weeks):
        """Run simulation for the given number of weeks."""

        # Begin with starting stats
        self.record_weekly_stats()
        i = 0
        # Do until end of simulation or until the syndicate has been taken down.
        while i <= weeks and not self.syndicate.bigg.jail:
            simulated_data.create_potential_heists()                        # Create random potential heists
            self.academy.assign_heists()                                    # The Academy assign detectives to the cases
            self.syndicate.initiate_heists()                                # Syndicate begins heists
            self.academy.review_recovered_assets()                          # Detectives review their recovered assets
            self.syndicate.bribes()                                         # Syndicate attempts to bribe detectives
            self.academy.fight_corruption()                                 # The Academy attempts to fight corruption
            self.record_weekly_stats()                                      # Log stats
            i += 1                                                          # Iterate the next week

    def record_weekly_stats(self):
        """Record stats for current week."""

        # Calculate total jailed
        total_jailed = simulated_data.data['jailed_thieves'].sum() + simulated_data.data['jailed_lieutenants'].sum()
        total_jailed += simulated_data.weekly_jailed_thieves + simulated_data.weekly_jailed_lieutenants

        total_actors = simulated_data.data['thieves'].sum() + simulated_data.data['lieutenants'].sum()
        total_actors += simulated_data.weekly_thieves + simulated_data.weekly_lieutenants - 1

        # Create new row to add to data frame
        new_row = {
            'actors': total_actors - total_jailed,
            'jailed': total_jailed,
            'wealth': self.syndicate.bigg.wallet,
            'thieves': simulated_data.weekly_thieves,
            'lieutenants': simulated_data.weekly_lieutenants,
            'jailed_thieves': simulated_data.weekly_jailed_thieves,
            'jailed_lieutenants': simulated_data.weekly_jailed_lieutenants,
            'collected': simulated_data.weekly_collection,
            'recovered': simulated_data.weekly_recovered_assets,
            'bribes': sum([self.academy.detectives[detective].bribe_assets for detective in self.academy.detectives])
        }

        # add new row to data frame
        simulated_data.add_new_row('data', new_row)

        # Reset weekly stats
        simulated_data.reset_stats()
        self.academy.reset_stats()
        self.syndicate.reset_stats()

if __name__ == '__main__':
    # Sample parameters1.csv file to upload data from.

    # weeks,num_thieves,heist_coefficient,promotion_wealth,jailed_thieves,num_detectives,solve_init,solve_cap,seizes_first,seizes_thereafter,initial_risk,initial_bribe
    # 500,7,1000,1000000,3,3,.25,0.75,1000000,1000000,0.05,0.1

    # Generate a twenty sided and ten sided die
    twenty_sided = TwentySidedDie()
    ten_sided = TenSidedDie()

    scenarios = ['parameters/parameters1.csv', 'parameters/parameters2.csv', 'parameters/parameters3.csv']

    for i in range(len(scenarios)):
        # Create Data handler
        simulated_data = SimulatedData()

        # load data parameters
        simulated_data.load_parameters(scenarios[i])

        # Create Crymland
        crymland = Crymland()

        # Run simulation
        crymland.run_sim(simulated_data.parameters['weeks'])

        # Plot results
        simulated_data.plot_time_series('Syndicate Size vs Total Apprehended', ['actors', 'jailed'])
        simulated_data.plot_time_series('Personal Wealth of Mr.Bigg', 'wealth')
        simulated_data.plot_time_series('Accepted Bribes', 'bribes')

        # Save results
        simulated_data.save_results('results/simulation_results{}.csv'.format(i))


import json
import logging
import boto3
from abc import ABC, abstractmethod

from django.conf import settings

logger = logging.getLogger(__name__)

class AWSBatchClient(ABC):

    def __init__(self):
        self.client =  boto3.client("batch", region_name=settings.AWS_REGION_NAME)

    @abstractmethod
    def get_job_definition(self):
        """Get the job definition name"""

class AWSBatchJobManager(AWSBatchClient):
    """
    AWS Batch job manager for scheduling Engage model runs.
    """
    def __init__(self, compute_environment):
        """Initialize the batch job manager

        Parameters
        ----------
        compute_environment : object | ComputeEnvironment
            The name of Batch compute environment
        """
        super().__init__()
        self.compute_environment = compute_environment

    def get_job_definition(self):
        """Store job definition name from compute environment"""
        return self.compute_environment.full_name

    def submit_job(self, job):
        response = self.client.submit_job(
            jobName=job["name"],
            jobQueue=settings.AWS_BATCH_JOB_QUEUE,
            jobDefinition=job["definition"],
            containerOverrides={"command": job["command"]},
            dependsOn=job["depends_on"]
        )
        logger.info("Engage batch job submitted - %s", job["name"])
        return response

    def generate_job_message(self, run_id, user_id, depends_on):
        command = [
            "engage", "solve-model",
            "--run-id", str(run_id),
            "--user-id", str(user_id)
        ]
        if self.compute_environment.cmd:
            command = json.loads(self.compute_environment.cmd) + command
        job = {
            "name": f"Engage-Model-Run-UserId-{user_id}-RunId-{run_id}",
            "command": command,
            "definition": self.get_job_definition(),
            "depends_on": [{"jobId":d,"type":"SEQUENTIAL"} for d in depends_on]
        }
        return job

    def describe_jobs(self, jobs: list) -> list:
        if not jobs:
            return [], True

        all_compelete = True
        uncomplete_status = ["SUBMITTED", "PENDING", "RUNNABLE", "STARTING", "RUNNING"]

        response = self.client.describe_jobs(jobs=jobs)
        result = {}
        for item in response["jobs"]:
            result[item["jobId"]] =  item["status"]
            if  item["status"] in uncomplete_status:
                all_compelete = False
        return result, all_compelete

    def terminate_job(self, job_id, reason):
        response = self.client.terminate_job(
            jobId=job_id,
            reason=reason
        )
        return response


from django.shortcuts import render, redirect
from .models import Book, Author

def index(request):

    context = {
        "books_table" : Book.objects.all()
    }

    return render(request, 'books_authors_app/addbook.html', context)

def add_book(request):
    if request.method == 'POST':

        Book.objects.create(title = request.POST['title'], description = request.POST['description'])

    return redirect("/")

def delete_book(request, id):
        book_id = int(id)

        delete_book = Book.objects.get(id= book_id )
        delete_book.delete()

        return redirect ("/")

def view_book(request, id):
    book_id = int(id)

    view_book = Book.objects.get(id=book_id)
    not_author = Author.objects.exclude(books = book_id)

    context = {
        'book': view_book,
        'not_author' : not_author,
    }
    print("$"*80)
    print(not_author)

    return render(request, 'books_authors_app/view_book.html', context)

def add_author_to_book(request, id):
    book_id = int(id)

    book = Book.objects.get(id = book_id)
    request = request.POST['add_author']
    first_name, last_name = request.split(" ")
    author = Author.objects.get(first_name = first_name, last_name = last_name)

    book.authors.add(author)

    return redirect(f"/view_book/{book_id}")

def show_authors(request):
    authors = Author.objects.all()
    print("*"*80)
    print(authors)

    context = {
        'authors': authors
    }

    return render(request,'books_authors_app/addauthor.html', context)

def add_author(request):
    if request.method == 'POST':
        Author.objects.create(first_name = request.POST['auth_first_name'], last_name = request.POST['auth_last_name'], notes = request.POST['notes'])
        return redirect('/show_authors')

    return redirect('/show_authors')

def delete_author(request, id):
    auth_id = int(id)
    author = Author.objects.get(id=auth_id)
    author.delete()

    return redirect('/show_authors')

def view_author(request, id):
    auth_id = int(id)
    author = Author.objects.get(id=auth_id)
    not_book = Book.objects.exclude(authors=auth_id)

    context = {
        'author': author,
        'not_book': not_book,
    }

    return render(request, 'books_authors_app/view_author.html', context)

def add_book_to_author(request, id):
    if request.method == 'POST':
        auth_id = int(id)

        author = Author.objects.get(id=auth_id)
        book = Book.objects.get(title = request.POST['add_author'])

        author.books.add(book)
        print(author)

        return redirect(f"/view_author/{auth_id}")
    return redirect(f"/view_author/{auth_id}")


"""
https://leetcode.com/problems/cousins-in-binary-tree/

"""

from typing import Optional
from utils import TreeNode

class Solution:
    def isCousins(self, root: Optional[TreeNode], x: int, y: int) -> bool:
        level = [root]
        while True:
            newLevel = []
            vals = set()
            for node in level:
                if node:
                    vals.add(node.val)
                    newLevel.append(node.left)
                    newLevel.append(node.right)

            xIn = x in vals
            yIn = y in vals
            if xIn ^ yIn:
                return False
            if xIn & yIn:
                return True
            level = newLevel

"""
okay.. siblings vs cousins
"""

class Solution:
    def isCousins(self, root: Optional[TreeNode], x: int, y: int) -> bool:
        encodings = {}

        def encode(root, pEncoding, left):
            if not root:
                return

            if left:
                thisNode = pEncoding+"0"
            else:
                thisNode = pEncoding+"1"

            encodings[root.val] = thisNode
            encode(root.left, thisNode, True)
            encode(root.right, thisNode, False)
        encode(root, "", False)
        return len(encodings[x]) == len(encodings[y]) and encodings[x][:-1] != encodings[y][:-1]

if __name__ == '__main__':
    s = Solution()

    print(s.isCousins(TreeNode(1,
                               TreeNode(2,
                                        None,
                                        TreeNode(4)),
                               TreeNode(3)), 2, 3))


from django import template
from django.utils.safestring import mark_safe
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.contrib.staticfiles.storage import staticfiles_storage

register = template.Library()

@register.simple_tag
def vertical_bar(value, max_value, display=None):
    if not max_value:
        return ""

    if display == "ratio":
        value_str = "%d/%d" % (value, max_value)
    else:
        value_str = str(value)

    # handle value == None
    value = value or 0

    try:
        ratio = min(float(value) / float(max_value), 1) * 100
    except ValueError:
        return ""

    if ratio > 40:
        left = format_html("{}", value_str)
        right = ""
    else:
        left = format_html(" ")
        right = format_html("{}", value_str)

    return format_html(
        '

{}
{}
',
        ratio,
        left,
        right,
    )

@register.filter()
def timeline(data):
    s = '\n'
    s += '\n"

    return mark_safe(s)


#! -*- coding:utf-8 -*-

import re
import gc
import codecs
import random
import warnings
import pandas as pd
from sklearn.metrics import f1_score
from nltk.metrics.distance import jaccard_distance
from sklearn.model_selection import StratifiedKFold
from keras.layers import *
from keras.callbacks import *
from keras.optimizers import *
from keras.initializers import *
from keras.models import Model
from keras.optimizers import Adam
from keras.utils import to_categorical
from keras_bert import AdamWarmup, calc_train_steps
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
from keras.backend.tensorflow_backend import set_session

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))

maxlen = 512
config_path = 'F://workplace/BERT/chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json'
checkpoint_path = 'F://workplace/BERT/chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_model.ckpt'
dict_path = 'F://workplace/BERT/chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt'

sub_dir = './result/bert_end2end_classfication_sub.csv'
train_a_dir = './data/a/Train_Data.csv'
test_a_dir = './data/a/Test_Data.csv'
train_b_dir = './data/b/round2_train.csv'
test_b_dir = './data/b/round2_test.csv'

def data_preprocess(x, s):
    data = []
    text_list = re.split("[。！!；;？]", x[s])
    entity_list = x['entity'].split(';')
    for text in text_list:
        for entity in entity_list:
            if entity != '' and entity in text:
                data.append(text)
                break
    if len(data) != 0:
        data = '。'.join(data)
    else:
        data = x[s]
    return data

class OurTokenizer(Tokenizer):
    def _tokenize(self, text):
        R = []
        for c in text:
            if c in self._token_dict:
                R.append(c)
            elif self._is_space(c):
                R.append('[unused1]')  # space类用未经训练的[unused1]表示
            else:
                R.append('[UNK]')  # 剩余的字符是[UNK]
        return R

def seq_padding(X, padding):
    return np.array([np.concatenate([x, [padding] * (maxlen - len(x))]) if len(x) < maxlen else x[:maxlen] for x in X])

class data_generator:
    def __init__(self, data, batch_size=8, shuffle=True):
        self.data = data
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.steps = len(self.data) // self.batch_size
        if len(self.data) % self.batch_size != 0:
            self.steps += 1

    def __len__(self):
        return self.steps

    def __iter__(self):
        while True:
            idxs = list(range(len(self.data)))

            if self.shuffle:
                np.random.shuffle(idxs)

            X1, X2, Y = [], [], []
            for i in idxs:
                d = self.data[i]
                x1, x2 = tokenizer.encode(first=d[0] + '_' + d[1] + '_' + d[2])
                y = d[3]
                X1.append(x1)
                X2.append(x2)
                Y.append(y)
                if len(X1) == self.batch_size or i == idxs[-1]:
                    X1 = seq_padding(X1, 0)
                    X2 = seq_padding(X2, 0)
                    Y = np.array(Y)
                    yield [X1, X2], Y
                    [X1, X2, Y] = [], [], []

def build_bert(nclass):
    bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)

    for l in bert_model.layers:
        l.trainable = True

    x1_in = Input(shape=(None,))
    x2_in = Input(shape=(None,))
    x = bert_model([x1_in, x2_in])
    x = Lambda(lambda x: x[:, 0])(x)
    x = Dense(nclass, activation='sigmoid')(x)

    model = Model([x1_in, x2_in], x)
    model.compile(loss='binary_crossentropy', optimizer=Adam(1e-5), metrics=['accuracy'])
    print(model.summary())
    return model

def run_cv(nfold, train_list, test_list):
    train_preds = np.zeros((len(train_list), 1))
    test_preds = np.zeros((len(test_list), 1))
    skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=1017)
    for i, (train_index, valid_index) in enumerate(skf.split(train_list, train_list[:, 3])):
        print('第%s折开始训练' % (i + 1))

        X_train, X_valid, = train_list[train_index, :], train_list[valid_index, :]
        train_D = data_generator(X_train, batch_size=4, shuffle=True)
        valid_D = data_generator(X_valid, batch_size=4, shuffle=True)
        test_D = data_generator(test_list, batch_size=4, shuffle=False)

        K.clear_session()
        model = build_bert(1)
        early_stopping = EarlyStopping(monitor='val_loss', patience=1, verbose=1, mode='min')
        plateau = ReduceLROnPlateau(monitor="val_loss", verbose=1, mode='min', factor=0.5, patience=2)
        checkpoint = ModelCheckpoint('./model/bert_end2end_classfication_fold_' + str(i + 1) + '.h5', monitor='val_loss',
                                     verbose=2, save_best_only=True, mode='min', save_weights_only=True)
        model.fit_generator(
            train_D.__iter__(),
            steps_per_epoch=len(train_D),
            epochs=10,
            validation_data=valid_D.__iter__(),
            validation_steps=len(valid_D),
            callbacks=[early_stopping, checkpoint],
        )
        model.save('./model/bert_end2end_classfication_fold_' + str(i + 1) + '.h5')
        train_preds[valid_index] = model.predict_generator(valid_D.__iter__(), steps=len(valid_D), verbose=1)
        test_preds += model.predict_generator(test_D.__iter__(), steps=len(test_D), verbose=1) / nfold

        del model
        gc.collect()

    return train_preds, test_preds

train_a = pd.read_csv(train_a_dir, index_col=None)
train_b = pd.read_csv(train_b_dir, index_col=None)
train = pd.concat([train_a, train_b], axis=0, ignore_index=True)
test = pd.read_csv(test_b_dir, index_col=None)

print('train', train.shape)
print('test', test.shape)
print(train.columns)
print(test.columns)

print('train content')
train.fillna('', inplace=True)
train['title'] = train.apply(lambda x: data_preprocess(x, 'title'), axis=1)
train['text'] = train.apply(lambda x: data_preprocess(x, 'text'), axis=1)
train['content'] = train.apply(lambda x: x['title'] + '_' + x['text'], axis=1)

print('test content')
test.fillna('', inplace=True)
test['title'] = test.apply(lambda x: data_preprocess(x, 'title'), axis=1)
test['text'] = test.apply(lambda x: data_preprocess(x, 'text'), axis=1)
test['content'] = test.apply(lambda x: x['title'] + '_' + x['text'], axis=1)

print('token_dict')
token_dict = {}
with codecs.open(dict_path, 'r', 'utf8') as reader:
    for line in reader:
        token = line.strip()
        token_dict[token] = len(token_dict)

print('tokenizer')
tokenizer = OurTokenizer(token_dict)

print('train_list')
train_id = []
train_list = []
for i in train.index:
    data_row = train.iloc[i]
    entitys = data_row['entity'].split(';')
    for entity in entitys:
        if len(entity) != 0:
            train_id.append(data_row['id'])
            if entity in data_row['key_entity'].split(';'):
                train_list.append((entity, data_row['entity'], data_row['content'], 1))
            else:
                train_list.append((entity, data_row['entity'], data_row['content'], 0))
train_list = np.array(train_list)

print('test_list')
test_id = []
test_list = []
for i in test.index:
    data_row = test.iloc[i]
    entitys = data_row['entity'].split(';')
    for entity in entitys:
        if len(entity) != 0:
            test_id.append(data_row['id'])
            test_list.append((entity, data_row['entity'], data_row['content'], 0))
test_list = np.array(test_list)

print('start train')
train_preds, test_preds = run_cv(5, train_list, test_list)

train_preds = train_preds[:, 0]
test_preds = test_preds[:, 0]
test_pre_result = [1 if test_pred > 0.5 else 0 for test_pred in test_preds]

id_list = []
negative_list = []
key_entity_list = []
tmp = pd.DataFrame({'id': test_id, 'entity': test_list[:, 0], 'pre_result': test_pre_result})
for id in tmp['id'].unique():
    df = tmp[(tmp['id'] == id) & (tmp['pre_result'] == 1)]
    entitys = df['entity'].tolist()
    if len(entitys) > 0:
        id_list.append(id)
        negative_list.append(1)
        key_entity_list.append(';'.join(entitys))
    else:
        id_list.append(id)
        negative_list.append(0)
        key_entity_list.append('')

sub = pd.DataFrame({'id': id_list, 'negative': negative_list, 'key_entity': key_entity_list})
sub = pd.merge(test[['id']], sub, on='id', how='left')
sub['negative'] = sub['negative'].fillna(0).astype(int)
sub['key_entity'] = sub['key_entity'].fillna('')
print(sub['negative'].value_counts())
sub.to_csv(sub_dir, index=None)