import torch
from torch.utils.data import Dataset, DataLoader
from random import randint
import pickle
import numpy as np
from PIL import Image

class EpicKitchensDataset(Dataset):
    def __init__(self, labels_path, is_flow=False, transforms=None):
        labels_df = load_pickle_data(labels_path)
        self.transforms = transforms
        self.labels = []
        self.narration_ids = []
        self.is_flow = is_flow
        self.input_names = []
        for index, row in labels_df.iterrows():
            seg_img_names = sample_train_segment(
                16,
                row["start_frame"],
                row["stop_frame"],
                row["participant_id"],
                row["video_id"],
                is_flow=is_flow
            )
            self.labels.append(row["verb_class"])
            self.narration_ids.append(f"{row['video_id']}_{row['uid']}")
            self.input_names.append(seg_img_names)

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, index):
        if self.is_flow:
            imgs = load_flow_frames(self.input_names[index])
            imgs = self.transforms(imgs)
            imgs = video_to_tensor(imgs)
        else:
            imgs = load_rgb_frames(self.input_names[index])
            imgs = self.transforms(imgs)
            imgs = video_to_tensor(imgs)
        return self.labels[index], imgs, self.narration_ids[index]

class SequentialClassKitchens(Dataset):
    def __init__(self, labels_path, class_num, temporal_window=16, is_flow=False):
        labels_df = load_pickle_data(labels_path)
        filtered_df = labels_df[labels_df["verb_class"] == class_num]
        self.labels = []
        self.input_names = []
        self.frame_start_numbers = []
        self.is_flow = is_flow
        for index, row in filtered_df.iterrows():
            clip_length = row["stop_frame"] - row["start_frame"]
            num_windows, total_window_len = get_num_seq_windows(clip_length, temporal_window)
            frame_start_numbers = get_start_frame_numbers(
                row["start_frame"],
                row["stop_frame"],
                total_window_len,
                num_windows
            )
            self.labels.append(f"{row['verb_class']}_{row['uid']}")
            self.input_names.append(
                sample_test_sequential_frames(
                    frame_start_numbers,
                    temporal_window,
                    self.is_flow,
                    row["participant_id"],
                    row["video_id"]
                )
            )

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, index):
        seq_window_imgs = []
        for frame_paths in self.input_names[index]:
            if self.is_flow:
                imgs = load_flow_frames(frame_paths)
            else:
                imgs = load_rgb_frames(frame_paths)
            imgs = video_to_tensor(imgs)
            seq_window_imgs.append(imgs)
        return self.labels[index], seq_window_imgs

def sample_test_sequential_frames(start_frames, temporal_window_len, is_flow, part_id, video_id):
    seq_frame_names = []
    for start_frame in start_frames:
        if is_flow:
            seq_frame_names.append([
                [
                    f"/user/work/rg16964/epic_kitchens_data/flow/{part_id}/{video_id}/u/frame_{str(int((start_frame+i)/2)).zfill(10)}.jpg",
                    f"/user/work/rg16964/epic_kitchens_data/flow/{part_id}/{video_id}/v/frame_{str(int((start_frame+i)/2)).zfill(10)}.jpg"
                ] for i in range(temporal_window_len)
            ])
        else:
            seq_frame_names.append(
                [f"/user/work/rg16964/epic_kitchens_data/rgb/{part_id}/{video_id}/frame_{str((start_frame+i)).zfill(10)}.jpg" for i in range(temporal_window_len)]
            )
    return seq_frame_names

def get_num_seq_windows(clip_length, window_len):
    num_windows = round(clip_length/window_len)
    window_total_len = (num_windows*window_len) - (2*(num_windows-1))
    if window_total_len > clip_length:
        num_windows -= 1
        window_total_len = (num_windows*window_len) - (2*(num_windows-1))
    return num_windows, window_total_len

def get_start_frame_numbers(start_frame, stop_frame, total_window_len, num_windows):
    centre_frame = int(start_frame + ((stop_frame - start_frame)/2))
    half_window_len = int(total_window_len / 2)
    frame_start_numbers = []
    for i in range(num_windows):
        frame_start_numbers.append((centre_frame-half_window_len) + (16*i) - (2*i))
    return frame_start_numbers

def load_pickle_data(file_name):
    with open(file_name, 'rb') as f:
        data = pickle.load(f)
        return data

def video_to_tensor(frames):
    return frames.transpose([3, 0, 1, 2])

def load_flow_frames(flow_filenames):
    for i, group in enumerate(flow_filenames):
        img_u = np.array(Image.open(group[0]))
        img_v = np.array(Image.open(group[1]))
        if i == 0:
            frames = np.array([[img_u, img_v]])
        else:
            frames = np.concatenate((frames, np.array([[img_u, img_v]])), axis=0)
    frames = frames.transpose([0, 2, 3, 1])
    return (((frames / 255) * 2) - 1)

def load_rgb_frames(rgb_filenames):
    for i, filename in enumerate(rgb_filenames):
        img_matrix = np.array(Image.open(filename))
        if i == 0:
            frames = np.array([img_matrix])
        else:
            frames = np.concatenate((frames, np.array([img_matrix])), axis=0)
    return (((frames / 255) * 2) - 1)

def sample_train_segment(temporal_window, start_frame, end_frame, part_id, video_id, is_flow):
    half_frame = int(temporal_window/2)
    step = 2
    seg_img_names = []
    segment_start = int(start_frame) + (step*half_frame)
    segment_end = int(end_frame) + 1 - (step*half_frame)
    # Write a comment to explain this weirdness
    if segment_start >= segment_end:
        segment_start = int(start_frame)
        segment_end = int(end_frame)
    if segment_start <= half_frame*step+1:
        segment_start <= half_frame*step+2
    centre_frame = randint(segment_start, segment_end)
    for i in range(centre_frame-(step*half_frame), centre_frame+(step*half_frame), step):
        if is_flow:
            seg_img_names.append([
                f"/user/work/rg16964/epic_kitchens_data/flow/{part_id}/{video_id}/u/frame_{str(int(i/2)).zfill(10)}.jpg",
                f"/user/work/rg16964/epic_kitchens_data/flow/{part_id}/{video_id}/v/frame_{str(int(i/2)).zfill(10)}.jpg"
            ])
        else:
            seg_img_names.append(f"/user/work/rg16964/epic_kitchens_data/rgb/{part_id}/{video_id}/frame_{str(i).zfill(10)}.jpg")
    return seg_img_names

if __name__ == "__main__":
    train_dataset = EpicKitchensDataset(labels_path="/user/work/rg16964/label_lookup/D2_train.pkl", is_flow=False)
    train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=False)
    for (labels, rgb_inputs) in train_dataloader:
        print(labels)
        print(labels.shape)
        break


# -*- coding: utf-8 -*-

import logging.config
import logging
import traceback

PLUGIN_NAME = 'SmojSubmit'

class SmojSubmitFormatter(logging.Formatter):
    def format(self, record):
        level = record.levelname[0]
        name = record.name
        if name.startswith(PLUGIN_NAME + '.'):
            name = name[len(PLUGIN_NAME) + 1 : ]
        if record.exc_info is not None:
            trace = ''.join(traceback.format_exception(*record.exc_info))
            s = PLUGIN_NAME + ': [{0} {1}:{2.lineno}] {2.msg}. Traceback:\n{3}'.format(level, name, record, trace)
        else:
            s = PLUGIN_NAME + ': [{0} {1}:{2.lineno}] {2.msg}'.format(level, name, record)
        return s

def init_logging():
    config = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'smojsubmit_fmt': {
                '()': __name__ + '.SmojSubmitFormatter'
            }
        },
        'handlers': {
            'smojsubmit_handler': {
                'level': 'WARNING',
                'formatter': 'smojsubmit_fmt',
                'class': 'logging.StreamHandler',
                'stream': 'ext://sys.stderr'
            }
        },
        'loggers': {
            'SmojSubmit': {
                'handlers': ['smojsubmit_handler'],
                'level': 'WARNING',
                'propagate': False
            }
        }
    }
    logging.config.dictConfig(config)

    logger = logging.getLogger(__name__)
    logger.info('logging initialized')


import weakref
from pyroute2.common import basestring
from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg

class Interface(dict):

    table = 'interfaces'

    def __init__(self, db, key):
        self.db = db
        self.event_map = {ifinfmsg: "load_ifinfmsg"}
        self.kspec = ('target', ) + db.indices[self.table]
        self.schema = ('target', ) + \
            tuple(db.schema[self.table].keys())
        self.names = tuple((ifinfmsg.nla2name(x) for x in self.schema))
        self.key = self.complete_key(key)
        self.changed = set()
        self.load_sql()

    def __setitem__(self, key, value):
        self.changed.add(key)
        dict.__setitem__(self, key, value)

    def snapshot(self):
        snp = type(self)(self.db, self.key)
        self.db.save_deps(self.table, id(snp), weakref.ref(snp))
        return snp

    def complete_key(self, key):
        if isinstance(key, dict):
            ret_key = key
        else:
            ret_key = {'target': 'localhost'}

        if isinstance(key, basestring):
            ret_key['IFLA_IFNAME'] = key
        elif isinstance(key, int):
            ret_key['index'] = key

        fetch = []
        for name in self.kspec:
            if name not in ret_key:
                fetch.append('f_%s' % name)

        if fetch:
            keys = []
            values = []
            for name, value in ret_key.items():
                keys.append('f_%s = ?' % name)
                values.append(value)
            spec = (self
                    .db
                    .execute('SELECT %s FROM interfaces WHERE %s' %
                             (' , '.join(fetch), ' AND '.join(keys)),
                             values)
                    .fetchone())
            for name, value in zip(fetch, spec):
                ret_key[name[2:]] = value

        return ret_key

    def update(self, data):
        for key, value in data.items():
            self.load_value(key, value)

    def load_value(self, key, value):
        if key not in self.changed:
            dict.__setitem__(self, key, value)

    def load_ifinfmsg(self, target, event):
        # TODO: partial match (object rename / restore)
        # ...

        # full match
        for name, value in self.key.items():
            if name == 'target':
                if value != target:
                    return
            elif value != (event.get_attr(name) or event.get(name)):
                return
        #
        # load the event
        for name in self.schema:
            value = event.get_attr(name) or event.get(name)
            if value is not None:
                self.load_value(ifinfmsg.nla2name(name), value)

    def load_sql(self):
        keys = []
        values = []
        for name, value in self.key.items():
            keys.append('f_%s = ?' % name)
            values.append(value)
        spec = (self
                .db
                .execute('SELECT * FROM interfaces WHERE %s' %
                         ' AND '.join(keys), values)
                .fetchone())
        self.update(dict(zip(self.names, spec)))
        return self


import sqlite3
from dataclasses import asdict

from nicegui import ui

from app.types import BattingAverage, BowlingAverage, Player, Season, SeasonRecord

def show_season(db: sqlite3.Connection, year: int) -> None:
    min_innings = 5
    min_wickets = 10
    players = Player.all(db)

    with ui.header(elevated=True).style("background-color: maroon"):
        ui.label(f"{year} Season").style("color: gold")

    with ui.left_drawer():
        with ui.column():
            ui.link("Hundreds", "/hundreds")

    with ui.row():
        season = Season.for_year(db, year)
        ui.table(rows=[asdict(season)], columns=Season.table_cols()).props("dense")
    with ui.row():
        records = [row.row_dict() for row in SeasonRecord.for_year(db, year)]
        ui.table(rows=records, columns=SeasonRecord.table_cols()).props("dense")
    with ui.row():
        with ui.card():
            averages, also_batted = BattingAverage.for_year(db, year, min_innings)
            show_batting(min_innings, players, averages, show_position=True)
            show_batting(min_innings, players, also_batted, show_position=False)
        with ui.card():
            bowl_aves, also_bowled = BowlingAverage.for_year(db, year, min_wickets)
            show_bowling(min_wickets, players, bowl_aves, show_position=True)
            show_bowling(min_wickets, players, also_bowled, show_position=False)
        with ui.card():
            ui.label("other stats")

def show_batting(min_innings, players, averages, show_position=True):
    for average in averages:
        average.name = players[average.player_id].name
    records = [asdict(row) for row in averages]
    table = ui.table(
        rows=records,
        columns=BattingAverage.table_cols(),
        title=f"Batting (min {min_innings} innings)"
        if show_position
        else "Also batted",
    ).props("dense")

    table.add_slot(
        "body-cell-name",
        r"""

                    {{props.row.name}}

                """,
    )
    if not show_position:
        table.add_slot(
            "body-cell-position",
            r""" 	""",
        )

def show_bowling(min_wickets, players, averages, show_position=True):
    for average in averages:
        average.name = players[average.player_id].name
    records = [asdict(row) for row in averages]
    with ui.table(
        rows=records,
        columns=BowlingAverage.table_cols(),
        title=f"Bowling (min {min_wickets} wickets)"
        if show_position
        else "Also bowled",
    ).props("dense") as table:
        table.add_slot(
            "body-cell-name",
            r"""

                    {{props.row.name}}

                """,
        )
        if not show_position:
            table.add_slot(
                "body-cell-position",
                r""" 	""",
            )


import discord
from discord import app_commands
import morsecode
import caesarcode

token = 'xxxxxxxxxxxxxxxxxxx'
intents = discord.Intents.default()
client = discord.Client(intents=intents)
tree = app_commands.CommandTree(client)

@client.event
async def on_ready():
    await tree.sync(guild=discord.Object(id=1109016227210350634))
    print("DisCipher up")

@tree.command(name = "morse-encode", description = "encodes from english to morse", guild=discord.Object(id=1109016227210350634))
async def encoding(interaction: discord.Interaction, input: str) -> None:
    await interaction.response.send_message(morsecode.encode(input))

@tree.command(name = "morse-decode", description = "decodes from morse  to english", guild=discord.Object(id=1109016227210350634))
async def decoding(interaction: discord.Interaction, input: str) -> None:
    await interaction.response.send_message(morsecode.decode(input))

@tree.command(name = "caesar-encode", description = "encodes from english to caesar cipher", guild=discord.Object(id=1109016227210350634))
async def encoding(interaction: discord.Interaction, shifting: int, *, input: str) -> None:
    await interaction.response.send_message(caesarcode.encode(shifting, input))

@tree.command(name = "caesar-decode", description = "decodes from caeser cipher to english", guild=discord.Object(id=1109016227210350634))
async def encoding(interaction: discord.Interaction, shifting: int, *, input: str) -> None:
    await interaction.response.send_message(caesarcode.decode(shifting, input))

@tree.command(name = "rot13-encode", description = "encodes from english to ROT13", guild=discord.Object(id=1109016227210350634))
async def encoding(interaction: discord.Interaction, input: str) -> None:
    await interaction.response.send_message(caesarcode.rot3encode(input))

@tree.command(name = "rot13-decode", description = "decodes from ROT13 to english", guild=discord.Object(id=1109016227210350634))
async def encoding(interaction: discord.Interaction, input: str) -> None:
    await interaction.response.send_message(caesarcode.rot3decode(input))

client.run(token)


import os
import inspect
import json
import sys
import platform

if platform.system() == 'Linux':
    configdir = os.path.expanduser('~') + '/.config/bassa/'
elif platform.system() == 'Windows':
    configdir = os.path.expanduser('~') + '/%app_data%/bassa/'
elif platform.system() == 'Darwin':
    configdir = os.path.expanduser('~') + '/.config/bassa/'

def check_conf_availability():
    """Checks whether the conf files have been properly set. Exits otherwise"""
    dlConf = get_conf_reader('dl.conf')
    emailConf = get_conf_reader('email.conf')

    if any(val == '' for val in dlConf.values()):
        sys.exit("Please set all the values in dl.conf")
    if any(val == '' for val in emailConf.values()):
        sys.exit("Please set all the values in email.conf")

def get_conf_reader(confFile):
    f = open(configdir + confFile)

    try:
        txt = ""
        for line in f:
            txt += line.strip()
        return json.loads(txt)
    finally:
        f.close()


#!/usr/bin/env python
"""
A complex-ish mox test example.

.. moduleauthor: Enji Cooper
.. date: April 2014
"""

import sqlite3
import unittest

import mox

from nose.tools import (
    assert_raises,
)

tmp_database = None

def setup():
    """Initialize an sqlite3 object in memory with a dummy table"""

    global tmp_database

    tmp_database = sqlite3.connect(':memory:')
    table_create_sql = \
        'CREATE TABLE ' \
        'perple_table(name STRING, age INTEGER, PRIMARY KEY(name ASC));'
    tmp_database.execute(table_create_sql)

def teardown():
    """Not strictly required, but close the db object when the test is done"""

    try:
        cursor = tmp_database.cursor()
        cursor.execute('SELECT * FROM perple_table')
        print(cursor.fetchall())
    finally:
        tmp_database.close()

class BadInsert(Exception):
    """Exception raised on failed PerpleDb.insert's"""

class RegistrationFailed(Exception):
    """Exception raised on failed PerpleManager.registers's"""

class PerpleManager(object):

    def __init__(self, db):
        self._db = db

    def register(self, perple):
        try:
            self._db.insert(perple)
        except BadInsert:
            raise RegistrationFailed

class PerpleDb(object):
    """Ermah-person class!"""

    def __init__(self, db):
        self._cursor = db.cursor()

    def insert(self, perple):
        try:
            data = (perple.name, perple.age, )
            insert_sql = 'INSERT INTO perple_table(name,age) VALUES (?,?);'

            self._cursor.execute(insert_sql, data)
            if self._cursor.rowcount == 0:
                raise Exception('The insert failed')
        except:
            raise BadInsert

class Perple(object):

    age = None
    name = None

    def __init__(self, name, age):
        self.age = age
        self.name = name

class TestPerpleManager(object):

    def setUp(self):
        self.mox = mox.Mox()
        self.dao = self.mox.CreateMock(PerpleDb)
        self.manager = PerpleManager(self.dao)
        self.perple = Perple('Kate', 42)

    def test_register_successful_insert(self):
        self.dao.insert(self.perple)
        self.mox.ReplayAll()
        self.manager.register(self.perple)
        self.mox.VerifyAll()

    def test_failed_insert(self):
        self.dao.insert(self.perple).AndRaise(BadInsert("I don't like Kate!"))
        self.mox.ReplayAll()
        assert_raises(RegistrationFailed, self.manager.register, self.perple)
        self.mox.VerifyAll()


import itertools
import re

from eth_abi.exceptions import (
    ParseError,
    ABITypeError,
)
from eth_abi.grammar import (
    parse as parse_type,
    normalize as normalize_type,
)

from func_sig_registry.utils.solidity import (
    get_arg_types,
    validate_standard_type,
)

DYNAMIC_TYPES = ['bytes', 'string']

STATIC_TYPE_ALIASES = [
    'uint',
    'int',
    'ufixed',
    'fixed',
    'byte',
]

BITS_REGEX = r'(?:{})'.format('|'.join(str(i) for i in range(8, 257, 8)))
PLACES_REGEX = r'(?:{})'.format('|'.join(str(i) for i in range(1, 81)))

STATIC_TYPES = [
    'uint{0}'.format(BITS_REGEX),
    'int{0}'.format(BITS_REGEX),
    'address',
    'bool',
    'fixed{0}x{1}'.format(BITS_REGEX, PLACES_REGEX),
    'ufixed{0}x{1}'.format(BITS_REGEX, PLACES_REGEX),
] + ['bytes{0}'.format(i) for i in range(1, 33)]

TYPE_REGEX = '|'.join((
    _type + r'(?![a-z0-9])'
    for _type
    in itertools.chain(STATIC_TYPES, STATIC_TYPE_ALIASES, DYNAMIC_TYPES)
))

NAME_REGEX = (
    r'[a-zA-Z_]'
    r'[a-zA-Z0-9_]*'
)

SUB_TYPE_REGEX = (
    r'\['
    r'[0-9]*'
    r'\]'
)

EVENT_ARGUMENT_REGEX = (
    r'(?:{type})'
    r'(?:(?:{sub_type})*)?'
    r'(?:\s+indexed)?'
    r'\s+'
    '{name}'
).format(
    type=TYPE_REGEX,
    sub_type=SUB_TYPE_REGEX,
    name=NAME_REGEX,
)

RAW_EVENT_RE = re.compile(r"""
event                             # leading "event" keyword
\s+
{name}                            # event name
\s*
\(                                # opening paren before arg list
    \s*
    (?:
        {arg}                     # first arg in arg list
        (?:
            (?:\s*,\s*{arg}\s*)*  # other args in arg list
        )?
        (?:,)?                    # optional trailing comma in arg list
    )?                            # optional arg list
    \s*
\)                                # closing paren after arg list
(?:\s*anonymous)?                 # optional anonymous
""".format(name=NAME_REGEX, arg=EVENT_ARGUMENT_REGEX), re.VERBOSE)

def extract_event_signatures(code):
    """
    Given a string of solidity code, extract all of the event declarations,
    including anonymous
    """
    matches = RAW_EVENT_RE.findall(code)
    return matches or []

EVENT_PARTS_RE = re.compile(r"""
^
\s*
(?:event\s+)?                     # optional leading "event" keyword
(?P<et_name>[a-zA-Z_][a-zA-Z0-9_]*)  # function name
\s*
\(                                   # opening paren before arg list
    (?P<arglist>.*)                  # argument list
\)                                   # closing paren after arg list
\s*
(?:anonymous)?                       # optional "anonymous" keyword
\s*
$
""", re.DOTALL | re.VERBOSE)

def extract_event_name(raw_signature):
    et_parts_match = EVENT_PARTS_RE.match(raw_signature)

    if et_parts_match is None:
        raise ValueError("Could not parse event signature")

    group_dict = et_parts_match.groupdict()
    et_name = group_dict['et_name']

    if et_name == 'event':
        raise ValueError("Bad event name")

    arglist = group_dict['arglist']

    return et_name, arglist

def validate_indexed_args(arglist: str):
    """
    Raise Exception if number of indexed arguments is greater than 3.
    Accepts only comma-delimited list of arguments
    """
    parts = arglist.split(',')
    indexed_amt = 0

    for p in parts:
        arg_parts = p.split()
        if(len(arg_parts) > 1 and arg_parts[1] == "indexed"):
            indexed_amt += 1

    if(indexed_amt > 3):
        raise ValueError("Too many indexed arguments")

def normalize_event_signature(raw_signature: str) -> str:
    """
    Return a normalized event signature from the input signature
    ``raw_signature``.  This function must return a normalized event
    signature or raise a ``ValueError``.
    """
    et_name, args = extract_event_name(raw_signature)

    # Check if number of indexed arguments is allowed
    validate_indexed_args(args)

    try:
        # Make arg list look like tuple and try to parse as normalized abi type
        args_tuple_type = parse_type(normalize_type(f'({args})'))
    except ParseError:
        # If not parseable, assume it's a solidity function signature, extract
        # types, and attempt to parse again
        arg_types = ','.join(get_arg_types(args))
        try:
            args_tuple_type = parse_type(normalize_type(f'({arg_types})'))
        except ParseError as e:
            raise ValueError('could not parse function args') from e

    # If args list parseable as abi type, do some validation and return string
    # representation
    try:
        # No custom ABI types
        validate_standard_type(args_tuple_type)
        # All basic types have expected properties
        args_tuple_type.validate()
    except ABITypeError as e:
        raise ValueError('event args contain non-standard types') from e
    else:
        return f'{et_name}{args_tuple_type.to_type_str()}'


import json
import csv

def json_csv(location_path):
    with open(location_path) as f:
        data = json.load(f)
    posts = data['posts']
    users = data['users']
    u = {}
    for i in users.keys():
        u.update({i:users[i]['username']})
    csvHeader = ['ID','Species Name','Count','Date','Time','Location','Details','Habitat Ans','ObsCount','ObsType']
    csvfile = "data3.csv"
    with open(csvfile, "a",encoding='utf-8',newline='') as fp:
        wr = csv.writer(fp, dialect='excel')
        wr.writerow(csvHeader)
    #spName,count,details = [],[],[]
    #uid,date,location,obsCount,obsType,time,habAns = "","","","","","",""
    for i in posts.keys():
        for j in posts[i].keys():
            if(j == 'spName'):
                spName = (posts[i][j].split(','))[:-1]
            elif(j == 'count'):
                count = (posts[i][j].split(','))[:-1]
            elif(j == 'details'):
                details = (posts[i][j].split(','))[:-1]
            elif(j == 'uid'):
                uid = posts[i][j]
            elif(j == 'date'):
                date = posts[i][j]
            elif(j == 'location'):
                location = posts[i][j]
            elif(j == 'obsCount'):
                obsCount = posts[i][j]
            elif(j == 'obsType'):
                obsType = posts[i][j]
            elif(j == 'time'):
                time = posts[i][j]
            elif(j == 'habAns'):
                habAns = posts[i][j]
        for k in range(len(spName)):
            csvRow = [u[uid],spName[k],count[k],date,time,location,details[k],habAns,obsCount,obsType]
            with open(csvfile, "a",encoding='utf-8',newline='') as fp:
                   wr = csv.writer(fp, dialect='excel')
                   wr.writerow(csvRow)
if __name__ == "__main__":
    location_path = 'data\dragonfly.json'
    json_csv(location_path)


from celery import task
from django.core.mail import send_mail
from .models import Order

@task
def order_created(order_id):
    order = Order.objects.get(id = order_id)
    subject = 'Order number {}'.format(order.id)
    message = 'Dear, {0}, Thank you for your purchase.' \
              'Your order number is: {1}'.format(order.name, order.id)

    mail = send_mail(subject, message, 'asdf@gmail.com', [order.email])
    return mail


from django.urls import path, include
from stock import views

app_name = 'stock'

urlpatterns = [
    path('', views.ShowStockList.as_view(), name="stock_list"),
    path('add-stock/', views.StockAddForm.as_view(), name="stock_add"),
    path('view-stock/<slug:product_id>', views.StockViewForm.as_view(), name="stock_view"),

    # AJAX URL
    path('search-product', views.SearchProduct, name="search_product"),
    path('view-product', views.ViewProduct, name="view_peoduct"),
]


import os
import logging

import jax
from mpi4py import MPI

from .api import easydist_shard, get_opt_strategy, set_device_mesh
from .sharding_interpreter import EDJaxShardingAnn
from .bridge import jax2md_bridge

__all__ = [
    "easydist_shard", "get_opt_strategy", "set_device_mesh", "EDJaxShardingAnn", "jax2md_bridge"
]

logger = logging.getLogger(__name__)

def is_jax_distributed_initialized():
    return jax._src.distributed.global_state.client is not None

def easydist_setup_jax(device, allow_tf32):
    os.environ["NVIDIA_TF32_OVERRIDE"] = "1" if allow_tf32 else "0"
    jax.config.update('jax_platforms', device)

    # setup distributed
    comm = MPI.COMM_WORLD
    size, rank = comm.Get_size(), comm.Get_rank()

    if not is_jax_distributed_initialized():

        jax.distributed.initialize(coordinator_address="localhost:19705",
                                   num_processes=size,
                                   process_id=rank,
                                   local_device_ids=rank)

        logging.info(
            f"[Rank {rank}], Global Devices: {jax.device_count()}, Local Devices: {jax.local_device_count()}"
        )


from os.path import join
import pickle
import copy
import numpy as np

import torch

class BaseRLAlgo:
    def __init__(
        self,
        actor,
        actor_optim,
        critic,
        critic_optim,
        action_range,
        n_step=4,
        gamma=0.99,
        device="cpu"
    ):
        self.actor = actor
        self.actor_optimizer = actor_optim

        self.critic = critic
        self.critic_optimizer = critic_optim

        self.n_step = n_step
        self.gamma = gamma

        self.device = device

        self.action_range = action_range
        self._action_scale = torch.tensor(
            (action_range[1] - action_range[0]) / 2.0, device=self.device)
        self._action_bias = torch.tensor(
            (action_range[1] + action_range[0]) / 2.0, device=self.device)

    def select_action(self, state, to_cpu=True):
        # use the actor network to compute the action
        raise NotImplementedError

    def train_rl(self, state, action, next_state, reward, not_done, gammas):
        # update actor and critic with the batch of transition samples
        raise NotImplementedError

    def train(self, replay_buffer, batch_size=256):
        transitions = replay_buffer.sample_transition(self.n_step, self.gamma, batch_size)
        loss_info = self.train_rl(*transitions)
        return loss_info

    def grad_norm(self, model):
        total_norm = 0
        for p in model.parameters():
            param_norm = p.grad.data.norm(2).item() if p.grad is not None else 0
            total_norm += param_norm ** 2
        total_norm = total_norm ** (1. / 2)
        return total_norm

    def save(self, dir, filename):
        self.actor.to("cpu")
        with open(join(dir, filename + "_actor"), "wb") as f:
            pickle.dump(self.actor.state_dict(), f)
        self.actor.to(self.device)

    def load(self, dir, filename):
        with open(join(dir, filename + "_actor"), "rb") as f:
            self.actor.load_state_dict(pickle.load(f))
            self.actor_target = copy.deepcopy(self.actor)

class ReplayBuffer(object):
    def __init__(self, state_dim, action_dim, max_size=int(1e6), device="cpu", safe_rl=False, reward_norm=False):
        self.max_size = max_size
        self.ptr = 0
        self.size = 0
        self.mean, self.std = 0.0, 1.0
        self.reward_norm = reward_norm

        self.safe_rl = safe_rl

        self.state = np.zeros((max_size, *state_dim))
        self.action = np.zeros((max_size, action_dim))
        self.next_state = np.zeros((max_size, *state_dim))
        self.reward = np.zeros((max_size, 1))
        self.collision_reward = np.zeros((max_size, 1))
        self.not_done = np.zeros((max_size, 1))
        self.task = np.zeros((max_size, 1))

        # Reward normalization
        self.mean = None
        self.std = None

        self.device = device

    def add(self, state, action, next_state, reward, done, task, collision_reward):
        self.state[self.ptr] = state
        self.action[self.ptr] = action
        self.next_state[self.ptr] = next_state
        self.reward[self.ptr] = reward # (reward - 0.02478) / 6.499
        self.not_done[self.ptr] = 1. - done
        self.task[self.ptr] = task
        self.collision_reward[self.ptr] = collision_reward

        self.ptr = (self.ptr + 1) % self.max_size
        self.size = min(self.size + 1, self.max_size)

        if self.ptr == 1000 and self.reward_norm:  # and self.mean is None:
            rew = self.reward[:1000]
            self.mean, self.std = rew.mean(), rew.std()
            if np.isclose(self.std, 0, 1e-2) or self.std is None:
                self.mean, self.std = 0.0, 1.0
        self.mean, self.std = 0.0, 1.0

    def sample(self, batch_size, start_idx=0):
        index = np.random.randint(start_idx, self.size, size=batch_size)

        return (
            torch.FloatTensor(self.state[index]).to(self.device),
            torch.FloatTensor(self.action[index]).to(self.device),
            torch.FloatTensor(self.next_state[index]).to(self.device),
            torch.FloatTensor(self.reward[index]).to(self.device),
            torch.FloatTensor(self.not_done[index]).to(self.device),
            torch.FloatTensor(self.task[index]).to(self.device),
            torch.FloatTensor(self.collision_reward[index]).to(self.device),
            index)

    def n_step_return(self, n_step, ind, gamma):
        reward = []
        not_done = []
        next_state = []
        gammas = []
        collision_reward = []

        for i in ind:
            n = 0
            r = 0
            c = 0
            for _ in range(n_step):
                idx = (i + n) % self.size
                assert self.mean is not None
                assert self.std is not None
                r += (self.reward[idx] - self.mean) / self.std * gamma ** n
                c += self.collision_reward[idx] * gamma ** n
                if not self.not_done[idx]:
                    break
                n = n + 1
            next_state.append(self.next_state[idx])
            not_done.append(self.not_done[idx])
            reward.append(r)
            gammas.append([gamma ** (n + 1)])
            collision_reward.append(c)

        next_state = torch.FloatTensor(np.array(next_state)).to(self.device)
        not_done = torch.FloatTensor(np.array(not_done)).to(self.device)
        reward = torch.FloatTensor(np.array(reward)).to(self.device)
        gammas = torch.FloatTensor(np.array(gammas)).to(self.device)

        collision_reward = torch.FloatTensor(
            np.array(collision_reward)
        ).to(self.device)
        return next_state, reward, not_done, gammas, collision_reward

    def sample_transition(self, n_step=4, gamma=0.99, batch_size=256):
        # Sample replay buffer ("task" for multi-task learning)
        state, action, next_state, reward, not_done, task, collision_reward, index = self.sample(batch_size)
        next_state, reward, not_done, gammas, collision_reward = self.n_step_return(n_step, index, gamma)
        return state, action, next_state, reward, not_done, gammas, collision_reward


import io
from os.path import abspath, join, dirname

from bddrest.authoring import when, status, response, Update
from sqlalchemy_media import StoreManager

from jaguar.models import Member, Message, Room
from jaguar.tests.helpers import AutoDocumentationBDDTest, cas_mockup_server

THIS_DIR = abspath(join(dirname(__file__)))
IMAGE_PATH = join(THIS_DIR, 'stuff', '150x150.png')

class TestGetMember(AutoDocumentationBDDTest):

    @classmethod
    def mockup(cls):
        session = cls.create_session()
        with StoreManager(session):
            with open(IMAGE_PATH, 'rb') as f:
                cls.message1 = Message(
                    body='This is message 1',
                    mimetype='image/png',
                    attachment=io.BytesIO(f.read()),
                )
                cls.message2 = Message(
                    body='This is message 2',
                    mimetype='text/plain'
                )
                cls.message3 = Message(
                    body='This is message 3',
                    mimetype='text/plain'
                )
                user1 = Member(
                    email='user1@example.com',
                    title='user1',
                    access_token='access token1',
                    reference_id=2,
                    messages=[cls.message1]
                )
                user2 = Member(
                    email='user2@example.com',
                    title='user2',
                    access_token='access token2',
                    reference_id=3,
                    messages=[cls.message3, cls.message2]
                )
                room1 = Room(
                    title='room1',
                    members=[user1, user2],
                    messages=[cls.message1, cls.message3]
                )
                room2 = Room(
                    title='room2',
                    members=[user2],
                    messages=[cls.message2]
                )
                session.add(user1)
                session.commit()

    def test_get_user_by_id(self):
        self.login('user1@example.com')

        with cas_mockup_server(), self.given(
            f'Get a message by id',
            f'/apiv1/messages/id:{self.message1.id}',
            f'GET',
        ):
            assert status == 200
            assert response.json['body'] == 'This is message 1'

            when(
                'Get The message sent by another user in the same room',
                url_parameters=Update(id=f'{self.message3.id}')
            )
            assert status == 200
            assert response.json['body'] == 'This is message 3'

            when('Invalid message id', url_parameters=Update(id='message1'))
            assert status == 404

            when('Message not found', url_parameters=Update(id=0))
            assert status == 404

            when('Try to pass unauthorize request', authorization=None)
            assert status == 401


from flask import Flask, render_template, flash, request, redirect, url_for
from flask_wtf.file import FileField, FileRequired
from werkzeug.utils import secure_filename
from werkzeug.datastructures import CombinedMultiDict
from wtforms import Form, TextField, SelectField, TextAreaField, validators, StringField,\
     SubmitField, RadioField
import decode_text
import logging
from local_defs import mydefs
import os
import time
from logging.handlers import RotatingFileHandler
# App config.
baseurl2 = 'doc.darkphoenix.net:5555'
DEBUG = True
app = Flask(__name__)
app.config.from_object(__name__)
app.config['SECRET_KEY'] = '7d441f27d441f27567d441f2b6176a'
app.config['UPLOAD_FOLDER'] = mydefs.UPLOAD_FOLDER

class text_decoder(Form):
    mapFile_1 = SelectField(u'Code Type', choices=mydefs.IMAGE_TYPE)
    mapFile_2 = SelectField(u'Version', choices=mydefs.RELEASE)
    mapFile_3 = SelectField(u'Version', choices=mydefs.PATCH_LETTER)
    mapFile_4 = SelectField(u'Version', choices=mydefs.PATCH_SPECIAL)
    # mapfile = TextField('Mapfile:', validators=[validators.required()])
    file = FileField('File:')

class text_decoder_nofile(Form):
    mapFile_1 = SelectField(u'Code Type', choices=mydefs.IMAGE_TYPE)
    mapFile_2 = SelectField(u'Version', choices=mydefs.RELEASE)
    mapFile_3 = SelectField(u'Version', choices=mydefs.PATCH_LETTER)
    mapFile_4 = SelectField(u'Version', choices=mydefs.PATCH_SPECIAL)
    stack_output = TextAreaField('stack_output', validators=[validators.Required()])

class gen_script(Form):
    # script_lan = SelectField(u'Script Language', choices=mydefs.SCRIPT_LANGUAGES)
    target_ip = TextField('Target IP', validators=[validators.Required()])
    access_type = RadioField('Access Type:', choices=[('ssh', 'ssh'),\
                        ('telnet','telnet')])
    authen_t = RadioField('Authentication:',choices=[\
    ('implicit','implicit'),('UserEnable','UserEnable'),\
    ('NoneEnable','NoneEnable'),('None','None')])
    command_trig = TextField('Trigger Command: ', validators=[validators.Required()])
    trig_type = RadioField('Trigger Type: ', choices=[\
    ('Text Pattern','Text Pattern'),('Conditional','Conditional')])
    trig_value = TextField('Trigger Value:')

@app.route("/", methods=['GET', 'POST'])
def base_page():
    return render_template('base.html')

@app.route("/text_decode", methods=['GET', 'POST'])
def text_decode():
    form = text_decoder(request.form)
    # print(form.errors)
    if request.method == 'POST':
        print('Got here 1')
        f = request.files['file']
        print('f is ', f)
        filename = secure_filename(f.filename)
        print('Got here 2')
        fs = f.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        stackfile = mydefs.UPLOAD_FOLDER + '/' + filename
        mapfile = mydefs.MAP_ROOT + request.form['mapFile_2'] +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        map_timestamp = time.ctime(os.path.getmtime(mapfile))
        temp_mappath =  request.form['mapFile_2'] +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        print('Got here 3')
        print('Got here 3.5')
        print('Form Validation ', form.validate())
        print('Form errors ', form.errors)
        print('filename is ', filename)
        print('fs is', fs)
        print('stackfile is', stackfile)
        print('mapfile is', mapfile)
        if form.validate():
            flash('Okay now will decode')
            print('Got here 4')
            d_result = decode_text.decode_t(mapfile, stackfile, '1',\
                                            map_timestamp, temp_mappath)
            print("Result is ", d_result)
            return render_template('results.html', result=d_result)
        else:
            flash('Error: All the form fields are required. ')
            return redirect('/text_decode')
    return render_template('t_decode.html', form=form)

@app.route("/text_decode_nofile", methods=['GET', 'POST'])
def text_decode_nofile():
    form = text_decoder_nofile(request.form)

    print(form.errors)
    if request.method == 'POST':
        print('Got here 1')

        print('Got here 2')
        stack_output = request.form['stack_output']
        if request.form['mapFile_3'] is 'UNDEFINED':
            MAP_LETTER = request.form['mapFile_4']
        else:
            MAP_LETTER = request.form['mapFile_3']

        mapfile = mydefs.MAP_ROOT + request.form['mapFile_2'] + MAP_LETTER +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        map_timestamp = time.ctime(os.path.getmtime(mapfile))
        temp_mappath =  request.form['mapFile_2'] +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        print('timestamp is ',map_timestamp)
        print('mypath is', temp_mappath)
        print('Got here 3')
        print('Got here 3.5')
        print('Form Validation ', form.validate())
        print('Form errors ', form.errors)
        print('mapfile is', mapfile)
        if form.validate():
            flash('Okay now will decode')
            print('Got here 4')
            d_result = decode_text.decode_t(mapfile, stack_output, 0,\
                                            map_timestamp, temp_mappath)
            print("Result is ", d_result)
            return render_template('results.html', result=d_result)
        else:
            flash('Error: All the form fields are required. ')
            return redirect('/text_decode_nofile')
    return render_template('t_decode_nofile.html', form=form)

@app.route("/icx_decoder", methods=['GET', 'POST'])
def stack_decodenew():
    form = text_decoder(CombinedMultiDict((request.form, request.files)))
    # print(form.errors)
    if request.method == 'POST':
        print('Got here 1')
        f = request.files['file']
        print('f is ', f)
        filename = secure_filename(f.filename)
        print('Got here 2')
        fs = f.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        stackfile = mydefs.UPLOAD_FOLDER + '/' + filename
        print('Got here 3')
        if request.form['mapFile_3'] is 'UNDEFINED':
            MAP_LETTER = request.form['mapFile_4']
        else:
            MAP_LETTER = request.form['mapFile_3']

        mapfile = mydefs.MAP_ROOT + request.form['mapFile_2'] + MAP_LETTER +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        if not os.path.isfile(mapfile):
            flash('Sorry this code image is not supported yet!')
            return redirect('/icx_decoder')

        map_timestamp = time.ctime(os.path.getmtime(mapfile))
        temp_mappath =  request.form['mapFile_2'] +\
                   '/LATEST_BUILD/dist' + request.form['mapFile_1'] +\
                   'prod.map'
        print('Got here 3.5')
        print('Form Validation ', form.validate())
        print('Form errors ', form.errors)
        print('filename is ', filename)
        print('fs is', fs)
        print('stackfile is', stackfile)
        print('mapfile is', mapfile)
        if form.validate():
            flash('Okay now will decode')
            print('Got here 4')
            d_result = decode_text.decode_corefile(mapfile, stackfile,\
                                                   map_timestamp, temp_mappath)
            print("Result is ", d_result)
            return render_template('results.html', result=d_result)
        else:
            flash('Error: All the form fields are required. ')
            return redirect('/icx_decoder')
    return render_template('stack_decodenew.html', form=form)

@app.route('/result', methods=['GET', 'POST'])
def result():
    if request.method == 'POST':
        result = request.form
        return render_template('results.html', result=d_result)
    else:
        return redirect('/')

if __name__ == "__main__":
    handler = RotatingFileHandler('web.log', maxBytes=10000, backupCount=1)
    handler.setLevel(logging.INFO)
    app.logger.addHandler(handler)
    app.run(host='0.0.0.0', port=5555)


#!/usr/bin/env python

from bucoffea.helpers.dataset import extract_year
from bucoffea.processor.executor import run_uproot_job_nanoaod
from bucoffea.helpers.cutflow import print_cutflow
from coffea.util import save
import coffea.processor as processor
import argparse

def parse_commandline():

    parser = argparse.ArgumentParser()
    parser.add_argument('processor', type=str, help='The processor to be run. (monojet or vbfhinv)')
    args = parser.parse_args()

    return args

def main():

    fileset = {
        "VBF_HToInvisible_M125_pow_pythia8_2017" : [
            "root://cmsxrootd.fnal.gov//store/user/aandreas/nanopost/03Sep20v7/VBF_HToInvisible_M125_13TeV_TuneCP5_powheg_pythia8/VBF_HToInvisible_M125_pow_pythia8_2017/200925_184136/0000/tree_1.root"
        ]
    }

    years = list(set(map(extract_year, fileset.keys())))
    assert(len(years)==1)

    args = parse_commandline()
    processor_class = args.processor

    if processor_class == 'monojet':
        from bucoffea.monojet import monojetProcessor
        processorInstance = monojetProcessor()
    elif processor_class == 'vbfhinv':
        from bucoffea.vbfhinv import vbfhinvProcessor
        processorInstance = vbfhinvProcessor()
    elif processor_class == 'lhe':
        from bucoffea.gen.lheVProcessor import lheVProcessor
        processorInstance = lheVProcessor()
    elif args.processor == 'purity':
        from bucoffea.photon_purity import photonPurityProcessor
        processorInstance = photonPurityProcessor()
    elif args.processor == 'sumw':
        from bucoffea.gen import mcSumwProcessor
        processorInstance = mcSumwProcessor()

    for dataset, filelist in fileset.items():
        newlist = []
        for file in filelist:
            if file.startswith("/store/"):
                newlist.append("root://cms-xrd-global.cern.ch//" + file)
            else: newlist.append(file)
        fileset[dataset] = newlist

    for dataset, filelist in fileset.items():
        tmp = {dataset:filelist}
        output = run_uproot_job_nanoaod(tmp,
                                    treename='Runs' if args.processor=='sumw' else 'Events',
                                    processor_instance=processorInstance,
                                    executor=processor.futures_executor,
                                    executor_args={'workers': 4, 'flatten': True},
                                    chunksize=500000,
                                    )
        save(output, f"{processor_class}_{dataset}.coffea")
        # Debugging / testing output
        # debug_plot_output(output)
        print_cutflow(output, outfile=f'{processor_class}_cutflow_{dataset}.txt')

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from flask import request, jsonify
from . import api
from application import db
from application.models.menu import Menu
from application.models.mixin import SerializableModelMixin
from application.lib.rest.auth_helper import required_token, required_admin

# create - name, category
@api.route('/menus', methods=['POST'])
@required_token
def create_menus():
    request_params = request.get_json()
    name = request_params.get('name')
    category = request_params.get('category')

    # name이 제대로 입력이 안된 경우
    if name is None:
        return jsonify(
            userMessage="메뉴명을 기입해주세요."
        ), 400

    # category가 제대로 입력이 안된 경우
    if category is None:
        return jsonify(
            userMessage="카테고리 정보를 기입해주세요."
        ), 400

    # 이미 등록되어있는지 확인
    q = db.session.query(Menu).filter(Menu.name == name, Menu.category == category)
    if q.count() > 0:
        return jsonify(
            userMessage="이미 등록되어있는 메뉴입니다."
        ), 409

    try:
        for key in request_params.keys():
            request_params[SerializableModelMixin.to_snakecase(key)] = request_params.pop(key)

        menu = Menu(**request_params)
        db.session.add(menu)
        db.session.commit()

        return jsonify(
            data=menu.serialize()
        ), 200
    except:
        return jsonify(
            userMessage="오류가 발생했습니다. 관리자에게 문의해주세요."
        ), 403

# read
@api.route('/menus/<int:menu_id>', methods=['GET'])
@required_token
def get_menu_by_id(menu_id):
    try:
        menu = db.session.query(Menu).get(menu_id)
        return jsonify(
            data=menu.serialize()
        ), 200

    except:
        return jsonify(
            userMessage="해당 메뉴를 찾을 수 없습니다."
        ), 404

# read
@api.route('/menus', methods=['GET'])
@required_token
def get_menus():
    q = db.session.query(Menu)

    name = request.args.get('name')
    category = request.args.get('category')

    if name is not None:
        q = q.filter(Menu.name == name)

    if category is not None:
        q = q.filter(Menu.category == category)

    return jsonify(
        data=map(lambda obj: obj.serialize(), q)
    ), 200

# update
@api.route('/menus/<int:menu_id>', methods=['PUT'])
@required_token
def update_menu(menu_id):
    menu = db.session.query(Menu).filter(Menu.id == menu_id).one()

    if menu is None:
        return jsonify(
            userMessage="메뉴를 찾을 수 없습니다."
        ), 404

    request_params = request.get_json()
    if request_params.get('name'):
        name = request_params.get('name')
    else:
        name = menu.name

    if request_params.get('category'):
        category = request_params.get('category')
    else:
        category = menu.category

    q = db.session.query(Menu).filter(Menu.name == name, Menu.category == category)
    if q.count() > 0:
        if q.one() == menu:
            pass
        else:
            return jsonify(
                    userMessage="기존에 동일한 메뉴가 있습니다."
                )
    else:
        menu.name = name
        menu.category = category
        db.session.commit()

    return get_menu_by_id(menu_id)

# delete 필요없을듯하당
@api.route('/menus/<int:menu_id>', methods=['DELETE'])
@required_admin
def delete_menu(menu_id):
    try:
        menu = db.session.query(Menu).get(menu_id)

        try:
            db.session.delete(menu)
            db.session.commit()
            return jsonify(
                userMessage="삭제가 완료되었습니다."
            ), 200
        except:
            return jsonify(
                userMessage="삭제 실패."
            ), 403

    except:
        return jsonify(
            userMessage="메뉴를 찾을 수 없습니다."
        ), 404


"""
Draw fields
===========
"""
# %%
# The objective here is to manipulate a multivariate stochastic process :math:`X: \Omega \times \mathcal{D} \rightarrow \mathbb{R}^d`,
# where :math:`\mathcal{D} \in \mathbb{R}^n` is discretized on the mesh :math:`\mathcal{M}`
# and exhibit some of the services exposed by the *Process* objects:
#
# - ask for the dimension, with the method getOutputDimension
# - ask for the mesh, with the method getMesh
# - ask for the mesh as regular 1-d mesh, with the getTimeGrid method
# - ask for a realization, with the method the getRealization method
# - ask for a continuous realization, with the getContinuousRealization method
# - ask for a sample of realizations, with the getSample method
# - ask for the normality of the process with the isNormal method
# - ask for the stationarity of the process with the isStationary method
#

# %%
import openturns as ot
import openturns.viewer as viewer
from matplotlib import pylab as plt

ot.Log.Show(ot.Log.NONE)

# %%
# We create a mesh -a time grid- which is a RegularGrid :
tMin = 0.0
timeStep = 0.1
n = 100
time_grid = ot.RegularGrid(tMin, timeStep, n)
time_grid.setName("time")

# %%
# We create a Normal process :math:`X_t = (X_t^0, X_t^1, X_t^2)` in dimension 3 with an exponential covariance model.
# We define the amplitude and the scale of the `ExponentialModel`
scale = [4.0]
amplitude = [1.0, 2.0, 3.0]

# %%
# We define a spatial correlation :
spatialCorrelation = ot.CorrelationMatrix(3)
spatialCorrelation[0, 1] = 0.8
spatialCorrelation[0, 2] = 0.6
spatialCorrelation[1, 2] = 0.1

# %%
# The covariance model is now created with :
myCovarianceModel = ot.ExponentialModel(scale, amplitude, spatialCorrelation)

# %%
# Eventually, the process is  built with :
process = ot.GaussianProcess(myCovarianceModel, time_grid)

# %%
# The dimension d of the process may be retrieved by
dim = process.getOutputDimension()
print("Dimension : %d" % dim)

# %%
# The underlying mesh of the process is obtained with the `getMesh` method :
mesh = process.getMesh()

# %%
# We have access to peculiar data of the mesh such as the corners :
minMesh = mesh.getVertices().getMin()[0]
maxMesh = mesh.getVertices().getMax()[0]

# %%
# We draw it :
graph = mesh.draw()
graph.setTitle("Time grid (mesh)")
graph.setXTitle("t")
graph.setYTitle("")
view = viewer.View(graph)

# %%
# We can get the time grid of the process when the mesh can be interpreted as a regular time grid :
print(process.getTimeGrid())

# %%
# A typical feature of a process is the generation of a realization of itself :
realization = process.getRealization()

# %%
# Here it is a sample of size :math:`100 \times 4` (100 time steps, 3 spatial cooordinates and the time variable).
# We are able to draw its marginals, for instance the first (index 0) one :math:`X_t^0`, against the time with no interpolation:
interpolate = False
graph = realization.drawMarginal(0, interpolate)
graph.setTitle("First marginal of a realization of the process")
graph.setXTitle("t")
graph.setYTitle(r"$X_t^0$")
view = viewer.View(graph)

# %%
# The same graph, but with interpolated values (default behaviour) :
graph = realization.drawMarginal(0)
graph.setTitle("First marginal of a realization of the process")
graph.setXTitle("t")
graph.setYTitle(r"$X_t^0$")
view = viewer.View(graph)

# %%
# We can build a function representing the process using P1-Lagrange interpolation (when not defined from a functional model).
continuousRealization = process.getContinuousRealization()

# %%
# Once again we draw its first marginal :
marginal0 = continuousRealization.getMarginal(0)
graph = marginal0.draw(minMesh, maxMesh)
graph.setTitle("First marginal of a P1-Lagrange continuous realization of the process")
graph.setXTitle("t")
view = viewer.View(graph)

# %%
# Please note that the `marginal0` object is a function. Consequently we can
# evaluate it at any point of the domain, say :math:`t_0=3.5678`
t0 = 3.5678
print(t0, marginal0([t0]))

# %%
# Several realizations of the process may be determined at once :
number = 10
fieldSample = process.getSample(number)

# %%
# Let us draw them the first marginal)
graph = fieldSample.drawMarginal(0, False)
graph.setTitle("First marginal of 10 realizations of the process")
graph.setXTitle("t")
graph.setYTitle(r"$X_t^0$")
view = viewer.View(graph)

# %%
# Same graph, but with interpolated values :
graph = fieldSample.drawMarginal(0)
graph.setTitle("First marginal of 10 realizations of the process")
graph.setXTitle("t")
graph.setYTitle(r"$X_t^0$")
view = viewer.View(graph)

# %%
# Miscellanies
# ------------
#
# We can extract any marginal of the process with the `getMarginal` method.
# Beware the numerotation begins at 0 ! It may be not implemented yet for
# some processes.
# The extracted marginal is a 1D gaussian process :
print(process.getMarginal([1]))

# %%
# If we extract simultaneously two indices we build a 2D gaussian process :
print(process.getMarginal([0, 2]))

# %%
# We can check whether the process is normal or not :
print("Is normal ? ", process.isNormal())

# %%
# and the stationarity as well :
print("Is stationary ? ", process.isStationary())

plt.show()


#!/usr/bin/env python3

from argparse import ArgumentParser

import pandas as pd

import risk_stratification
import extract_features

def annotate_with_diagnoses_used(risk_df, diags_df, cutoff, codes):
    # dates within their test date
    pruned_diags_df = diags_df[diags_df.date >= cutoff]
    pruned_diags_df["UID"] = pruned_diags_df["UID"].astype(str)
    # now only diagnoses we considered when modeling
    pruned_diags_df = pruned_diags_df[pruned_diags_df["diag_cd"].isin(codes)]
    # add what outcome we have for each
    pruned_diags_df = pd.merge(pruned_diags_df, risk_df, how="inner", on="UID")
    return pruned_diags_df

def annotate_with_model_weights_info(diags_df, model_weights_df):
    # sort in descending order
    model_weights_df = model_weights_df.sort_values("odds_ratio",
                                                    ascending=False)
    model_weights_df["feature"] = model_weights_df["feature"].astype(str)
    sorted_codes = model_weights_df["feature"].astype(str).values.tolist()

    # code position: position of diagnosis code in codes
    # sorted by model weights (i.e. lower index -> more highly associated
    # with PDAC diagnosis in our model)
    sorted_codes_map = {code: ix for ix, code in enumerate(sorted_codes)}
    diags_df["code_position"] = diags_df["diag_cd"].map(
        lambda c: sorted_codes_map[c])

    # code description and weight
    diags_df = pd.merge(
        diags_df,
        model_weights_df[[
            "feature",
            "feature_desc",
            "weight",
        ]],
        how="left",
        left_on="diag_cd",
        right_on="feature",
    )
    return diags_df

def get_args():
    parser = ArgumentParser(
        description=
        "Take test-data risk group predictions and add observed diagnosis info"
    )
    parser.add_argument(
        "--risk_train_path",
        type=str,
        help="Risk pROC data (train)",
    )
    parser.add_argument(
        "--risk_test_path",
        type=str,
        help="Risk pROC data (test) -- will add diagnosis info here",
    )
    parser.add_argument(
        "--risk_percentiles",
        type=float,
        nargs="+",
        help="Risk group percentiles",
    )
    parser.add_argument(
        "--risk_group_names",
        type=str,
        nargs="+",
        help="Risk group names (aligned with --risk_percentiles)",
    )
    parser.add_argument(
        "--model_name",
        type=str,
        help="Model name to filter test risk data",
    )
    parser.add_argument(
        "--cutoff",
        type=int,
        help="Cutoff to filter test risk data",
    )
    parser.add_argument(
        "--diagnoses",
        type=str,
        help="Path to .feather diagnoses data",
    )
    parser.add_argument(
        "--model_weights",
        type=str,
        help="Path to .csv features (diagnoses) used by model and weights",
    )
    parser.add_argument(
        "--output",
        type=str,
        help="Path to output .csv",
    )
    return parser.parse_args()

def main():
    args = get_args()
    risk_df = risk_stratification.load_and_assign_risk_groups(
        args.risk_train_path,
        args.risk_test_path,
        args.risk_percentiles,
        args.risk_group_names,
        args.model_name,
    )
    risk_df = risk_df[risk_df["cutoff"] == args.cutoff]
    risk_df = risk_df.drop(columns=["percentile_75", "percentile_99"])
    high_risk_df = risk_df[risk_df["risk_group"] == "High"].copy()

    df_diags = pd.read_feather(args.diagnoses)

    model_weights_df = pd.read_csv(args.model_weights)
    model_weights_df["feature"] = model_weights_df["feature"].astype(str)

    high_risk_df = annotate_with_diagnoses_used(
        high_risk_df,
        df_diags,
        args.cutoff,
        model_weights_df["feature"].values.tolist(),
    )
    high_risk_df = annotate_with_model_weights_info(
        high_risk_df,
        model_weights_df,
    )
    # for each patient/code entry take single entry
    high_risk_df = high_risk_df.groupby(["UID", "diag_cd"]).head(1)
    # have highest confidence predictions first
    high_risk_df = high_risk_df.sort_values(
        ["yprob", "UID", "weight"],
        ascending=[False, True, False],
    )
    # clean up column names
    high_risk_df = high_risk_df.drop(columns=[
        "feature",
        "Code",
        "cancer",
        "yprob",
        "date",
    ])
    clean_col_names = {
        "diag_cd": "Diagnosis_code",
        "model": "Model",
        "cutoff": "Cutoff",
        "ytrue": "Cancer",
        "odds": "Risk_score",
        "risk_group": "Risk_group",
        "code_position": "Diagnosis_code_position",
        "feature_desc": "Diagnosis_code_description",
        "weight": "Diagnosis_code_model_weight",
    }
    high_risk_df = high_risk_df.rename(columns=clean_col_names)
    high_risk_df.to_csv(args.output, index=False)

if __name__ == "__main__":
    try:
        main()
    except Exception as err:
        import pdb
        pdb.post_mortem()


from PyQt4 import QtGui
import pyqtgraph as pg
import numpy
from math import isinf
import os.path
import csv

class FiberPlot(QtGui.QFrame):

    """

    """

    def __init__(self, parent=None):
        super().__init__(parent)

        self.dataFile = ""
        self.fiberData = None
        toolbar = self.initToolbar()

        self.plot = pg.PlotWidget()
        self.plot.getPlotItem().sigXRangeChanged.connect(self.updateXRange)
        self.plot.scene().sigMouseClicked.connect(self.mouseClickEvent)
        self.dataCurve = self.plot.plot()
        self.curve = self.plot.plot()
        layout = QtGui.QVBoxLayout()
        layout.addWidget(toolbar)
        layout.addWidget(self.plot)
        self.setLayout(layout)

    def initToolbar(self):
        toolbar = QtGui.QToolBar()

        self.foregroundColorButton = pg.ColorButton(color=(255, 255, 255))
        self.foregroundColorButton.sigColorChanged.connect(self.updateCurve)
        self.foregroundColorButton.setToolTip(self.tr("Line color"))
        toolbar.addWidget(self.foregroundColorButton)

        self.backgroundColorButton = pg.ColorButton(color=(0, 0, 0))
        self.backgroundColorButton.sigColorChanged.connect(self.updateBgColor)
        self.backgroundColorButton.setToolTip(self.tr("Background color"))
        toolbar.addWidget(self.backgroundColorButton)

        self.lineWidthSpinBox = QtGui.QDoubleSpinBox()
        self.lineWidthSpinBox.setRange(0, 10)
        self.lineWidthSpinBox.setValue(1)
        self.lineWidthSpinBox.setDecimals(1)
        self.lineWidthSpinBox.setSingleStep(1)
        self.lineWidthSpinBox.valueChanged.connect(self.updateCurve)
        self.lineWidthSpinBox.setToolTip(self.tr("Line width"))
        toolbar.addWidget(self.lineWidthSpinBox)

        toolbar.addSeparator()

        self.selectDataAction = toolbar.addAction(
            QtGui.QIcon.fromTheme('x-office-spreadsheet'),
            self.tr("Import data"),
            self.importData)

        self.dataFgColorButton = pg.ColorButton(color=(0, 255, 0))
        self.dataFgColorButton.sigColorChanged.connect(self.updateDataCurve)
        self.dataFgColorButton.setToolTip(self.tr("Imported data color"))
        self.dataFgColorButton.setEnabled(False)
        toolbar.addWidget(self.dataFgColorButton)

        self.dataLwSpinBox = QtGui.QDoubleSpinBox()
        self.dataLwSpinBox.setRange(0, 10)
        self.dataLwSpinBox.setValue(2)
        self.dataLwSpinBox.setDecimals(1)
        self.dataLwSpinBox.setSingleStep(1)
        self.dataLwSpinBox.valueChanged.connect(self.updateDataCurve)
        self.dataLwSpinBox.setToolTip(self.tr("Imported data line width"))
        self.dataLwSpinBox.setEnabled(False)
        toolbar.addWidget(self.dataLwSpinBox)

        self.deleteDataAction = toolbar.addAction(
            QtGui.QIcon.fromTheme('edit-delete'),
            self.tr("Delete data"),
            self.deleteData)
        self.deleteDataAction.setEnabled(False)

        return toolbar

    def autoScale(self):
        r = self.fiber.innerRadius(-1) * 1.5e6
        viewBox = self.plot.getPlotItem().getViewBox()
        viewBox.setXRange(-r, r, 0)
        nmin = self.fiber.minIndex(-1, self.wl)
        nmax = max(layer.maxIndex(self.wl) for layer in self.fiber.layers)
        viewBox.setYRange(nmin, nmax)
        self.updateXRange(viewBox)

    def updateInfo(self, fiber, wl):
        self.fiber = fiber
        self.wl = wl
        self.autoScale()

    def updateXRange(self, viewbox):
        xmin, xmax = viewbox.viewRange()[0]
        xmin *= 1e-6
        xmax *= 1e-6

        R = numpy.empty(0)
        N = numpy.empty(0)

        if xmax > 0:
            for i, layer in enumerate(self.fiber.layers):
                inr = self.fiber.innerRadius(i)
                out = self.fiber.outerRadius(i)
                if inr > xmax and not isinf(out):
                    break
                if out < xmin:
                    continue
                if inr < xmin:
                    inr = xmin
                if out > xmax:
                    out = xmax
                rr = numpy.linspace(inr, out)
                nn = numpy.fromiter((layer.index(r, self.wl) for r in rr),
                                    dtype=float, count=rr.size)
                R = numpy.concatenate((R, rr))
                N = numpy.concatenate((N, nn))
        if xmin < 0:
            for i, layer in enumerate(self.fiber.layers):
                inr = -self.fiber.innerRadius(i)
                out = -self.fiber.outerRadius(i)
                if inr < xmin and not isinf(out):
                    break
                if out > xmax:
                    continue
                if inr > xmax:
                    inr = xmax
                if out < xmin:
                    out = xmin
                rr = numpy.linspace(out, inr)
                nn = numpy.fromiter((layer.index(r, self.wl) for r in rr),
                                    dtype=float, count=rr.size)
                R = numpy.concatenate((rr, R))
                N = numpy.concatenate((nn, N))

        self.curve.setData(R * 1e6, N)

    def mouseClickEvent(self, event):
        if event.double():
            self.autoScale()

    def updateCurve(self, *args, **kwargs):
        self.curve.setPen(color=self.foregroundColorButton.color(),
                          width=self.lineWidthSpinBox.value())

    def updateDataCurve(self, *args, **kwargs):
        self.dataCurve.setPen(color=self.dataFgColorButton.color(),
                              width=self.dataLwSpinBox.value())

    def updateBgColor(self):
        self.plot.setBackground(self.backgroundColorButton.color())

    def importData(self):
        lastDir = os.path.dirname(self.dataFile)

        filename, selectedFilter = QtGui.QFileDialog.getOpenFileName(
            self,
            self.tr("Select data file"),
            dir=lastDir,
            filter=self.tr("Comma Separated Values (*.csv);;"
                           "Tab Separated Values (*.tab)"))
        if not filename:
            return

        self.dataFile = filename
        dialect = 'excel-tab' if "(*.tab)" in selectedFilter else 'excel'

        data = []
        with open(filename, newline='') as csvfile:
            reader = csv.reader(csvfile, dialect=dialect)
            for row in reader:
                data.append([float(x.replace(',', '.')) for x in row])
        self.fiberData = numpy.array(data).T

        self.dataCurve.setData(self.fiberData[0], self.fiberData[1])
        self.dataFgColorButton.setEnabled(True)
        self.dataLwSpinBox.setEnabled(True)
        self.deleteDataAction.setEnabled(True)
        self.updateDataCurve()

    def deleteData(self):
        ret = QtGui.QMessageBox.warning(
            self,
            self.tr("Remove fiber data"),
            self.tr("Are you sure you want to remove imported fiber data?"),
            QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
            QtGui.QMessageBox.Yes)

        if ret == QtGui.QMessageBox.Yes:
            self.fiberData = None
            self.dataCurve.setData([], [])
            self.dataFgColorButton.setEnabled(False)
            self.dataLwSpinBox.setEnabled(False)
            self.deleteDataAction.setEnabled(False)


from os import replace
from gspread.client import Client
import jsonpickle
from karim import LOCALHOST
from gspread.models import Spreadsheet, Worksheet
from oauth2client.service_account import ServiceAccountCredentials
import gspread
import os, re
import json
from datetime import datetime
from karim.secrets import secrets

def auth():
    creds_string = secrets.get_var('GSPREAD_CREDS')
    if creds_string == None:
        # use creds to create a client to interact with the Google Drive API
        scope = [
            'https://spreadsheets.google.com/feeds',
            'https://www.googleapis.com/auth/drive',
            'https://www.googleapis.com/auth/drive.file',
            'https://www.googleapis.com/auth/drive']
        # CREDENTIALS HAVE NOT BEEN INITIALIZED BEFORE
        client_secret = os.environ.get('GCLIENT_SECRET')
        if LOCALHOST:
            # CODE RUNNING LOCALLY
            print('DATABASE: Resorted to local JSON file')
            with open('karim/secrets/client_secret.json') as json_file:
                client_secret_dict = json.load(json_file)
        else:
            # CODE RUNNING ON SERVER
            client_secret_dict = json.loads(client_secret)

        creds = ServiceAccountCredentials.from_json_keyfile_dict(
            client_secret_dict, scope)
        creds_string = jsonpickle.encode(creds)
        secrets.set_var('GSPREAD_CREDS', creds_string)
    creds = jsonpickle.decode(creds_string)
    client = gspread.authorize(creds)

    # IF NO SPREADSHEET ENV VARIABLE HAS BEEN SET, SET UP NEW SPREADSHEET
    if secrets.get_var('SPREADSHEET') == None:
        spreadsheet = set_sheet(client)
        return spreadsheet
    else:
        SPREADSHEET = secrets.get_var('SPREADSHEET')
        spreadsheet = client.open_by_key(SPREADSHEET)
        return spreadsheet

def log(timestamp:datetime, user_id:int or str, action:str):
    spreadsheet = auth()
    logs = spreadsheet.get_worksheet(2)
    logs.append_row([str(timestamp), user_id, action])

def add_subscriber(id:int or str):
    """
    Add subscriber's id to the GSheet database

    :param id: Telegram id of the user
    :type id: intorstr
    """
    try:
        spreadsheet = auth()
        subscribers = spreadsheet.get_worksheet(0)
        subscribers.append_row([id])
        # LOG
        log(datetime.utcnow(), id, 'SUBSCRIBE')
        return True
    except:
        return False

def remove_subscriber(id:int or str):
    """
    Attempt to remove a subscriber from the newsletter's database

    :param id: Telegram id of the user
    :type id: intorstr
    :return: True if subscriber has been deleted successfully, False is an error occurs
    :rtype: boolean
    """
    spreadsheet = auth()
    subscribers = spreadsheet.get_worksheet(0)
    row = find_by_username(username=str(id), sheet=subscribers)[0]
    subscribers.delete_row(row)
    # LOG
    log(datetime.utcnow(),id, 'UNSUBSCRIBE')
    return True

def is_subscriber(id:int or str):
    """
    is_subscriber [summary]

    [extended_summary]

    :param id: [description]
    :type id: intorstr
    :return: True if id is found, False if id does not exist
    :rtype: boolean
    """
    spreadsheet = auth()
    subscribers = spreadsheet.get_worksheet(0)
    row:int = find_by_username(str(id), sheet=subscribers)[0]
    if row:
        return True
    else:
        return False

def get_subscribers():
    """
    Get a list of Telegram ids of the subscribers of the newsletter

    :return: List of Telegram ids of the subscribers
    :rtype: list
    """
    spreadsheet = auth()
    subscribers = spreadsheet.get_worksheet(0)
    rows:list = get_rows(subscribers)
    simplified = []
    for row in rows:
        simplified.append(row[0])
    return simplified

def add_scrape(username:str, name:str, scraped:list):
    """
    Adds a new row in the Google Sheets Database with the username, name and scraped follower's usernames.
    Extends the scraped list of an already existing entry if the username is already in the database.

    :param username: The username of the user
    :type username: str
    :param name: The name of the scraped selection (can be custumed by user)
    :type name: str
    :param scraped: List of usernames of the user's followers
    :type scraped: list
    """
    string_scraped = str(scraped)
    string_scraped = string_scraped.replace('[', '')
    string_scraped = string_scraped.replace(']', '')
    string_scraped = string_scraped.replace("'", '')
    spreadsheet = auth()
    sheet:Worksheet = spreadsheet.get_worksheet(1)
    last_scrape = find_by_username(username, sheet)
    if not last_scrape[0]:
        sheet.append_row([username, name, string_scraped])
    else:
        followers = get_targets(username)
        for item in scraped:
            if item not in followers:
                followers.append(item)
        sheet.delete_row(last_scrape[0])
        followers_str = str(followers)
        followers_str = followers_str.replace('[', '')
        followers_str = followers_str.replace(']', '')
        followers_str = followers_str.replace("'", '')
        sheet.append_row([username, name, followers_str])

def get_targets(username:str):
    """
    Return the scraped username of the followers of a user matching the username - and present in the GSheet Database.

    :param username: Username to look up in the database
    :type username: str
    :return: List of usernames (str) of the followers of the user
    :rtype: list
    """
    spreadsheet:Spreadsheet = auth()
    sheet = spreadsheet.get_worksheet(1)
    scraped = get_by_username(username, sheet)
    targets_str = scraped[2]
    targets_str = re.sub(r"\s+", "", targets_str)
    targets = targets_str.split(',')
    print('SHEET TARGETS: ', targets)
    return targets

def get_all_scraped():
    """
    Get a list of the rows' content from the Google Sheets database.

    :return: List of lists, where each sub-list contains a row's contents.
    :rtype: list
    """
    spreadsheet = auth()
    sheet:Worksheet = spreadsheet.get_worksheet(1)
    scraped = get_rows(sheet)
    return scraped

def get_scraped(username:str):
    """
    Get the information of a scraped selection matching the username

    :param username: The userename to look up for
    :type username: str
    :return: List containing the scrape selection's information | None
    :rtype: list | None
    """
    scraped:list = get_all_scraped()
    for item in scraped:
        if item[0] == username:
            return item
    return None

def find_by_username(username:str, sheet:Worksheet, col:int=1):
    """
    Return the GSheet index of the row matching the username.

    :param username: The username to match
    :type username: str
    :param sheet: The GSheet worksheet to get data from
    :type sheet: Worksheet
    :param col: Column to look up for matching, defaults to 1
    :type col: int, optional
    :return: List of indexes of the rows that match the username
    :rtype: list
    """
    if not sheet:
        spreadsheet = auth()
        sheet = spreadsheet.get_worksheet(0)
    column = sheet.col_values(col)
    rows = []
    for num, cell in enumerate(column):
        if str(cell) == str(username):
            rows.append(num + 1)
    if rows == []:
        rows.append(None)
    return rows

def get_by_username(username:str, sheet:Worksheet):
    """
    Return the contents of the GSheet Database row matching the username.

    :param username: The username to match
    :type username: str
    :param sheet: GSheet worksheet to get data from
    :type sheet: Worksheet
    :return: A list with the contents of the row matching the username
    :rtype: list
    """
    row = find_by_username(username, sheet)[0]
    rows = get_rows(sheet)
    return rows[row-1]

def get_rows(sheet:Worksheet):
    """
    Get a list of the rows' content from the Google Sheets database.

    :param sheet: GSheets worksheet to get data from
    :type sheet: Worksheet

    :return: List of lists, where each sub-list contains a row's contents.
    :rtype: list
    """
    rows:list = sheet.get_all_values()
    return rows

def get_sheet_url(index:int=0):
    """
    Returns the link of a worksheet

    Args:
        index (int, optional): Index of the sheet to get. Can be either 0, 1 or 2. Defaults to 0.

    Returns:
        str: Url of the selected worksheet
    """
    spreadsheet = auth()
    sheet:Worksheet = spreadsheet.get_worksheet(index)
    url = 'https://docs.google.com/spreadsheets/d/{}/edit#gid={}'.format(spreadsheet.id, sheet.id)
    return url

def set_sheet(client:Client):
    """
    Setup spreadsheet database if none exists yet.
    Will save the spreadsheet ID to Heroku Env Variables or to secrets.json file
    The service email you created throught the Google API will create the new spreadsheet and share it with the email you indicated in the GDRIVE_EMAIL enviroment variable. You will find the spreadsheet database in your google drive shared folder.
    Don't change the order of the worksheets or it will break the code.

    :param client: GSpread client to utilize
    :type client: Client
    :return: The newly created spreadsheet
    :rtype: Spreadsheet
    """
    # CREATE SPREADSHEET
    spreadsheet:Spreadsheet = client.create('KARIM NEWSLETTER')
    secrets.set_var('SPREADSHEET', spreadsheet.id)

    # CREATE GROUP CHATS SHEET
    subscribers = spreadsheet.add_worksheet(title="Subscribers", rows="150", cols="1")
    scraped = spreadsheet.add_worksheet(title='IG Scraped', rows = '150', cols='3')

    # CREATE LOGS SHEET
    logs = spreadsheet.add_worksheet(title="Logs", rows="500", cols="3")
    logs.append_row(["TIMESTAMP", "USER ID", "ACTION"])

    # DELETE PRE-EXISTING SHEET
    sheet = spreadsheet.get_worksheet(0)
    spreadsheet.del_worksheet(sheet)

    # SHARE SPREADSHEET
    spreadsheet.share(value=secrets.get_var('GDRIVE_EMAIL'),
                      perm_type="user", role="owner")
    return spreadsheet