import pickle
import tkinter as tk
from tkinter import filedialog

import pandas as pd
# from PIL import Image
from PIL import Image, ImageTk

df_all_values = pd.read_csv('./modeling.csv')
df_article_cluster = pd.read_csv('./articles_clusters.csv')
df_TRS_TR_OneCus = pd.read_csv('./Customer_Articles.csv')
similar_items_df = pd.read_csv('./similar_items.csv')

print("Loading Model")
# Load the model from the file
with open('lgbm_model.pkl', 'rb') as file:
    lgbm_model = pickle.load(file)

df_all_values = df_all_values.reset_index(drop=True)

PATH = './data/images/'

# def showPrediction(predictionsDF,
#                    choosen_product,
#                    rows=2,
#                    columns=7,
#                    figsize=(20, 10)):
#
#     predictions = list(predictionsDF)
#     bought_items = list(choosen_product['article_id'])
#
#     print(bought_items)
#     print(predictions)
#
#     f, ax = plt.subplots(rows, columns, figsize=figsize)
#     for i in range(rows):
#         index = 0
#         for j in range(columns):
#             if i == 0:
#                 try:
#                     img_path = f'{PATH}0{str(bought_items[index])[:2]}/0{int(bought_items[index])}.jpg'
#                     img = Image.open(img_path)
#                     ax[i, j].imshow(img)
#                     ax[i, j].set_xticks([], [])
#                     ax[i, j].set_yticks([], [])
#                     ax[i, j].grid(False)
#                     ax[i, j].set_title("Bought")
#                     index += 1
#                 except IndexError:
#                     continue
#             else:
#                 try:
#                     img_path = f'{PATH}0{str(predictions[index])[:2]}/0{int(predictions[index])}.jpg'
#                     img = Image.open(img_path)
#                     ax[i, j].imshow(img)
#                     ax[i, j].set_xticks([], [])
#                     ax[i, j].set_yticks([], [])
#                     ax[i, j].grid(False)
#                     ax[i, j].set_title("Prediction")
#                     index += 1
#                 except IndexError:
#                     continue
#     plt.tight_layout()
#     plt.show()

# def show_similar_to_predictions(predictionsDF,
#                                 similar_items_df,
#                                 rows=10,
#                                 columns=8,
#                                 figsize=(20, 10)):
#     predictions = list(predictionsDF)
#     print(predictions)
#
#     # Find similar items for each predicted item
#     similar_items = []
#     for item_id in predictions:
#         similar_items.append(similar_items_df[
#             similar_items_df["item"] == item_id]["similar_items"].iloc[0])
#
#     similar_items = [
#         list(map(int, s[1:-1].split(', '))) for s in similar_items
#     ]
#
#     # Flatten the list of similar items and remove duplicates
#     similar_items = list(
#         set([item for sublist in similar_items for item in sublist]))
#
#
#     # print(f"Recommended items for customer {customer_id}: {similar_items}")
#
#     # Show the images of the recommended items
#     f, ax = plt.subplots(rows, columns, figsize=figsize)
#     for i in range(rows):
#         for j in range(columns):
#             index = i * columns + j
#             if index < len(similar_items):
#                 try:
#                     img_path = f'{PATH}0{str(similar_items[index])[:2]}/0{int(similar_items[index])}.jpg'
#                     img = Image.open(img_path)
#                     ax[i, j].imshow(img)
#                     ax[i, j].set_xticks([], [])
#                     ax[i, j].set_yticks([], [])
#                     ax[i, j].grid(False)
#                     ax[i, j].set_title("Recommended")
#                 except IndexError:
#                     continue
#             else:
#                 break
#
#     plt.tight_layout()
#     plt.show()

# show_similar_to_predictions(predictions, similar_items_df)

# def adjust_id(x):
#     '''Adjusts article ID code.'''
#     x = str(x)
#     if len(x) == 9:
#         x = "0" + x
#
#     return x

def find_article_by_image():
    def process_image():
        # Clear the output widget
        output.delete(1.0, tk.END)

        # Get the uploaded file
        file_path = filedialog.askopenfilename(
            filetypes=[('JPEG Image', '*.jpg'), ('JPEG Image', '*.jpeg')])
        if file_path:
            with open(file_path, 'rb'):
                # uploaded_content = file.read()
                # uploaded_image = Image.open(io.BytesIO(uploaded_content))

                # Get the filename without the extension
                filename = file_path.split('/')[-1].split('.')[0][1:]
                filename = f'0{filename}'

                product = df_all_values[df_all_values['article_id'] == int(filename)]

                product = product.drop('article_id', axis=1)
                print("Predicting")
                prediction = lgbm_model.predict(product.to_numpy().reshape(1, -1))
                predictions = df_article_cluster[df_article_cluster.cluster ==
                                 prediction[0]][:7]['article_id'].values

                predictions = list(predictions)

                # Find the article in the articles dataframe
                # article = articles.loc[int(filename)]
                # articles["article_id"] = articles["article_id"].apply(
                #     lambda x: adjust_id(x))
                # article = articles[articles['article_id'] == filename]

                # Display the article information
                if len(predictions) > 0:
                    # Find similar items for each predicted item
                    similar_items = []
                    for file in predictions:
                        print(f'0{file}')
                        similar_items.append(
                            similar_items_df[similar_items_df["item"] == int(f'0{file}')]["similar_items"].iloc[0])

                    # similar_items = similar_items[0]
                    # Convert the string to a list using eval()
                    new_similar_items = []
                    for product in similar_items:
                        new_similar_items.append(eval(product))

                    similar_items = new_similar_items
                    similar_items = list(set([item for sublist in similar_items for item in sublist]))

                    print(similar_items)

                    # Show the images of the recommended items
                    rows, columns = 10, 8
                    f = tk.Frame(root)
                    f.pack()
                    for i in range(rows):
                        for j in range(columns):
                            index = i * columns + j
                            if index < len(similar_items):
                                try:
                                    img_path = f'{PATH}0{str(similar_items[index])[:2]}/0{int(similar_items[index])}.jpg'
                                    img = Image.open(img_path)
                                    img = img.resize(
                                        (100, 100)
                                    )  # Adjust the size of the image as needed
                                    photo = ImageTk.PhotoImage(img)
                                    label = tk.Label(f, image=photo)
                                    label.image = photo
                                    label.grid(row=i, column=j)
                                    label.bind('<Button-1>',
                                               lambda event, index=index:
                                               show_recommended(event, index))
                                except IndexError:
                                    continue
                            else:
                                break
                    if len(similar_items) > 0:
                        button = tk.Button(root,
                                           text="Show Recommended",
                                           command=show_all_recommended)
                        button.pack()
                    else:
                        label = tk.Label(root, text="No similar items found.")
                        label.pack()
                else:
                    label = tk.Label(root, text="No matching article found.")
                    label.pack()

    def show_recommended(event, index):
        # Display the clicked image in a new window or perform any other action
        print(f"Clicked image index: {index}")

    def show_all_recommended():
        # Display all recommended images or perform any other action
        print("Showing all recommended images")

    # Create the main tkinter window
    root = tk.Tk()
    root.title("Find Article by Image")

    # Create the button to start processing
    button = tk.Button(root, text="Process Image", command=process_image)
    button.pack()

    # Create the output widget
    output = tk.Text(root, height=20, width=50)
    output.pack()

    # Start the tkinter event loop
    root.mainloop()

find_article_by_image()


import os
import traceback

from fastapi import FastAPI, Path, Query, Body, HTTPException
from starlette.requests import Request
from starlette.responses import FileResponse, StreamingResponse
from starlette.staticfiles import StaticFiles

from node import Node, File, Dir, NodeType
from utils import get_user
from model import Target, StorageModel, SyncModel, MoveModel
from transfer import Transfer
import storage as storage_module

app = FastAPI(
    title='stome',
    openapi_url='/api/openapi.json',
    docs_url='/api/docs',
    redoc_url='/api/redoc',
)

@app.get('/api/file/{path:path}')
async def download_file(path, request: Request):
    f = File(path)
    ensure_existed(f)
    ensure_not_type(f, NodeType.Dir)
    inode = f.inode
    return StreamingResponse(inode.stream, media_type=inode.mime)

@app.post('/api/file/{path:path}')
async def upload_file(
        *,
        path,
        force: bool = Query(
            False,
            title='Force create even existed, old file will be deleted',
        ),
        transfer_id: str = Query(
            None,
            alias='transfer',
            title='Transfer UUIDv4',
        ),
        request: Request,
):
    ensure_me(request)
    f = File(path)
    ensure_not_type(f, NodeType.Dir)
    if f and not force:
        raise HTTPException(409, 'Existed')
    if transfer_id:
        transfer = Transfer(transfer_id, int(request.headers['content-length']))
    else:
        transfer = None
    await f.create(request.stream(), transfer=transfer)

@app.get('/api/meta/{path:path}')
async def get_meta(path, request: Request):
    node = Node(path)
    ensure_existed(node)
    meta = node.meta
    meta.update({
        'path': '/' + node.path,
        'type': node.type,
    })
    return meta

@app.post('/api/meta/{path:path}')
def update_meta(path, request: Request):
    ensure_me(request)

@app.get('/api/dir/{path:path}')
def list_directory(path):
    node = Dir(path)
    ensure_existed(node)
    meta = node.meta
    dirs, files = node.list()
    meta['dirs'] = [dict(d) for d in dirs]
    meta['files'] = [dict(f) for f in files]
    return meta

@app.post('/api/dir/{path:path}')
def create_directory(path, request: Request = None):
    ensure_me(request)
    d = Dir(path)
    if d:
        raise HTTPException(409, 'Conflict')
    d.create(request)

@app.post('/api/mv')
def move(spec: MoveModel, request: Request):
    """
    1. Rename
        {
            "src_paths": [
                ("/foo.jpg", "/bar.jpg"),
                ("/big.flv", "/giant.flv"),
            ],
        }
    2. Move to directory
        {
            "src_paths": ["/1.jpg", "/2.jpg"],
            "dst_path": '/img',
        }
    """
    ensure_me(request)
    errors = []
    for src_path, dst_path in normalized_move_paths(spec):
        src = Node(src_path)
        dst = Node(dst_path)
        if not src:
            errors.append({
                'err': 'not found',
                'src': src_path,
            })
        if dst:
            errors.append({
                'err': 'existed',
                'src': src_path,
                'dst': dst_path,
            })
        try:
            src.move(dst)
        except OSError:
            errors.append({
                'err': 'exception',
                'detail': traceback.format_exc(),
                'src': src_path,
                'dst': dst_path,
            })
    return {'errors': errors}

@app.post('/api/rm')
def delete(target: Target, request: Request):
    ensure_me(request)
    for path in target.paths:
        node = Node(path)
        if not node:
            raise HTTPException(404, 'Not found')
        node.delete()

@app.get('/api/storages')
def get_storages(request: Request):
    ensure_me(request)
    return {
        'data': [
            {
                'name': 's3',
                'id': '1',
                'template_id': '1',
                'size': '19.3GB',
            },
            {
                'name': 's3 2019',
                'id': '2',
                'template_id': '1',
                'size': '213MB',
            },
            {
                'name': 'dropbox',
                'id': '3',
                'template_id': '2',
                'size': '0',
            },
        ],
    }

@app.post('/api/storages')
def create_storage(storage: StorageModel, request: Request):
    ensure_me(request)
    storage_module.create(storage)

@app.get('/api/storage-templates')
def get_storage_templates():
    return {'data': storage_module.TEMPLATES}

@app.post('/api/sync-to-storage')
def sync_to_storage(sync: SyncModel, request: Request):
    ensure_me(request)
    print('sync_to_storage', dict(sync))
    return sync

@app.post('/api/sync-from-storage')
def sync_from_storage(sync: SyncModel, request: Request):
    ensure_me(request)
    print('sync_from_storage', dict(sync))

@app.get('/api/stream/transfer')
def transfer_event_source(request: Request):
    """
    SSE (Server Sent Event) endpoint for transfer events.
    """
    return StreamingResponse(
        Transfer.stream(request),
        media_type='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'X-Accel-Buffering': 'no',
        },
    )

app.mount('/_next', StaticFiles(directory='../frontend/out/_next', html=True))

@app.get('/{path:path}')
async def path(path, request: Request):
    node = Node(path)
    if not node:
        raise HTTPException(404, 'Not found')
    if node.type == NodeType.Dir:
        return FileResponse('../frontend/out/index.html')
    else:
        return await download_file(path, request)

def ensure_existed(node):
    if not node:
        raise HTTPException(404, 'Not found')

def ensure_not_type(node, node_type):
    if node.type == node_type:
        raise HTTPException(400, f'Is {node_type.value}')

def ensure_type(node, node_type):
    if node.type != node_type:
        raise HTTPException(400, f'Is not {node_type.value}')

def ensure_me(request):
    user = get_user(request)
    if user['username'] != 'fans656':
        raise HTTPException(401, 'require fans656 login')

def normalized_move_paths(spec):
    move_paths = []
    for src_path in spec.src_paths:
        if isinstance(src_path, tuple):
            src_path, dst_path = src_path
        else:
            fname = os.path.basename(src_path)
            dst_path = os.path.join(spec.dst_path, fname)
        move_paths.append((src_path, dst_path))
    return move_paths


from crypt import methods
from unicodedata import name
from urllib import request
from flask import Flask,request,json,jsonify
import json

app = Flask(__name__)

@app.route('/service', methods=['POST','GET'])
def home():
    data = json.loads(request.data)
    name = data.get("name")
    age = data.get("age")

    json_object = json.dumps(data, indent=4)
    with open("sample.json", "w") as outfile:
        outfile.write(json_object)

    return jsonify({"name":name,"age":age})

if __name__ ==('__main__'):
    app.run(debug=True)


# Models and related classes and functions
import torch
from torch.utils.data import Dataset
import transformers
from sklearn.metrics import precision_recall_fscore_support, confusion_matrix
import numpy as np
import time

def n_tokens_in_core_sentences(idx_first_sent, idx_last_sent, tokenized_sentences):
    '''
    Computes the number of tokens from a set of core sentences.
    Arguments:
        idx_first_sent: index of the first core sentence (integer).
        idx_last_sent: index of the last core sentence (integer).
        tokenized_sentences: the representation of a tokenized chunk as a list (sentences) of list of integers (token IDs).
    Returns:
        The number of tokens in the sequence of sentences indicated by the idx_first_sent and idx_last_sent arguments.
    '''
    n = 0
    for i in range(idx_first_sent, idx_last_sent + 1):
        n += (len(tokenized_sentences[i]) + 1) # +1 because future SEP token
    return n

def core_sentences_idx(tokenized_sentences, max_seq_len, min_context_len):
    '''
    Sets the number of chunks and their respective core sentences.
    Arguments:
        max_seq_len: maximum number of tokens in a chunk.
        min_context_len: desired minimum number of edge tokens.
        tokenized_sentences: list (sentences) of list (tokens in a sentence) of integers (token IDs).
    Returns:
        List of tuples. Each tuple represents a chunk and it has two
        integers which are the index of first and last core sentence in the chunk.
    '''
    threshold_ctx = max_seq_len - min_context_len - 2 # -2 because CLS and first SEP

    n_sentences = len(tokenized_sentences)
    idx_core_sentences = [] # list of tuple of 2 integers (indexes of first and last core sentences in a chunk)
    idx_current_sent = 0
    while idx_current_sent < n_sentences:
        # new chunk
        idx_start = idx_current_sent # begining of current chunk
        idx_end = idx_current_sent   # end of current chunk
        idx_current_sent += 1
        len_core_sent_in_chunk = n_tokens_in_core_sentences(idx_start, idx_end, tokenized_sentences)
        while len_core_sent_in_chunk < threshold_ctx and idx_current_sent < n_sentences:
            # checking if current sentence fits into the current chunk while maintain minimum context length
            if len_core_sent_in_chunk + len(tokenized_sentences[idx_current_sent]) < threshold_ctx:
                idx_end = idx_current_sent
                idx_current_sent += 1
                len_core_sent_in_chunk = n_tokens_in_core_sentences(idx_start, idx_end, tokenized_sentences)
            else:
                break
        idx_core_sentences.append((idx_start, idx_end))
    return idx_core_sentences

def tokenize_sentences(sentences, tokenizer, max_seq_len, min_context_len):
    '''
    This function assembles a set of DFCSCs for a sequence of sentences (document). The sentences
    are tokenized and each chunk is created as a sequence of token IDs. The chunks include the core
    sentences and the edge sentence. The chunks' length
    is limited by max_seq_len argument. This function aims to entirely encode the core
    sentences while it try to maintain a minimum number of tokens in the left and right
    edges. The min_context_len parameter sets the minimum number of edge
    tokens. The number of core sentences is dynamically set and it depends on the length
    of the core sentences and on the min_context_len param.
    Arguments:
        sentences: list of strings. Each string represents a sentence and the list represents a document.
        tokenizer: tokenizer of respective sentence encoder.
        max_seq_len: maximum number of tokens in a chunk.
        min_context_len: desired minimum number of edge tokens.
    Returns:
        List of list of integers. The first level list represents a DFCSC. The sublist holds the IDs of the
        tokens in the chunk. A sublist will have the following
        layout: [CLS] left edge tokens [SEP] core sentence 1 [SEP] core sentence 2 [SEP] ...
        core sentence M [SEP] right edge tokens [SEP]
    '''
    # tokenizing sentences to get their number of tokens
    tokenized_sentences = tokenizer(
        sentences,
        add_special_tokens=False,
        padding=False,
        return_token_type_ids=False,
        return_attention_mask=False,
        truncation=True,
        max_length=max_seq_len - 4 # -4 because of special tokens
    )['input_ids']

    # chunks with indexes of respective core sentences
    idx_chunks = core_sentences_idx(tokenized_sentences, max_seq_len, min_context_len)

    # assembling DFCSCs
    chunks = []
    for core_start, core_end in idx_chunks:
        current_chunk = []
        # inserting core sentences
        for i in range(core_start, core_end + 1):
            sent = tokenized_sentences[i].copy()
            sent.append(tokenizer.sep_token_id)
            current_chunk.extend(sent)
        # inserting edges
        current_chunk.insert(0, tokenizer.sep_token_id)
        len_ctx = max_seq_len - len(current_chunk) - 2 # -2 because [CLS] and final [SEP]

        if len_ctx > 0:
            # flattening left edge tokens in a unique list
            left_tokens = []
            for j in range(core_start):
                left_tokens.extend(tokenized_sentences[j])
            # flattening right edge tokens in a unique list
            right_tokens = []
            for j in range(core_end + 1, len(tokenized_sentences)):
                right_tokens.extend(tokenized_sentences[j])
            # inserting edge tokens
            count = 0
            idx_left = len(left_tokens) - 1
            idx_right = 0
            while count < len_ctx and (idx_left >= 0 or idx_right < len(right_tokens)):
                if idx_left >= 0:
                    current_chunk.insert(0, left_tokens[idx_left])
                    idx_left -= 1
                    count += 1
                if idx_right < len(right_tokens) and count < len_ctx:
                    current_chunk.append(right_tokens[idx_right])
                    idx_right += 1
                    count += 1
        # inserting special tokens
        current_chunk.insert(0, tokenizer.cls_token_id)
        current_chunk.append(tokenizer.sep_token_id)

        chunks.append(current_chunk)

    return chunks

def pad_sentences(sentences_as_ids, pad_token_id, max_seq_len):
    """
    Inserts [PAD] tokens in tokenized sentences. The tokens are inserted at right
    untill each tokenized sentence reachs max_seq_len. sentences_as_ids is
    modified in place.
    Arguments:
        sentences_as_ids : list of token ids as returned by the tokenize_sentences function.
        pad_token_id : id of [PAD] token.
        max_seq_len : the desired final length for all sentences.
    """
    for s in sentences_as_ids:
        diff = max_seq_len - len(s)
        if diff > 0:
            for i in range(diff):
                s.append(pad_token_id)

class DFCSC_Dataset(torch.utils.data.Dataset):
    """
    A dataset object representing a set of documents structured as DFCSCs.
    Each item of the dataset represents a DFCSC, i.e., a set of core sentences and edge tokens (check
    the tokenize_sentences function).
    The beginning and end of documents are lost after creation of DFCSCs.
    """
    def __init__(self, dic_docs, labels_to_idx, tokenizer, max_seq_len, min_context_len):
        """
        Arguments:
            dic_docs: a dictionary whose each item is a document (key: docId, value: pandas DataFrame).
            labels_to_idx : dictionary that maps each label (string) to a index (integer).
            tokenizer : tokenizer of the encoder model.
            max_seq_len: maximum sequence length (the maximum number of tokens in a DFCSC).
            min_context_len: desired minimum number of edge tokens.
        """
        self.labels = []    # list of list of strings (n chunks, n core sentences in chunks)
        self.targets = []   # list of unidimensional tensors (n chunks, n core sentences in chunks)
        self.input_ids = [] # tensor of shape (n chunks, max_seq_len)
        self.len = 0

        # tokenizing and spliting sentences in DFCSCs
        for df in dic_docs.values():
            # chunks
            chunks = tokenize_sentences(df['sentence'].tolist(), tokenizer, max_seq_len, min_context_len)
            pad_sentences(chunks, tokenizer.pad_token_id, max_seq_len)
            chunks = torch.tensor(chunks, dtype=torch.long)
            self.input_ids.append(chunks)

            # labels and targets
            labels_in_doc = df['label'].tolist()
            idx_start = 0
            for i in range(chunks.shape[0]):
                n_sep_tokens = torch.count_nonzero(chunks[i] == tokenizer.sep_token_id)
                n_labels_in_chunk = n_sep_tokens - 2 # -2 because SEP token of each context side
                idx_end = idx_start + n_labels_in_chunk
                chunk_labels = labels_in_doc[idx_start:idx_end]
                assert n_labels_in_chunk == len(chunk_labels)
                self.labels.append(chunk_labels)
                self.targets.append(torch.tensor(
                    [labels_to_idx[l] for l in chunk_labels],
                    dtype=torch.long
                ))
                idx_start = idx_end
        self.input_ids = torch.vstack(self.input_ids)
        self.len = self.input_ids.shape[0]

        # creating attention masks
        self.masks = torch.ones(self.input_ids.shape)
        self.masks[self.input_ids == tokenizer.pad_token_id] = 0

    def __getitem__(self, index):
        return {
            'ids': self.input_ids[index],       # shape: (max_seq_len)
            'mask': self.masks[index],          # shape: (max_seq_len)
            'targets': self.targets[index],     # shape: (num of core sentences in current sequence)
            'labels': self.labels[index]        # shape: (num of core sentences in current sequence)
        }

    def __len__(self):
        return self.len

def collate_batch(batch):
    '''
    Prepares a batch of dataset items.
    Arguments:
        batch: list of dataset items (dictionaries).
    Returns:
        A dictionary with following items:
            'ids': tensor of input ids. Shape: (n sequences in batch, max_seq_len)
            'masks': tensor of attention masks. Shape: (n sequences in batch, max_tokens)
            'targets': tensor of golden class ids. Shape: (n core sentences in batch)
            'labels': list of golden labels. Shape: (n core sentences in batch)
    '''
    labels = []
    targets = []
    input_ids = None
    masks = None
    for entry in batch:
        labels.extend(entry['labels'])
        targets.extend(entry['targets'])
        if input_ids is None:
            input_ids = entry['ids'].reshape(1,-1) # reshape to assure a 2-D tensor when the batch contains just one element
            masks = entry['mask'].reshape(1,-1)
        else:
            input_ids = torch.vstack((input_ids, entry['ids']))
            masks = torch.vstack((masks, entry['mask']))
    return {
        'ids': input_ids,
        'mask': masks,
        'targets': torch.tensor(targets, dtype=torch.long),
        'labels': labels
    }

def count_labels(ds, labels):
    """
    Returns the number of sentences by label for a provided DFCSC_Dataset.
    Arguments:
        ds : a DFCSC_Dataset.
        labels: list of strings. The set of possible labels.
    Returns:
       A dictionary mapping each label (string) to its number of sentences (integer).
    """
    count_by_label = {l: 0 for l in labels}
    for sublist in ds.labels:
        for l in sublist:
            count_by_label[l] = count_by_label[l] + 1
    return count_by_label

class MockDFCSC_BERT(torch.nn.Module):
    '''
    A mock of DFCSC_BERT_Classifier. It's usefull to accelerate the validation
    of the training loop.
    '''
    def __init__(self, n_classes, sep_token_id):
        super(MockDFCSC_BERT, self).__init__()
        self.classifier = torch.nn.Linear(10, n_classes) # 10 => random choice
        self.sep_token_id = sep_token_id

    def forward(self, input_ids, attention_mask):
        # getting number of labels/targets
        n_chunks = input_ids.shape[0]
        n_sep_tokens = torch.count_nonzero(input_ids == self.sep_token_id).item()
        n_targets = n_sep_tokens - 2 * n_chunks

        mock_data = torch.rand((n_targets, 10), device=input_ids.device)
        logits = self.classifier(mock_data)    # shape: (n_targets, n_classes)

        return logits

class DFCSC_BERT_Classifier(torch.nn.Module):
    """
    Sentence Classifier based on a BERT kind encoder. This model expects to
    get as inputs sentence representations from a DFCSC_Dataset object.
    The sentence encoder must be a pre-trained model based on BERT's architecture
    like BERT, RoBERTa and ALBERT.
    """
    def __init__(self, encoder_id, sep_token_id, n_classes, dropout_rate, embedding_dim):
        '''
        This model comprises a pre-trained sentence encoder and a classification head.
        The sentence encoder must be a model following BERT architecture.
        The classification head is a single feedforward layer.
        Arguments:
            encoder_id: ID (string) of the encoder model in Hugging Faces repository.
            sep_token_id: ID (integer) of the [SEP] token.
            n_classes: number of classes.
            dropout_rate: dropout rate of classification layer.
            embedding_dim: dimension of hidden units in the sentence encoder (e.g., 768 for BERT).
        '''
        super(DFCSC_BERT_Classifier, self).__init__()
        self.sep_token_id = sep_token_id
        self.encoder = transformers.AutoModel.from_pretrained(encoder_id)

        dropout = torch.nn.Dropout(dropout_rate)
        dense_out = torch.nn.Linear(embedding_dim * 2, n_classes) # multiplication by because we concatenate [CLS] and [SEP] embeddings
        torch.nn.init.xavier_uniform_(dense_out.weight)
        self.classifier = torch.nn.Sequential(
            dropout, dense_out
        )

    def forward(self, input_ids, attention_mask):
        '''
        Each call to this method process a batch of DFCSCs. The model exploits the
        edge tokens and the core sentences to produce embeddings for the core sentences,
        but it doesn't produce embeddings for edge tokens.
        This method returns one logit tensor for each core sentence in the batch.
        Arguments:
            input_ids : tensor of shape (batch_size, seq_len)
            attention_mask : tensor of shape (batch_size, seq_len)
        Returns:
            logits : tensor of shape (n of core sentences in batch, n of classes)
        '''
        output_1 = self.encoder(
            input_ids=input_ids,             # input_ids.shape: (batch_size, seq_len)
            attention_mask=attention_mask    # attention_mask.shape: (batch_size, seq_len)
        )
        hidden_state = output_1.last_hidden_state  # hidden states of last encoder's layer => shape: (batch_size, seq_len, embedd_dim)

        ### In this approach we concatenate [CLS] embedding with [SEP] embedding
        # getting embeddings of core [SEP] and [CLS]tokens
        batch_size = input_ids.shape[0]
        core_embeddings = [] # core_embeddings.shape: (n core sentences in batch, embedding_dim)
        for i in range(batch_size): # iterates chunks
            cls_embedding = hidden_state[i, 0, :] # cls_embedding.shape: (embedd_dim)
            idx_seps = torch.nonzero(input_ids[i] == self.sep_token_id, as_tuple=True)[0] # indexes of all [SEP] tokens in currrent sentence
            # we dont want the embeddings of the 1st and last [SEP] tokens
            idx_core_sep = idx_seps[1:-1] # index of core [SEP] tokens in current chunk
            core_sep_emb = hidden_state[i, idx_core_sep, :] # embeddings of core [SEP]. shape: (n core sentences in chunk, embedd_dim)
            for j in range(core_sep_emb.shape[0]):
                core_embeddings.append(
                    torch.hstack((cls_embedding, core_sep_emb[j]))
                )
        core_embeddings = torch.vstack(core_embeddings)

        logits = self.classifier(core_embeddings)   # logits.shape: (n core sentences in batch, num of classes)

        return logits

class LongformerEncoder(torch.nn.Module):
    '''
    Uses a Longformer model to encode DFCSCs.
    '''

    def __init__(self, encoder_id, sep_token_id):
        super(LongformerEncoder, self).__init__()
        self.sep_token_id = sep_token_id
        self.encoder = transformers.AutoModel.from_pretrained(encoder_id)

    def forward(self, input_ids, attention_mask):
        '''
        Encodes a batch of DFCSCs.
        Arguments:
            input_ids: PyTorch tensor with shape (batch_size, chunk_len).
            attention_mask: PyTorch tensor with shape (batch_size, chunk_len).
        Returns:
            PyTorch tensor with shape (n core sentences in batch, embedding_dim * 2). This tensor
            holds the embeddings of the core sentences from all chunks in the batch. Each embedding
            is the concatenation of the last hidden state of the <s> token of the respective chunk and
            the respective last hidden state of the </s> token of the respective core sentence.
        '''
        # The global_attention_mask tells which tokens must have global attention.
        # Here, <s> and </s> have global attention
        global_attention_mask = torch.zeros(
            input_ids.shape,
            dtype=torch.long,
            device=input_ids.device
        )
        global_attention_mask[:, 0] = 1 # global attention for <s> since it must be the first token in a chunk
        idx_sep = torch.nonzero(input_ids == self.sep_token_id, as_tuple=True)
        for i in range(idx_sep[0].shape[0]):
            global_attention_mask[idx_sep[0][i], idx_sep[1][i]] = 1 # global attention for </s>

        # Encoding
        output_1 = self.encoder(
            input_ids=input_ids,             # input_ids.shape: (batch_size, chunk_len)
            attention_mask=attention_mask    # attention_mask.shape: (batch_size, chunk_len)
        )
        hidden_state = output_1.last_hidden_state  # hidden states of last encoder's layer => shape: (batch_size, chunk_len, embedd_dim)

        ### In this approach we concatenate <s> embedding with </s> embedding
        # getting embeddings of core </s> and <s> tokens
        batch_size = input_ids.shape[0]
        core_embeddings = []
        for i in range(batch_size): # iterates chunks
            cls_embedding = hidden_state[i, 0, :] # cls_embedding.shape: (embedd_dim)
            idx_seps = torch.nonzero(input_ids[i] == self.sep_token_id, as_tuple=True)[0] # indexes of all </s> tokens in currrent sentence
            # we dont want the embeddings of the 1st and last </s> tokens
            idx_core_sep = idx_seps[1:-1] # index of core </s> tokens in current chunk
            core_sep_emb = hidden_state[i, idx_core_sep, :] # embeddings of core </s>. shape: (n core sentences in chunk, embedd_dim)
            for j in range(core_sep_emb.shape[0]):
                core_embeddings.append(
                    torch.hstack((cls_embedding, core_sep_emb[j]))
                )
        core_embeddings = torch.vstack(core_embeddings) # core_embeddings.shape: (n core sentences in batch, embedding_dim * 2)
        return core_embeddings

class DFCSC_Longformer_Classifier(torch.nn.Module):
    """
    Sentence Classifier based on a Longformer kind encoder. This model expects to
    get as inputs sentence represenations from a DFCSC_Dataset object.
    The sentence encoder must be a pre-trained model based on Longformer's architecture.
    """
    def __init__(self, encoder_id, sep_token_id, n_classes, dropout_rate, embedding_dim):
        '''
        This model comprises a pre-trained sentence encoder and a classification head.
        The sentence encoder must be a model following Longformer architecture.
        The classification head is a linear classifier (a single feedforward layer).
        Arguments:
            encoder_id: ID (string) of the encoder model in Hugging Faces repository.
            sep_token_id: ID (integer) of the [SEP] token.
            n_classes: number of classes.
            dropout_rate: dropout rate of classification layer.
            embedding_dim: dimension of hidden units in the sentence encoder (e.g., 768 for Longformer).
        '''
        super(DFCSC_Longformer_Classifier, self).__init__()
        self.encoder = LongformerEncoder(encoder_id, sep_token_id)

        dropout = torch.nn.Dropout(dropout_rate)
        dense_out = torch.nn.Linear(embedding_dim * 2, n_classes)
        torch.nn.init.xavier_uniform_(dense_out.weight)
        self.classifier = torch.nn.Sequential(
            dropout, dense_out
        )

    def forward(self, input_ids, attention_mask):
        '''
        Each call to this method process a batch of DFCSCs. The model exploits the
        edge tokens and the core sentences to produce embeddings for the core sentences,
        but it doesn't produce embeddings for edge tokens.
        This method returns one logit tensor for each core sentence in the batch.
        Arguments:
            input_ids : tensor of shape (batch_size, seq_len)
            attention_mask : tensor of shape (batch_size, seq_len)
        Returns:
            logits : tensor of shape (n of core sentences in batch, n of classes)
        '''
        core_embeddings = self.encoder(input_ids, attention_mask)

        logits = self.classifier(core_embeddings)   # logits.shape: (n core sentences in batch, num of classes)

        return logits

def evaluate(model, test_dataloader, loss_function, device):
    """
    Evaluates a provided DFCSC_Classifier model.
    Arguments:
        model: the model to be evaluated.
        test_dataloader: torch.utils.data.DataLoader instance containing the test data.
        loss_function: instance of the loss function used to train the model.
        device: device where the model is located.
    Returns:
        eval_loss (float): the computed test loss score.
        precision (float): the computed test macro Precision score.
        recall (float): the computed test macro Recall score.
        f1 (float): the computed test macro F1 score.
        confusion_matrix: the computed test confusion matrix.
    """
    predictions = torch.tensor([]).to(device)
    y_true = torch.tensor([]).to(device)
    eval_loss = 0
    model.eval()
    with torch.no_grad():
        for data in test_dataloader:
            ids = data['ids'].to(device)
            mask = data['mask'].to(device)
            y_true_batch = data['targets'].to(device)
            y_hat = model(ids, mask)
            # ignores classes with negative ID
            idx_valid = (y_true_batch >= 0).nonzero().squeeze()
            y_true_batch_valid = y_true_batch[idx_valid]
            y_hat_valid = y_hat[idx_valid]
            loss = loss_function(y_hat_valid, y_true_batch_valid)
            eval_loss += loss.item()
            predictions_batch = y_hat_valid.argmax(dim=1)
            predictions = torch.cat((predictions, predictions_batch))
            y_true = torch.cat((y_true, y_true_batch_valid))
        predictions = predictions.detach().to('cpu').numpy()
        y_true = y_true.detach().to('cpu').numpy()
    eval_loss = eval_loss / len(test_dataloader)
    t_metrics_macro = precision_recall_fscore_support(
        y_true,
        predictions,
        average='macro',
        zero_division=0
    )
    cm = confusion_matrix(
        y_true,
        predictions
    )

    return eval_loss, t_metrics_macro[0], t_metrics_macro[1], t_metrics_macro[2], cm

def fit(train_params, ds_train, ds_test, device):
    """
    Creates and train an instance of DFCSC_Classifier.
    Arguments:
        train_params: dictionary storing the training params.
        ds_train: instance of DFCSC_Dataset storing the train data.
        tokenizer: the tokenizer of the chosen pre-trained sentence encoder.
        device: device where the model is located.
    """
    learning_rate = train_params['learning_rate']
    weight_decay = train_params['weight_decay']
    n_epochs = train_params['n_epochs']
    batch_size = train_params['batch_size']
    encoder_id = train_params['encoder_id']
    n_classes = train_params['n_classes']
    dropout_rate = train_params['dropout_rate']
    embedding_dim = train_params['embedding_dim']
    use_mock = train_params['use_mock']

    dl_train = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, collate_fn=collate_batch)
    dl_test = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=True, collate_fn=collate_batch)

    if use_mock:
        sentence_classifier = MockDFCSC_BERT(n_classes, train_params['sep_token_id']).to(device)
    elif encoder_id.lower().find('longformer') > -1:
        sentence_classifier = DFCSC_Longformer_Classifier(
            encoder_id,
            train_params['sep_token_id'],
            n_classes,
            dropout_rate,
            embedding_dim
        ).to(device)
    else:
        sentence_classifier = DFCSC_BERT_Classifier(
            encoder_id,
            train_params['sep_token_id'],
            n_classes,
            dropout_rate,
            embedding_dim
        ).to(device)

    criterion = torch.nn.CrossEntropyLoss().to(device)
    optimizer = torch.optim.Adam(
        sentence_classifier.parameters(),
        lr=learning_rate,
        betas=(0.9, 0.999),
        eps=train_params['eps'],
        weight_decay=weight_decay
    )
    lr_scheduler = transformers.get_linear_schedule_with_warmup(
        optimizer,
        num_warmup_steps = 0,
        num_training_steps = len(dl_train) * n_epochs
    )

    metrics = {} # key: epoch number, value: numpy tensor storing train loss, test loss, Precision (macro), Recall (macro), F1 (macro)
    confusion_matrices = {} # key: epoch number, value: scikit-learn's confusion matrix
    start_train = time.perf_counter()
    for epoch in range(1, n_epochs + 1):
        print(f'Starting epoch {epoch}... ', end='')
        start_epoch = time.perf_counter()
        epoch_loss = 0
        sentence_classifier.train()
        for train_data in dl_train:
            optimizer.zero_grad()
            ids = train_data['ids'].to(device)
            mask = train_data['mask'].to(device)
            y_hat = sentence_classifier(ids, mask)
            y_true = train_data['targets'].to(device)
            # ignores classes with negative ID
            idx_valid = (y_true >= 0).nonzero().squeeze()
            y_true_valid = y_true[idx_valid]
            y_hat_valid = y_hat[idx_valid]
            loss = criterion(y_hat_valid, y_true_valid)
            epoch_loss += loss.item()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(sentence_classifier.parameters(), 1.0)
            optimizer.step()
            lr_scheduler.step()
        epoch_loss = epoch_loss / len(dl_train)
        # evaluation
        optimizer.zero_grad()
        eval_loss, p_macro, r_macro, f1_macro, cm = evaluate(
            sentence_classifier,
            dl_test,
            criterion,
            device
        )
        #storing metrics
        metrics[epoch] = np.array([epoch_loss, eval_loss, p_macro, r_macro, f1_macro])
        confusion_matrices[epoch] = cm
        end_epoch = time.perf_counter()
        print('finished! Time: ', time.strftime("%Hh%Mm%Ss", time.gmtime(end_epoch - start_epoch)))

    end_train = time.perf_counter()

    return metrics, confusion_matrices, time.strftime("%Hh%Mm%Ss", time.gmtime(end_train - start_train))


import unittest2
import dulwich.object_store
import dulwich.objects
import logging

from quantumkeep.tree import *

class TestTree(unittest2.TestCase):

    def setUp(self):
        self.git_store = dulwich.object_store.MemoryObjectStore()

    def test_tree_append(self):
        tree = Tree(git_store=self.git_store)
        self.assertEqual(list(tree), [])
        msg1 = "Hello World"
        tree.append((
            "hi",
            TreeEntry(
                mode=tree_entry_modes.raw_blob,
                target=msg1,
            ),
        ))
        (k, entry) = tree[0]
        self.assertEqual(k, "hi")
        self.assertEqual(entry.target, msg1)
        self.assertEqual(tree["hi"].target, msg1)

    def test_tree_slice(self):
        tree = Tree(git_store=self.git_store)
        self.assertEqual(list(tree), [])
        for i in xrange(0, 11):
            tree.append((
                "key%i" % i,
                TreeEntry(
                    mode=tree_entry_modes.raw_blob,
                    target="value%i" % i
                ),
            ))

        def row_tuples(iterable):
            return [(key, entry.target) for key, entry in iterable]

        def expected_tuples(iterable):
            return [("key%i" % i, "value%i" % i) for i in iterable]

        s1 = tree[2:]
        self.assertEqual(
            row_tuples(s1),
            expected_tuples(xrange(2, 11)),
        )

        del tree[4:]
        self.assertEqual(
            row_tuples(tree),
            expected_tuples(xrange(0, 4))
        )

        tree[2:4] = [(
            "key16",
            TreeEntry(
                mode=tree_entry_modes.raw_blob,
                target="value16",
            ),
        )]
        self.assertEqual(
            row_tuples(tree),
            expected_tuples((0, 1, 16))
        )

    def test_subtree(self):
        tree = Tree(git_store=self.git_store)
        subtree = Tree()
        subtree.append((
            "hi",
            TreeEntry(
                mode=tree_entry_modes.raw_blob,
                target="Hello World",
            ),
        ))
        tree["foo"] = subtree
        entry = tree["foo"]
        self.assertEqual(type(entry.target), Tree)
        self.assertEqual(entry.mode, tree_entry_modes.tree)
        self.assertEqual(entry.target["hi"].target, "Hello World")

    def test_write(self):
        tree = Tree(git_store=self.git_store)
        subtree = Tree()
        subtree.append((
            "hi",
            TreeEntry(
                mode=tree_entry_modes.raw_blob,
                target="Hello World",
            ),
        ))
        tree["foo"] = subtree
        git_tree_id = tree.write_git_tree()
        git_tree = self.git_store[git_tree_id]
        git_subtree_id = git_tree["foo"][1]
        git_subtree = self.git_store[git_subtree_id]
        git_blob_id = git_subtree["hi"][1]
        git_blob = self.git_store[git_blob_id]
        self.assertEqual(git_blob.as_raw_string(), "Hello World")


import datetime, time, playsound

hour = input("Enter hour : ")
min = input("Enter min : ")
sec = input("Enter sec : ")
set_alarm_timer = f"{hour} : {min} : {sec} "
while True:
    time.sleep(1)
    current_time = datetime.datetime.now()
    now = current_time.strftime("%H:%M:%S")
    split_alarm_time = set_alarm_timer.split(":")
    split_now_time = now.split(":")
    if int(split_now_time[0]) == int(split_alarm_time[0]):
        if int(split_now_time[1]) == int(split_alarm_time[1]):
            if int(split_now_time[2]) == int(split_alarm_time[2]):
                print("Time to wake up")
                playsound('/alarm.mp3')  # for playing note.mp3 file
                break


from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import os
from PIL import Image
import numpy as np
import os.path as osp
import lmdb
import io

import torch
from torch.utils.data import Dataset

def read_image(img_path, mode='RGB'):
    """Keep reading image until succeed.
    This can avoid IOError incurred by heavy IO process."""
    got_img = False
    if not osp.exists(img_path):
        raise IOError("{} does not exist".format(img_path))
    while not got_img:
        try:
            img = Image.open(img_path).convert(mode)
            got_img = True
        except IOError:
            print("IOError incurred when reading '{}'. Will redo. Don't worry. Just chill.".format(img_path))
            pass
    return img

class ImageDataset(Dataset):
    """Image Person ReID Dataset"""
    def __init__(self, dataset, transform=None):
        self.dataset = dataset
        self.transform = transform

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, index):
        img_path, pid, camid = self.dataset[index]
        img = read_image(img_path)
        if self.transform is not None:
            img = self.transform(img)
        return img, pid, camid

class ImageDataset_seg(Dataset):
    """Image Person ReID Dataset"""
    def __init__(self, dataset, transform=None):
        self.dataset = dataset
        self.transform = transform

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, index):
        path, pid, camid = self.dataset[index]
        img_path, head_path, upper_body_path, lower_body_path, shoes_path, foreground_path = path

        img = read_image(img_path, mode='RGB')
        head = read_image(head_path, mode='L')
        upper_body = read_image(upper_body_path, mode='L')
        lower_body = read_image(lower_body_path, mode='L')
        shoes = read_image(shoes_path, mode='L')
        foreground = read_image(foreground_path, mode='L')

        sequence = [img, head, upper_body, lower_body, shoes, foreground]

        if self.transform is not None:
            self.transform.randomize_parameters()
            sequence = [self.transform(img) for img in sequence]

        sequence = torch.cat(sequence, 0) #[3+1+1+1, h, w]

        return sequence, pid, camid


from fastapi import APIRouter, Depends

from app.config import Settings, get_settings

router = APIRouter()

@router.get("/ping")
async def ping(settings: Settings = Depends(get_settings)):
    return {
        "ENVIRONMENT": settings.environment,
        "TESTING": settings.testing,
        "ping": "PONGGGGG!",
    }


import numpy as np
import yfinance as yf
import pandas as pd
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, LSTM
import os
# Lets save as csv and then upload to view and change to numpy. store all data in 1 file

symbols_file = pd.read_csv('static/mysite/nasdaq-listed-symbols_csv.csv')
symbols = symbols_file.pop('Symbol')
company_names = symbols_file.pop('Company Name')
data = []
preddata = []
# symbol_data = []
# prediction_data = []
count = 1

for symbol in range(len(symbols)):
    predict = []
    to_remove = []

    try:
        data = yf.download(tickers=symbols[symbol], period='max', interval='1d', group_by='ticker', auto_adjust=True,
                           prepost=True, proxy=None)
        print(symbols[symbol], count)
        OPEN, high, low, close, volume = data.pop('Open'), data.pop('High'), data.pop('Low'), data.pop(
            'Close'), data.pop('Volume')
        y = close
        y = y[1:]

        for i in range(len(close)):
            if np.isnan(close[i]):
                to_remove.append(i)
        for i in range(len(to_remove)):
            OPEN = OPEN[:to_remove[i] - i] + OPEN[to_remove[i] - i + 1:]
            high = high[:to_remove[i] - i] + high[to_remove[i] - i + 1:]
            low = low[:to_remove[i] - i] + low[to_remove[i] - i + 1:]
            close = close[:to_remove[i] - i] + close[to_remove[i] - i + 1:]
            volume = volume[:to_remove[i] - i] + volume[to_remove[i] - i + 1:]
            y = y[:to_remove[i] - i] + y[to_remove[i] - i + 1:]

        OPEN = (OPEN - min(OPEN)) / (max(OPEN) - min(OPEN))
        high = (high - min(high)) / (max(high) - min(high))
        low = (low - min(low)) / (max(low) - min(low))
        close = (close - min(close)) / (max(close) - min(close))
        volume = (volume - min(volume)) / (max(volume) - min(volume))

        for i in range(len(close)):
            predict.append([OPEN[i], high[i], low[i], close[i], volume[i]])

        predict, train_labels = np.array(predict), np.array(y)
        mx, mn = max(train_labels), min(train_labels)
        tl = train_labels
        train_labels = (train_labels - min(train_labels)) / (max(train_labels) - min(train_labels))
        train_data = predict[:-1]
        train_data = train_data.reshape(-1, 5, 1)

        model = Sequential()
        model.add(LSTM(256, activation='sigmoid', kernel_initializer='he_normal', input_shape=(train_data.shape[1:]),
                       return_sequences=True))
        model.add(LSTM(64, activation='sigmoid', kernel_initializer='he_normal', return_sequences=True))
        model.add(LSTM(64, activation='sigmoid', kernel_initializer='he_normal'))
        model.add(Dense(1, activation='linear'))

        model.compile(optimizer='adam', loss='mse', metrics=['mse'])
        model.fit(train_data, train_labels, epochs=10, batch_size=256, verbose=0)
        mse, mae = model.evaluate(train_data, train_labels, verbose=0)
        print('MSE: %.3f,  MAE: %.3f' % (mse, mae))

        predictions = model.predict(predict.reshape(-1, 5, 1))
        prediction = float(predictions[-1] * (mx - mn) + mn)
        print('Previous:', tl[-1], 'Prediction:', prediction)
        preddata.append([symbols[symbol], prediction, mse, np.array2string(np.flipud(tl[-7:]), separator=', ')[1:-1]])

        df = pd.DataFrame(np.array(preddata))
        for i in os.listdir('static/mysite/'):
            if i == 'data.csv':
                os.unlink('static/mysite/data.csv')
        df.to_csv('static/mysite/data.csv', index=False)
        # change to csv
        # data_file = open('static/mysite/prediction_data.txt', 'w')
        # for i in prediction_data:
        #     data_file.write(str(i))
        #     data_file.write(' ')
        # data_file.close()
        #
        # data_file = open('static/mysite/symbol_data.txt', 'w')
        # for i in symbol_data:
        #     data_file.write(str(i))
        #     data_file.write(' ')
        # data_file.close()
        # print('Predictions saved')

    except ValueError:
        print('Err~~')
        count -= 1

    count += 1

########################################################################################################################
# import numpy as np
# import yfinance as yf
# import pandas as pd
# from matplotlib.figure import Figure
# from tensorflow.keras import Sequential
# from tensorflow.keras.layers import Dense, LSTM
# import os
# import shutil
#
# symbols_file = pd.read_csv('static/mysite/nasdaq-listed-symbols_csv.csv')
# symbols = symbols_file.pop('Symbol')
# company_names = symbols_file.pop('Company Name')
#
# for file in os.listdir('static/mysite/saved_model_weights/'):
#     file_path = os.path.join('static/mysite/saved_model_weights/', file)
#     if os.path.isfile(file_path):
#         os.unlink(file_path)
#     else:
#         shutil.rmtree(file_path)
#
# for file in os.listdir('static/mysite/plots/'):
#     file_path = os.path.join('static/mysite/plots/', file)
#     os.unlink(file_path)
#
# for i in range(len(symbols)):
#     to_remove = []
#     count = 0
#     try:
#         data = yf.download(tickers=symbols[i], period='max', interval='1d', group_by='ticker', auto_adjust=True,
#                            prepost=True, proxy=None)
#         print(symbols[i], i)
#
#         train_data = np.array(data.pop('Open'))
#         train_labels = np.array(data.pop('Close'))
#         for _ in range(len(train_data)):
#             if np.isnan(train_data[_]):
#                 to_remove.append(_)
#
#         for _ in to_remove:
#             train_data = np.delete(train_data, _ - count)
#             count += 1
#         to_remove.clear()
#         count = 0
#
#         for _ in range(len(train_labels)):
#             if np.isnan(train_labels[_]):
#                 to_remove.append(_)
#
#         for _ in to_remove:
#             train_labels = np.delete(train_labels, _ - count)
#             count += 1
#
#         test = train_data
#         graph = train_data
#         close = train_labels
#         test = test.reshape(-1, 1, 1)
#         graph = graph.reshape(-1, 1, 1)
#         mx = max(train_data)
#         mn = min(train_data)
#         graph = (graph - mn) / (mx - mn)
#         train_data, train_labels = (train_data - mn) / (mx - mn), (train_labels - mn) / (mx - mn)
#         train_data = train_data.reshape(-1, 1, 1)
#
#         model = Sequential()
#         model.add(LSTM(256, activation='sigmoid', kernel_initializer='he_normal', input_shape=(train_data.shape[1:])))
#         model.add(Dense(128, activation='sigmoid', kernel_initializer='he_normal'))
#         model.add(Dense(64, activation='sigmoid', kernel_initializer='he_normal'))
#         model.add(Dense(1, activation='linear'))
#
#         model.compile(optimizer='adam', loss='mse', metrics=['mse'])
#         model.fit(train_data, train_labels, epochs=18, batch_size=256)
#         mse, mae = model.evaluate(train_data, train_labels, verbose=0)
#         print('MSE: %.3f,  MAE: %.3f' % (mse, mae))
#
#         predictions = model.predict(graph)
#         for _ in range(len(predictions)):
#             predictions[_] = predictions[_] * (mx - mn) + mn
#
#         fig = Figure()
#         ax = fig.subplots()
#         ax.plot(predictions, linewidth=0.6, label='Predicted Closing Price')
#         ax.plot(close, linewidth=0.6, label='Actual Closing Price')
#         ax.set_title(symbols[i] + ' (' + company_names[i] + ')' + ' Results From Model Training')
#         ax.set_xlabel('Days since IPO')
#         ax.set_ylabel('Price ($)')
#         ax.legend()
#         fig.savefig('static/mysite/plots/' + symbols[i] + '.PNG', dpi=300)
#
#         model.save('static/mysite/saved_model_weights/' + symbols[i] + '_saved_model.h5')
#         data_file = open(symbols[i] + ' data.txt', 'w')
#         for _ in test:
#             data_file.write(str(_[0][0]))
#             data_file.write(' ')
#         data_file.close()
#         os.rename(symbols[i] + ' data.txt', 'static/mysite/saved_model_weights/' + symbols[i] + ' data.txt')
#         print('Model saved')
#
#     except:
#         pass


# -*- coding: utf-8 -*-

from PyQt5 import QtGui, QtWidgets, QtCore
from PyQt5.QtCore import Qt

from forms.CM_EditForm import Ui_EditForm
from logic.CM_Abstracts import AbstractLogic
from model.CM_classes import InterfactorLink, AbstractFactor, QuantitativeFactor, OrdinalFactor, NominalFactor
from model.CM_funcs import number_2_quality, quality_2_number

class EditWindow(AbstractLogic, Ui_EditForm):

    def __init__(self, cognitive_model):
        super(EditWindow, self).__init__(cognitive_model)
        self.setupUi(self)
        self.model_changed()

        # Set form params
        self.spinRanks.setMaximum(20)
        self.spinNominals.setMaximum(20)
        self.spinNominals.setValue(3)
        self.spinRanks.setValue(3)
        self.ranks_changed(3)
        self.cats_changed(3)
        self.current_link = None
        self.current_factor = None

        # Set text parameters
        self.textEdit.setPlaceholderText('Не более 300 символов')
        self.nameEdit.setPlaceholderText('Не более 50 символов')
        self.nameEditFactor.setPlaceholderText('Не более 50 символов')
        reg = QtCore.QRegExp("[а-яА-Я0-9a-zA-ZёЁ \\/\\.\\,\\*\\(\\)\\[\\]\\{\\}\\$\\\\\\%\\№\\#\\!\\?\\-\\+]{50}")
        validator = QtGui.QRegExpValidator(reg)
        self.nameEdit.setValidator(validator)
        self.nameEditFactor.setValidator(validator)

        # Connect events on signals for description group
        self.btnSort.clicked.connect(self.clicked_sort)
        self.btnChangeText.clicked.connect(self.clicked_change_text)
        # Connect events on signals for factors group
        self.listFactors.itemClicked.connect(self.factor_selected)
        self.spinRanks.valueChanged.connect(self.ranks_changed)
        self.spinNominals.valueChanged.connect(self.cats_changed)
        self.btnDeleteFactor.clicked.connect(self.clicked_delete_factor)
        self.btnAddFactor.clicked.connect(self.clicked_add_factor)
        # Connect events on signals for links group
        self.comboFactorFrom.currentIndexChanged.connect(self.check_factors_scales)
        self.comboFactorTo.currentIndexChanged.connect(self.check_factors_scales)
        self.listLinks.itemClicked.connect(self.link_selected)
        self.spinStrength.valueChanged.connect(self.check_quality)
        self.comboQuality.activated.connect(self.check_strength)
        self.btnDeleteLink.clicked.connect(self.clicked_delete_link)
        self.btnAcceptLink.clicked.connect(self.clicked_accept_link)

    def model_changed(self):
        # Change form
        self.comboFactorTo.clear()
        self.comboFactorFrom.clear()
        self.listLinks.clear()
        self.listFactors.clear()
        self.btnDeleteLink.setDisabled(True)
        self.btnDeleteFactor.setDisabled(True)
        # Text
        self.nameEdit.setText(self.cognitive_model.name)
        self.textEdit.setPlainText(self.cognitive_model.description)
        # Change factors
        factors: list[AbstractFactor] = self.cognitive_model.get_factors()
        for f in factors:
            item = QtWidgets.QListWidgetItem(str(f))
            item.setData(Qt.UserRole + 1, f)
            self.listFactors.addItem(item)
            self.comboFactorTo.addItem(str(f.get_id()), f)
            self.comboFactorFrom.addItem(str(f.get_id()), f)
        # Change links
        links = self.cognitive_model.get_links()
        for L in links:
            item = QtWidgets.QListWidgetItem(str(L))
            item.setData(Qt.UserRole + 1, L)
            self.listLinks.addItem(item)

    def check_factors_scales(self, ind):
        i = self.comboFactorFrom.currentIndex()
        j = self.comboFactorTo.currentIndex()
        if i < 0 or j < 0:
            pass
        factor_from = self.comboFactorFrom.itemData(i)
        factor_to = self.comboFactorTo.itemData(j)
        if factor_from and factor_to:
            self.check_nominals_link(factor_from, factor_to)

    def link_selected(self, item: QtWidgets.QListWidgetItem):
        link: InterfactorLink = item.data(Qt.UserRole + 1)
        self.current_link = link
        self.btnDeleteLink.setEnabled(True)
        self.spinStrength.setValue(abs(round(link.strength * 100)))
        if link.strength > 0:
            self.radioPositive.setChecked(True)
        else:
            self.radioNegative.setChecked(True)
        self.check_nominals_link(link.factor_from, link.factor_to)
        i, j = link.get_factors_numbers()
        self.comboFactorFrom.setCurrentIndex(i - 1)
        self.comboFactorTo.setCurrentIndex(j - 1)

    def check_nominals_link(self, factor_from, factor_to):
        scale = max(factor_to.scale, factor_from.scale)
        if scale == 2:
            self.radioPositive.setChecked(True)
            self.radioPositive.setDisabled(True)
            self.radioNegative.setDisabled(True)
        else:
            self.radioPositive.setEnabled(True)
            self.radioNegative.setEnabled(True)

    def check_quality(self, strength: int):
        self.comboQuality.setCurrentIndex(number_2_quality(strength))

    def check_strength(self, index: int):
        self.spinStrength.setValue(quality_2_number(index))

    def clicked_delete_link(self):
        if not self.current_link:
            self.btnDeleteLink.setDisabled(True)
        self.cognitive_model.del_link(self.current_link)
        self.notify_main()

    def clicked_accept_link(self):
        i = self.comboFactorFrom.currentIndex()
        j = self.comboFactorTo.currentIndex()
        if i < 0 or j < 0:
            pass
        if i == j:
            QtWidgets.QMessageBox.information(self,
                                              "Неправильная связь",
                                              "Петли запрещены",
                                              buttons=QtWidgets.QMessageBox.Ok)
            return
        factor_from = self.comboFactorFrom.itemData(i)
        factor_to = self.comboFactorTo.itemData(j)
        if factor_to.role < 2:
            QtWidgets.QMessageBox.information(self,
                                              "Неправильная связь",
                                              "Входной фактор не может быть зависимым",
                                              buttons=QtWidgets.QMessageBox.Ok)
            return
        strength = self.spinStrength.value()
        sign = 1 if self.radioPositive.isChecked() else -1
        self.cognitive_model.add_link(factor_from, factor_to, strength * sign / 100)
        self.notify_main()

    def clicked_sort(self):
        if self.cognitive_model.check_cycles():
            QtWidgets.QMessageBox.information(self,
                                              "Обнаружен цикл",
                                              "В модели обнаружены циклы, поэтому топологическая "
                                              "сортировка факторов невозможна!",
                                              buttons=QtWidgets.QMessageBox.Ok)
            return
        self.cognitive_model.sort_factors()
        self.notify_main()

    def clicked_change_text(self):
        name = self.nameEdit.text()
        if not name:
            QtWidgets.QMessageBox.information(self,
                                              "Ошибка ввода",
                                              "Отсутствует имя модели",
                                              buttons=QtWidgets.QMessageBox.Ok)
            return
        self.cognitive_model.name = name
        self.cognitive_model.description = self.textEdit.document().toPlainText()[:300]

    def factor_selected(self, item: QtWidgets.QListWidgetItem):
        factor: AbstractFactor = item.data(Qt.UserRole + 1)
        self.current_factor = factor
        self.btnDeleteFactor.setEnabled(True)
        self.comboRole.setCurrentIndex(factor.role)
        self.comboScale.setCurrentIndex(factor.scale)
        self.nameEditFactor.setText(factor.name)
        if factor.scale == 0:
            factor: QuantitativeFactor
            self.spinMax.setValue(factor.max_value)
            self.spinMin.setValue(factor.min_value)
            self.spinMean.setValue(factor.mean)
            self.spinDeviation.setValue(factor.standard_deviation)
        elif factor.scale == 1:
            factor: OrdinalFactor
            self.spinRanks.setValue(factor.max_value)
            self.ranks_changed(factor.max_value)
            for i, prob in enumerate(factor.ranks_probabilities):
                self.tableRanks.cellWidget(i, 0).setValue(round(prob * 100))
        elif factor.scale == 2:
            factor: NominalFactor
            self.spinNominals.setValue(len(factor.cats))
            self.cats_changed(len(factor.cats))
            for i, prob in enumerate(factor.cats_probabilities):
                self.tableNominals.cellWidget(i, 1).setValue(round(prob * 100))
                self.tableNominals.cellWidget(i, 0).setText(factor.cats[i])

    def ranks_changed(self, max_rank: int):
        n = self.tableRanks.rowCount()
        if self.tableRanks.rowCount() == max_rank:
            return
        self.tableRanks.setRowCount(max_rank)
        for i in range(n, max_rank):
            spin = QtWidgets.QSpinBox()
            spin.setMaximum(99)
            spin.setMinimum(1)
            spin.setValue(100//max_rank)
            self.tableRanks.setCellWidget(i, 0, spin)

    def cats_changed(self, cats_count: int):
        n = self.tableNominals.rowCount()
        if self.tableNominals.rowCount() == cats_count:
            return
        self.tableNominals.setRowCount(cats_count)
        for i in range(n, cats_count):
            # Prob of nominal
            spin = QtWidgets.QSpinBox()
            spin.setMaximum(99)
            spin.setMinimum(1)
            spin.setValue(100//cats_count)
            self.tableNominals.setCellWidget(i, 1, spin)
            # Name of nominal
            line = QtWidgets.QLineEdit()
            reg = QtCore.QRegExp("[а-яА-Я0-9a-zA-ZёЁ \\/\\.\\,\\*\\(\\)\\[\\]\\{\\}\\$\\\\\\%\\№\\#\\!\\?\\-\\+]{50}")
            validator = QtGui.QRegExpValidator(reg)
            line.setValidator(validator)
            line.setText(chr(ord('A') + i))
            self.tableNominals.setCellWidget(i, 0, line)

    def clicked_delete_factor(self):
        if not self.current_factor:
            self.btnDeleteFactor.setDisabled(True)
        self.cognitive_model.del_factor(self.current_factor)
        self.notify_main()

    def clicked_add_factor(self):
        name = self.nameEditFactor.text()
        if not name:
            QtWidgets.QMessageBox.information(self,
                                              "Ошибка ввода",
                                              "Отсутствует имя фактора",
                                              buttons=QtWidgets.QMessageBox.Ok)
            return
        role = self.comboRole.currentIndex()
        scale = self.comboScale.currentIndex()
        if role < 0 or role < 0:
            return
        if scale == 0:
            min_value = self.spinMin.value()
            max_value = self.spinMax.value()
            mean = self.spinMean.value()
            deviation = self.spinDeviation.value()
            factor = QuantitativeFactor(name, role, max_value=max_value, min_value=min_value, mean=mean,
                                        standard_deviation=deviation)
        elif scale == 1:
            max_value = self.spinRanks.value()
            ranks_probabilities = [self.tableRanks.cellWidget(i, 0).value()/100 for i in range(max_value)]
            factor = OrdinalFactor(name, role, max_value=max_value, ranks_probabilities=ranks_probabilities)
        else:
            count = self.spinNominals.value()
            cats = []
            cats_probabilities = []
            for i in range(count):
                text = self.tableNominals.cellWidget(i, 0).text().strip()
                prob = self.tableNominals.cellWidget(i, 1).value()/100
                if not text:
                    QtWidgets.QMessageBox.information(self,
                                                      "Ошибка ввода",
                                                      "Отсутствует номинал",
                                                      buttons=QtWidgets.QMessageBox.Ok)
                    return
                cats.append(text)
                cats_probabilities.append(prob)
            if len(set(cats)) != count:
                QtWidgets.QMessageBox.information(self,
                                                  "Повторяющиеся номиналы",
                                                  "Среди номниалов есть одинаковые, "
                                                  "поэтому добавление фактора невозможно",
                                                  buttons=QtWidgets.QMessageBox.Ok)
                return
            factor = NominalFactor(name, role, cats=cats, cats_probabilities=cats_probabilities)
        if not self.cognitive_model.add_factor(factor):
            QtWidgets.QMessageBox.information(self,
                                              "Имя уже занято",
                                              "Имя этого фактора уже занято, выберите другое",
                                              buttons=QtWidgets.QMessageBox.Ok)
        self.notify_main()


"""
Contains the script for evaluating the baseline models
On the newly unseen data
Preprocessing done in Notebook "2020.12.3_genotype_baselines_newdataset.ipynb"

M. Togninalli, 01.2021
"""

import numpy as np
import pandas as pd
import os
import json
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from scipy.stats import pearsonr
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler

from sklearn.linear_model import LassoCV

from sklearn.linear_model import ElasticNet, Lasso, Ridge
from sklearn.ensemble import RandomForestRegressor
import pickle

# Reduced set of features available for newer data set, to get same order
FEATURE_COLUMNS = ['BLUE_Mean', 'GNDVI_Mean',
       'GREEN_Mean', 'NDRE_Mean', 'NDVI_Mean',
       'NIR_Mean','REDEDGE_Mean', 'RED_Mean']

# Train and test
def get_xy_df(df):
    x = df.drop(columns=['GRYLD', 'Plot_ID']).values
    # x = df[FEATURE_COLUMNS].values
    y = df['GRYLD'].values
    return x,y

def main():
    # OPTION:
    optimize_params = True

    # Load the genotypes of new plots.
    df_inference = pd.read_csv('/links/groups/borgwardt/Data/Jesse_2018/202010_new_testset/csv/'\
                            'df_20201110_numpy_MIL_npy_coordinates_geno_filtered.csv')
    # 2. Load genotypes
    # Alternative way of loading genotypes
    base_path = '/links/groups/borgwardt/Data/Jesse_2018/numpy_MIL_resized/genotypes'
    # Remove known genotypes
    old_gids = np.loadtxt('/links/groups/borgwardt/Data/Jesse_2018/genotypes/gids.txt', dtype=str)

    index = df_inference.query(
        "gid not in @old_gids")['gid'].unique()
    X = np.array([np.load(f'{base_path}/{x}.npy') for x in df_inference.query(
        "gid not in @old_gids")['gid'].unique()]).astype(np.float32)
    df_genotype_processed = pd.DataFrame(X, index=index)

    # 3. Load VIs
    df_vis = pd.read_csv('/links/groups/borgwardt/Data/Jesse_2018/202010_new_testset/csv/df_20210125_VIs.csv')
    date_group = 2 # Change this potentially
    # Only keep the relevant group
    df_vis = df_vis[df_vis['date_group']==date_group]

    # Concatenate the dataframes
    df_vis['gid'] = df_vis['Plot_ID'].map(df_inference.set_index('plot_id').gid)
    df_vis = df_vis.dropna()
    df_vis = df_vis.join(df_genotype_processed, on='gid', how='left').dropna()
    # Remove unnecessary infor
    gids = df_vis['gid']
    plot_ids = df_vis['Plot_ID']
    df_vis = df_vis.drop(columns=['date_group', 'gid'])
    del df_genotype_processed
    print(f'DFs loaded and concatenated: dims {df_vis.shape}')

    results = []
    split_id_range = 5

    # Obtain X and y
    X_test, y_test = get_xy_df(df_vis)
    print(f'{len(X_test)} inference samples.')

    # Iterate over splits
    output_path = f'/home/tomatteo/Projects/yield_prediction/results/baselines_202012/VIs'
    for i in range(split_id_range):
        # Retrieve the splitted data
        print(f'=============== VIs - Genotype baseline - Data Preprocessing - split {i} ===============')
        yscaler = pickle.load(open(os.path.join(output_path,f'y_scaler_fold{i}.pkl'), 'rb'))
        scaler = pickle.load(open(os.path.join(output_path,f'scaler_fold{i}.pkl'), 'rb'))
        lasso = pickle.load(open(os.path.join(output_path,f'trained_geno_vis_lasso_fold{i}_hyperopt_{optimize_params}.pkl'), 'rb'))
        rf = pickle.load(open(os.path.join(output_path,f'trained_geno_vis_rf_fold{i}_hyperopt_{optimize_params}.pkl'), 'rb'))

        X_test = scaler.transform(X_test)

        print('\tData loaded and scaled.')

        print(f'=============== VIs - Genotype baseline - Random Forest - split {i} ===============')
        # Evaluation
        y_pred = yscaler.inverse_transform(rf.predict(X_test))

        r2_sc = r2_score(y_test, y_pred)
        mse = mean_squared_error(y_test, y_pred)
        mae = mean_absolute_error(y_test, y_pred)
        pears = pearsonr(y_test, y_pred)[0]

        results.append(['full_randomforest', i, mae, mse, r2_sc, pears, date_group])
        print('MSE: {:.4f}, R2_SCORE: {:.4f}, Pearson: {:.4f}'.format(mse, r2_sc, pears))
        print()

        pd.DataFrame(np.array([plot_ids,gids,y_test,y_pred]).T, columns=['plot_id','gid','target_yield', 'predicted_yield']).to_csv(
                 os.path.join(output_path, f'results_rf_hyperopt_{optimize_params}_split{i}.csv'), index=False)

        print(f'=============== VIs - Genotype baseline - Lasso - split {i} ===============')

        y_pred = yscaler.inverse_transform(lasso.predict(X_test))

        r2_sc = r2_score(y_test, y_pred)
        mse = mean_squared_error(y_test, y_pred)
        mae = mean_absolute_error(y_test, y_pred)
        pears = pearsonr(y_test, y_pred)[0]

        results.append(['Lasso', i, mae, mse, r2_sc, pears, date_group])
        print('MSE: {:.4f}, R2_SCORE: {:.4f}, Pearson: {:.4f}'.format(mse, r2_sc, pears))
        print()

        pd.DataFrame(np.array([plot_ids,gids,y_test,y_pred]).T, columns=['plot_id','gis','target_yield', 'predicted_yield']).to_csv(
                 os.path.join(output_path, f'results_lasso_hyperopt_{optimize_params}_split{i}.csv'), index=False)

    # Save results
    pd.DataFrame(results, columns=['method', 'split_id', 'mae', 'mse', 'r2_sc', 'pears', 'date_group']).to_csv(
        f'/home/tomatteo/Projects/yield_prediction/results/baselines_202012/VIs/new_testset_results_20210126_hyperopt_{optimize_params}.csv', index=False
    )
    return

if __name__== '__main__':
    main()


import unittest
import itertools

from typing import List
from dbt.exceptions import VersionsNotCompatibleError
from dbt.semver import (
    VersionSpecifier,
    UnboundedVersionSpecifier,
    VersionRange,
    reduce_versions,
    versions_compatible,
    resolve_to_specific_version,
    filter_installable,
)

def semver_regex_versioning(versions: List[str]) -> bool:
    for version_string in versions:
        try:
            VersionSpecifier.from_version_string(version_string)
        except Exception:
            return False
    return True

def create_range(start_version_string, end_version_string):
    start = UnboundedVersionSpecifier()
    end = UnboundedVersionSpecifier()

    if start_version_string is not None:
        start = VersionSpecifier.from_version_string(start_version_string)

    if end_version_string is not None:
        end = VersionSpecifier.from_version_string(end_version_string)

    return VersionRange(start=start, end=end)

class TestSemver(unittest.TestCase):
    def assertVersionSetResult(self, inputs, output_range):
        expected = create_range(*output_range)

        for permutation in itertools.permutations(inputs):
            self.assertEqual(reduce_versions(*permutation), expected)

    def assertInvalidVersionSet(self, inputs):
        for permutation in itertools.permutations(inputs):
            with self.assertRaises(VersionsNotCompatibleError):
                reduce_versions(*permutation)

    def test__versions_compatible(self):
        self.assertTrue(versions_compatible("0.0.1", "0.0.1"))
        self.assertFalse(versions_compatible("0.0.1", "0.0.2"))
        self.assertTrue(versions_compatible(">0.0.1", "0.0.2"))
        self.assertFalse(versions_compatible("0.4.5a1", "0.4.5a2"))

    def test__semver_regex_versions(self):
        self.assertTrue(
            semver_regex_versioning(
                [
                    "0.0.4",
                    "1.2.3",
                    "10.20.30",
                    "1.1.2-prerelease+meta",
                    "1.1.2+meta",
                    "1.1.2+meta-valid",
                    "1.0.0-alpha",
                    "1.0.0-beta",
                    "1.0.0-alpha.beta",
                    "1.0.0-alpha.beta.1",
                    "1.0.0-alpha.1",
                    "1.0.0-alpha0.valid",
                    "1.0.0-alpha.0valid",
                    "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay",
                    "1.0.0-rc.1+build.1",
                    "2.0.0-rc.1+build.123",
                    "1.2.3-beta",
                    "10.2.3-DEV-SNAPSHOT",
                    "1.2.3-SNAPSHOT-123",
                    "1.0.0",
                    "2.0.0",
                    "1.1.7",
                    "2.0.0+build.1848",
                    "2.0.1-alpha.1227",
                    "1.0.0-alpha+beta",
                    "1.2.3----RC-SNAPSHOT.12.9.1--.12+788",
                    "1.2.3----R-S.12.9.1--.12+meta",
                    "1.2.3----RC-SNAPSHOT.12.9.1--.12",
                    "1.0.0+0.build.1-rc.10000aaa-kk-0.1",
                    "99999999999999999999999.999999999999999999.99999999999999999",
                    "1.0.0-0A.is.legal",
                ]
            )
        )

        self.assertFalse(
            semver_regex_versioning(
                [
                    "1",
                    "1.2",
                    "1.2.3-0123",
                    "1.2.3-0123.0123",
                    "1.1.2+.123",
                    "+invalid",
                    "-invalid",
                    "-invalid+invalid",
                    "-invalid.01",
                    "alpha",
                    "alpha.beta",
                    "alpha.beta.1",
                    "alpha.1",
                    "alpha+beta",
                    "alpha_beta",
                    "alpha.",
                    "alpha..",
                    "beta",
                    "1.0.0-alpha_beta",
                    "-alpha.",
                    "1.0.0-alpha..",
                    "1.0.0-alpha..1",
                    "1.0.0-alpha...1",
                    "1.0.0-alpha....1",
                    "1.0.0-alpha.....1",
                    "1.0.0-alpha......1",
                    "1.0.0-alpha.......1",
                    "01.1.1",
                    "1.01.1",
                    "1.1.01",
                    "1.2",
                    "1.2.3.DEV",
                    "1.2-SNAPSHOT",
                    "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788",
                    "1.2-RC-SNAPSHOT",
                    "-1.0.3-gamma+b7718",
                    "+justmeta",
                    "9.8.7+meta+meta",
                    "9.8.7-whatever+meta+meta",
                    "99999999999999999999999.999999999999999999.99999999999999999----RC-SNAPSHOT.12.09.1--------------------------------..12",
                ]
            )
        )

    def test__reduce_versions(self):
        self.assertVersionSetResult(["0.0.1", "0.0.1"], ["=0.0.1", "=0.0.1"])

        self.assertVersionSetResult(["0.0.1"], ["=0.0.1", "=0.0.1"])

        self.assertVersionSetResult([">0.0.1"], [">0.0.1", None])

        self.assertVersionSetResult(["<0.0.1"], [None, "<0.0.1"])

        self.assertVersionSetResult([">0.0.1", "0.0.2"], ["=0.0.2", "=0.0.2"])

        self.assertVersionSetResult(["0.0.2", ">=0.0.2"], ["=0.0.2", "=0.0.2"])

        self.assertVersionSetResult([">0.0.1", ">0.0.2", ">0.0.3"], [">0.0.3", None])

        self.assertVersionSetResult([">0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"])

        self.assertVersionSetResult([">0.0.1", "0.0.2", "<0.0.3"], ["=0.0.2", "=0.0.2"])

        self.assertVersionSetResult([">0.0.1", ">=0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"])

        self.assertVersionSetResult([">0.0.1", "<0.0.3", "<=0.0.3"], [">0.0.1", "<0.0.3"])

        self.assertVersionSetResult([">0.0.1", ">0.0.2", "<0.0.3", "<0.0.4"], [">0.0.2", "<0.0.3"])

        self.assertVersionSetResult(["<=0.0.3", ">=0.0.3"], [">=0.0.3", "<=0.0.3"])

        self.assertInvalidVersionSet([">0.0.2", "0.0.1"])
        self.assertInvalidVersionSet([">0.0.2", "0.0.2"])
        self.assertInvalidVersionSet(["<0.0.2", "0.0.2"])
        self.assertInvalidVersionSet(["<0.0.2", ">0.0.3"])
        self.assertInvalidVersionSet(["<=0.0.3", ">0.0.3"])
        self.assertInvalidVersionSet(["<0.0.3", ">=0.0.3"])
        self.assertInvalidVersionSet(["<0.0.3", ">0.0.3"])

    def test__resolve_to_specific_version(self):
        self.assertEqual(
            resolve_to_specific_version(create_range(">0.0.1", None), ["0.0.1", "0.0.2"]), "0.0.2"
        )

        self.assertEqual(
            resolve_to_specific_version(create_range(">=0.0.2", None), ["0.0.1", "0.0.2"]), "0.0.2"
        )

        self.assertEqual(
            resolve_to_specific_version(create_range(">=0.0.3", None), ["0.0.1", "0.0.2"]), None
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(">=0.0.3", "<0.0.5"), ["0.0.3", "0.0.4", "0.0.5"]
            ),
            "0.0.4",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(None, "<=0.0.5"), ["0.0.3", "0.1.4", "0.0.5"]
            ),
            "0.0.5",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range("=0.4.5a2", "=0.4.5a2"), ["0.4.5a1", "0.4.5a2"]
            ),
            "0.4.5a2",
        )

        self.assertEqual(
            resolve_to_specific_version(create_range("=0.7.6", "=0.7.6"), ["0.7.6-b1", "0.7.6"]),
            "0.7.6",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"]
            ),
            "1.2.0a1",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(">=1.0.0", "<1.2.0"), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"]
            ),
            "1.1.0",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"]
            ),
            "1.2.0",
        )

        self.assertEqual(
            resolve_to_specific_version(
                create_range(">=1.0.0", "<1.2.0"),
                ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"],
            ),
            "1.1.0",
        )

        self.assertEqual(
            resolve_to_specific_version(
                # https://github.com/dbt-labs/dbt-core/issues/7039
                # 10 is greater than 9
                create_range(">0.9.0", "<0.10.0"),
                ["0.9.0", "0.9.1", "0.10.0"],
            ),
            "0.9.1",
        )

    def test__filter_installable(self):
        installable = filter_installable(
            [
                "1.1.0",
                "1.2.0a1",
                "1.0.0",
                "2.1.0-alpha",
                "2.2.0asdf",
                "2.1.0",
                "2.2.0",
                "2.2.0-fishtown-beta",
                "2.2.0-2",
            ],
            install_prerelease=True,
        )
        expected = [
            "1.0.0",
            "1.1.0",
            "1.2.0a1",
            "2.1.0-alpha",
            "2.1.0",
            "2.2.0-2",
            "2.2.0asdf",
            "2.2.0-fishtown-beta",
            "2.2.0",
        ]
        assert installable == expected

        installable = filter_installable(
            [
                "1.1.0",
                "1.2.0a1",
                "1.0.0",
                "2.1.0-alpha",
                "2.2.0asdf",
                "2.1.0",
                "2.2.0",
                "2.2.0-fishtown-beta",
            ],
            install_prerelease=False,
        )
        expected = ["1.0.0", "1.1.0", "2.1.0", "2.2.0"]
        assert installable == expected


import cv2
import dlib
import torch
import argparse
import torchvision
import numpy as np
import torch.nn as nn
from skimage.transform import resize

# image = cv2.imread('./test_pic/AF1.jpg')
# image = cv2.imread('./test_pic/AF1675.jpg')
# image = cv2.imread('./test_pic/a.png')
# image = cv2.imread('./test_pic/b.png')
image = cv2.imread('./test_pic/AF1053.jpg')
detector = dlib.get_frontal_face_detector()
rects = detector(image, 1)

def key_points(img):
    points_keys = []
    PREDICTOR_PATH = "shape_predictor_68_face_landmarks.dat"
    predictor = dlib.shape_predictor(PREDICTOR_PATH)

    # for i in range(len(rects)):
    # len(rects)代表人脸个数
    landmarks = np.matrix([[p.x,p.y] for p in predictor(img,rects[0]).parts()])
    maxv = np.amax(landmarks, axis=0)
    minv = np.amin(landmarks, axis=0)
    img = img.copy()
    for idx, point in enumerate(landmarks):
        pos = (point[0,0],point[0,1])
        points_keys.append(pos)
        cv2.circle(img,pos,2,(255,0,0),-1)
    return img, maxv, minv

face_key, maxv, minv = key_points(image)
lefttop_x = minv.A[0][0]
lefttop_y = minv.A[0][1]
rightbottom_x = maxv.A[0][0]
rightbottom_y = maxv.A[0][1]
# print(lefttop_x,lefttop_y,rightbottom_x,rightbottom_y)
cv2.rectangle(face_key, (lefttop_x, lefttop_y), (rightbottom_x, rightbottom_y), (0, 255, 0), 2)

cv2.namedWindow('result', cv2.WINDOW_AUTOSIZE)
cv2.imshow('result', face_key)
cv2.waitKey(0)
cv2.destroyAllWindows()


from datetime import datetime

#armazenamento de compras
compras = {
  "Pedro": {
    "cliente": "Pedro",
    "produtos": ["1", "5"],
    "total": 35.00,
    "created_at": datetime.today().strftime('%d/%m/%Y')
    },
    "Sergio": {
    "cliente": "Sergio",
    "produtos": ["3"],
    "total": 18.00,
    "created_at": datetime.today().strftime('%d/%m/%Y')
    },
}

class SalesRepo:

  #adicionar uma compra
  def add(cliente: str, codigosProdutos: list, precoTotal: float) -> dict:
    #cria dicionario de dados da compra
    compra = {"cliente": cliente, "produtos": codigosProdutos, "total": precoTotal, "created_at": datetime.today().strftime('%d/%m/%Y')}
    #adiciona a compra ao armazenamento
    compras['cliente'] = compra
    return compra

  #retorna o dicionario de compras
  def getAll() -> dict:
    return compras


import sys
import base64

def get_license_key(name, license):

    # Compute the base64 license key. This key will be provided to you by Vade Secure,
    # and has the following format: <CUSTOMER_NAME>:<CUSTOMER_LICENSE>.
    url_key = u'{0}:{1}'.format(name, license)

    # It must be Base64-encoded. Handled different on Python 2 vs 3.
    if sys.version_info[0] == 2:
        auth_token = base64.b64encode(bytes(url_key).encode("utf-8"))
    else:
        auth_token = base64.b64encode(bytes(url_key, 'ascii')).decode('ascii')

    return auth_token


import requests
from lxml import etree

import os
import time

import conf

def getHtml(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    return resp.text

def parseChapterURL(html):
    root = etree.HTML(html)
    title = root.xpath('//*[(@id = "list")]//a/text()')
    # print(title)
    chapterUrlList = root.xpath('//*[(@id = "list")]//a/@href')
    print('title:', len(title))
    print('url:', len(chapterUrlList))
    result = []
    for i in zip(title, chapterUrlList):
        # 判断是否为章节链接：使用正则表达式
        if i[0].startswith('第'):
            result.append(i)
        # result.append(i)
    return result

def parseChapterContent(html):
    root = etree.HTML(html)
    content = []
    title = ''.join(root.xpath('//*[(@id = "neirongDiv")]//h1/text()')).strip()
    content.append(title)
    paragraph = root.xpath('//*[(@id = "zjneirong")]/p/text()') if root.xpath('//*[(@id = "zjneirong")]/p/text()') else root.xpath('//*[(@id = "zjneirong")]/text()')
    for text in paragraph:
        content.append(text.strip())
    return (os.linesep * 2).join(content)

def saveTxt(content, fileName=conf.FILE_NAME, mode='a', encoding='utf-8'):
    with open(file=fileName, mode=mode, encoding=encoding) as fw:
        fw.write(content)
        fw.write(os.linesep*2)

def test():
    # print(parseChapterURL(getHtml(conf.TXT_URL)))
    # print(parseChapterContent(getHtml(conf.CHAPTER_TEST_URL)))
    chapterUrlList =parseChapterURL(getHtml(conf.TXT_URL))
    print(len(chapterUrlList))
    for url in chapterUrlList:
        content = parseChapterContent(getHtml(url[1]))
        saveTxt(content)
        print(url[0], '下载完成')
        time.sleep(1)
    print('全部下载完成')

def main():
    test()

if __name__ == '__main__':
    main()