#!/usr/bin/python3
# -*- coding: utf-8 -*-

'''
Working from left-to-right if no digit is exceeded by the digit to its left it is called an increasing number; for example, 134468.

Similarly if no digit is exceeded by the digit to its right it is called a decreasing number; for example, 66420.

We shall call a positive integer that is neither increasing nor decreasing a "bouncy" number; for example, 155349.

Clearly there cannot be any bouncy numbers below one-hundred, but just over half of the numbers below one-thousand (525) are bouncy.
In fact, the least number for which the proportion of bouncy numbers first reaches 50% is 538.

As n increases, the proportion of bouncy numbers below n increases such that
there are only 12951 numbers below one-million that are not bouncy and only 277032 non-bouncy numbers below 10**10.

How many numbers below a googol (10**100) are not bouncy?

Notes:
* compute number of constant numbers
* use table [L][10] to compute increasing numbers
* use table [L][10] to compute decreasing numbers
* result = inc + dec - const

'''

LEN = 100
#LEN=10 # 277032
#LEN=6 # 12951
#LEN=3  # 474
#LEN=2 # 99

def computeInc(tab):
   for d in range(10):
      tab[0][d] = 1

   for l in range(1,LEN):
     for d in range(10):
      s = 0
      for i in range(d+1):
         s = s + tab[l-1][i]
      tab[l][d] = s

def removeZeroes(incTab):
    for i in range(LEN):
      incTab[i][0] = 0

def computeDec(tab):
   for d in range(10):
      tab[0][d] = 1

   for l in range(1,LEN):
     for d in range(10):
      s = 0
      for i in range(d, 10):
         s = s + tab[l-1][i]
      tab[l][d] = s

def tableSum(tab):
    s = 0
    for i in range(LEN):
      s = s + sum(tab[i])
    return s

def main():
    incTab = [[0 for y in range(10)] for x in range(LEN)]
    decTab = [[0 for y in range(10)] for x in range(LEN)]
    computeInc(incTab)
    removeZeroes(incTab)
    computeDec(decTab)
    removeZeroes(decTab)
    for r in incTab:print(r)
    print('-')
    for r in decTab:print(r)
    si = tableSum(incTab)
    sd = tableSum(decTab)
    md = 9 * (LEN)
    print('%i + %i - %i = %i' % (si, sd, md, si+sd-md))

if  __name__ =='__main__':main()


class Washer:  # Subsystem 1
    def wash(self, clothes):
        print("Washing {}".format(','.join(clothes)))

class Dryer:  # Subsystem 2
    def dry(self, clothes):
        print("Drying {}".format(','.join(clothes)))

class Folder:  # Subsystem 3
    def fold(self, clothes):
        print("Folding {}".format(','.join(clothes)))

class CleaningMachine:  # Facade
    washer = None
    dryer = None
    folder = None

    def __init__(self, washer, dryer, folder):
        self.washer = washer
        self.dryer = dryer
        self.folder = folder

    def clean_clothes(self, clothes):  # Easy to use combining methods from subsystem
        self.washer.wash(clothes)
        self.dryer.dry(clothes)
        self.folder.fold(clothes)

if __name__ == "__main__":
    clothes = ['T-Shirt', 'Pants', 'Jeans', "Jacket"]
    washer = Washer()
    dryer = Dryer()
    folder = Folder()

    cleaner = CleaningMachine(washer=washer, dryer=dryer, folder=folder)
    cleaner.clean_clothes(clothes)


from dotenv import load_dotenv
import os

from pymongo import MongoClient

class MongoDB:
    def __init__(self):
        load_dotenv()
        self.string_connection = os.getenv("MONGODB_STRING_CONNECTION")
        self.database = os.getenv("MONGODB_DATABASE")
        self.collection_name = os.getenv("MONGODB_COLLECTION")
        self.__get_connection()

    def __get_connection(self):
        client = MongoClient(self.string_connection)
        database = client[self.database]
        self.collection = database[self.collection_name]

    def get_data(self, criterial):
        return self.collection.find(criterial)


import logging
import os
from collections import Counter
from heapq import nlargest
from string import punctuation
import numpy as np
import pandas as pd
import spacy
import streamlit as st
import plotly.graph_objs as go
import plotly.figure_factory as ff
import plotly.express as px
import nltk
import pyLDAvis
import pyLDAvis.gensim_models
import pyLDAvis.sklearn
import textattack.models.wrappers
import torch
import tensorflow as tf
import matplotlib.pyplot as plt
import transformers

from io import StringIO
from fastapi import APIRouter, HTTPException, File, UploadFile
from fastapi.encoders import jsonable_encoder
from operator import itemgetter
from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline, AutoModelForSequenceClassification
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English
from spacy import displacy
from wordcloud import WordCloud
from textblob import TextBlob

# API router
router = APIRouter(prefix='/endpoints/toolkit',
                   tags=['toolkit'],
                   responses={200: {'description': 'OK'},
                              404: {'description': 'Resource Not Found'},
                              415: {'description': 'Unsupported Media Type'}})

# file counter
fc = 0

def summarise(text, stopwords, pos_tag, nlp, sent_count):
    """
    This function summarise the text dataframe

    Parameters
    ----------
    text:                               DataFrame
    nlp:                                NLP model
    pos_tag:                            Text pos tag
    stopwords:                          Stopwords
    sent_count:                         Number of sentences to summarise to
    ----------
    """

    try:
        # DEFINE LISTS AND DICTS
        keyword = []
        sent_strength = {}
        data = nlp(str(text))

        # EXTRACT KEYWORDS FROM TEXT
        for token in data:
            if token.text in stopwords or token.text in punctuation:
                continue
            if token.pos_ in pos_tag:
                keyword.append(token.text)

        # COUNT THE FREQUENCY OF WORDS
        freq_word = Counter(keyword)
        max_freq = Counter(keyword).most_common(1)[0][1]
        for word in freq_word.keys():
            freq_word[word] = (freq_word[word] / max_freq)

        # CALCULATE SENTENCE SCORES
        for sent in data.sents:
            for word in sent:
                if word.text in freq_word.keys():
                    if sent in sent_strength.keys():
                        sent_strength[sent] += freq_word[word.text]
                    else:
                        sent_strength[sent] = freq_word[word.text]

        # CONCATENATE THE STRINGS IN THE LIST TO A LARGER STRING
        summarized_sentences = nlargest(sent_count, sent_strength, key=sent_strength.get)
        final_sentences = [w.text for w in summarized_sentences]
        summary = ' '.join(final_sentences)
    except Exception:
        return text
    else:
        return summary

def modelIterator(model, vectoriser, top_n, vb=True):
    """
    This function prints out and returns the extracted topics for the NLP model passed on to it

    Parameters
    ----------
    model:                              NLP Model
    vectoriser:                         Vectorised text
    top_n:                              Number of Topics to return
    vb:                                 Verbose tag (will print out the topics if set to True
    ---------
    """
    frame_list = []

    for id_, topic in enumerate(model.components_):
        lister = [(vectoriser.get_feature_names()[i], topic[i]) for i in topic.argsort()[:-top_n - 1:-1]]
        df = pd.DataFrame(data=lister,
                          index=range(len(lister)),
                          columns=['word', 'weight'])

        if vb:
            st.markdown(f'### Topic {id_}')
            st.dataframe(df)

        frame_list.append(df)

    return frame_list

def dominantTopic(vect, model, n_words):
    """
    Returns the topic text

    Parameters
    ----------
    vect:                               Vectorizer used
    model:                              NLP Model
    n_words:                            Number of Topics to return
    ----------
    """
    kw = np.array(vect.get_feature_names())
    topic_kw = []
    for weights in model.components_:
        top_kw = (-weights).argsort()[:n_words]
        topic_kw.append(kw.take(top_kw))

    return topic_kw

@router.post('/wordcloud')
async def wordcloud(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data', max_word: int = 200,
                    contour: int = 3, width: int = 800, height: int = 400, colour: str = 'steelblue'):
    """
    Wordcloud creation

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **max_word**: Max number of words to render in the wordcloud image

    **contour**: Contour width

    **width**: Width of the wordcloud image

    **height**: Height of the wordcloud image

    **colour**: Colour of the background
    """

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        if not raw_data.empty:
            raw_data = raw_data[[data_column]]
            wc = WordCloud(background_color='white',
                           max_words=max_word,
                           contour_width=contour,
                           width=width,
                           height=height,
                           contour_color=colour)
            wc.generate(' '.join(raw_data[data_column]))
            img = wc.to_image()
            data = {
                'image': str(img.tobytes())
            }

            return jsonable_encoder(data)
        else:
            raise HTTPException(status_code=404, detail='Error: Document-Term Matrix was not properly prepared. Try '
                                                        'again.')

@router.post('/ner')
async def ner(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data',
              model: str = 'en_core_web_sm', one_datapoint_analyser: int = None):
    """
    Conduct NER analysis

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **model**: spaCy model to load

    **one_datapoint_analyser**: The datapoint to render into HTML format
    """

    NLP = None

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        if not raw_data.empty:
            # init the required columns
            raw_data = raw_data[[data_column]]
            raw_data['NER'] = ''
            raw_data['COMPILED_LABELS'] = ''
            raw_data = raw_data.astype(str)

            if model == 'en_core_web_sm':
                try:
                    NLP = spacy.load('en_core_web_sm')
                except OSError:
                    logging.warning('Model not found, downloading...')
                    try:
                        os.system('python -m spacy download en_core_web_sm')
                    except Exception as ex:
                        logging.error(f'Unable to download Model. Error: {ex}')
                        raise HTTPException(status_code=415, detail=ex)
                except Exception as ex:
                    raise HTTPException(status_code=415, detail=ex)
            elif model == 'en_core_web_lg':
                try:
                    NLP = spacy.load('en_core_web_lg')
                except OSError:
                    logging.warning('Model not found, downloading...')
                    try:
                        os.system('python -m spacy download en_core_web_lg')
                    except Exception as ex:
                        logging.error(f'Unable to download Model. Error: {ex}')
                        raise HTTPException(status_code=415, detail=ex)
                except Exception as ex:
                    raise HTTPException(status_code=415, detail=ex)

            for index in range(len(raw_data)):
                temp_nlp = NLP(raw_data[data_column][index])
                raw_data.at[index, 'NER'] = str(list(zip([word.text for word in temp_nlp.ents],
                                                         [word.label_ for word in temp_nlp.ents])))
                raw_data.at[index, 'COMPILED_LABELS'] = str(list(set([word.label_ for word in temp_nlp.ents])))

            if one_datapoint_analyser is not None:
                cpy = raw_data.copy()
                temp = cpy[data_column][one_datapoint_analyser]
                render = displacy.render(list(NLP(str(temp)).sents),
                                         style='ent',
                                         page=True)
                data = {
                    'data': raw_data.to_json(),
                    'render': render
                }
                return jsonable_encoder(data)
            else:
                data = {
                    'data': raw_data.to_json()
                }
                return jsonable_encoder(data)
        else:
            raise HTTPException(status_code=404, detail='Error: Data not loaded properly. Try again.')

@router.post('/pos')
async def pos(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = '',
              model: str = 'en_core_web_sm', one_datapoint_analyser: int = None, compact: bool = True,
              colour: str = 'steelblue', bg: str = 'white'):
    """
    Conduct POS tagging

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **model**: spaCy model to load

    **one_datapoint_analyser**: The datapoint to render into HTML format

    **compact**: Compact the renders

    **colour**: Colour of the words in the render

    **bg**: Colour of the background
    """

    NLP = None

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        if not raw_data.empty:
            raw_data = raw_data[[data_column]]
            raw_data['POS'] = ''
            raw_data = raw_data.astype(str)

            if model == 'en_core_web_sm':
                try:
                    NLP = spacy.load('en_core_web_sm')
                except OSError:
                    logging.warning('Model not found, downloading...')
                    try:
                        os.system('python -m spacy download en_core_web_sm')
                    except Exception as ex:
                        logging.error(f'Unable to download Model. Error: {ex}')
                        raise HTTPException(status_code=415, detail=ex)
                except Exception as ex:
                    raise HTTPException(status_code=415, detail=ex)
            elif model == 'en_core_web_lg':
                try:
                    NLP = spacy.load('en_core_web_lg')
                except OSError:
                    logging.warning('Model not found, downloading...')
                    try:
                        os.system('python -m spacy download en_core_web_lg')
                    except Exception as ex:
                        logging.error(f'Unable to download Model. Error: {ex}')
                        raise HTTPException(status_code=415, detail=ex)
                except Exception as ex:
                    raise HTTPException(status_code=415, detail=ex)

            for index in range(len(raw_data)):
                temp_nlp = NLP(raw_data[data_column][index])
                raw_data.at[index, 'POS'] = str(list(zip([str(word) for word in temp_nlp],
                                                         [word.pos_ for word in temp_nlp])))
                raw_data.at[index, 'COMPILED_LABELS'] = str(list(set([word.pos_ for word in temp_nlp])))

            if one_datapoint_analyser is not None:
                cpy = raw_data.copy()
                temp = cpy[data_column][one_datapoint_analyser]
                render = displacy.render(list(NLP(str(temp)).sents),
                                         style='dep',
                                         options={
                                             'compact': compact,
                                             'color': colour,
                                             'bg': bg,
                                         })
                data = {
                    'data': raw_data.to_json(),
                    'render': render
                }
                return jsonable_encoder(data)
            else:
                data = {
                    'data': raw_data.to_json()
                }
                return jsonable_encoder(data)
        else:
            raise HTTPException(status_code=415, detail='Error: Data not loaded properly. Try again.')

@router.post('/summarise')
def summarise(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data', mode: str = 'basic',
              model: str = 'en_core_web_sm', sentence_len: int = 3, min_words: int = 80, max_words: str = 150,
              max_tensor: int = 512):
    """
    Summarise texts

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **mode**: Define whether or not to conduct 'basic' or 'advanced' summarisation on input data

    **model**: spaCy model to load

    **sentence_len**: The maximum length of sentence to return

    **min_words**: The minimum number of words to include in the summary

    **max_words**: The maximum number of words to include in the summary

    **max_tensor**: The maximum number of input tensors for advanced summarisation process
    """

    NLP = None

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        # load up the data first
        raw_data = raw_data[[data_column]]
        raw_data['SUMMARY'] = np.nan
        raw_data = raw_data.astype(str)

        if not raw_data.empty:
            if mode == 'basic':
                if model == 'en_core_web_sm':
                    try:
                        NLP = spacy.load('en_core_web_sm')
                    except OSError:
                        logging.warning('Model not found, downloading...')
                        try:
                            os.system('python -m spacy download en_core_web_sm')
                        except Exception as ex:
                            logging.error(f'Unable to download Model. Error: {ex}')
                            raise HTTPException(status_code=415, detail=ex)
                    except Exception as ex:
                        raise HTTPException(status_code=415, detail=ex)
                elif model == 'en_core_web_lg':
                    try:
                        NLP = spacy.load('en_core_web_lg')
                    except OSError:
                        logging.warning('Model not found, downloading...')
                        try:
                            os.system('python -m spacy download en_core_web_lg')
                        except Exception as ex:
                            logging.error(f'Unable to download Model. Error: {ex}')
                            raise HTTPException(status_code=415, detail=ex)
                    except Exception as ex:
                        raise HTTPException(status_code=415, detail=ex)

                stopwords = list(STOP_WORDS)
                pos_tag = ['PROPN', 'ADJ', 'NOUN', 'VERB']
                raw_data['SUMMARY'] = raw_data[data_column]. \
                    apply(lambda x: summarise(x, stopwords, pos_tag, NLP, sentence_len))
                data = {
                    'data': raw_data.to_json()
                }
                return jsonable_encoder(data)

            elif mode == 'advanced':
                if torch.cuda.is_available():
                    try:
                        torch.cuda.get_device_name(torch.cuda.current_device())
                    except AssertionError:
                        raise HTTPException(status_code=415, detail='Error: CUDA Device is not enabled. Try again.')
                    except Exception as ex:
                        raise HTTPException(status_code=415, detail=ex)
                    else:
                        tokenizer = AutoTokenizer.from_pretrained('t5-base')
                        model = AutoModelWithLMHead.from_pretrained('t5-base', return_dict=True)
                        raw_data = raw_data.astype(object)
                        raw_data['ENCODED'] = raw_data[data_column]. \
                            apply(lambda x: tokenizer.encode('summarize: ' + x,
                                                             return_tensors='pt',
                                                             max_length=max_tensor,
                                                             truncation=True))
                        raw_data['OUTPUTS'] = raw_data['ENCODED']. \
                            apply(lambda x: model.generate(x,
                                                           max_length=max_words,
                                                           min_length=min_words,
                                                           length_penalty=5.0,
                                                           num_beams=2))
                        raw_data['SUMMARISED'] = raw_data['OUTPUTS'].apply(
                            lambda x: tokenizer.decode(x[0]))
                        raw_data.drop(columns=['ENCODED', 'OUTPUTS'], inplace=True)
                        raw_data['SUMMARISED'] = raw_data['SUMMARISED']. \
                            str.replace('<pad> ', '').str.replace('</s>', '')
                        raw_data = raw_data.astype(str)
                        data = {
                            'data': raw_data.to_json()
                        }
                        return jsonable_encoder(data)

        else:
            raise HTTPException(status_code=415, detail='Error: Data not loaded properly. Try again.')

@router.post('/sentiment')
def sentiment(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data', model: str = 'vader',
              colour: str = '#2ACAEA'):
    """
    Conduct Sentiment Analysis

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **model**: spaCy model to load

    **colour**: Colour of plots generated
    """

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        if not raw_data.empty:
            if model == 'vader':
                replacer = {
                    r"'": '',
                    r'[^\w\s]': ' ',
                    r' \d+': ' ',
                    r' +': ' '
                }

                raw_data['VADER SENTIMENT TEXT'] = raw_data[data_column]. \
                    replace(to_replace=replacer, regex=True)

                vader_analyser = SentimentIntensityAnalyzer()
                sent_score_list = []
                sent_label_list = []

                # scoring
                for i in raw_data['VADER SENTIMENT TEXT'].tolist():
                    sent_score = vader_analyser.polarity_scores(i)

                    if sent_score['compound'] > 0:
                        sent_score_list.append(sent_score['compound'])
                        sent_label_list.append('Positive')
                    elif sent_score['compound'] == 0:
                        sent_score_list.append(sent_score['compound'])
                        sent_label_list.append('Neutral')
                    elif sent_score['compound'] < 0:
                        sent_score_list.append(sent_score['compound'])
                        sent_label_list.append('Negative')

                raw_data['VADER OVERALL SENTIMENT'] = sent_label_list
                raw_data['VADER OVERALL SCORE'] = sent_score_list
                raw_data['VADER POSITIVE SCORING'] = [vader_analyser.polarity_scores(doc)['pos'] for doc in
                                                      raw_data['VADER SENTIMENT TEXT'].values.tolist()]
                raw_data['VADER NEUTRAL SCORING'] = [vader_analyser.polarity_scores(doc)['neu'] for doc in
                                                     raw_data['VADER SENTIMENT TEXT'].values.tolist()]
                raw_data['VADER NEGATIVE SCORING'] = [vader_analyser.polarity_scores(doc)['neg'] for doc in
                                                      raw_data['VADER SENTIMENT TEXT'].values.tolist()]

                # create plots
                hac_plot = ff.create_distplot([raw_data['VADER OVERALL SCORE'].tolist()],
                                              ['VADER'],
                                              colors=[colour],
                                              bin_size=0.25,
                                              curve_type='normal',
                                              show_rug=False,
                                              show_hist=False)
                hac_plot.update_layout(title_text='Distribution Plot',
                                       xaxis_title='VADER Score',
                                       yaxis_title='Frequency Density',
                                       legend_title='Frequency Density')
                data = {
                    'data': raw_data.to_json(),
                    'dot_image': hac_plot.to_image(format="png")

                }
                return jsonable_encoder(data)

            elif model == 'textblob':
                pol_list = []
                sub_list = []

                # scoring: polarity
                raw_data['POLARITY SCORE'] = raw_data[data_column]. \
                    apply(lambda x: TextBlob(x).sentiment.polarity)
                for i in raw_data['POLARITY SCORE'].tolist():
                    if float(i) > 0:
                        pol_list.append('Positive')
                    elif float(i) < 0:
                        pol_list.append('Negative')
                    elif float(i) == 0:
                        pol_list.append('Neutral')
                raw_data['POLARITY SENTIMENT'] = pol_list

                # scoring: subjectivity
                raw_data['SUBJECTIVITY SCORE'] = raw_data[data_column].apply(
                    lambda x: TextBlob(x).sentiment.subjectivity
                )
                for i in raw_data['SUBJECTIVITY SCORE'].tolist():
                    if float(i) < 0.5:
                        sub_list.append('Objective')
                    elif float(i) > 0.5:
                        sub_list.append('Subjective')
                    elif float(i) == 0.5:
                        sub_list.append('Neutral')
                raw_data['SUBJECTIVITY SENTIMENT'] = sub_list
                hac_plot = px.scatter(raw_data[['SUBJECTIVITY SCORE', 'POLARITY SCORE']],
                                      x='SUBJECTIVITY SCORE',
                                      y='POLARITY SCORE',
                                      labels={
                                          'SUBJECTIVITY SCORE': 'Subjectivity',
                                          'POLARITY SCORE': 'Polarity'
                                      })
                hac_plot1 = ff.create_distplot([raw_data['SUBJECTIVITY SCORE'].tolist(),
                                                raw_data['POLARITY SCORE'].tolist()],
                                               ['Subjectivity', 'Polarity'],
                                               curve_type='normal',
                                               show_rug=False,
                                               show_hist=False)
                data = {
                    'data': raw_data.to_json(),
                    'dot_image': hac_plot.to_image(format="png"),
                    'word_image': hac_plot1.to_image(format="png")
                }
                return jsonable_encoder(data)
        else:
            raise HTTPException(status_code=415, detail='Error: Data not loaded properly. Try again.')

@router.post('/modelling')
def topic_modelling(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data', model: str = 'lda',
                    num_topics: int = 10, max_features: int = 5000, max_iter: int = 10, min_df: int = 5,
                    max_df: float = 0.90, worker: int = 1, colour: str = 'steelblue', alpha: float = 0.10,
                    l1_ratio: float = 0.50):
    """
    Topic Modelling

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **model**: spaCy model to load

    **num_topics**: Number of topics to model

    **max_features**: Maximum number of features to consider

    **max_iter**: Maximum number of epochs to fit data

    **min_df**: Minimum length of words

    **max_df**: Maximum length of words

    **worker**: Number of workers

    **colour**: Colour of the plots

    **alpha**: Alpha value

    **l1_ratio**: L1 ratio value
    """

    global fc

    try:
        if ftype == 'csv':
            raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(str)
        elif ftype == 'xlsx':
            raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(str)
        elif ftype == 'json':
            raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(str)
        else:
            raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
    except Exception as ex:
        raise HTTPException(status_code=415, detail=ex)
    else:
        if not raw_data.empty:
            try:
                cv = CountVectorizer(min_df=min_df,
                                     max_df=max_df,
                                     stop_words='english',
                                     lowercase=True,
                                     token_pattern=r'[a-zA-Z\-][a-zA-Z\-]{2,}',
                                     max_features=max_features)
                vectorised = cv.fit_transform(raw_data[data_column])
            except ValueError:
                raise HTTPException(status_code=415, detail='Error: The column loaded is empty or has invalid data'
                                                            ' points. Try again.')
            except Exception as ex:
                raise HTTPException(status_code=415, detail=ex)
            else:
                if model == 'lda':
                    LDA = LatentDirichletAllocation(n_components=num_topics,
                                                    max_iter=max_iter,
                                                    learning_method='online',
                                                    n_jobs=worker)
                    LDA_data = LDA.fit_transform(vectorised)
                    topic_text = modelIterator(LDA, cv, top_n=num_topics,
                                               vb=False)
                    keywords = pd.DataFrame(dominantTopic(vect=cv, model=LDA,
                                                          n_words=num_topics))
                    keywords.columns = [f'word_{i}' for i in range(keywords.shape[1])]
                    keywords.index = [f'topic_{i}' for i in range(keywords.shape[0])]
                    LDA_vis = pyLDAvis.sklearn.prepare(LDA, vectorised, cv, mds='tsne')
                    pyLDAvis.save_html(LDA_vis,
                                       str(os.path.join(os.getcwd(), f'lda_id{fc}.html')))
                    with open(os.path.join(os.getcwd(), f'lda_id{fc}.html')) as f:
                        render = f.read()
                    fc += 1

                    data = {
                        'topic_text': {i: (topic_text[i].to_json()) for i
                                       in range(len(topic_text))},
                        'data': raw_data.to_json(),
                        'keywords': keywords.to_json(),
                        'render': render
                    }

                    return jsonable_encoder(data)

                elif model == 'nmf':
                    TFIDF = TfidfVectorizer(max_df=max_df,
                                            min_df=min_df,
                                            max_features=max_features,
                                            stop_words='english')
                    TFIDF_vectorised = TFIDF.fit_transform(raw_data
                                                           [data_column]
                                                           .values.astype(str))
                    NMF_model = NMF(n_components=num_topics,
                                    max_iter=max_iter,
                                    random_state=1,
                                    alpha=alpha,
                                    l1_ratio=l1_ratio).fit(TFIDF_vectorised)
                    topic_text = modelIterator(model=NMF_model,
                                               vectoriser=TFIDF,
                                               top_n=num_topics,
                                               vb=False)
                    keywords = pd.DataFrame(dominantTopic(model=NMF_model,
                                                          vect=TFIDF,
                                                          n_words=num_topics))
                    keywords.columns = [f'word_{i}' for i in range(keywords.shape[1])]
                    keywords.index = [f'topic_{i}' for i in range(keywords.shape[0])]
                    data = {
                        'topic_text': {i: (topic_text[i].to_json()) for i
                                       in range(len(topic_text))},
                        'data': raw_data.to_json(),
                        'keywords': keywords.to_json()
                    }
                    return jsonable_encoder(data)

                elif model == 'lsi':
                    LSI = TruncatedSVD(n_components=num_topics, n_iter=max_iter)
                    LSI_data = LSI.fit_transform(vectorised)
                    topic_text = modelIterator(LSI, cv,
                                               top_n=num_topics, vb=False)
                    keywords = pd.DataFrame(dominantTopic(model=LSI, vect=cv,
                                                          n_words=num_topics))
                    keywords.columns = [f'word_{i}' for i in range(keywords.shape[1])]
                    keywords.index = [f'topic_{i}' for i in range(keywords.shape[0])]

                    # SVD
                    svd_2d = TruncatedSVD(n_components=2)
                    data_2d = svd_2d.fit_transform(vectorised)

                    mar_fig = go.Scattergl(
                        x=data_2d[:, 0],
                        y=data_2d[:, 1],
                        mode='markers',
                        marker=dict(
                            color=colour,
                            line=dict(width=1)
                        ),
                        text=cv.get_feature_names(),
                        hovertext=cv.get_feature_names(),
                        hoverinfo='text'
                    )
                    mar_fig = [mar_fig]
                    mar_fig = go.Figure(data=mar_fig, layout=go.Layout(title='Scatter Plot'))
                    word_fig = go.Scattergl(
                        x=data_2d[:, 0],
                        y=data_2d[:, 1],
                        mode='text',
                        marker=dict(
                            color=colour,
                            line=dict(width=1)
                        ),
                        text=cv.get_feature_names(),
                    )
                    word_fig = [word_fig]
                    word_fig = go.Figure(data=word_fig, layout=go.Layout(title='Scatter Word Plot'))

                    data = {
                        'topic_text': {i: (topic_text[i].to_json()) for i
                                       in range(len(topic_text))},
                        'data': raw_data.to_json(),
                        'keywords': keywords.to_json(),
                        'point_figure': mar_fig.to_image(format='png'),
                        'word_figure': word_fig.to_image(format='png')
                    }

                    return jsonable_encoder(data)
        else:
            raise HTTPException(status_code=415, detail='Error: Data not loaded properly. Try again.')

@router.post('/classification')
def classification(file: UploadFile = File(...), ftype: str = 'csv', data_column: str = 'data', topics: str = ''):
    """
    Conduct Text Classification

    **file**: Data

    **ftype**: The file format to read the input data as

    **data_column**: Column in the pandas DataFrame to process

    **topics**: A string (delimited by commas) or a list of topics to classify data into
    """

    if torch.cuda.is_available():
        try:
            torch.cuda.get_device_name(torch.cuda.current_device())
        except AssertionError:
            raise HTTPException(status_code=415, detail='Error: CUDA Device is not enabled. Try again.')
        except Exception as ex:
            raise HTTPException(status_code=415, detail=ex)
        else:
            try:
                if ftype == 'csv':
                    raw_data = pd.read_csv(StringIO(str(file.file.read(), 'latin1')), encoding='latin1').astype(object)
                elif ftype == 'xlsx':
                    raw_data = pd.read_excel(StringIO(str(file.file.read(), 'utf-8')), engine='openpyxl').astype(object)
                elif ftype == 'json':
                    raw_data = pd.read_json(StringIO(str(file.file.read(), 'utf-8'))).astype(object)
                else:
                    raise HTTPException(status_code=415, detail='Error: File format input is not supported. Try again.')
            except Exception as ex:
                raise HTTPException(status_code=415, detail=ex)
            else:
                if type(topics) == str:
                    topics = [word.strip().lower() for word in topics.split(sep=',')]
                elif type(topics) == list:
                    topics = topics
                else:
                    raise HTTPException(status_code=415, detail='Error: Invalid data type for topics.')

                classifier = pipeline('zero-shot-classification')
                raw_data['TEST'] = raw_data[data_column].apply(lambda x: classifier(x, topics))
                raw_data['CLASSIFIED'] = raw_data['TEST']. \
                    apply(lambda x: list(zip(x['labels'].tolist(), x['scores'].tolist())))
                raw_data['MOST PROBABLE TOPIC'] = raw_data['CLASSIFIED'].apply(lambda x: max(x, key=itemgetter[1])[0])
                raw_data = raw_data.astype(str)

                data = {
                    'data': raw_data.to_json()
                }
                return jsonable_encoder(data)
    else:
        raise HTTPException(status_code=404, detail='Error: CUDA Device is not detected. Try again.')


#!/usr/bin/env python

import argparse
import csv
from Bio import SeqIO
import time

parser = argparse.ArgumentParser(description='writes a table of repeat matches content in a genome given an assembly a gff file containing repeats.')
parser.add_argument("-s","--seqfile",
					type=str,
					default='',
					help="Sequence file in fasta format")
parser.add_argument("-g","--gff",
					type=str,
					default='',
					help="gff3 file containing repeats as a type")
parser.add_argument("-wsi","--windowsize",
					type=int,
					default=10000,
					help="window size in basepairs")
parser.add_argument("-wst","--windowstep",
					type=int,
					default=10000,
					help="window step length in basepairs")
parser.add_argument("-o","--output",
					type=str,
					default='',
					help="name of cytoband output file")
args=parser.parse_args()

seqfile = args.seqfile
gff = args.gff
windowsize = args.windowsize
windowstep = args.windowstep
output = args.output

#########################################################################################

class GFF:
	def __init__(self, gff_entry):
		self.seqid = gff_entry[0]
		self.source = gff_entry[1]
		self.type = gff_entry[2]
		self.start = int(gff_entry[3])
		self.end = int(gff_entry[4])
		self.score = gff_entry[5]
		self.strand = gff_entry[6]
		self.phase = gff_entry[7]
		self.attributes = gff_entry[8]

def GFF_parse(gff) :
	gff_list = []
	with open(gff) as OF:
		reader = csv.reader(OF, delimiter='\t')
		for row in reader :
			if (row[0][0] != '#') and (len(row) > 1) :
				gff_entry = GFF(row)
				gff_list.append(gff_entry)
	return(gff_list)

def Remove_encompassed_repeats(reduced_gff):
	if len(reduced_gff) < 2 :
		return(reduced_gff)
	else:
		#start_time = time.time()
		reduced_gff.sort(key=lambda x: (x.end),reverse=True)
		reduced_gff.sort(key=lambda x: (x.start))
		reduced_gff = reduced_gff + ['end']
		entry_to_keep = []
		focal_entry = reduced_gff[0]
		if len(reduced_gff) > 1:
			test_entry = reduced_gff[1]
			#print(focal_entry.start, focal_entry.end)
			while test_entry != 'end':
				#print(test_entry.start, test_entry.end)
				if test_entry.end <= focal_entry.end :
					index = reduced_gff.index(test_entry)
					test_entry = reduced_gff[(index + 1)]
				else:
					entry_to_keep.append(focal_entry)
					focal_entry = test_entry
					#print(reduced_gff.index(focal_entry))
					index = reduced_gff.index(test_entry)
					test_entry = reduced_gff[(index + 1)]
		else:
			entry_to_keep.append(focal_entry)
		#end_time = time.time()
		#print((end_time - start_time))
		return(entry_to_keep)

def Remove_overlapping_repeats(reduced_gff):
	if len(reduced_gff) < 2 :
		return(reduced_gff)
	else:
		#start_time = time.time()
		#if len(reduced_gff) == 1:
		#	print(reduced_gff)
		reduced_gff.sort(key=lambda x: (x.end),reverse=True)
		reduced_gff.sort(key=lambda x: (x.start))
		reduced_gff = reduced_gff + ['end']
		entry_to_keep = []
		focal_entry = reduced_gff[0]
		if len(reduced_gff) > 1:
			i = 1
			#entry_to_keep.append(focal_entry)
			while reduced_gff[i] != 'end':
				if reduced_gff[i].start < focal_entry.end:
					gff_entry = GFF([reduced_gff[i].seqid, reduced_gff[i].source, reduced_gff[i].type, focal_entry.start, reduced_gff[i].end, '','','',''])
					focal_entry = gff_entry
					i = i + 1
				else:
					entry_to_keep.append(focal_entry)
					focal_entry = reduced_gff[i]
					i = i + 1
		entry_to_keep.append(focal_entry)
		#end_time = time.time()
		#print((end_time - start_time))
		return(entry_to_keep)

def Calc_repeat_content(sequence_list, gff_list, windowsize, windowstep) :
	start_time = time.time()
	content_list = []
	for seq in sequence_list:
		print(seq.id)
		reduced_gff = [repeat for repeat in gff_list if repeat.seqid == seq.id]
		endtime1 = time.time()
		print('time to reduce list = ' + str(endtime1 - start_time))
		#print(len(reduced_gff))
		reduced_gff = Remove_encompassed_repeats(reduced_gff)
		endtime2 = time.time()
		print('time to remove encompassing = ' + str(endtime2 - endtime1))
		#print(len(reduced_gff))
		#print(reduced_gff[-1])
		reduced_gff = Remove_overlapping_repeats(reduced_gff)
		endtime3 = time.time()
		print('time to remove overlap = ' + str(endtime3 - endtime2))
		if len(seq.seq) < windowsize:
			sum_repeat_length = 0
			for repeat in reduced_gff :
				repeat_length = abs(repeat.end - repeat.start)
				sum_repeat_length = sum_repeat_length + repeat_length
			repeat_content = sum_repeat_length / len(seq.seq)
			start = 1
			end = len(seq.seq)
			entry = [seq.id, start, end, repeat_content]
			print(repeat_content, start, end, seq.id)
			content_list.append(entry)
		else:
			start = 1
			end = windowsize
			while end <= len(seq.seq) :
				sum_repeat_length = 0
				for repeat in reduced_gff:
					if repeat.start >= start and repeat.end <= end :
						repeat_length = abs((repeat.end + 1) - repeat.start)
						sum_repeat_length = sum_repeat_length + repeat_length
					elif repeat.start < start and repeat.end > start and repeat.end <= end :
						repeat_length = abs((repeat.end + 1) - start)
						sum_repeat_length = sum_repeat_length + repeat_length
					elif repeat.start >= start and repeat.start < end and repeat.end > end :
						repeat_length = abs((end + 1) - repeat.start)
						sum_repeat_length = sum_repeat_length + repeat_length
				repeat_content = sum_repeat_length / windowsize
				entry = [seq.id, start, end, repeat_content]
				print(repeat_content,start,end, seq.id)
				content_list.append(entry)
				start = start + windowstep
				end = end + windowstep
		print()
	end_time = time.time()
	print((end_time - start_time))
	return(content_list)

#########################################################################################

gff_list = GFF_parse(gff)

gff_list = [ entry for entry in gff_list if entry.source == 'repeat_gff:repeatmasker' and entry.type == 'match']

sequences = list(SeqIO.parse(seqfile,"fasta"))

repeat_list = Calc_repeat_content(sequences, gff_list, windowsize, windowstep)
#test = Calc_repeat_content(reduced_sequences, gff_list, windowsize, windowstep)

with open(output,'w') as outFile:
	outfile_writer = csv.writer(outFile, delimiter = '\t')
	outfile_writer.writerow(['scaffold', 'start', 'end', 'repeat_content'])
	for entry in repeat_list:
		outfile_writer.writerow(entry)

outFile.close()

#########################################################################################


from scipy.spatial.qhull import Voronoi

from .base import mse
from .layers import *

class LayerVoronoiImage(Image):
    def __init__(self, layers, original_image, pts_number, size):
        super().__init__(original_image)
        self.a_im = None
        self.layers = layers
        self.im = None
        self.pts_number = pts_number
        self.radius = 4  # size[0] // int(np.sqrt(pts_number)) // 2
        self.size = size
        self.rgb = [None] * 3
        self.polys = [None] * 3
        self.ch_l_point = None

    def __deepcopy__(self, memo):
        copy = type(self)(None, self.original_image, self.pts_number, self.size)
        memo[id(self)] = copy
        copy.layers = deepcopy(self.layers, memo)
        copy.rgb = deepcopy(self.rgb)
        copy.polys = [self.polys[i] for i in range(len(self.polys))]
        copy.ch_l_point = None
        return copy

    def draw(self, canvas=None, make_beautiful=False):
        if self.im is None:
            layers = [np.zeros(self.size, dtype=np.uint8) for _ in range(3)] if canvas is None else cv2.split(
                np.copy(canvas))
            for i, layer in enumerate(self.layers):
                # if there is no change in color
                if self.ch_l_point is None or self.ch_l_point[0] != i or self.rgb[i] is None:

                    # if change is in structure
                    if self.rgb[i] is None or make_beautiful:
                        if self.polys[i] is None:
                            vor = Voronoi(layer[0], incremental=False)
                            self.polys[i] = vor
                        else:
                            vor = self.polys[i]

                        # draw from scratch
                        ver = np.int32(vor.vertices)
                        for j in range(0, len(vor.point_region)):
                            ri = vor.point_region[j]
                            if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                                cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                             int(self.layers[i][1][j]), cv2.LINE_AA if make_beautiful else cv2.LINE_8)

                    # nothing changed this time
                    else:
                        layers[i] = self.rgb[i]
                else:
                    # change only color
                    vor = self.polys[i]
                    layers[i] = self.rgb[i]
                    j = self.ch_l_point[1]
                    ri = vor.point_region[j]
                    ver = np.int32(vor.vertices)
                    if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                        cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                     int(self.layers[i][1][j]))
            self.rgb = layers
            self.im = cv2.merge(layers)
        return self.im

    # draw centers
    def draw_approximated(self):
        if self.a_im is None:
            self.a_im = LayerVoronoiImage.approximated(self.layers, self.radius, self.size)
        return self.a_im

    # fast method of drawing
    @staticmethod
    def approximated(pts, r, size):
        layers = np.zeros((3, size[0], size[1]), dtype=np.uint8)
        for i in range(len(pts)):
            layer = pts[i]
            for j in range(len(layer[0])):
                x, y = pts[i][0][j]
                c = pts[i][1][j]
                layers[i, y - r: y + r, x - r: x + r] = int(c)
        return cv2.merge(layers)

    def fitness(self, image=None, gen=0, **kwargs):
        if self.fit is None:
            if image is None:
                image = self.original_image
            self.fit = mse(image, self.draw())
            # self.draw_approximated() if gen < 3000 else self.draw())  # 500 / np.linalg.norm(self.draw() - image)
        return self.fit

    def mutate(self, gen):
        mutated = deepcopy(self)
        mutated.im = None
        self._mutate(mutated, gen)
        return mutated

    def _chance(self, gen):
        if gen < 500_000:
            return 0.5
        else:
            return 0.5 + (gen - 500_000) * 0.5 / 500_000

    def _mutate(self, mutated, gen):
        changed_layer = rd.randint(0, 3)
        layer = mutated.layers[changed_layer]
        point = rd.randint(0, len(layer[0]))

        if rd.rand() > 0.5:#self._chance(gen):
            mutated.rgb[changed_layer] = None
            mutated.polys[changed_layer] = None
            xy = rd.randint(0, 2)
            if gen < 50000:
                layer[0][point][xy] = rd.randint(0, self.size[xy])
            else:
                layer[0][point][xy] = layer[0][point][xy] + rd.randn() * 10
        else:
            layer[1][point] = rd.randint(0, 256)
            mutated.ch_l_point = (changed_layer, point)

    def draw_bfl(self):
        layers = [np.zeros(self.size, dtype=np.uint8) for _ in range(3)]
        for i, layer in enumerate(self.layers):
            vor = Voronoi(layer[0], incremental=False)
            # draw from scratch
            ver = np.int32(vor.vertices)
            for j in range(0, len(vor.point_region)):
                ri = vor.point_region[j]
                if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                    cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                 int(self.layers[i][1][j]), cv2.LINE_AA)

        return cv2.merge(layers)

    def save(self, name):
        cv2.imwrite(name, self.draw_bfl())

class VoronoiLayerImageGenerator(Generator):
    def __init__(self, original_image):
        super().__init__(original_image)

    def generate(self, n=1, pts_numbers=35*35):
        ims = []
        for _ in range(n):
            pts = [[[], []] for _ in range(3)]
            for i in range(3):
                for _ in range(3):
                    pts[i][0] = rd.randint(0, self._original_image.shape[0], size=(pts_numbers, 2))
                    pts[i][1] = rd.randint(0, 256, (pts_numbers))
            ims.append(LayerVoronoiImage(pts, self._original_image, pts_numbers, self._original_image.shape[:2]))
        return np.array(ims)


import csv
import re
import docx
from docx.shared import Inches, Pt, RGBColor
from docx.oxml.ns import nsdecls
from docx.oxml import parse_xml
from docx.enum.text import WD_COLOR_INDEX

# Global Variables
# test suite header
TEST_SUITE_HEADER = r"\d{1,2}\.\d{1,2}"
# Input Variables
# Name of CSV to use for test output
CSVFILES = [
    "../Arista Agora Test Plan V9.3 - EOS Test Items.csv",
    "../Arista Agora Test Plan V9.3 - CVA_CVP Test Items.csv",
]
DOCFILE = "../test_summary_results.docx"
# EOS version
EOS = "4.29.1F-DPE"
# Column name of interesting values
TITLE_ROW = 0
HEADER_ROW = 1
TC_ID = r".*Test Case ID.*"
CASE_NAME = r".*Test Case Description.*"
PROCEDURE = r".*Test Steps.*"
EXPECTED_RESULT = r".*Criteria.*"
PASS_FAIL = r"Pass/Fail"
OBSERVATION = r".*Observations.*"
TC_TYPE = r".*Test Case Type.*"
# Column Width
WIDTHS = (
    Inches(0.5),
    Inches(1.17),
    Inches(2.50),
    Inches(1.25),
    Inches(0.75),
    Inches(1.50),
)

def parse_csv():
    """Iterate through a list of CSV files.  Open and parse CSV data
       and find test data

    Returns:
        list: List of interesting test data
    """
    test_data = []

    # iterate through CSV files
    for csvfile in CSVFILES:
        print(f"Parsing CSV file: {csvfile} for test data...")

        # open csv
        with open(csvfile, "r", encoding="utf-8") as file:
            csv_reader = csv.reader(file)
            # regex for matching a test suite header
            inspect_csv_data(test_data, csv_reader)

    return test_data

def set_header_columns(row):
    """Use regex to find a column index for interesting data

    Args:
        row (list): CSV row

    Returns:
        dict: key is column name and value is column index
    """

    header_columns = {}

    # iterate through cells to find column headers
    for count, _ in enumerate(row):
        if re.match(TC_ID, row[count]):
            header_columns["tc_id"] = count
        elif re.match(CASE_NAME, row[count]):
            header_columns["case_name"] = count
        elif re.match(PROCEDURE, row[count]):
            header_columns["procedure"] = count
        elif re.match(EXPECTED_RESULT, row[count]):
            header_columns["expected_result"] = count
        elif re.match(PASS_FAIL, row[count]):
            header_columns["pass_fail"] = count
        elif re.match(OBSERVATION, row[count]):
            header_columns["observation"] = count
        elif re.match(TC_TYPE, row[count]):
            header_columns["tc_type"] = count

    return header_columns

def inspect_csv_data(test_data, csv_reader):
    """Inspect each row of CSV data and determine if its interesting

    Args:
        test_data (list): Use CSV file to update list with interesting test data
        csv_reader (obj): CSV object to iterate through
    """

    for count, row in enumerate(csv_reader):
        test_data_entry = []

        # skip title row
        if count == TITLE_ROW:
            pass
        # inspect header and find correct column numbers
        elif count == HEADER_ROW:
            header_columns = set_header_columns(row)
        # parse csv row for a test case ids
        elif "TN" in row[header_columns["tc_id"]]:
            test_data_entry.append(row[header_columns["tc_id"]])
            test_data_entry.append(row[header_columns["case_name"]])
            test_data_entry.append(row[header_columns["procedure"]])
            test_data_entry.append(row[header_columns["expected_result"]])
            test_data_entry.append(row[header_columns["pass_fail"]])
            test_data_entry.append(row[header_columns["observation"]])
            test_data.append(test_data_entry)
        # parse csv row for header
        elif re.match(TEST_SUITE_HEADER, row[header_columns["case_name"]]):
            test_data_entry.append(row[header_columns["case_name"]])
            test_data.append(test_data_entry)
        # parse csv row for sub-section
        elif (not row[header_columns["tc_id"]] and not row[header_columns["tc_type"]]) and row[
            header_columns["case_name"]
        ] != "":
            test_data_entry.append("")
            test_data_entry.append(row[header_columns["case_name"]])
            test_data.append(test_data_entry)

def write_report(test_data):
    """Write word doc with test data collected from CSV

    Args:
        test_data (list): List of interesting test data
    """
    print(f"Writing report doc: {DOCFILE}")

    # Open doc
    document = docx.Document()
    # Change doc margins
    section = document.sections[0]
    section.left_margin = Inches(0.5)
    section.right_margin = Inches(0.5)

    write_header(document)
    write_results_table(document, test_data)

    doc_text = (
        "\n\n\nPlease note – for more details on test outputs please "
        "reference the test case output appendix docs.\n"
    )
    write_text(document, doc_text)

    # Close doc
    document.save(DOCFILE)

def write_header(document):
    """Write test section header to word doc

    Args:
        document (obj): Test report Word doc
        test_data (list): List of interesting test data
    """

    # Write summary header
    heading = document.add_heading(level=2)
    run = heading.add_run("4	Test Summary Result")
    run.font.size = Pt(16)
    run.font.color.rgb = RGBColor(204, 0, 0)

    # Header text
    doc_text = (
        f"\nTest Cases for Arista BAU Hardware & EOS {EOS} Certification. "
        "Please note – for more details on test outputs please reference "
        "the test case output appendix docs.\n\n\n"
    )
    write_text(document, doc_text)

def write_text(document, doc_text, font_pt=10, font="Arial"):
    """Write a paragraph to Word Doc

    Args:
        document (obj): Word doc object
        doc_text (str): Text to write to doc
        pt (int): Word font size
        font (string): Word font style
    """
    para = document.add_paragraph()
    run = para.add_run(doc_text)
    run.font.size = Pt(font_pt)
    run.font.name = font

def write_header_row(table):
    """Write the header row for the result table

    Args:
        table (obj): Word doc table object
    """

    headers = [
        "Test Case",
        "Case Name",
        "Procedure",
        "Expected result",
        "Pass/Fail",
        "Observation",
    ]

    # Write header row in table
    for count, header in enumerate(headers):
        set_cell_widths(table, 0)
        para = table.rows[0].cells[count].paragraphs[0]

        run = para.add_run(header)
        run.font.name = "Arial"
        run.font.size = Pt(9)
        run.font.bold = True

        shade_cell(count, table, 0, "00FFFF")

def write_results_table(document, test_data):
    """Write results table to Word doc

    Args:
        document (obj): Word doc object
        test_data (list): List of interesting test data
    """

    # create table
    table = document.add_table(rows=1, cols=6, style="Table Grid")
    table.autofit = False

    # create table rows
    write_header_row(table)
    write_data_row(table, test_data)

def write_data_row(table, test_data):
    """Write a test case result row in table

    Args:
        table (obj): Word doc table object
        test_data (list): List of interesting test data
    """

    # Iterate through test data and write table rows
    for count, test_case_entry in enumerate(test_data):
        count += 1
        row_cells = table.add_row().cells
        set_cell_widths(table, count)

        # Identify test suite sub-header row
        if re.match(TEST_SUITE_HEADER, test_case_entry[0]):
            row_cells[0].merge(row_cells[5])
            row_cells[0].text = ""
            run = row_cells[0].paragraphs[0].add_run(test_case_entry)
            run.font.name = "Arial"
            run.font.size = Pt(9)
            run.bold = True
            shade_cell(0, table, count, "CCFFFF")

        # Identify test case result data row
        elif "TN" in test_case_entry[0]:
            for counter, test_data_entry in enumerate(test_case_entry):
                row_cells[counter].text = ""
                run.font.name = "Arial"
                run.font.size = Pt(9)

                # Test Case Identifier cell
                if counter == 0:
                    test_data_entry = format_test_id(test_data_entry)
                    run = row_cells[counter].paragraphs[0].add_run(test_data_entry)
                # Test Case Pass/Fail cell
                elif counter == 4:
                    format_pass_fail(row_cells, counter, test_data_entry)
                # Cells not needing special formatting
                else:
                    run = row_cells[counter].paragraphs[0].add_run(test_data_entry)

                run.font.name = "Arial"
                run.font.size = Pt(9)

        # identify test suite sub section
        elif test_case_entry[0] == "":
            row_cells[2].merge(row_cells[5])

            run = row_cells[1].paragraphs[0].add_run(test_case_entry[1])
            run.font.name = "Arial"
            run.font.size = Pt(9)

            for cell_idx in [0, 1, 2]:
                shade_cell(cell_idx, table, count, "CCFFFF")

def shade_cell(cell_idx, table, count, shade):
    """Shade a cell in word doc table

    Args:
        cell_idx (int): Column index for cell to shade
        table (obj): Word doc table object
        count (int): Row index for cell to shade
        shade (str): hexadecimal color representation
    """
    cell = table.cell(count, cell_idx)
    color = parse_xml(
        # pylint: disable-next=consider-using-f-string
        r'<w:shd {} w:fill="{}"/>'.format(nsdecls("w"), shade)
    )
    # pylint: disable-next=protected-access
    cell._tc.get_or_add_tcPr().append(color)

def format_test_id(test_data_entry):
    """Format the test case id

    Args:
        test_data_entry (str): Text for table cell

    Returns:
        str: Formatted text for table cell
    """
    if "TN" in test_data_entry:
        test_data_entry = test_data_entry.split("TN")[1]

    return test_data_entry

def format_pass_fail(row_cells, counter, test_data_entry):
    """Format pass fail cell

    Args:
        row_cells (obj): Table cell
        counter (int): Table cell number
        test_data_entry (str): Text for table cell
    """
    run = row_cells[counter].paragraphs[0].add_run(test_data_entry.upper())

    if test_data_entry.upper() == "PASS":
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.BRIGHT_GREEN
        run.font.bold = True
    elif test_data_entry.upper() == "FAIL":
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.RED
        run.font.bold = True
    else:
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.YELLOW
        run.font.bold = True

def set_cell_widths(table, row):
    """Set width of Word table cell

    Args:
        table (obj): Word doc table object
        row (int): Table row number for operations
    """

    for column, width in enumerate(WIDTHS):
        table.rows[row].cells[column].width = width

def main():
    """Main Python function"""

    print("Starting...")
    test_data = parse_csv()
    write_report(test_data)
    print("Complete")

if __name__ == "__main__":
    main()


import sys
from heapq import *
input = sys.stdin.readline

N, K = map(int, input().split())
jewel = []
bags = []

for _ in range(N):
    heappush(jewel, list(map(int, input().split())))

for _ in range(K):
    heappush(bags, int(input()))

ans = []
answer = 0
while len(bags) > 0:
    bag = heappop(bags)
    while len(jewel) > 0:
        if jewel[0][0] <= bag:
            heappush(ans, -jewel[0][1])
            heappop(jewel)
        else: break

    if len(ans) > 0:
        answer += -heappop(ans)
print(answer)


import json

def save(data: dict) -> None:
    """Save data to json file."""
    print("== Save to json file ==")
    with open('tree.json', 'w') as f:
        json.dump(data, f)

def load() -> None:
    """Load data from json file."""
    print("== Load data from json file ==")
    with open('tree.json', 'r', encoding='utf-8') as f:
        json_file = json.load(f)

    for line in json_file:
        print(f"{line['name']} {line['parent']} {line['type']}"
              f" {line['size']}")