import nltk
import pandas as pd
import numpy as np
import re
import codecs
import json

import re
import numpy as np
import pandas as pd
from pprint import pprint
import pickle
# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
from gensim.models import Phrases
from gensim.corpora import Dictionary, MmCorpus
from gensim.models.word2vec import LineSentence
from gensim.models.ldamulticore import LdaMulticore
# spacy for lemmatization
import spacy
# NLTK for text cleaning
from nltk.tokenize import sent_tokenize, word_tokenize, RegexpTokenizer
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import stopwords, names
from nltk.tokenize import RegexpTokenizer
from nltk import tokenize
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
# TextBlob package for translation and spelling correction
from textblob import TextBlob

nlp = spacy.load('en')
# Plotting tools
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
# %matplotlib inline

# Enable logging for gensim - optional
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)

import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)

### Step 1: Initialization.

# Set up tokenizer.
tokenizer = RegexpTokenizer(r'\w+')
# Set up stop words.
en_stop = set(stopwords.words('english'))
# Set up stemmer.
p_stemmer = PorterStemmer()
# For sentiment analysis
senti = SentimentIntensityAnalyzer()

# Load saved models.
# I trained three LDA models from step 5, they have slightly different topics.
# I trained three models because there is randomness in LDA training.
ldamodel1 = pickle.load(open("PATH/model/lda_1.pickle", "rb"))
dict1 = pickle.load(open("PATH/model/dictionary_1.pickle", "rb"))
corpus1 = pickle.load(open("PATH/model/corpus_1.pickle", "rb"))

# # Topic dictionary
# lda_topics={
#                 0:'Parking Service',
#                 1:'Sleep Quality',
#                 2:'WiFi',
#                 3:'Online Service',
#                 4:'Overall Experience',
#                 5:'Value for Money',
#                 6:'Swimming Pool/Spa',
#                 7:'Front Desk Service',
#                 8:'Food', ''
#                 9:'Cleanliness',
#                 10:'Surroundings',
#                 11:'Distance from Transportation',
#                 12:'Booking Experience',
#                 13:'Hotel Staff',
#                 14:'Early Check-in/Late Check-out',
#                 15:'Noise'
#             }

mallet_lda_topics={
                0:'Hotel Staff',
                1:'Accessibility',
                2:'Food',
                3:'Overall Experience',
                4:'Noise',
                5:'Value for Money',
                6:'Room Amenities',
                7:'Location in the city',
                8:'Overall Experience',
                9:'Cleanliness',
                10:'Early Check-in/Late Check-out',
                11:'Health and Wellness Amenities',
                12:'Booking Experience',
                13:'Sleep Quality',
                14:'Parking Facility'
            }

# helper functions for text preprocessing & LDA modeling:

def punct_space(token):
    """
    helper function to eliminate tokens
    that are pure punctuation or whitespace
    """

    return token.is_punct or token.is_space or token.like_num or token.is_digit

def line_review(filename):
    """
    generator function to read in reviews from Pandas Series
    and un-escape the original line breaks in the text
    """

    #with codecs.open(filename, encoding='utf_8') as f:
    for review in filename:
        yield review.replace('\\n', '\n')

def lemmatized_sentence_corpus(filename):
    """
    generator function to use spaCy to parse reviews,
    lemmatize the text, and yield sentences
    """

    for parsed_review in nlp.pipe(line_review(filename), batch_size=10000, n_threads=10):
        for sent in parsed_review.sents:
            yield u' '.join([token.lemma_ for token in sent
                             if not punct_space(token)])

def trigram_bow_generator(filepath):
    """
    generator function to read reviews from a file
    and yield a bag-of-words representation
    """
    # load finished dictionary from disk
    trigram_dictionary = Dictionary.load('./models2/trigram_dict_all.dict')

    for review in LineSentence(filepath):
        yield trigram_dictionary.doc2bow(review)

def LDA_Review(review_df,min_topic_freq=0):
    """
    Takes Pandas series as input,
    consisting of one review as
    text string per row
    """
    from tqdm import tqdm
    """
    accept the original text of a review and (1) parse it with spaCy,
    (2) apply text pre-proccessing steps, (3) create a bag-of-words
    representation, (4) create an LDA representation, and
    (5) print a sorted list of the top topics in the LDA representation
    """
    text = review_df['FullReview']
    # parse the review text with spaCy
    with codecs.open('./uni_temporary.txt', 'w', encoding='utf_8') as f:
        for sentence in tqdm(lemmatized_sentence_corpus(text)):
            # print(sentence)
            f.write(sentence + '\n')
    f.close()
    # load and apply the first-order and secord-order phrase models
    bigram_model = Phrases.load('./models2/bigram_model.txt')
    trigram_model = Phrases.load('./models2/trigram_model.txt')

    unigram_review = LineSentence('./uni_temporary.txt')
    bigram_review = bigram_model[unigram_review]
    trigram_review = trigram_model[bigram_review]
    # remove any remaining stopwords
    trigram_review = [term for term in trigram_review
                        if term not in spacy.lang.en.stop_words.STOP_WORDS]
    with codecs.open('./tri_temporary.txt', 'w', encoding='utf_8') as ftri:
        for sentence in trigram_review:
            sentence = u' '.join(sentence)
            ftri.write(sentence + '\n')
    ftri.close()

    trigram_dictionary = Dictionary.load('./models2/trigram_dict_all.dict')
    lda = LdaMulticore.load('./models2/lda_model')
    trigram_review = LineSentence('./tri_temporary.txt')
    # create a bag-of-words representation
    review_bow = trigram_dictionary.doc2bow(trigram_review)
    # create an LDA representation
    review_lda = lda.get_document_topics(review_bow)
    review_lda = sorted(review_lda, key=itemgetter(1),reverse=True)
    for topic_number, freq in review_lda:
        if freq < min_topic_freq:
            break
        # print the most highly related topic names and frequencies
        print('{:25} {}'.format(lda_topics[topic_number], round(freq, 3)))

    ### Step 2: Generate the contents of the doctors' snapshots.

    counter = 0
    # The temporary string that stores all of the review highlights in each round of the for loop below.
    big_str = []
    # For every doctor, find two things:
    #     1. The most mentioned FIVE topics in their reviews.
    #         1.1 The sentiments of these topics.
    #     2. The 3 most positive sentences and the 3 most negative sentences.
    #         2.1 Rank all sentences according to sentiment analysis.
    # I do NOT keep info about individual reviews. All sentences are stored in a
    # long list regardless of whether they are from the same reviews or not!
    ###########################################################################
    # Build sentence dataframe for the current doctor.
    ###########################################################################
    this_hotel = pd.DataFrame(columns = ["HotelName","Sentence", "Sentiment_neg",
                            "Sentiment_neu","Sentiment_pos",
                            "Sentiment_compound", "topic_1", "topic1_score",
                            "topic_2", "topic2_score"])
    sent_count = 0

    # For every review sentence
    for sentence in unigram_review:
        # Assess sentiment.
        sentiments = senti.polarity_scores(sentence)
        sentiment_neg = sentiments["neg"]
        sentiment_neu = sentiments["neu"]
        sentiment_pos = sentiments["pos"]
        sentiment_compound = sentiments["compound"]
        # Assign topic.
        # Default topic to -1.
        this_topic = -1
        # Preprocess sentence.
        sent_tokens = tokenizer.tokenize(str(sentence).lower())
        cleaned_sent = [p_stemmer.stem(i) for i in sent_tokens]
        # Evaluate for topic.
        sent_topics = []
        for mod_id in range(0, mod_num):
            model = ldamodel[mod_id]
            dicti = dictionary[mod_id]
            lda_score = model[dicti.doc2bow(cleaned_sent)]
            for item in lda_score:
                sent_topics.append((mod_id, item[0], item[1]))
        sent_topics =  sorted(sent_topics, key=lambda x: x[2], reverse=True)
        # Assign the most relevant topic to a sentence only if the topic is more than 70% dominant.
        if sent_topics[0][2] > 0.7:
            this_topic = topics_matrix[sent_topics[0][0]][sent_topics[0][1]]

        # Add procressed sentence and its meta information to the sentence dataframe.
        this_doc.loc[sent_count] = [sentence, sentiment, this_topic, sent_topics[0][2]]
        sent_count += 1

    ###########################################################################
    # Compiling results for a hotel.
    ###########################################################################
    # Review highlights.
    # Save the most positive and negative sentiments.
    this_doc2 = this_doc.sort_values(["sentiment"], ascending=[0]).reset_index(drop=True)
    this_doc2 = this_doc2.loc[this_doc2["topic"] != -1].reset_index(drop=True)
    this_doc2 = this_doc2.loc[this_doc2["topic_score"] > 0.5].reset_index(drop=True)
    sent_count_2 = len(this_doc2)
    composite = "NONE"
    # Save the most polarizing sentiments only if there are at least 6 sentences.
    if sent_count_2 > 5:
        sent1 = sent2 = sent3 = sent4 = sent5 = sent6 = ""
        # Only keep positive sentiment if its score is above 0.4 (within [-1, 1]).
        if this_doc2.loc[0]["sentiment"] > 0.4:
            sent1 = this_doc2.loc[0]["sentence"]
        if this_doc2.loc[1]["sentiment"] > 0.4:
            sent2 = this_doc2.loc[1]["sentence"]
        if this_doc2.loc[2]["sentiment"] > 0.4:
            sent3 = this_doc2.loc[2]["sentence"]
        # Only keep positive sentiment if its score is below -0.2 (within [-1, 1]).
        if this_doc2.loc[sent_count_2-1]["sentiment"] < -0.2:
            sent4 = this_doc2.loc[sent_count_2-1]["sentence"]
        if this_doc2.loc[sent_count_2-2]["sentiment"] < -0.2:
            sent5 = this_doc2.loc[sent_count_2-2]["sentence"]
        if this_doc2.loc[sent_count_2-3]["sentiment"] < -0.2:
            sent6 = this_doc2.loc[sent_count_2-3]["sentence"]
        composite = sent1 + "SSEEPP" + sent2 + "SSEEPP" + sent3 + "SSEEPP" + sent4 + "SSEEPP" + sent5 + "SSEEPP" + sent6 + "SSEEPP" + str(sent_count)
    # Add review highlights to the doctor dataframe.
    doctor_info.set_value(doctor_id, "summary", composite)

    # Top topics and their ratings.
    # Ratings are the percent positive sentences belonging to a topic.
    doc_topics = [ [ 0 for i in range(2) ] for j in range(topic_num) ]  # [total count, count positive]
    for index2 in range(0, len(this_doc2)):
        topic_index = this_doc2.loc[index2]["topic"]
        if topic_index != -1:
            doc_topics[topic_index][0] += 1
            topic_sentiment = this_doc2.loc[index2]["sentiment"]
            # A topic sentence if positive if its sentiment is bigger than 0.1.
            if topic_sentiment > 0.1:
                doc_topics[topic_index][1] += 1
    # Do not display dentist stuff for non-dentist
    if not is_dentist:
        doc_topics[3][0] = 0
    # Do not output "positive comment" as a topic. It is non-informative.
    doc_topics[0][0] = 0

    # Putting the results into a format to be sparsed by the webapp.
    doc_topic_tuples = []
    for index3, item in enumerate(doc_topics):
        doc_topic_tuples.append((index3, item[0], item[1]))
    doc_topic_tuples =  sorted(doc_topic_tuples, key=lambda x: x[1], reverse=True)
    for index4 in range(0, 5):
        if doc_topic_tuples[index4][1] >= 10:
            topic_name = topics[doc_topic_tuples[index4][0]][0]
            percent_positive = str(int(doc_topic_tuples[index4][2]/doc_topic_tuples[index4][1] * 100))
            composite = topic_name + "SSEEPP" + percent_positive + "SSEEPP" + str(doc_topic_tuples[index4][1])
            doctor_info.set_value(doctor_id, "percent{0}".format(str(index4+1)), composite)

            print(topic_name, "XXXXXX", doctor_info.loc[doctor_id]["specialty"])
            big_str.append(topic_name + "XXXXXX" + str(doctor_info.loc[doctor_id]["specialty"]))
        else:
            doctor_info.set_value(doctor_id, "percent{0}".format(str(index4+1)), "NONE")

    # Print progress.
    print(counter/5088)
    counter += doctor_review_count
    del this_doc
    del this_doc2

# Save the updated doctor dataframe containing snapshot information.
doctor_info.to_csv("PATH/result/all_doctors.csv")


from setuptools import setup
from setuptools import find_packages

PACKAGES = find_packages(where=".")

setup(
    name="clif",
    version="0.2.2",
    description="CLImate Fingerprinting methods",
    long_description="",
    url="",
    author=["Kenny Chowdhary", "Jake Nichol"],
    author_email="kchowdh@sandia.gov",
    license="BSD3",
    packages=PACKAGES,
    package_data={"clif": ["folder/*.txt"]},
    test_suite="nose.collector",
    tests_required=["nose"],
    install_requires=[
        "numpy",
        "scikit-learn",
        "tqdm",
        "xarray",
        "netCDF4",
        "statsmodels",
        "cftime",
        "matplotlib",
        "dask",
    ],
    include_package_data=True,
    zip_safe=False,
)


# coding: utf-8
# for autocomplete
import re
import operator

from django import forms
from django.db import models
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.conf.urls.defaults import patterns
from django.db.models.query import QuerySet
from django.http import HttpResponse, HttpResponseNotFound
from django.conf import settings
from django.template.defaultfilters import striptags, force_escape

# ModelAdmin Mixin to enable autocompletion
# based and inspired by http://jannisleidel.com/2008/11/autocomplete-form-widget-foreignkey-model-fields/
class AutocompleteFKMixin(object):
    # Add an autocomplete search URL in the ModelAdmin
    def get_urls(self):
        urls = super(AutocompleteFKMixin, self).get_urls()
        search_url = patterns('', (r'^search/$', self.search))
        return search_url + urls
    # Base queryset
    def get_autocomplete_queryset(self, request, field_name):
        """ Returns the basic queryset to filter results for an autocomplete field `field_name`.
        If None is returned - all db objects are used for a queryset. """
        return None
    # Search view
    def search(self, request):
        """ Searches in the fields of the given related model and returns the
        result as a simple string to be used by the jQuery Autocomplete plugin """
        query         = request.GET.get('q', None)
        app_label     = request.GET.get('app_label', None)
        model_name    = request.GET.get('model_name', None)
        search_fields = request.GET.get('search_fields', None)
        field_name    = request.GET.get('field_name', None)
        related_fields = [field[:-3] for field in request.GET
            if field[-3:] == "_pk" and request.GET[field]]

        # remove all garbage and injections
        from django.template.defaultfilters import slugify
        search_fields = [f for f in search_fields.split(',') if f==slugify(f)]

        if search_fields and app_label and model_name and query:
            def construct_search(field_name):
                # use different lookup methods depending on the notation
                if field_name.startswith('^'):
                    return "%s__istartswith" % field_name[1:]
                elif field_name.startswith('='):
                    return "%s__iexact" % field_name[1:]
                elif field_name.startswith('@'):
                    return "%s__search" % field_name[1:]
                else:
                    return "%s__icontains" % field_name

            model = models.get_model(app_label, model_name)
            qs = self.get_autocomplete_queryset(request, field_name)
            if qs is None:
                qs = model._default_manager.all()
            for bit in query.split():
                or_queries = [models.Q(**{construct_search(
                    smart_str(field_name)): smart_str(bit)})
                        for field_name in search_fields]

                other_qs = QuerySet(model)
                other_qs.dup_select_related(qs)
                other_qs = other_qs.filter(reduce(operator.or_, or_queries))
                qs = qs & other_qs
            # filter out by related ids
            qs = qs.filter(**dict([(str(field), request.GET[field+"_pk"]) for field in related_fields]))
            # sort rows directly containing query be the first
            from django.utils.datastructures import SortedDict

            # we do not was to dig into __ relations here. Only first model fields go for smart sorting.
            smartsorted_fields = []
            for field_name in search_fields:
                # add qualifying table
                try:
                    model._meta.get_field_by_name(field_name)
                except:
                    continue
                smartsorted_fields.append(field_name)

            for field_name in smartsorted_fields:
                full_field_name = model._meta.db_table+"."+field_name
                qs = qs.extra(select=SortedDict([
                    ('_%s_match' % field_name,
                        "CASE WHEN lower("+full_field_name+")=lower(%s) THEN 1 ELSE 0 END"),
                    ('_%s_contains' % field_name,
                    # can not put percents here because of http://code.djangoproject.com/ticket/6400
                        "CASE WHEN "+full_field_name+" LIKE %s THEN 1 ELSE 0 END"),
                    ]), select_params = [query, '%'+query+'%'])
            ordering = [ ['-_%s_match' % field_name, '-_%s_contains' % field_name]
                for field_name in smartsorted_fields]
            # flatten list of sorting options
            ordering = sum(ordering, [])
            qs = qs.order_by(*ordering)
            qs = qs[:25]

            # sometimes we want to present suggestions in a special way
            def ajax_str(obj):
                if hasattr(obj, 'ajax_str') and callable(obj.ajax_str):
                    return obj.ajax_str()
                else:
                    return unicode(obj)

            data = ''.join([u'%s|%s\n' % (ajax_str(f), f.pk) for f in qs])
            return HttpResponse(data, mimetype='text/plain')
        return HttpResponseNotFound()

    # Change form field for the related search fields
    def formfield_for_dbfield(self, db_field, **kwargs):
        """
        Overrides the default widget for Foreignkey fields if they are
        specified in the related_search_fields class attribute.
        """
        if isinstance(db_field, models.ForeignKey) and hasattr(self, 'related_search_fields') \
                and db_field.name in self.related_search_fields:
            kwargs['widget'] = ForeignKeySearchInput(db_field.rel, self.related_search_fields[db_field.name])
        return super(AutocompleteFKMixin, self).formfield_for_dbfield(db_field, **kwargs)

class ForeignKeySearchInput(forms.TextInput):
    """
    A Widget for displaying ForeignKeys in an autocomplete search input
    instead in a ``select`` box.
    """
    class Media:
        css = {
            'all': ('admin-autocomplete/jquery.autocomplete.css',)
        }
        js = (
            # Breaks our current
        ''') % {
            'search_fields': ','.join(self.search_fields),
            'admin_media_prefix': admin_media_prefix,
            'model_name': self.rel.to._meta.module_name,
            'app_label': self.rel.to._meta.app_label,
            'label': label,
            'name': name,
            'extra_params': extra_params,
        }


"""Utilities for handling language tokens."""

from tiktoken import encoding_for_model, get_encoding

from readmeai.core import logger

logger = logger.Logger(__name__)

def adjust_max_tokens(
    max_tokens: int, prompt: str, target: str = "Hello!"
) -> int:
    """Adjust the maximum number of tokens based on the specific prompt."""
    is_valid_prompt = prompt.strip().startswith(target.strip())
    adjusted_max_tokens = max_tokens if is_valid_prompt else max_tokens // 5
    return adjusted_max_tokens

def get_token_count(text: str, encoding_name: str) -> int:
    """Returns the number of tokens in a text string."""
    encoding = get_encoding(encoding_name)
    num_tokens = len(encoding.encode(text))
    return num_tokens

def get_token_encoder() -> str:
    """The token encoder to use for the model."""
    return (
        encoding_for_model("gpt-3.5-turbo")
        if "gpt"
        else get_encoding("cl100k_base")
    )

def truncate_tokens(text: str, max_tokens: int) -> str:
    """Truncate a text string to a maximum number of tokens."""
    if not text:
        return text
    try:
        encoder = get_token_encoder()

        prompt_token_total = len(encoder.encode(text))
        if prompt_token_total <= max_tokens:
            return text

        char_total = len(text)
        chars_per_token = char_total / prompt_token_total
        truncated_total = int(chars_per_token * max_tokens)
        return text[:truncated_total]

    except Exception as excinfo:
        logger.error(f"Error truncating tokens: {excinfo}")
        return text


import glob
import logging
import os
from collections import OrderedDict
from subprocess import Popen, PIPE, CalledProcessError

import cyvcf2
import pandas as pd
from django.conf import settings
from django.utils import timezone

from library.django_utils.django_file_utils import get_import_processing_filename
from library.utils.file_utils import name_from_filename
from upload.models import ModifiedImportedVariants, ToolVersion, UploadStep, \
    UploadStepTaskType, VCFSkippedContigs, VCFSkippedContig, UploadStepMultiFileOutput, VCFPipelineStage, \
    SimpleVCFImportInfo
from upload.tasks.vcf.unknown_variants_task import SeparateUnknownVariantsTask, AnnotateImportedVCFTask

def get_vt_tool_version(vt_command):
    p = Popen([vt_command, "--version"], stderr=PIPE)
    _, stderr = p.communicate()
    vt_version = stderr.decode().split('\n')[0]
    tool_version, _ = ToolVersion.objects.get_or_create(name='vt', version=vt_version)
    return tool_version

def create_sub_step(upload_step, sub_step_name, sub_step_commands):
    start_date = timezone.now()
    tool_version = get_vt_tool_version(settings.VCF_IMPORT_VT_COMMAND)
    command_line = ' '.join(sub_step_commands)
    return UploadStep.objects.create(name=sub_step_name,
                                     script=command_line,
                                     upload_pipeline=upload_step.upload_pipeline,
                                     sort_order=upload_step.sort_order,
                                     parent_upload_step=upload_step,
                                     tool_version=tool_version,
                                     start_date=start_date,
                                     task_type=UploadStepTaskType.TOOL)

def _write_split_headers(vcf_filename: str, remove_info: bool, split_headers_filename: str):
    """ Add VT normalize INFO to header, strip if remove_info = True """
    VT_HEADERS = [
        '##INFO=<ID=OLD_MULTIALLELIC,Number=1,Type=String,Description="Original chr:pos:ref:alt encoding">',
        '##INFO=<ID=OLD_VARIANT,Number=.,Type=String,Description="Original chr:pos:ref:alt encoding">',
    ]
    with open(split_headers_filename, "w") as f:
        written_vt_headers = False
        for line in cyvcf2.Reader(vcf_filename).raw_header.split("\n"):
            info_line = line.startswith("##INFO")
            if info_line or line.startswith("#CHROM"):
                if not written_vt_headers:
                    for vt_line in VT_HEADERS:
                        f.write(vt_line + "\n")
                    written_vt_headers = True

                if info_line and remove_info:
                    continue
            if line:
                f.write(line + "\n")

def preprocess_vcf(upload_step, remove_info=False, annotate_gnomad_af=False):
    MAX_STDERR_OUTPUT = 5000  # How much stderr output per process to store in DB

    VCF_CLEAN_AND_FILTER_SUB_STEP = "vcf_clean_and_filter"
    DECOMPOSE_SUB_STEP = "decompose"
    NORMALIZE_SUB_STEP = "normalize"
    UNIQ_SUB_STEP = "uniq"
    REMOVE_HEADER_SUB_STEP = "remove_header"
    SPLIT_VCF_SUB_STEP = "split_vcf"

    genome_build = upload_step.genome_build
    _ = genome_build.reference_fasta  # Fails if not available.

    vcf_filename = upload_step.input_filename
    if not os.path.exists(vcf_filename):
        raise FileNotFoundError(f"Can't access vcf: '{vcf_filename}'")

    pipe_commands = OrderedDict()  # Insert in pipeline order
    sub_steps = {}
    if vcf_filename.endswith(".gz"):
        pipe_commands["zcat"] = [settings.BASH_ZCAT, vcf_filename]
    else:
        pipe_commands["cat"] = ["cat", vcf_filename]

    upload_pipeline = upload_step.upload_pipeline
    vcf_name = name_from_filename(vcf_filename, remove_gz=True)
    skipped_contigs_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_contigs.tsv")
    skipped_records_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_records.tsv")
    skipped_filters_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_filters.tsv")

    manage_command = settings.MANAGE_COMMAND
    read_variants_cmd = manage_command + ["vcf_clean_and_filter",
                                          "--genome-build",
                                          genome_build.name,
                                          "--skipped-contigs-stats-file",
                                          skipped_contigs_stats_file,
                                          "--skipped-records-stats-file",
                                          skipped_records_stats_file,
                                          "--skipped-filters-stats-file",
                                          skipped_filters_stats_file]
    if remove_info:
        read_variants_cmd.append("--remove-info")
    pipe_commands[VCF_CLEAN_AND_FILTER_SUB_STEP] = read_variants_cmd
    sub_steps[VCF_CLEAN_AND_FILTER_SUB_STEP] = create_sub_step(upload_step, VCF_CLEAN_AND_FILTER_SUB_STEP, read_variants_cmd)

    # VT isn't the bottleneck here, it's my programs - so no speed advantage to using "+" for Uncompressed BCF streams
    pipe_commands[DECOMPOSE_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "decompose", "-s", "-"]
    pipe_commands[NORMALIZE_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "normalize", "-n", "-r", genome_build.reference_fasta, "-"]
    pipe_commands[UNIQ_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "uniq", "-"]

    # Split up the VCF
    split_vcf_dir = upload_pipeline.get_pipeline_processing_subdir("split_vcf")
    # We'll cat split_headers_filename on top of every split VCF
    split_headers_filename = os.path.join(split_vcf_dir, name_from_filename(vcf_filename, remove_gz=True))
    _write_split_headers(vcf_filename, remove_info, split_headers_filename)

    pipe_commands[REMOVE_HEADER_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "view", "-"]
    pipe_commands[SPLIT_VCF_SUB_STEP] = ["split", "-", vcf_name, "--additional-suffix=.vcf.gz", "--numeric-suffixes",
                                         "--lines", str(settings.VCF_IMPORT_FILE_SPLIT_ROWS),
                                         f"--filter='sh -c \"{{ cat {split_headers_filename}; cat; }} | bgzip -c > {split_vcf_dir}/$FILE\"'"]

    for sub_step_name in [DECOMPOSE_SUB_STEP, NORMALIZE_SUB_STEP, UNIQ_SUB_STEP]:
        sub_step_commands = pipe_commands[sub_step_name]
        sub_steps[sub_step_name] = create_sub_step(upload_step, sub_step_name, sub_step_commands)

    piped_command = ' | '.join([' '.join(command_with_args) for command_with_args in pipe_commands.values()])

    # if POPEN_SHELL=True we're running this all as one command as native shell text
    # this is not recommended, less portable and if commands have errors
    if settings.POPEN_SHELL:
        print("single_commands: %s" % " | ".join([' '.join(x) for x in pipe_commands.values()]))
        p = Popen(
            piped_command,
            shell=True
        )
        _, p_stderr = p.communicate()
        if p_stderr:
            logging.error("%s stderr:", piped_command)
            logging.error(p_stderr)

        if p.returncode:
            raise CalledProcessError(p.returncode, piped_command, output=p_stderr)

    else:
        pipes = {}
        stderr_filenames = {}
        p = None
        for sub_step_name, cmd in pipe_commands.items():
            stderr_filename = get_import_processing_filename(upload_pipeline.pk, f"stderr_out_{sub_step_name}.log")
            stderr_f = open(stderr_filename, "wb")
            kwargs = {"stdout": PIPE,
                      "stderr": stderr_f}
            if p:
                kwargs["stdin"] = p.stdout
            p = Popen(cmd, **kwargs)
            pipes[sub_step_name] = p
            stderr_filenames[sub_step_name] = stderr_filename

        print("pipe_commands: %s" % " | ".join([' '.join(x) for x in pipe_commands.values()]))

        for sub_step_name, p_cmd in reversed(pipe_commands.items()):
            p = pipes[sub_step_name]
            p.communicate()
            stderr_output = None
            stderr_filename = stderr_filenames.get(sub_step_name)
            if stderr_filename:
                with open(stderr_filename, "r") as sf:
                    stderr_output = sf.read()
                if stderr_output and len(stderr_output) > MAX_STDERR_OUTPUT:
                    half = MAX_STDERR_OUTPUT // 2
                    stderr_output = stderr_output[:half] + "\n ... \n" + stderr_output[-half:]
                sub_step = sub_steps.get(sub_step_name)
                if sub_step:
                    sub_step.output_text = stderr_output
                    sub_step.save()
                else:
                    logging.warning("%s stderr:", p_cmd)
                    logging.warning(stderr_output)

            if p.returncode:
                raise CalledProcessError(p.returncode, p_cmd, output=stderr_output)

    clean_sub_step = sub_steps[VCF_CLEAN_AND_FILTER_SUB_STEP]
    if os.path.exists(skipped_contigs_stats_file):
        df = pd.read_csv(skipped_contigs_stats_file, header=None, sep='\t', index_col=0)
        if not df.empty:
            import_info = VCFSkippedContigs.objects.create(upload_step=clean_sub_step)

            for contig, count in df.iloc[:, 0].items():
                VCFSkippedContig.objects.create(import_info=import_info,
                                                contig=contig,
                                                num_skipped=count)

    _store_vcf_skip_stats(skipped_records_stats_file, clean_sub_step, "records")
    _store_vcf_skip_stats(skipped_filters_stats_file, clean_sub_step, "FILTER")

    # Create this here so downstream tasks can add modified imported variant messages
    import_info, _ = ModifiedImportedVariants.objects.get_or_create(upload_step=upload_step)
    vcf_import_annotate_dir = upload_pipeline.get_pipeline_processing_subdir("vcf_import_annotate")
    sort_order = upload_pipeline.get_max_step_sort_order()
    for split_vcf_filename in glob.glob(f"{split_vcf_dir}/*.vcf.gz"):
        sort_order += 1
        separate_upload_step = UploadStep.objects.create(upload_pipeline=upload_pipeline,
                                                         name="Separate Unknown Variants Task",
                                                         sort_order=sort_order,
                                                         task_type=UploadStepTaskType.CELERY,
                                                         pipeline_stage=VCFPipelineStage.INSERT_UNKNOWN_VARIANTS,
                                                         input_filename=split_vcf_filename)
        separate_upload_step.launch_task(SeparateUnknownVariantsTask)
        output_filename = split_vcf_filename

        # If we annotate, that file will be processed in UploadStepMultiFileOutput
        if annotate_gnomad_af:
            sort_order += 1
            name = name_from_filename(split_vcf_filename, remove_gz=True)
            output_filename = os.path.join(vcf_import_annotate_dir, f"{name}.annotated.vcf.gz")
            gnomad_step = UploadStep.objects.create(upload_pipeline=upload_pipeline,
                                                    name="Annotate gnomAD AF",
                                                    sort_order=sort_order,
                                                    task_type=UploadStepTaskType.CELERY,
                                                    pipeline_stage=VCFPipelineStage.INSERT_UNKNOWN_VARIANTS,
                                                    input_filename=split_vcf_filename,
                                                    output_filename=output_filename)
            gnomad_step.launch_task(AnnotateImportedVCFTask)

        # We don't know how big the last split file is, so leave items_to_process as null so no check
        UploadStepMultiFileOutput.objects.create(upload_step=upload_step,
                                                 output_filename=output_filename)

def _store_vcf_skip_stats(filename, upload_set, description):
    if os.path.exists(filename):
        df = pd.read_csv(filename, header=None, sep='\t', index_col=0)
        if not df.empty:
            for name, count in df.iloc[:, 0].items():
                message_string = f"Skipped {count} '{name}' {description}"
                SimpleVCFImportInfo.objects.create(upload_step=upload_set, message_string=message_string)


import logging
from docarray import Document
from pywebio import session, config, start_server
from pywebio.output import *
from pywebio.pin import *
import os, io, uuid
from discord import Webhook, RequestsWebhookAdapter, Embed, File
import urllib.parse

_discord_webhook_url = os.environ["DISCORD_WEBHOOK"]
_webhook = Webhook.from_url(_discord_webhook_url, adapter=RequestsWebhookAdapter())
os.environ["JINA_HUB_ROOT"] = "/tmp/.jina"

server_url = 'grpc://dalle-flow.jina.ai:51005'

file_dir = os.path.dirname(os.path.abspath(__file__))

server_error_text = 'The demo server returned an error. Please come back in 5 mins and try again.'
image_gen_text = 'Generating images...\nThanks to JinaAI for providing the image generation service to public for free. This process may take up to 10 mins. Please be patient.'
step1 = "Step 1/3: Write down what you want to draw, and hit the `Draw` button. Up to 16 candidate images will be generated."
step2 = "Step 2/3: Click on your favorite image to fine tune it."
step3 = "Step 3/3: Choose one to generate a high-res image (1024 x 1024 px)"
credits = r'''### App source code:
https://github.com/pywebio/demos/tree/main/jina_dalle-flow

### Generated Art Gallery
Arts generated here are shared with PyWebIO community on our Discord server. Come see it: https://discord.gg/MvaCcg76Z7

**User privacy is fully respected as the sharing is anonymous.**

### Credits
This app is inspired by [JinaAI's DALLÂ·E Flow project](https://github.com/jina-ai/dalle-flow): A Human-in-the-Loop workflow for creating HD images from text.

**JinaAI**
* [Dalle-flow](https://github.com/jina-ai/dalle-flow)
* [Clip as service](https://github.com/jina-ai/clip-as-service)

**Boris Dayma**
* [DALLÂ·E Mini](https://github.com/borisdayma/dalle-mini)

**Jack Qiao**
* [Glid-3-xl](https://github.com/Jack000/glid-3-xl)

**Jingyun Liang**
* [SwinIR](https://github.com/JingyunLiang/SwinIR)
'''

def send_discord_result(prompt, image_data, stage):
    filename = uuid.uuid4().hex + '.png'
    file_path = os.path.join(file_dir, 'temp/', filename)

    if stage == 'preview':
        discord_msg = 'ðŸŽ¨ New art has just been created on https://demo.pyweb.io/live/jina_dalle-flow/app/ from a text input: `' + prompt + '`\n'
    else:
        discord_msg = 'After fine-tuning, the final result of `' + prompt + ':`\n'

    msg = Embed(description=discord_msg, color=10312083)

    data = image_data.split(',')[1]
    b = urllib.parse.unquote_to_bytes(data)
    bo = io.BytesIO(b)

    with open(file_path, "wb+") as f:
        f.write(bo.getbuffer())

    with open(file_path, "rb") as f:
        my_file = File(f)

    _webhook.send(embed=msg)
    _webhook.send(file=my_file)

@use_scope('images', clear=True)
def upscale_gen(prompt, diffused, dfav_id):
    global server_url
    fav = diffused[dfav_id]
    send_discord_result(prompt, diffused[dfav_id, 'uri'], 'diffused')
    put_text(image_gen_text)
    try:
        with put_loading():
            fav = fav.post(f'{server_url}/upscale', target_executor='upscaler')
        clear()
        image_data = fav.uri
        put_image(image_data)
    except Exception as e:
        logging.exception(e)
        clear()
        popup(server_error_text)

@use_scope('images', clear=True)
def diffused_gen(prompt, da, fav_id):
    global server_url
    fav = da[fav_id]
    send_discord_result(prompt, da[fav_id, 'uri'], 'preview')
    put_text(image_gen_text)
    try:
        with put_loading():
            diffused = fav.post(f'{server_url}', parameters={'skip_rate': 0.5, 'num_images': 9},
                                target_executor='diffusion').matches
        clear()
        put_text(step3)
        for i in range(0, 9):
            image_data = diffused[i, 'uri']
            put_image(image_data).onclick(lambda x=i: upscale_gen(prompt, diffused, x))
    except Exception as e:
        logging.exception(e)
        clear()
        popup(server_error_text)

@use_scope('images', clear=True)
def preview_image_gen(prompt):
    global server_url
    if prompt.replace(" ", "") == '':
        toast('Please input a valid sentence', color='error', duration=10)
    else:
        put_text(image_gen_text)
        try:
            with put_loading():
                da = Document(text=prompt).post(server_url, parameters={'num_images': 8}).matches
            clear()
            put_text(step2)
            for i in range(0, 16):
                image_data = da[i, 'uri']
                put_image(image_data).onclick(lambda x=i: diffused_gen(prompt, da, x))
        except Exception as e:
            logging.exception(e)
            clear()
            popup(server_error_text)

css = """
#pywebio-scope-images {
    height: calc(100vh - 150px);
    overflow-y: hidden;
}
#pywebio-scope-images:hover {
    overflow-y: scroll;
}
#pywebio-scope-input {
    height: calc(100vh - 150px);
    overflow-y: hidden;
}
#pywebio-scope-input:hover {
    overflow-y: scroll;
}
/* Works on Firefox */
* {
  scrollbar-width: thin;
}
/* Works on Chrome, Edge, and Safari */
*::-webkit-scrollbar {
  width: 7px;
}
*::-webkit-scrollbar-track {
  background: transparent;
}
*::-webkit-scrollbar-thumb {
  background-color: gray;
  border-radius: 20px;
  border: 2px
}
"""

@config(theme="minty", css_style=css)
def main():
    session.set_env(title='Dall-E Flow Web App', output_max_width='100%')

    put_markdown('# Dall-E Flow Web App')
    put_row(
        [put_scope('input'), None, put_scope('images')],
        size="2fr 40px minmax(60%, 6fr)",
    )

    with use_scope('input'):
        put_textarea('description',
            placeholder='E.g., An oil painting of a humanoid robot playing chess in the style of Matisse',
            rows=5,
        ),
        put_button('Draw', onclick=lambda: preview_image_gen(pin['description']))
        put_markdown(credits).style('font-size: 12px; color: gray; background: #f6f6f6;')

    with use_scope('images'):
        put_markdown('### How it works')
        for step in [step1, step2, step3]:
            put_text(step)

if __name__ == '__main__':
    start_server(main, debug=True, port=9999)


"""

"""

import fractions

def max_fraction(a,b):
    #print(a,b)
    if len(a)==1:
        return a[0],b[0]
    else:
        max_num = a[0]
        max_den = b[0]
        for i in range(1,len(a)):
            max_num = max_num + a[i]
            max_den = max_den + b[i]
        return max_num,max_den

def main():
    t = int(input())
    for i in range(t):
        n = int(input())
        a = []
        b = []
        for j in range(n):
            num,den = input().split("/")
            a.append(int(num))
            b.append(int(den))
        for j in range(n):
            max_num,max_den = max_fraction(a[j:],b[j:])
            gcd = fractions.gcd(max_num,max_den)
            print(str(max_num//gcd)+"/"+str(max_den//gcd))
        print()

if __name__ == '__main__':
    main()


import os
import sys
import argparse
import multiprocessing
import subprocess

def preprocess_rgb(sequence):
    sys.stderr.write(('Processing sequence {}\n'.format(sequence)))
    command = 'imgp -x 640x480 -wf {}/*.color.jpg'.format(sequence)
    subprocess.run(command, shell=True, stdout=subprocess.DEVNULL, check=True)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', help='Path to directory containing sequences to preprocess', required=True)
    parser.add_argument('--nproc', default=8, help='Number of processes to spawn')
    args = parser.parse_args()

    print("Finding all sequences to process")
    sequences = next(os.walk(args.data_dir))[1]
    sequences.sort()
    sequences = [os.path.join(args.data_dir, seq) for seq in sequences]

    print('Preprocess start')
    pool = multiprocessing.Pool(processes=args.nproc)
    pool.map(preprocess_rgb, sequences)

    pool.close()
    pool.join()
    print('Done')

if __name__ == "__main__":
    main()


from __future__ import print_function
import datetime
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']

def all_events():
    accounts = ["2u","my"]
    events = []
    for acc in accounts:
        events.extend(events_by_account(acc))
    return events

def events_by_account(acc):
    pickle_file_name = f"{acc}-token.pickle"
    creds_file_name = f"{acc}-credentials.json"
    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists(pickle_file_name):
        with open(pickle_file_name, 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                creds_file_name, SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open(pickle_file_name, 'wb') as token:
            pickle.dump(creds, token)

    service = build('calendar', 'v3', credentials=creds)

    # Call the Calendar API
    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time
    events_result = service.events().list(calendarId='primary', timeMin=now,
                                        maxResults=10, singleEvents=True,
                                        orderBy='startTime').execute()
    return events_result.get('items', [])

if __name__ == '__main__':
    all_events()


# PROGRAMMER: Humberto Jimenez
# DATE CREATED: 21 Feb 2021
#Purpose: to create a model training object that acts as an image classifier.
#Parts of this model are inspired directly from the pytorch exercises taught in the Udacity sessions,
#I have added a note to indicate where parts of the code is similar to what was taught, any similarities with other student's work is purely coincidental
#as we have all learned from the same source (examples and videos).

#basic imports
import argparse
import numpy as np
from PIL import Image
import json
import os
import random

#torch and torchvision import
import torch
from torchvision import transforms, models
import torch.nn.functional as F

from tools import load_checkpoint, load_cat_names

#Define arguments loader
#Learning source https://pymotw.com/3/argparse/
def parse_args ():
    parser = argparse.ArgumentParser(description = "Image classifier calculations")
    parser.add_argument('checkpoint', action='store', default='checkpoint.pth')
    parser.add_argument('--topk', dest='topk', default='5', type=int)
    parser.add_argument('--filepath', dest='filepath', type=str)
    parser.add_argument('--category_names', dest='category_names', default='cat_to_name.json')
    parser.add_argument('--gpu', action='store', default='gpu') #Added gpu/cpu argument
    return parser.parse_args()

#Define image processor
#This section was inspired by browsing the web and with the aid of this learnign source: https://pillow.readthedocs.io/en/latest/reference/Image.html
def process_image(image):
    #Open the image using PIL
    pil_image = Image.open(image)

    #perform transformations to make it suitable for the model
    adjust_image = transforms.Compose([transforms.Resize(256),
                                      transforms.CenterCrop(224),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406],
                                                           [0.229, 0.224, 0.225])
                                      ])

    #adjust image using the transformations
    image = adjust_image(pil_image)

    return image

#Define predictor
def predict(image_path, model, topk):
    #Invoke the arguments
    args = parse_args() #Adding to fix args not defined issue
    #move the model to the gpu environment
    device = 'cuda' if torch.cuda.is_available() and args.gpu else 'cpu' #readjusted the cuda statement
    model.to(device)

    # transform the image by running it through the process_image method,
    #unsqueeze to return the tensor with an extra dimension and convert it to float.
    # Reference for unsqueeze method: https://pytorch.org/docs/stable/generated/torch.unsqueeze.html

    image = torch.unsqueeze(process_image(image_path),0).float()

    #Enable no grad to prevent a backward step and reduce memory consumption
    with torch.no_grad():
        #Send image and label to the gpu/device
        image = image.to(device)
        #push the image into the network
        logps = model.forward(image)
    #Calculate the probability using the softmax function
    probability = F.softmax(logps.data, dim=1)
    #Create the array of top probabilities
    probs = np.array(probability.topk(topk)[0][0])

    #Determine top classes and create an array with the top possible answers
    index_to_class = {val: key for key, val in model.class_to_idx.items()}
    top_classes = [np.int(index_to_class[each]) for each in np.array(probability.topk(topk)[1][0])]

    return probs, top_classes

#Define main part of the program to run
def main():
    #invoke all arguments for the function
    args = parse_args()
    model = load_checkpoint(args.checkpoint)
    cat_to_name = load_cat_names(args.category_names)

    #Choose a random image from the test folder #10 if the user does not specify a file
    if args.filepath is None:
        test_image = random.choice(os.listdir('flowers/test/10/'))
        test_img_path= 'flowers/test/10/'+ test_image
    else:
        test_img_path = args.filepath

    #Calculate probability and classes using model
    probs, classes = predict(test_img_path, model, args.topk) #Added the topk to allow the user to choose their no. of classes
    labels = [cat_to_name[str(index)] for index in classes]

    #Print results to display file, class labels and probability
    print ('selected file:' + test_img_path)
    print('the image is likely to be a: {} with a confidence interval of: {:.3f} %'.format(cat_to_name[str(classes[0])] , probs[0]))
    print('\n other alternatives are: \n {} \n with {} probability'.format(labels, probs))

if __name__ == "__main__":
    main()


import numpy as np
import os
import pickle

from collections import defaultdict
from ..dataset.qa import Database
from ..utilities.answer_utils import EvalAIAnswerPreprocessor, compute_answers_scores
from ..utilities.config import *
from ..utilities.vocab import VocabDict

def filter_answers(answers_dset, evalai_answer_processor, num_ans):
    print("[FILTER ANSWERS] Top %s answers..." % num_ans)
    answers_dict = defaultdict(lambda: 0)
    for ans in answers_dset:
        gtruth = ans["multiple_choice_answer"]
        gtruth = evalai_answer_processor(gtruth)
        answers_dict[gtruth] += 1
    answers_list = sorted(answers_dict, key=answers_dict.get, reverse=True)
    answers_list = [t.strip() for t in answers_list if len(t.strip()) > 0]
    assert "<unk>" not in answers_list
    answers_list = ["<unk>"] + answers_list[: num_ans - 1]
    print("[FILTER ANSWERS] Top %s answers...DONE" % num_ans)
    return answers_list

def extract_answers(annotations, valid_answers_list, evalai_answer_processor):
    print("[EXTRACT ANSWERS] ...")
    answers = [None] * len(annotations)
    for i in range(len(annotations)):
        question_id = annotations[i]["question_id"]
        all_answers = [
            evalai_answer_processor(answer["answer"])
            for answer in annotations[i]["answers"]
        ]

        valid_answers = [None] * len(all_answers)
        for j, answer in enumerate(all_answers):
            if answer in valid_answers_list:
                valid_answers[j] = answer
            else:
                valid_answers[j] = "<unk>"

        answers[i] = dict(question_id=question_id, valid_answers=valid_answers)
    print("[EXTRACT ANSWERS] ...DONE")
    return answers

def compute_target(dataroot, outputs_dir, evalai_answer_processor, num_ans, split):
    print("[COMPUTE TARGET] %s set." % split)
    train2014 = Database(dataroot, "train2014")
    train_annotations = train2014.load("annotations")
    val2014 = Database(dataroot, "val2014")
    val_annotations = val2014.load("annotations")

    answers_dset = train_annotations + val_annotations
    vocab_entry = filter_answers(answers_dset, evalai_answer_processor, num_ans)

    answer_file_name = "answers_top%s_vocab.txt" % num_ans
    answer_file = (os.path.join(outputs_dir, answer_file_name),)
    with open(answer_file, "w") as f:
        f.writelines([w + "\n" for w in vocab_entry])

    answer_vocab = VocabDict(vocab_entry)
    valid_answers_list = answer_vocab.word_lists

    if split == "train":
        ans = extract_answers(
            train_annotations, valid_answers_list, evalai_answer_processor
        )
    elif split == "val":
        ans = extract_answers(
            val_annotations, valid_answers_list, evalai_answer_processor
        )

    targets = []
    for i in range(len(ans)):
        question_id = ans[i]["question_id"]
        valid_answers = ans[i]["valid_answers"]
        answers_indices = [answer_vocab.word2idx(ans) for ans in valid_answers]
        answers_scores = compute_answers_scores(
            answers_indices, answer_vocab.num_vocab, answer_vocab.UNK_idx
        )

        targets.append(
            {
                "question_id": question_id,
                "answers_indices": answers_indices,
                "answers_scores": answers_scores,
            }
        )
    target_file = os.path.join(outputs_dir, split + "_target.pkl")
    pickle.dump(targets, open(target_file, "wb"))
    print("[COMPUTE TARGET] %s set....DONE" % split)
    return

if __name__ == "__main__":
    evalai_answer_processor = EvalAIAnswerPreprocessor()
    compute_target(dataroot, outputs_dir, evalai_answer_processor, num_answers, train)
    compute_target(dataroot, outputs_dir, evalai_answer_processor, num_answers, val)


import logging

from os import path, mkdir
from uuid import UUID

logger = logging.getLogger(__name__)

def ensureFolderExists(theFolder):
    """Check if a folder exists, and if not, create it.
    """
    if not path.isdir(theFolder):
        mkdir(theFolder)
        logger.info("Created folder: %s" % theFolder)
    return path.isdir(theFolder)

def isValidUUID4(theUUID):
    """Check if a string is a valid UUID4.
    """
    if not isinstance(theUUID, str):
        return False
    try:
        _ = UUID(theUUID, version=4)
    except ValueError:
        return False
    return True

def parseSizeString(theSize):
    """Parses a size string into an integer byte value.
    """
    if not isinstance(theSize, str):
        return -1
    theBits = theSize.split()
    if len(theBits) == 1:
        try:
            return int(round(float(theBits[0])))
        except Exception:
            return -1
    elif len(theBits) > 1:
        if theBits[1].lower() == "kb":
            dScale = 1024
        elif theBits[1].lower() == "mb":
            dScale = 1024*1024
        elif theBits[1].lower() == "gb":
            dScale = 1024*1024*1024
        else:
            dScale = 1
        try:
            return int(round(float(theBits[0])*dScale))
        except Exception:
            return -1
    return -1


import os
import subprocess
import pathlib
import sys

EXTENSIONS = [
    '.cpp',
    '.h',
]

def main():
    clang_format = pathlib.Path(sys.prefix) / 'bin' / 'clang-format'
    root = pathlib.Path(os.getenv('BUILD_WORKSPACE_DIRECTORY')) / 'risc0'
    for root, _, files in os.walk(root):
        for file in files:
            path = pathlib.Path(root) / file
            if path.suffix in EXTENSIONS:
                print('.', end='', flush=True)
                subprocess.run([clang_format, '-i', path])
    print()

if __name__ == "__main__":
    main()


import urllib.request
import zipfile
import os
import shutil
import yaml

from datetime import datetime

def read_yaml(file_path):
    with open(file_path, "r") as f:
        return yaml.safe_load(f)

def download():
    url = "https://github.com/Dimbreath/GenshinData/archive/refs/heads/master.zip"
    print ("Downloading...")
    print(datetime.now().time())
    filename, headers = urllib.request.urlretrieve(url, filename="./DimbreathDatamine.zip")
    print ("Download complete!")
    print(datetime.now().time())

def unzip():
    if os.path.exists("./DimbreathDatamine.zip"):
        print("\nUnzipping...")
        print(datetime.now().time())
        with zipfile.ZipFile("./DimbreathDatamine.zip", 'r') as zip_ref:
            try:
                zip_ref.extractall("./DimbreathDatamine")
            except FileNotFoundError:
                print(
"""Error: FileNotFoundError

Please do not tamper with the files mid download.
Run this program again to fix it.""")
                input("\n\nPress Enter to Exit")
                exit()
        print("Unzipped!")
        print(datetime.now().time())
    else:
        print("No file to UNZIP.")

def delZIP():
    print("\nRemoving ZIP...")
    print(datetime.now().time())
    if os.path.exists("./DimbreathDatamine.zip"):
        os.remove("./DimbreathDatamine.zip")
        print("Zip removed!")
        print(datetime.now().time())
    else:
        print("No ZIP file to delete.")

def moveAndDelExtra():
    print("\nRemoving extra files...")
    print(datetime.now().time())
    if os.path.exists(".\DimbreathDatamine\GenshinData-master"):
        try:
            shutil.rmtree(".\DimbreathDatamine\ExcelBinOutput", ignore_errors=True)
            shutil.move(".\DimbreathDatamine\GenshinData-master\ExcelBinOutput",".\DimbreathDatamine")

            shutil.rmtree(".\DimbreathDatamine\TextMap", ignore_errors=True)
            shutil.move(".\DimbreathDatamine\GenshinData-master\TextMap",".\DimbreathDatamine")
        except:
            print(
"""Ignoring Errors AHHHHHHHHHHHHH!!!!!!!!!

Lowkey though, run the file again to fix :).

Error: shutil.Error: Destination not found.""")
            input("\n\nPress Enter to Exit")
            exit()

        shutil.rmtree(".\DimbreathDatamine\GenshinData-master", ignore_errors=True)
        print("Extra files removed!")
    else:
        print("No extra files found.")
    print(datetime.now().time())
    print("\nFile Extraction Complete!")

def extractLang():
    print("\nExtracting languages...")
    print(datetime.now().time())

    for lang in read_yaml("config.yaml")[list(read_yaml("config.yaml").keys())[3]].keys():
        if not read_yaml("config.yaml")[list(read_yaml("config.yaml").keys())[3]][lang]:
            path = ".\DimbreathDatamine\GenshinData-master\TextMap\%s" % lang
            if os.path.exists(path):
                os.remove(path)

    print("\nLanguages Extracted!")
    print(datetime.now().time())

download()
unzip()
delZIP()
extractLang()
moveAndDelExtra()

input("\n\nPress Enter to Exit")
exit()


from fpdf import FPDF
import matplotlib.pyplot as plt

# pdf.add_font('custom_font', fname = './copy.ttf')

n = int(input('ENTER COUNT : '))
names = []

for i in range(0, n) :
    name = input('ENTER NAME : ')
    names.append(name)

def make_pdf(name, dir) :
    pdf = FPDF(orientation = 'L', unit = 'mm', format = 'A4')
    pdf.add_page()
    pdf.image('./template.png', h = 210, w = 300, x = 0, y = 0)
    pdf.set_font('Helvetica', 'B', 48)
    pdf.set_text_color(255, 157, 125)
    pdf.set_xy(0,100)
    pdf.cell(w = 300, h = 10, txt = name, align = 'C')
    pdf.output(dir + '/' + name + '.pdf', 'F')

for name in names :
    make_pdf(name, './outputs')

# orange 255, 157, 125
# yellow 237, 255, 122

#1414 x 2000


import json
import numpy as np
from sklearn.model_selection import train_test_split
import time
import torch
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader, TensorDataset

def load_data(data_path):
    """
    Load data from JSON file and return features and labels

    Parameters:
    - data_path: path to JSON file (string)

    Return:
    - X: features from data (numpy array)
    - y: labels for data (numpy array)

    Resource:
        https://github.com/musikalkemist/DeepLearningForAudioWithPython/blob/master/19-%20How%20to%20implement%20an%20RNN-LSTM%20for%20music%20genre%20classification/code/19-%20How%20to%20implement%20an%20RNN-LSTM%20for%20music%20genre%20classification.py
    """

    # open file for reading
    with open(data_path, "r") as fp:
        data = json.load(fp)

    # store features and labels as np arrays
    X = np.array(data["mfcc"])
    y = np.array(data["labels"])

    return X, y

def prepare_datasets(data_path, test_size, validation_size):
    """
    Loads data and splits it into train, validation and test sets.

    Parameters:
    - test_size: percentage of data set to allocate for testing (float)
    - validation_size: percentages of training set to allocate for validation (float)

    Return:
    - X_train: training set (numpy array)
    - X_validation: training validation set (numpy array)
    - X_test: testing set (numpy array)
    - y_train: labels for training set (numpy array)
    - y_validation: labels for training validation set (numpy array)
    - y_test: labels for testing set (numpy array)

    Resource:
        https://github.com/musikalkemist/DeepLearningForAudioWithPython/blob/master/19-%20How%20to%20implement%20an%20RNN-LSTM%20for%20music%20genre%20classification/code/19-%20How%20to%20implement%20an%20RNN-LSTM%20for%20music%20genre%20classification.py
    """
    # load data
    X, y = load_data(data_path)

    # create train, validation and test split
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size=validation_size)

    return X_train, X_validation, X_test, y_train, y_validation, y_test

def get_data_loaders(data_path):
    """
    Create and return PyTorch DataLoader objects for the training, validation, and test data

    Parameters:
    - data_path: Path to JSON file with data (string)

    Return:
    - train_loader: training data (DataLoader)
    - val_loader: val data (DataLoader)
    - test_loader: test data (DataLoader)
    """
    batch_size = 32

    # get train, validation, test splits
    X_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(data_path, 0.2, 0.2)

    train_features = torch.Tensor(X_train)
    train_targets = torch.Tensor(y_train)
    val_features = torch.Tensor(X_validation)
    val_targets = torch.Tensor(y_validation)
    test_features = torch.Tensor(X_test)
    test_targets = torch.Tensor(y_test)

    train_dataset = TensorDataset(train_features, train_targets)
    val_dataset = TensorDataset(val_features, val_targets)
    test_dataset = TensorDataset(test_features, test_targets)

    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, drop_last=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, drop_last=True)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, drop_last=True)

    return train_loader, val_loader, test_loader

def train(model, train_loader, val_loader, k_epochs, device):
    """
    Time and train model for k_epochs using the train_loader while regulating performance
    using the val_loader

    Parameters:
    - model: model to train with
    - train_loader: data to train on (DataLoader)
    - val_loader: data to validate and regulate performance (DataLoader)
    - k_epochs: number of epochs to run training on (int)
    - device: device to move to

    Guidance from CS451 lecture notes
    """

    begin = time.time()

    loss_fn = nn.BCEWithLogitsLoss() # Loss function for binary classification
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # Printing purposes
    last_epoch = len(model.training_info['train_losses'])

    for epoch in range(k_epochs):

        # Training
        model.train() #Set model to training mode
        train_loss = 0
        correct = 0
        total = 0

        for i, data in enumerate(train_loader, 0):
            X, y = data
            X = X.to(device)
            y = y.to(device)

            optimizer.zero_grad()

            output = model(X)

            # Training Loss
            loss = loss_fn(output.squeeze(), y)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()

            # Training Accuracy
            predicted = (output.sigmoid() >= 0.5).float()
            predicted = predicted.view(-1) # modify dimensions
            total += y.size(0)
            correct += (predicted == y.float()).sum().item()

        # Calculate training loss and accuracy
        train_loss /= len(train_loader)
        train_accuracy = 100. * (correct / total)

        # Add training loss and accuracy to model's history
        model.training_info['train_losses'].append(train_loss)
        model.training_info['train_accuracies'].append(train_accuracy)

        # Validation
        model.eval() # Set model to evaluation mode
        val_loss = 0
        correct = 0
        total = 0

        # Do not store computational graph since we are simulating testing
        with torch.no_grad():
            for i, data in enumerate(val_loader, 0):
                X, y = data
                X = X.to(device)
                y = y.to(device)

                output = model(X)

                # Validation Loss
                loss = loss_fn(output.squeeze(), y)
                val_loss += loss.item()

                # Validation Accuracy
                predicted = (output.sigmoid() >= 0.5).float()
                predicted = predicted.view(-1) # modify dimensions
                total += y.size(0)
                correct += (predicted == y.float()).sum().item()

            # Calculate validation loss and accuracy
            val_loss /= len(val_loader)
            val_accuracy = 100. * (correct / total)

            # Add validation loss and accuracy to model's history
            model.training_info['val_losses'].append(val_loss)
            model.training_info['val_accuracies'].append(val_accuracy)

        print(f'Epoch {last_epoch+epoch+1}:')
        print(f'\tTrain Loss={train_loss:.4f} Train Accuracy={train_accuracy:.2f}%')
        print(f'\tVal Loss  ={val_loss:.4f} Val Accuracy  ={val_accuracy:.2f}%')

    end = time.time()
    print(f'Finished training in {round(end - begin)}s')

def test(model, test_loader, device):
    """
    Test performance of model based on test_loader

    Parameters:
    - model: model to test
    - test_loader: test data to assess performance (DataLoader)
    - device: device to move to

    Guidance from CS451 lecture notes
    """
    correct = 0
    total = 0

    # Do no store computational graph since we are testing
    with torch.no_grad():
        for data in test_loader:
            X, y = data
            X = X.to(device)
            y = y.to(device)

            y_hat = model(X)

            predicted = (y_hat.sigmoid() >= 0.5).float()
            predicted = predicted.view(-1) # modify dimensions

            # compute the accuracy
            total += y.size(0)
            correct += (predicted == y).sum().item()

    print(f'Test accuracy: {100 * correct // total} %')

def plot_history(title, model, graph_path):
    """
    Plot the models history of training/validation accuracies and
    training/validation losses on separate graphs based on what is
    stored in its training info. Save the created graphs at graph_path.

    Parameters:
    - title: title to give graph (string)
    - model: model whose history to inspect
    - graph_path: path for storing images of graphs (string)
    """

    fig, axs = plt.subplots(2)

    fig.suptitle(title, fontweight='heavy')

    # create accuracy sublpot
    axs[0].plot(model.training_info["train_accuracies"], label="train accuracy")
    axs[0].plot(model.training_info["val_accuracies"], label="val accuracy")
    axs[0].set_ylabel("Accuracy")
    axs[0].set_xlabel("Epoch")
    axs[0].legend(loc="lower right")
    axs[0].set_title("Accuracy eval")

    # create error sublpot
    axs[1].plot(model.training_info["train_losses"], label="train error")
    axs[1].plot(model.training_info["val_losses"], label="val error")
    axs[1].set_ylabel("Error")
    axs[1].set_xlabel("Epoch")
    axs[1].legend(loc="upper right")
    axs[1].set_title("Error eval")

    fig.tight_layout()

    plt.savefig(graph_path)
    plt.show()


# -*- coding: utf-8 -*-
# @Author  : MengYangD
# @FileName: unet.py

# refer:
# https://github.com/milesial/Pytorch-UNet/blob/master/unet/unet_model.py
# https://github.com/usuyama/pytorch-unet/blob/master/pytorch_unet.py
# https://github.com/jvanvugt/pytorch-unet/blob/master/unet.py

import torch.nn as nn
import torch.nn.functional as F
from .unet_parts import *
from maskrcnn_benchmark.utils.registry import Registry
from maskrcnn_benchmark.layers import FrozenBatchNorm2d

class BaseStem(nn.Module):
    def __init__(self, cfg, norm_func):
        super(BaseStem, self).__init__()

        out_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS

        self.conv1 = nn.Conv2d(
            3, out_channels, kernel_size=7, stride=2, padding=3, bias=False
        )
        self.bn1 = norm_func(out_channels)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = F.relu_(x)
        x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1)
        return x

class StemWithFixedBatchNorm(BaseStem):
    def __init__(self, cfg):
        super(StemWithFixedBatchNorm, self).__init__(
            cfg, norm_func=FrozenBatchNorm2d
        )

class UNet(nn.Module):
    def __init__(self, cfg):
        super(UNet, self).__init__()
        # Translate string names to implementations
        stem_module = _STEM_MODULES[cfg.MODEL.RESNETS.STEM_FUNC]
        self.stem = stem_module(cfg)
        in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS

        self.inc = inconv(in_channels, 64)
        self.down1 = down_sample(64, 128)
        self.down2 = down_sample(128, 256)
        self.down3 = down_sample(256, 512)
        self.down4 = down_sample(512, 1024)

        self.up1 = up_sample(1024, 512, padding=1, up_mode='upsample')
        self.up2 = up_sample(512, 256, padding=1, up_mode='upsample')
        self.up3 = up_sample(256, 128, padding=1, up_mode='upsample')
        self.up4 = up_sample(128, 64, padding=1, up_mode='upsample')

        # self.outc = outconv(64,2)

    def forward(self, x):
        # x1 = self.inc(x)
        x1 = self.stem(x)
        print('------------')
        print('x1,', x1.shape)
        x2 = self.inc(x1)
        print('------------')
        print('x2,', x2.shape)
        down1 = self.down1(x2)
        down2 = self.down2(down1)
        down3 = self.down3(down2)
        down4 = self.down4(down3)
        print('------------')
        print('d1,', down1.shape)
        print('------------')
        print('d2,', down2.shape)
        print('------------')
        print('d3,', down3.shape)
        print('------------')
        print('d4,', down4.shape)

        up1 = self.up1(down4, down3)
        up2 = self.up2(up1, down2)
        up3 = self.up3(up2, down1)
        up4 = self.up4(up3, x1)
        # out = self.outc(up4)

        outputs_res = [up4, up2, up3, up1]

        # return F.sigmoid(out)
        return outputs_res

_STAGE_SPECS = Registry({
    "UNet-FPN": UNet,
})

_STEM_MODULES = Registry({
    "StemWithFixedBatchNorm": StemWithFixedBatchNorm,
})


import openai
from config import api_key
from dbInit import db

openai.api_key  = api_key
def get_completion(prompt, model="gpt-3.5-turbo"):
    messages = [{"role": "user", "content": prompt}]
    response = openai.ChatCompletion.create(
        model=model,
        messages=messages,
        temperature=0.5, # this is the degree of randomness of the model's output
    )
    return response.choices[0].message["content"]

data_cursor = db.inventario.find()

data_list = list(data_cursor)

while True:
    message = input("Escribe tu prompt ✍  (o 'salir' ❌ para terminar):")

    if message.lower() == 'salir':
        print("Saliendo del programa. ¡Hasta luego! 👋")
        break

    prompt = f"""
     {message}
    ```{data_list}```
    """
    response = get_completion(prompt)
    print(response)


from setuptools import find_packages, setup

with open("README.md", "r") as fh:
    long_description = fh.read()
setup(
    name="Mountain_Scott_ds12",
    version="1.0",
    author="Mountain Scott",
    author_email="mountainscott@yahoo.com",
    description="Helper functions for splitting and adding columns to dataframes",
    long_description=long_description,
    long_description_content_type="text/markdown", # required if using a md file for long desc
    license="MIT",
    url="https://github.com/Elswayre/Ray_lambda_ds12",
    #keywords="",
    packages=find_packages() # ["lambdata_mountain"]
)


import threading, datetime, socket, select, os
from io import SEEK_SET
from subprocess import Popen, PIPE
from numbers import Integral

HOST = "10.51.16.78"
PORT = 10007
BUFF_S = 5
fw = -1
offset = 0

list_keys = []
list_values = []

def write_in_place (partscount, text, outfile, num):
    offset_ = (partscount - num - 1)*BUFF_S
    print("Offset: " + str(offset_))
    print(text)
    if offset_ != 0:
        offset_ -= offset
        outfile.seek(offset_, SEEK_SET)
        outfile.write(text)
    else:
        outfile.seek(0, SEEK_SET)
        outfile.write(text)
    return

indata = open('test.txt', 'r')
indata.seek(0, os.SEEK_END)
file_size = indata.tell()
print ("File size -> " + str(file_size))
indata.seek(0, os.SEEK_SET)
cout_parts = int(file_size / BUFF_S)

offset = BUFF_S - file_size % BUFF_S
if offset != BUFF_S:
    cout_parts += 1
else:
    offset = 0
print ("Offset -> " + str(offset))

connected = []
server = socket.socket()
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind((HOST, PORT))
server.listen(cout_parts*2)
connected.append(server)

print ("Count parts -> " + str(cout_parts))
out = open('out.txt', 'w')
arr_proc = []
z = 0
c = 0
flag = 0
while not flag:
    read_sockets, a, b = select.select(connected, [], [])
    for sock in read_sockets:
        if sock == server:
            curr_data = indata.read(BUFF_S)
            sockt, addr = server.accept()
            connected.append(sockt)
            list_keys.append(0)
            list_values.append(addr[1])
            print(str(z) + "\n")
            if fw == -1:
                fw = z
            print("Client (%s, %s) connected" % addr)
            sockt.send((str(z) + curr_data).encode())
            z += 1
            c += 1
        else:
            data = sock.recv(BUFF_S + 1).decode()
            id = int(data[0])
            print(str(id))
            print ("Incoming data -> " + data[1: ])
            write_in_place(cout_parts , data[1: ], out, id)
            c += 1
            sock.close()
            connected.remove(sock)
            if c > cout_parts * 2 - 1:
                flag = 1
                print ("End cycle")
out.close()
server.close()


from unittest import TestCase

from mock import call, patch, Mock

from gofer.rmi.model import protocol
from gofer.rmi.model.parent import Monitor, Call
from gofer.rmi.model.parent import Result, Progress, Error, Raised, Ping

MODULE = 'gofer.rmi.model.parent'

class TestMonitor(TestCase):

    @patch(MODULE + '.sleep')
    def test_run(self, sleep):
        context = Mock()
        context.cancelled.side_effect = [False, True]
        child = Mock()

        # test
        m = Monitor(context, child)
        m.run()

        # validation
        child.terminate.assert_called_once_with()
        sleep.assert_called_once_with(0.10)

    @patch(MODULE + '.Monitor.join')
    def test_stop(self, join):
        context = Mock()
        child = Mock()

        # test
        m = Monitor(context, child)
        m.stop()

        # validation
        self.assertEqual(m.poll, False)
        join.assert_called_once_with()

class TestReplies(TestCase):

    def test_result(self):
        payload = 'done'
        reply = Result(payload)

        # test
        try:
            reply()
            self.fail(msg='End not raised')
        except protocol.End as end:
            self.assertEqual(end.result, payload)

    @patch(MODULE + '.Context.current')
    def test_progress(self, current):
        class P(object):
            report = Mock()
        context = Mock()
        context.progress = P()
        current.return_value = context
        payload = protocol.ProgressPayload(1, 2, 3)
        reply = Progress(payload)

        # test
        reply()

        context.progress.report.assert_called_once_with()
        self.assertEqual(context.progress.__dict__, payload.__dict__)

    def test_error(self):
        payload = 18
        reply = Error(payload)
        self.assertRaises(Exception, reply)

    def test_raised(self):
        payload = ValueError()

        # test
        reply = Raised(payload)
        try:
            reply()
            self.fail(msg='ValueError not raised')
        except ValueError as e:
            self.assertEqual(e, payload)

    def test_ping(self):
        payload = 1234
        reply = Ping(payload)
        reply()

class TestCall(TestCase):

    @patch(MODULE + '.Call.read')
    @patch(MODULE + '.Target')
    @patch(MODULE + '.Context')
    @patch(MODULE + '.Process')
    @patch(MODULE + '.Pipe')
    @patch(MODULE + '.Monitor')
    def test_call(self, monitor, pipe, process, context, target, read):
        pipe.return_value = Mock(reader=Mock(), writer=Mock())

        # test
        _call = Call(Mock(), 1, 2, a=1, b=2)
        _call()

        # validation
        pipe.assert_called_once_with()
        pipe.return_value.writer.close.assert_called_once_with()
        target.assert_called_once_with(_call.method, *_call.args, **_call.kwargs)
        process.assert_called_once_with(target.return_value, pipe.return_value)
        monitor.assert_called_once_with(context.current.return_value, process.return_value)
        monitor.return_value.start.assert_called_once_with()
        read.assert_called_once_with(pipe.return_value.reader)
        monitor.return_value.stop.assert_called_once_with()
        process.return_value.wait.assert_called_once_with()

    @patch(MODULE + '.protocol.Reply')
    def test_read(self, reply):
        replies = [Mock(), Mock(side_effect=protocol.End(18))]
        reply.read.side_effect = replies
        _call = Call(Mock())
        pipe = Mock()

        # test
        retval = _call.read(pipe)

        # validation
        self.assertEqual(
            reply.read.call_args_list,
            [
                call(pipe),
                call(pipe)
            ])
        self.assertEqual(retval, 18)


"""makeNovel Build Command

makeNovel – Build Command
=========================
Main function for the build command

File History:
Created:   2018-03-15 [0.1.0]

"""

import logging
import getopt
import mknov   as mn

from os import path, getcwd, pardir

from mknov.error   import ErrHandler, ErrCodes
from mknov.config  import Config
from mknov.main    import MakeNovel

logger = logging.getLogger(__name__)

def buildProject(sysArgs):

    # Valid Input Options
    shortOpt = "hm:at"
    longOpt  = [
        "help",
        "master=",
        "all",
        "timeline",
    ]

    helpMsg = (
        "makenovel {version} build\n"
        "\n"
        "Usage:\n"
        " -h, --help      Print this message.\n"
        " -m, --master=   Specify the master document. If none is specified,\n"
        "                   the previously used file will be used.\n"
        " -a, --all       Build all items.\n"
        " -t, --timeline  Build novel timeline.\n"
    ).format(
        version   = mn.__version__,
        status    = mn.__status__,
        copyright = mn.__copyright__
    )

    # Default Values
    masterFile    = mn.CFG.masterFile
    buildTimeLine = False

    # Parse Options
    try:
        inOpts, inArgs = getopt.getopt(sysArgs,shortOpt,longOpt)
    except getopt.GetoptError:
        print(helpMsg)
        exit(2)

    # Parse Input
    for inOpt, inArg in inOpts:
        if inOpt in ("-h","--help"):
            print(helpMsg)
            return True
        elif inOpt in ("-m","--master"):
            masterFile = inArg

    #
    # Execute Build
    #

    if masterFile == "":
        mn.OUT.errMsg("No master file specified")
        ErrHandler.terminateExec(ErrCodes.ERR_MISSINGINFO)
        exit(2)

    # Command Header
    mn.OUT.printHeader("This is %s build - version %s" % (
        mn.__appname__,mn.__version__), 72
    )

    # Echo Settings
    mn.OUT.infMsg("Master file: %s" % masterFile)

    theNovel = MakeNovel()
    theNovel.setMasterFile(masterFile)
    theNovel.buildInit()

    mn.OUT.infMsg("")
    mn.CFG.saveConfig()

    return False


#!/usr/bin/env python
import rospy
from imutils.video import VideoStream
from imutils.video import FPS
import numpy as np
import cv2
import time
from camera import RSD435

def measure_walloutlet(sensor, outlet_cascade, fps):
    img = sensor.color_image()
    H,W = sensor.image_size()

    # detect outles in gray image
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    gray = cv2.GaussianBlur(gray, (3,3), 0)
    outlets = outlet_cascade.detectMultiScale(gray, 1.1, 20, minSize=(10,10))

    text_horizontal = 0
    for (x,y,w,h) in outlets:
        img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
        u = int(x + 0.5*w)
        v = int(y + 0.5*h)
        point3d = sensor.point3d(u,v)
        # print(point3d)
        # update the FPS counter
        fps.update()
        fps.stop()
        # information will be displayed on the frame
        info = [
            ("FPS", "{:.2f}".format(fps.fps())),
            ("x", "{:.2f}".format(point3d[0])),
            ("y", "{:.2f}".format(point3d[1])),
            ("z", "{:.2f}".format(point3d[2]))
        ]

        # loop over the info tuples and draw them on our frame
        for (i, (k, v)) in enumerate(info):
            text = "{}: {}".format(k, v)
            cv2.putText(img, text, (10 + text_horizontal*100, H - ((i * 20) + 20)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)

        text_horizontal += 1

    cv2.namedWindow("wall outlet measurement")
    img = cv2.resize(img, None, fx=1.5, fy=1.5)
    cv2.imshow('wall outlet measurement',img)
    cv2.waitKey(3)

if __name__ == '__main__':
    rospy.init_node("measure_walloutlet", anonymous=True, log_level=rospy.INFO)
    sensor = RSD435()
    outlet_cascade = cv2.CascadeClassifier('../classifier_usa/cascade.xml')
    fps = FPS().start()
    rate = rospy.Rate(10)
    try:
        while not rospy.is_shutdown():
            if sensor.ready():
                measure_walloutlet(sensor,outlet_cascade,fps)
            rate.sleep()
    except rospy.ROSInterruptException:
        pass
    cv2.destroyAllWindows()


import time
from inbox import constants
from inbox.models import PrimaryDomain
from tests import AppEngineFlaskTestCase
from inbox.helpers import IMAPHelper, DriveHelper, MigrationHelper
from secret_keys import (
    TEST_LOGIN, TEST_PASS, TEST_OAUTH2_CONSUMER_KEY,
    TEST_OAUTH2_CONSUMER_SECRET, TEST_PRIMARY_CREDENTIALS,
    TEST_PRIMARY_ADMIN_EMAIL, TEST_PRIMARY_REFRESH_TOKEN,
    TEST_FAKE_PRIMARY_CREDENTIALS
)
from inbox.util import OAuthEntity
import mock
from fixtures import RFC_822_TEST
import email

class ImapHelperTestCase(AppEngineFlaskTestCase):
    def test_login(self):
        imap = IMAPHelper()
        result, data = imap.login(TEST_LOGIN, TEST_PASS)
        self.assertEquals('OK', result, "Couldn't login successfully")
        imap.close()

    def init(self, key, secret):
        self.key = TEST_OAUTH2_CONSUMER_KEY
        self.secret = TEST_OAUTH2_CONSUMER_SECRET

    @mock.patch.object(OAuthEntity, '__init__', init)
    def test_oauth1_2lo_login(self):
        imap = IMAPHelper()
        imap.oauth1_2lo_login("administrador@eforcers.com.co")
        imap.close()

    def test_list_messages(self):
        imap = IMAPHelper()
        result, data = imap.login(TEST_LOGIN, TEST_PASS)
        self.assertEquals('OK', result, "Couldn't login successfully")

        data = imap.list_messages()
        self.assertNotEquals(0, len(data), "Couldn't list messages successfully")
        imap.close()

    def test_get_message(self):
        imap = IMAPHelper()
        result, data = imap.login(TEST_LOGIN, TEST_PASS)
        self.assertEquals('OK', result, "Couldn't login successfully")

        data = imap.list_messages()
        self.assertNotEquals(0, len(data), "Couldn't list messages successfully")

        result, data = imap.get_message(data[0])
        self.assertEquals('OK', result, "Couldn't get message successfully")
        imap.close()

    def test_get_localized_labels(self):
        imap = IMAPHelper()
        result, data = imap.login(TEST_LOGIN, TEST_PASS)
        imap.get_localized_labels()
        self.assertNotEqual("",
                            imap.all_labels[constants.GMAIL_TRASH_KEY],
                            "Trash label is missing")
        self.assertNotEqual("",
                            imap.all_labels[constants.GMAIL_ALL_KEY],
                            "Trash label is missing")
        imap.close()

    def test_copy_message(self):
        imap = IMAPHelper()
        result, data = imap.login(TEST_LOGIN, TEST_PASS)
        self.assertEquals('OK', result, "Couldn't login successfully")

        imap.create_label('testlabel')

        prev_number_messages = len(imap.list_messages(criteria='label:testlabel'))

        messages = imap.list_messages()

        imap.copy_message(msg_id=messages[0], destination_label='testlabel')
        new_number_messages = len(imap.list_messages(criteria='label:testlabel'))

        self.assertEquals(prev_number_messages,
                          new_number_messages - 1,
                          "The message wasn't copied")

        imap.remove_message_label(msg_id=messages[0], prev_label='testlabel')

        new_number_messages = len(imap.list_messages(criteria='label:testlabel'))

        self.assertEquals(prev_number_messages,
                          new_number_messages,
                          "The message label wasn't removed")

class DriveHelperTestCase(AppEngineFlaskTestCase):

    def test_init(self):
        admin_email = TEST_PRIMARY_ADMIN_EMAIL

        drive = DriveHelper(credentials_json=TEST_PRIMARY_CREDENTIALS,
                    admin_email=admin_email,
                    refresh_token=TEST_PRIMARY_REFRESH_TOKEN)
        folder = drive.get_folder('anything')

    def test_get_and_create_folder(self):
        admin_email = TEST_PRIMARY_ADMIN_EMAIL

        drive = DriveHelper(credentials_json=TEST_PRIMARY_CREDENTIALS,
                    admin_email=admin_email,
                    refresh_token=TEST_PRIMARY_REFRESH_TOKEN)

        # Create and check folder
        folder = drive.get_folder('integration-test-folder')
        self.assertEquals(None, folder, "Folder shouldn't exist")

        folder = drive.create_folder('integration-test-folder')
        self.assertNotEquals(None, folder, "Folder exists now")

        created_folder = drive.get_folder('integration-test-folder')
        self.assertEquals(created_folder, folder, "Folder wasn't created")

        # Create and check subfolder
        sub_folder = drive.get_folder('integration-test-subfolder')
        self.assertEquals(None, sub_folder, "Folder shouldn't exist")

        parents = [{'id': folder['id']}]
        sub_folder = drive.create_folder(
            'integration-test-subfolder', parents=parents)
        self.assertNotEquals(None, sub_folder, "Subfolder exists now")

        created_subfolder = drive.get_folder(
            'integration-test-subfolder', parent_id=folder['id'])
        self.assertEquals(created_subfolder, sub_folder, "Subfolder wasn't created")

        # Garbage collect
        drive.service.files().delete(fileId=folder['id']).execute()
        drive.service.files().delete(fileId=sub_folder['id']).execute()

    def test_insert_file_and_get_metadata(self):
        admin_email = TEST_PRIMARY_ADMIN_EMAIL

        drive = DriveHelper(credentials_json=TEST_PRIMARY_CREDENTIALS,
                    admin_email=admin_email,
                    refresh_token=TEST_PRIMARY_REFRESH_TOKEN)

        # Create in root
        created_file = drive.insert_file("integration-test.txt", 'plain/text', 'lorem ipsum')
        self.assertNotEquals(None, created_file, "File wasn't created")

        saved_file = drive.get_metadata(title="integration-test.txt")
        self.assertEquals(created_file['md5Checksum'],
                          saved_file['md5Checksum'], "The file wasn't created")

        # Create in a folder
        created_folder = drive.create_folder("integration-text-folder")
        self.assertNotEquals(None, created_folder, "Folder wasn't created")

        created_folder_file = drive.insert_file(
            "integration-foldertest.txt", 'plain/text', 'lorem ipsum',
            parent_id=created_folder['id'])
        self.assertNotEquals(None, created_folder_file,
                             "File wasn't created")

        saved_folder_file = drive.get_metadata(
            "integration-foldertest.txt", parent_id=created_folder['id'])
        self.assertEquals(created_folder_file['md5Checksum'],
                          saved_folder_file['md5Checksum'],
                          "File in folder wasnn't created")

        # Garbage Collect
        drive.service.files().delete(fileId=created_folder['id']).execute()
        drive.service.files().delete(fileId=created_file['id']).execute()
        drive.service.files().delete(fileId=created_folder_file['id']).execute()

    def test_insert_permission(self):
        admin_email = TEST_PRIMARY_ADMIN_EMAIL

        drive = DriveHelper(credentials_json=TEST_PRIMARY_CREDENTIALS,
                    admin_email=admin_email,
                    refresh_token=TEST_PRIMARY_REFRESH_TOKEN)

        # Create in root
        created_file = drive.insert_file("integration-test.txt", 'plain/text', 'lorem ipsum')
        self.assertNotEquals(None, created_file, "File wasn't created")

        permission = drive.insert_permission(file_id=created_file['id'],
                                value='dev@eforcers.com',
                                type='user', role='reader')

        self.assertNotEquals(None, permission, "Permission not set")

        drive.service.files().delete(fileId=created_file['id']).execute()

class MigrationHelperTestCase(AppEngineFlaskTestCase):

    def test_init_and_insert(self):
        msg = email.message_from_string(RFC_822_TEST)

        imap = IMAPHelper()
        imap.oauth1_2lo_login(user_email='administrador@eforcers.com.co')

        messages = imap.list_messages(
                           criteria='pruebamigration-uniquehash-34lkj3lk5j3l4kj3lk4j')

        # Delete messages from previous tests
        for message in messages:
            result, label_data = imap.get_message_labels(msg_id=message)
            if not result:
                assert False
                imap.remove_message_label(
                    msg_id=message, prev_label='integration-tests')
            imap.delete_message(msg_id=message)

        messages = imap.list_messages(
                           criteria='pruebamigration-uniquehash-34lkj3lk5j3l4kj3lk4j')
        prev_number_messages = len(messages)

        self.assertEquals(0, prev_number_messages,
                          "Message cleanup wasn't executed")

        time.sleep(1)

        migration = MigrationHelper(
                credentials_json=TEST_PRIMARY_CREDENTIALS,
                refresh_token=TEST_PRIMARY_REFRESH_TOKEN)
        result = migration.migrate_mail(
            user_email='administrador@eforcers.com.co',
            msg=msg, labels=['integration-tests'])

        if not result:
            assert False

        time.sleep(15)

        messages = imap.list_messages(
                           criteria='pruebamigration-uniquehash-34lkj3lk5j3l4kj3lk4j')
        number_messages = len(messages)

        if len(messages) == 0:
            messages = imap.list_messages(
                           criteria='pruebamigration-uniquehash-34lkj3lk5j3l4kj3lk4j',
                           only_from_trash=True)
            number_messages = len(messages)

        self.assertEquals(1, number_messages,
                          "Message cleanup wasn't executed")

        for message in messages:
            result, label_data = imap.get_message_labels(msg_id=message)
            if not result:
                assert False
            labels = (((label_data[0].split('('))[2].split(')'))[0]).split()
            self.assertIn('integration-tests', labels,
                          "Label wasn't preserved")

        imap.close()


import csv
import os

# Load the specific libraries
from flask import Flask, request
from pydantic import BaseModel

# Initiate the web server
app = Flask(__name__)

# Create a class "User" (cf. CSV header)
class User(BaseModel):
    uid: int
    first_name: str
    last_name: str
    department: str

# Get the current path of the file "user.csv"
filepath = os.path.join(
    os.path.dirname(os.path.realpath(__file__)), "..", "..", "user.csv"
)

# Load the user from the CSV in local memory
users = []
with open(filepath, "r") as f:
    titles = ["uid", "first_name", "last_name", "department"]
    with open(filepath, "r") as file:
        reader = csv.DictReader(file, titles)
        next(reader)  # skip header
        for csv_user in reader:
            users.append(User(**csv_user))

# Route (path of the URL) declaration
@app.route("/")
def hello_world():
    # Print hello world HTML web page
    return "<title>Hello world</title>
Hello, World!
"

@app.route("/user/<int:uid>", methods=["PUT", "GET"])
def get_user_by_id(uid):
    # Get or Update a user by it's Id
    user = users[uid].__dict__

    # Check if it's the verb is GET (means return a user)
    if request.method == "GET":
        return user

    # Update a user by it's Id
    users[uid] = User(
        **{
            "uid": uid,
            "first_name": request.args.get("first_name"),
            "last_name": request.args.get("last_name"),
            "department": request.args.get("department"),
        }
    )
    return users[uid].__dict__

@app.route("/user", methods=["POST", "GET"])
def mgt_user():
    # Get and Insert user

    # Check if it's the verb is POST (means insert new resource)
    if request.method == "POST":
        # 5.1 Add new user
        users.append(
            User(
                **{
                    "uid": len(users) + 1,
                    "first_name": request.args.get("first_name"),
                    "last_name": request.args.get("last_name"),
                    "department": request.args.get("department"),
                }
            )
        )
        return users[-1].__dict__

    # Get a user with the help of filtering
    args = request.args
    user_filtering = args.get("first_name")
    if user_filtering is None:
        return f"Invalid filter, got {args}, expect first_name=John"
    return [usr.__dict__ for usr in users if usr.first_name == user_filtering]

@app.route("/users")
def get_users():
    # Get the list of user
    return [usr.__dict__ for usr in users]

if __name__ == "__main__":
    # Start the Web server
    app.run(debug=True, port=5100)


import os
import json

from gtts import gTTS
from pydub import AudioSegment

from classifier.classifier import PatchClassifier
from sonifier.sonified_result import SonifiedResult

class Sonifier:
    def __init__(self, spec_filename, patch_size=96, use_gpu=True, verbose=False):
        self.patch_size = patch_size
        with open(spec_filename, "r") as f:
            self.spec = json.loads(f.read())
        self.verbose = verbose
        self.tags = []
        self.doc_features = []

        root = self.spec["root"]
        for data in self.spec["data"]:
            self.doc_features.append(data["doc_features"])
            folder = root + ('%s/' % data["id"])
            width, height = data["width"], data["height"]
            verbose_temp = 0.1
            aggregated_tags = { "text": 0, "image": 0, "graph": 0, "ad": 0 }
            num_patch = len(data["patches"])
            for patch in data["patches"]:
                tags = patch["tags"]
                aggregated_tags["text"] += tags[0] / num_patch
                aggregated_tags["image"] += tags[1] / num_patch
                aggregated_tags["graph"] += tags[2] / num_patch
                aggregated_tags["ad"] += tags[3] / num_patch
            self.tags.append(aggregated_tags)

    def generate_glance_mp3(self, data_id, output_filename):
        self.save_tts_mp3("구성요소 비율", "glance_temp.mp3", 1.5)
        scale = min(5, self.spec["data"][data_id]["height"] / 1500)
        duration_text = self.tags[data_id]["text"] * scale
        duration_image = (self.tags[data_id]["image"] + self.tags[data_id]["graph"]) * scale
        duration_ad = self.tags[data_id]["ad"] * scale

        freqs_text = [261.63]
        freqs_image = [293.66, 392.00]
        freqs_ad = [493.88, 523.25, 587.33]

        sr = SonifiedResult()
        sr.append_sinewave(freqs_text, duration_milliseconds=duration_text * 1000) #volume can be also adjusted by volume=0.0~1.0
        #sr.append_silence(duration_milliseconds=500)
        sr.append_sinewave(freqs_image, duration_milliseconds=duration_image * 1000) #volume can be also adjusted by volume=0.0~1.0
        #sr.append_silence(duration_milliseconds=500)
        sr.append_sinewave(freqs_ad, duration_milliseconds=duration_ad * 1000) #volume can be also adjusted by volume=0.0~1.0
        sr.append_silence(duration_milliseconds=200)
        sr.save_wav(output_filename)

        glance_0 = AudioSegment.from_mp3("glance_temp.mp3")
        glance_1 = AudioSegment.from_wav(output_filename)
        (glance_0 + glance_1).export(output_filename, format="mp3")

    def generate_keywords_mp3(self, data_id, output_filename):
        sentence = " ".join(self.doc_features[data_id]["keywords"])
        self.save_tts_mp3(sentence, output_filename, 1.5)

    def generate_summary_mp3(self, data_id, output_filename):
        doc_feature = self.doc_features[data_id]
        num_imgs = int(doc_feature["num_imgs"])
        num_links = int(doc_feature["num_links"])
        num_headers = int(doc_feature["num_headers"])
        num_tables = int(doc_feature["num_tables"])
        sentence = f"헤더 {num_headers}개 링크 {num_links}개 이미지 {num_imgs}개 표 {num_tables}개"
        self.save_tts_mp3(sentence, output_filename, 1.5)

    def generate_defaults_mp3(self, data_id, output_filename):
        doc_feature = self.doc_features[data_id]
        title = doc_feature['title']
        navs = " ".join(doc_feature['navs']) if len(doc_feature['navs']) > 0 else "이 없습니다."
        headers = " ".join(doc_feature['headers'][:5]) if len(doc_feature['headers']) > 0 else "가 없습니다."
        sentence = f"페이지 제목 {title} 네비게이션 {navs} 헤더 {headers}"
        self.save_tts_mp3(sentence, output_filename, 1.5)

    def generate_merged_sg_mp3(self, glance_filename, keyword_filename, default_filename, output_filename):
        glance = AudioSegment.from_mp3(glance_filename)
        keyword = AudioSegment.from_mp3(keyword_filename)
        default = AudioSegment.from_mp3(default_filename)
        (glance + keyword + default)[:15*1000].export(output_filename, format="mp3")

    def generate_merged_ss_mp3(self, summary_filename, default_filename, output_filename):
        summary = AudioSegment.from_mp3(summary_filename)
        default = AudioSegment.from_mp3(default_filename)
        (summary + default)[:15*1000].export(output_filename, format="mp3")

    @staticmethod
    def speed_change(sound, speed=1.0):
        # Manually override the frame_rate. This tells the computer how many
        # samples to play per second
        sound_with_altered_frame_rate = sound._spawn(sound.raw_data, overrides={
            "frame_rate": int(sound.frame_rate * speed)
        })
        # convert the sound with altered frame rate to a standard frame rate
        # so that regular playback programs will work right. They often only
        # know how to play audio at standard frame rate (like 44.1k)
        return sound_with_altered_frame_rate.set_frame_rate(sound.frame_rate)

    def save_tts_mp3(self, sentence, output_filename, speed=1):
        tts = gTTS(sentence, lang='ko')
        tts.save(output_filename)
        if speed != 1:
            reopened = AudioSegment.from_mp3(output_filename)
            self.speed_change(reopened, speed=1.3).export(output_filename, format="mp3")