import pandas as pd
import scipy.stats as stats
import operator
import numpy as np
from time import sleep as sl
import argparse
from sklearn.metrics import pairwise_distances,pairwise_distances_chunked
from sklearn.cluster import AgglomerativeClustering,DBSCAN
import time
from datetime import timedelta
import sys
from datetime import date
def parseargs():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-e','--mgt_epi_data',help = 'csv file with MGT and epidemiological information')
    parser.add_argument('-m','--mgt_level',help='the level of mgt for clustering, e.g. MGT9 ')
    parser.add_argument('-n','--number_iso_threshold',help = 'the number of isolates threshold for each type in the tested MGT level')
    parser.add_argument('-o','--outpath',help='the path of outfiles, e.g. /srv/scratch')
    parser.add_argument('-f','--mgt_flags',help='the csv file of the flag MGT-STs for meta data')
    parser.add_argument('-p','--prefix',help='the prefix for the outfile names')
    parser.add_argument('-c','--country',help='country for filtering isolates for pairwise distance analysis, e.g. -c Australia')
    parser.add_argument('-t','--transtype',help='international or national cluster, or a list of ODC10 STs in a .txt file without heading, e.g. -t international')
    parser.add_argument('-a','--whole_allele_profile',help='Allele profile of MGT9, e.g. /srv/scratch/mgt9_alleprofile.txt')
    parser.add_argument("-d", "--distances",
                        help="file containing distances corresponding to the alleleprofiles file (from previous run of this script if applicable)")
    parser.add_argument("-l", "--dist_limits",
                        help="comma separated list of cluster cutoffs or range or both i.e 1,2,5 or 1-10 or 1,2,5-10, note ODC0/MGT9 were automatically given",default="1,2,5,10")
    parser.add_argument("-j", "--no_jobs",
                        help="num jobs to split distance calc into", default=1)
    args = parser.parse_args()
    return  args
def main():
    t1 = time.time()
    args = parseargs()
    mgt_epi = pd.read_csv(args.mgt_epi_data)
    threshod_dic = {args.mgt_level: int(args.number_iso_threshold)}
    if args.mgt_flags == None:
        mgt_threshod, type_dic, isolate_dic, type_isolist_dic = transmi_tracking(threshod_dic, mgt_epi)
    if args.mgt_flags != None:
        flags = pd.read_csv(args.mgt_flags)
        mgt_threshod, type_dic, isolate_dic, type_isolist_dic = transmi_tracking_flags(threshod_dic, mgt_epi, flags)
    typedf = pd.DataFrame.from_dict(type_dic, orient='index')
    typedf.index.name = mgt_threshod
    isolate_df = pd.DataFrame.from_dict(isolate_dic, orient='index')
    isolate_df.index.name = 'Accession'
    typedflist = typedf.columns.tolist() + typedf.values.tolist()
    isolate_df.to_csv(args.outpath + '/' + args.prefix + args.mgt_level + '_isolate_transmission_link.csv')
    typedf.to_csv(args.outpath + '/' + args.prefix + args.mgt_level + '_mgttype_transmission_link.csv')

    ###### allele profile getting for pairwise calculation,
    if args.country != None:
        odc_pairwise_list = []
        if args.transtype == 'international':
            typedf2 = typedf[typedf['no_country'] >= 2] ### >=2 is international
            type_country_dic = typedf2['country_detail'].to_dict()

            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 1 and args.mgt_level == "ODC10":  ####### to set the threshold of >= 1 for each cluster in Australia.
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype == 'national':
            typedf2 = typedf[typedf['no_country'] == 1] ### >=2 is international; == 1 is national
            type_country_dic = typedf2['country_detail'].to_dict()

            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 2 and args.mgt_level == "ODC10":  ####### >=2 isolates for national transmission
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype == None:
            typedf2 = typedf[typedf['no_country'] >= 1] ### including both international and national
            type_country_dic = typedf2['country_detail'].to_dict()
            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 2 and args.mgt_level == "ODC10":  ####### >=2 isolates for national transmission
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype != None and ".txt" in args.transtype:
            odc_pairwise_list=open(args.transtype,'r').read().splitlines()
        # odc_pairwise_list = ['4969']
        for type in odc_pairwise_list :
            if type in type_isolist_dic:
                print(args.mgt_level + '_' + type)
                # time_pw(args, mgt_epi, args.mgt_level, type, args.outpath)
                ### to save the type correlated acc list
                isolatelistfile = open(args.outpath + '/' + args.mgt_level + '_' + type + '_' + args.country + '_correlated_isolatelist.txt','w')
                isolatelistfile.write(args.mgt_level + '_' + type + '\n')
                for acc in type_isolist_dic[type]:
                    isolatelistfile.write(acc + '\n')
                ### to calculate the pairwise distance of alleles
                if args.whole_allele_profile != "":
                    allele_prof = open(args.whole_allele_profile, "r").read().splitlines()
                    allele_proflist = get_part_alleleprofil(allele_prof, type_isolist_dic[type])
                    allele_prof_outfile = open(args.outpath + '/' + args.mgt_level + '_' + type + '_alleleprof.txt', 'w')
                    allele_prof_outlist=[]
                    for a in allele_proflist:
                        allele_prof_outlist.append(a)
                        allele_prof_outfile.write(a + '\n')
                    profs, id_to_strain = process_profiles(allele_prof_outlist)
                    pairw_outfrefix=args.outpath + '/' + args.mgt_level + '_' + type + '_'
                    pairwise_process(args, profs, id_to_strain, pairw_outfrefix)
                    t2 = timecal(t1)
                    ##  iso_process_profiles() and  iso_pairwise_process() are for pairwise distance of isolates in Australia
                    # iso_profs, iso_id_to_strain = iso_process_profiles(allele_prof_outlist)
                    # print(iso_id_to_strain)
                    # iso_pairwise_process(args, iso_profs, iso_id_to_strain, pairw_outfrefix)
def timecal(uptime):
    timespan = time.time() - uptime
    print(timedelta(seconds=timespan))
    return time.time()
def time_metric(a, b):
    match = 0
    missmatch = 0
    a = [int(x) for x in a]
    b = [int(x) for x in b]
    d0 = date(a[0], a[1], a[2])
    d1 = date(b[0], b[1], b[2])
    dayinterv = abs((d1 - d0).days)
    return dayinterv
###note columns have to include 'Accession','Collection Year','Collection Month','Collection Day'.
def time_pw(args, metadf,odc,type,outfrefixout):
    # metadf = pd.read_csv(metapath)
    metadf[odc] = metadf[odc].astype(str)
    metadf = metadf[(metadf[odc]==type) | (metadf[odc]== str(type))]
    timedf = pd.DataFrame(metadf, columns=['Accession','Collection Year','Collection Month','Collection Day'])
    # timedf = pd.read_csv('E:/2018/2019-06-14-Australia_SEN/test/time.csv')
    # # timedf['d'] = pd.to_datetime(timedf['Date'],format = '%Y/%m/%d')
    timedf['Collection Day']=timedf['Collection Day'].replace(np.nan,15)
    timedf = timedf[timedf['Collection Month'].notnull()]
    print({'time_input_dfsize': timedf.shape[0]})
    datedf=pd.DataFrame(timedf,columns=['Collection Year','Collection Month','Collection Day'])
    acclist = timedf['Accession'].values.tolist()
    start_time = time.time()
    if datedf.shape[0]>1:
        dayinterv = pairwise_distances(datedf, metric=time_metric, n_jobs=int(args.no_jobs))
        # pairw_outfrefix = 'E:/2018/2019-06-14-Australia_SEN/test/time_pairwise_'+ type + '_'
        if  len(dayinterv) >=2 :
            print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
            np.savetxt(outfrefixout + odc +'_' +type + '_'+"time_pwdistances.txt", dayinterv.astype(int), fmt='%i', header=",".join(acclist), delimiter=",")
def unneg(a):
    if "-" in a:
        return a.split("_")[0][1:]
    else:
        return a
def mgt_dist_metric(a, b):
    match = 0
    missmatch = 0

    for i in range(len(a)):
        aAllele = a[i]
        bAllele = b[i]
        # print(aAllele,bAllele)
        # sl(0.1)
        if aAllele == 0 or bAllele == 0 or aAllele == bAllele:
            match += 1
        else:
            missmatch += 1
            # print(aAllele,bAllele)

    return missmatch
def process_profiles(inprofiles, s=False):
    profs = {}
    id_to_strain = {}

    for line in inprofiles[1:]:
        col = line.split("\t")
        if s:
            if col[0] in s:
                # print(col[0])

                if col[1] not in profs:
                    noneg = [unneg(x) for x in col[3:]]
                    profs[col[1]] = noneg
                    id_to_strain[col[1]] = [str(col[0])]
                else:
                    id_to_strain[col[1]].append(str(col[0]))
        else:
            # print(col[0])
            if col[1] not in profs:
                noneg = [unneg(x) for x in col[3:]]
                profs[col[1]] = noneg
                id_to_strain[col[1]] = [str(col[0])]
            else:
                id_to_strain[col[1]].append(str(col[0]))

    return profs, id_to_strain
def pairwise_process(args,profs,id_to_strain, pairw_outfrefix):
    idlist = list(profs.keys())
    inprofs = [profs[x] for x in idlist]

    dfo = pd.DataFrame(inprofs)
    # distances only calculated if args.distances not set
    lend = ""
    if args.distances:
        # read in distances previosly calculated
        d = np.loadtxt(args.distances)
        lend = len(d) ### number of MGT9 STs in this cluster
    else:
        start_time = time.time()
        d = pairwise_distances(inprofs, metric=mgt_dist_metric, n_jobs=int(args.no_jobs))
        lend = len(d)
        # if  len(d) >=2 :
        print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
        np.savetxt(pairw_outfrefix + "mgt9_distances.txt", d.astype(int), fmt='%i', header=",".join(idlist), delimiter=",")
    # distance cutoffs to calculate
    if lend >=2:
        pairw_outfile = open(pairw_outfrefix + 'iso_odc_recal.txt','w')
        diststring = args.dist_limits
        dists = diststring.split(",")
        distances = []
        for i in dists:
            if "-" in i:
                n = i.split("-")
                nlist = list(range(int(n[0]) + 1, int(n[1]) + 2))
                # distance cutoffs seems to be non inclusive i.e. cutoff of 3 means max distance is 2
                # therefore need to add 1 to all values
            else:
                nlist = [int(i) + 1]
            distances += nlist
        clusterlists = {}
        preference = []
        for id in idlist:
            preference.append(len(id_to_strain[id]))
        start_time = time.time()
        for dist in distances:
            clusters = AgglomerativeClustering(n_clusters=None, distance_threshold=dist, affinity="precomputed",
                                               linkage="single").fit_predict(d)
            clusterls = list(clusters)
            clusterlists[dist] = clusterls
        print("clustering time", (" --- %s seconds ---" % (time.time() - start_time)))
        realdists = ["ODC" + str(x - 1) for x in distances]

        pairw_outfile.write("Strain\tMGT9\t{}\n".format("\t".join(realdists)))
        for i in range(len(idlist)):
            id = idlist[i]
            for strain in id_to_strain[id]:
                pairw_outfile.write(strain + '\t' + str(id))
                for d in distances:
                    clust = clusterlists[d][i]
                    pairw_outfile.write("\t" + str(clust + 1))
                pairw_outfile.write("\n")
        pairw_outfile.close()
    if lend < 2:  ### belong to the same MGT9 ST
        pairw_outfile = open(pairw_outfrefix + 'iso_odc_recal.txt','w')
        pairw_outfile.write('Strain' + '\t' + 'MGT9' + '\n')
        for st, isolist in id_to_strain.items():
            for iso in isolist:
                pairw_outfile.write(str(iso) + '\t' + str(st) + '\n')
        return
        ##### pairwise distance calculation
###  iso_process_profiles() and  iso_pairwise_process() are for pairwise distance of isolates in Australia
def iso_process_profiles(inprofiles, s=False):
    profs = {}
    id_to_strain = {}
    for line in inprofiles[1:]:
        col = line.split("\t")
        if s:
            if col[0] in s:
                # print(col[0])
                if col[0] not in profs:
                    noneg = [unneg(x) for x in col[3:]]
                    profs[col[0]] = noneg
                    id_to_strain[col[0]] = [str(col[1])]
                else:
                    id_to_strain[col[0]].append(str(col[1]))
        else:
            # print(col[0])

            if col[0] not in profs:
                noneg = [unneg(x) for x in col[3:]]
                profs[col[0]] = noneg
                id_to_strain[col[0]] = [str(col[1])]
            else:
                id_to_strain[col[0]].append(str(col[1]))
    return profs, id_to_strain
def iso_pairwise_process(args,profs,id_to_strain, pairw_outfrefix):
    idlist = list(profs.keys())
    # print(idlist)
    inprofs = [profs[x] for x in idlist]
    # distances only calculated if args.distances not set
    lend = ""
    if args.distances:
        # read in distances previosly calculated
        d = np.loadtxt(args.distances)
        lend = len(d)
    else:
        start_time = time.time()
        d = pairwise_distances(inprofs, metric=mgt_dist_metric, n_jobs=int(args.no_jobs))
        lend = len(d)
        if  len(d) >=2 :
            print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
            np.savetxt(pairw_outfrefix + "iso_distances.txt", d.astype(int), fmt='%i', header=",".join(idlist), delimiter=",")
def epi_filt(mgt_epi,fi_dic):
    col_name = mgt_epi.columns.values.tolist()
    mgt_epi = mgt_epi.values.tolist()
    for key in fi_dic:
        epi_filter_out = []
        for line in mgt_epi:
            line_index = col_name.index(key)
            line_value = str(line[line_index])
            if line_value in fi_dic[key]:
                epi_filter_out.append(line)
        mgt_epi = epi_filter_out
    mgt_epi = pd.DataFrame(mgt_epi)
    mgt_epi.columns = col_name
    print(mgt_epi.shape)
    return mgt_epi
def flag_reprot(flag_input, test_file,key_list):
    flag_input = flag_input.set_index('MGT_type')
    dic_file = flag_input.to_dict()
    mgt_levels_list = test_file.columns.tolist()
    mgt_levels_list = [a for a in mgt_levels_list if "MGT" in a]
    for level in mgt_levels_list:
        test_file[level] = level + test_file[level].astype(str)
    test_list = test_file.values.tolist()
    keyflag_dic = {}
    for k1 in dic_file:
        if k1 in key_list:
            # outfile = open(outpath + '/' + k1 + '.txt','w')
            dic_file2 = {k:v for k, v in dic_file[k1].items() if "nan" not in str(v)}
            mgtst_list = []
            for line in test_list:
                for value in line:
                    if value in dic_file2.keys():
                        mgtst = value
                        mgtst_list.append(mgtst)
                        strain_name = line [0]
                        predict_types = dic_file2[value]
                        # output = "{}\t{}\t{}".format(strain_name,predict_types,mgtst)
                        # print(output)
                        # outfile.write(output + '\n')
            mgtst_ser = pd.Series(mgtst_list)
            keyflag_dic[k1] = mgtst_ser.value_counts().to_dict()
    return keyflag_dic
def transmi_tracking_flags(threshod_dic, mgt_epi,flags):
    for mgt in threshod_dic:
        gp = mgt_epi.groupby([ mgt])['Strain'].count().fillna(0)
        pass_filter_type_dic = gp [gp>= threshod_dic[mgt]].to_dict()
        if 0 in pass_filter_type_dic.keys():
            pass_filter_type_dic.pop(0)
        mgt_threshod = mgt + '_>=_' + str(threshod_dic[mgt])
        type_dic = {}
        type_isolist_dic = {}
        isolate_dic = {}
        interspread = 0
        limited_year = 0
        large_sclale = 0
        for type in pass_filter_type_dic.keys():
            type_isolist_dic[type] = []
            subdf = mgt_epi[mgt_epi[mgt]== type]
            key_list = ['Population_Structure', 'MDR', 'AR2_1', 'Top_MGT4_STs']
            keyflag_dic = flag_reprot(flags, subdf, key_list)
            country_dic = subdf.groupby(['Country'])['Strain'].count().to_dict()
            year_dic = subdf.groupby(['Collection Year'])['Strain'].count().to_dict()
            source_dic = subdf.groupby(['Source Type'])['Strain'].count().to_dict()
            type_dic[type] = keyflag_dic

            # type_dic[type]={"no_isolates":{}}
            type_dic[type]['no_isolates'] = pass_filter_type_dic[type]
            type_dic[type]['country_detail'] = country_dic
            if 'None' in country_dic.keys():
                type_dic[type]['no_country'] = len(country_dic) - 1
            else:
                type_dic[type]['no_country'] = len(country_dic)

            type_dic[type]['year_detail'] = year_dic
            if 'None' in year_dic.keys():
                type_dic[type]['no_year'] = len(year_dic) - 1
            else:
                type_dic[type]['no_year'] = len(year_dic)
            if len(year_dic) <= 2 and  len(year_dic)> 0:
                limited_year =limited_year+1
            if len(country_dic) >= 2 :
                interspread = interspread + 1
            if len(year_dic) > 1 and len(year_dic) > 0 and len(country_dic) >= 2 and pass_filter_type_dic[type] > 50:
                large_sclale = large_sclale + 1

            type_dic[type]['source'] = source_dic
            if 'None' in source_dic.keys():
                type_dic[type]['no_source'] = len(source_dic) - 1
            else:
                type_dic[type]['no_source'] = len(source_dic)
            ########### to product isolate_dic
            acclist = subdf['Accession'].tolist()
            for acc in acclist:
                type_isolist_dic[type].append(acc)
                isolate_dic[acc] = {}
                isolate_dic[acc] = type_dic[type]
                isolate_dic[acc][mgt_threshod] = type

        print("No. of passed types: " + str(len(pass_filter_type_dic)))
        print('No. of potential international spreading clusters: ' + str(interspread))
        print('No. of potential international spreading clusters within years: ' + str(limited_year))
        print('No. of potential international spreading large clusters within years >50: ' + str(large_sclale))

        return mgt_threshod,type_dic, isolate_dic, type_isolist_dic
def transmi_tracking(threshod_dic, mgt_epi):
    for mgt in threshod_dic:
        gp = mgt_epi.groupby([mgt])['Strain'].count().fillna(0)
        pass_filter_type_dic = gp [gp>= threshod_dic[mgt]].to_dict()
        if 0 in pass_filter_type_dic.keys():
            pass_filter_type_dic.pop(0)
        mgt_threshod = mgt + '_>=_' + str(threshod_dic[mgt])
        type_dic = {}
        isolate_dic = {}
        type_isolist_dic ={}
        interspread = 0
        limited_year = 0
        large_sclale = 0
        for type in pass_filter_type_dic.keys():
            type_isolist_dic[type]=[]
            subdf = mgt_epi[mgt_epi[mgt]== type]
            key_list = ['Population_Structure', 'MDR', 'AR2_1', 'Top_MGT4_STs']
            # keyflag_dic = flag_reprot(flags, subdf, key_list)
            country_dic = subdf.groupby(['Country'])['Strain'].count().to_dict()
            year_dic = subdf.groupby(['Collection Year'])['Strain'].count().to_dict()
            source_dic = subdf.groupby(['Source Type'])['Strain'].count().to_dict()
            type_dic[type] = {}
            type_dic[type]['no_isolates'] = pass_filter_type_dic[type]
            type_dic[type]['country_detail'] = country_dic
            if 'None' in country_dic.keys():
                type_dic[type]['no_country'] = len(country_dic) - 1
            else:
                type_dic[type]['no_country'] = len(country_dic)

            type_dic[type]['year_detail'] = year_dic
            if 'None' in year_dic.keys():
                type_dic[type]['no_year'] = len(year_dic) - 1
            else:
                type_dic[type]['no_year'] = len(year_dic)
            if len(year_dic) <= 2 and  len(year_dic)> 0:
                limited_year =limited_year+1
            if len(country_dic) >= 2 :
                interspread = interspread + 1
            if len(year_dic) > 1 and len(year_dic) > 0 and len(country_dic) >= 2 and pass_filter_type_dic[type] > 50:
                large_sclale = large_sclale + 1
            type_dic[type]['source'] = source_dic
            if 'None' in source_dic.keys():
                type_dic[type]['no_source'] = len(source_dic) - 1
            else:
                type_dic[type]['no_source'] = len(source_dic)

            ########### to product isolate_dic
            acclist = subdf['Accession'].tolist()
            for acc in acclist:
                type_isolist_dic[type].append(acc)
                isolate_dic[acc] = {}
                isolate_dic[acc] = type_dic[type]
                isolate_dic[acc][mgt_threshod] = type
        print("No. of passed types: " + str(len(pass_filter_type_dic)))
        print('No. of potential international spreading clusters: ' + str(interspread))
        print('No. of potential international spreading clusters within years: ' + str(limited_year))
        print('No. of potential international spreading large clusters within years >50: '+ str(large_sclale))
        return mgt_threshod,type_dic, isolate_dic, type_isolist_dic
def get_part_alleleprofil(whol_alleprof, isolist):
    outlist = []
    outlist.append(whol_alleprof[0])
    for line in whol_alleprof:
        col = line.split('\t')
        for acc in isolist:
            if acc == col[0]:           # or acc + '_cladeC.fasta' == col[0]:
                outlist.append(line)
    return outlist
if __name__ == "__main__":
    main()


from __future__ import absolute_import
import logging
import string
from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
import re

# py2 vs py3 transition
from ..six import text_type as unicode
from ..six import string_types as basestring
from ..six import ensure_binary
from io import BytesIO

## XML isn't as forgiving as HTML, so rather than generate as strings,
## use DOM to generate the XML files.
from xml.dom.minidom import getDOMImplementation

import bs4

from .base_writer import BaseStoryWriter
from ..htmlcleanup import stripHTML,removeEntities
from ..story import commaGroups

logger = logging.getLogger(__name__)

class EpubWriter(BaseStoryWriter):

    @staticmethod
    def getFormatName():
        return 'epub'

    @staticmethod
    def getFormatExt():
        return '.epub'

    def __init__(self, config, story):
        BaseStoryWriter.__init__(self, config, story)

        self.EPUB_CSS = string.Template('''${output_css}''')

        self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>

<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" rel="stylesheet"/>

${title} by ${authorHTML}

''')

        self.EPUB_TITLE_ENTRY = string.Template('''
${label}: ${value}

''')

        self.EPUB_NO_TITLE_ENTRY = string.Template('''
${value}

''')

        self.EPUB_TITLE_PAGE_END = string.Template('''

''')

        self.EPUB_TABLE_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>

<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" rel="stylesheet"/>

${title} by ${authorHTML}

''')

        self.EPUB_TABLE_TITLE_ENTRY = string.Template('''

${label}:	${value}

''')

        self.EPUB_TABLE_TITLE_WIDE_ENTRY = string.Template('''

${label}: ${value}

''')

        self.EPUB_TABLE_NO_TITLE_ENTRY = string.Template('''

${label}${value}

''')

        self.EPUB_TABLE_TITLE_PAGE_END = string.Template('''

''')

        self.EPUB_TOC_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>

<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" rel="stylesheet"/>

Table of Contents

''')

        self.EPUB_TOC_ENTRY = string.Template('''
${chapter}

''')

        self.EPUB_TOC_PAGE_END = string.Template('''

''')

        self.EPUB_CHAPTER_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>

<title>${chapter}</title>
<link href="stylesheet.css" type="text/css" rel="stylesheet"/>
<meta name="chapterurl" content="${url}" />
<meta name="chapterorigtitle" content="${origchapter}" />
<meta name="chaptertoctitle" content="${tocchapter}" />
<meta name="chaptertitle" content="${chapter}" />

${chapter}

''')

        self.EPUB_CHAPTER_END = string.Template('''

''')

        self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>

<title>Update Log</title>
<link href="stylesheet.css" type="text/css" rel="stylesheet"/>

Update Log

''')

        self.EPUB_LOG_UPDATE_START = string.Template('''

''')

        self.EPUB_LOG_ENTRY = string.Template('''
${label}: ${value}
''')

        self.EPUB_LOG_UPDATE_END = string.Template('''

''')

        self.EPUB_LOG_PAGE_END = string.Template('''

''')

        self.EPUB_LOG_PAGE_END = string.Template('''

''')

        self.EPUB_COVER = string.Template('''
<title>Cover</title>

''')

    def writeLogPage(self, out):
        """
        Write the log page, but only include entries that there's
        metadata for.  START, ENTRY and END are expected to already be
        string.Template().  START and END are expected to use the same
        names as Story.metadata, but ENTRY should use id, label and value.
        """
        if self.hasConfig("logpage_start"):
            START = string.Template(self.getConfig("logpage_start"))
        else:
            START = self.EPUB_LOG_PAGE_START

        if self.hasConfig("logpage_end"):
            END = string.Template(self.getConfig("logpage_end"))
        else:
            END = self.EPUB_LOG_PAGE_END

        # if there's a self.story.logfile, there's an existing log
        # to add to.
        if self.story.logfile:
            logger.debug("existing logfile found, appending")
            # logger.debug("existing data:%s"%self._getLastLogData(self.story.logfile))
            replace_string = "" # "
"
            self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
        else:
            # otherwise, write a new one.
            self._write(out,START.substitute(self.story.getAllMetadata()))
            self._write(out,self._makeLogEntry())
            self._write(out,END.substitute(self.story.getAllMetadata()))

    # self parsing instead of Soup because it should be simple and not
    # worth the overhead.
    def _getLastLogData(self,logfile):
        """
        Make a dict() of the most recent(last) log entry for each piece of metadata.
        Switch rindex to index to search from top instead of bottom.
        """
        values = {}
        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
            try:
                # 1975-04-15
                span = ''%entry
                idx = logfile.rindex(span)+len(span)
                values[entry] = logfile[idx:logfile.index('\n',idx)]
            except Exception as e:
                #print("e:%s"%e)
                pass

        return values

    def _makeLogEntry(self, oldvalues={}):
        if self.hasConfig("logpage_update_start"):
            START = string.Template(self.getConfig("logpage_update_start"))
        else:
            START = self.EPUB_LOG_UPDATE_START

        if self.hasConfig("logpage_entry"):
            ENTRY = string.Template(self.getConfig("logpage_entry"))
        else:
            ENTRY = self.EPUB_LOG_ENTRY

        if self.hasConfig("logpage_update_end"):
            END = string.Template(self.getConfig("logpage_update_end"))
        else:
            END = self.EPUB_LOG_UPDATE_END

        retval = START.substitute(self.story.getAllMetadata())

        ## words_added is only used in logpage because it's the only
        ## place we know the previous version's word count.
        if 'words_added' in (self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries")):
            new_words = self.story.getMetadata('numWords')
            old_words = oldvalues.get('numWords',None)
            if new_words and old_words:
                self.story.setMetadata('words_added',commaGroups(unicode(int(new_words.replace(',',''))-int(old_words.replace(',','')))))

        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
            if self.isValidMetaEntry(entry):
                val = self.story.getMetadata(entry)
                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
                    label=self.get_label(entry)
                    # if self.hasConfig(entry+"_label"):
                    #     label=self.getConfig(entry+"_label")
                    # elif entry in self.titleLabels:
                    #     logger.debug("Using fallback label for %s_label"%entry)
                    #     label=self.titleLabels[entry]
                    # else:
                    #     label="%s"%entry.title()
                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))

                    retval = retval + ENTRY.substitute({'id':entry,
                                                        'label':label,
                                                        'value':val})
            else:
                # could be useful for introducing extra text, but
                # mostly it makes it easy to tell when you get the
                # keyword wrong.
                retval = retval + entry

        retval = retval + END.substitute(self.story.getAllMetadata())

        if self.getConfig('replace_hr'):
            # replacing a self-closing tag with a container tag in the
            # soup is more difficult than it first appears.  So cheat.
            retval = re.sub("
]*>","
* * *
",retval)

        return retval

    def writeStoryImpl(self, out):

        if self.story.oldcover and \
                ( (self.getConfig('use_old_cover') and
                   self.story.getMetadata('cover_image') != 'force' ) or not self.story.cover ):
            # logger.debug("use_old_cover:%s"%self.getConfig('use_old_cover'))
            self.use_oldcover = True
            self.story.setMetadata('cover_image','old')
        else:
            self.use_oldcover = False

        ## Python 2.5 ZipFile is rather more primative than later
        ## versions.  It can operate on a file, or on a BytesIO, but
        ## not on an open stream.  OTOH, I suspect we would have had
        ## problems with closing and opening again to change the
        ## compression type anyway.
        zipio = BytesIO()

        ## mimetype must be first file and uncompressed.  Python 2.5
        ## ZipFile can't change compression type file-by-file, so we
        ## have to close and re-open
        outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
        outputepub.debug=3
        outputepub.writestr('mimetype','application/epub+zip')
        outputepub.close()

        ## Re-open file for content.
        outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED)
        outputepub.debug=3

        ## Create META-INF/container.xml file.  The only thing it does is
        ## point to content.opf
        containerdom = getDOMImplementation().createDocument(None, "container", None)
        containertop = containerdom.documentElement
        containertop.setAttribute("version","1.0")
        containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
        rootfiles = containerdom.createElement("rootfiles")
        containertop.appendChild(rootfiles)
        rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
                                                              "media-type":"application/oebps-package+xml"}))
        outputepub.writestr("META-INF/container.xml",containerdom.toxml(encoding='utf-8'))
        containerdom.unlink()
        del containerdom

        ## Epub has two metadata files with real data.  We're putting
        ## them in content.opf (pointed to by META-INF/container.xml)
        ## and toc.ncx (pointed to by content.opf)

        ## content.opf contains metadata, a 'manifest' list of all
        ## other included files, and another 'spine' list of the items in the
        ## file

        uniqueid= 'fanficfare-uid:%s-u%s-s%s' % (
            self.getMetadata('site'),
            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))

        contentdom = getDOMImplementation().createDocument(None, "package", None)
        package = contentdom.documentElement
        ## might want 3.1 or something in future.
        epub3 = self.getConfig("epub_version",default="2.0").startswith("3")
        if epub3:
            package.setAttribute("version","3.0")
        else:
            package.setAttribute("version","2.0")
        logger.info("Saving EPUB Version "+package.getAttribute("version"))
        package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
        package.setAttribute("unique-identifier","fanficfare-uid")
        metadata=newTag(contentdom,"metadata",
                        attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
                               "xmlns:opf":"http://www.idpf.org/2007/opf"})
        package.appendChild(metadata)

        metadata.appendChild(newTag(contentdom,"dc:identifier",
                                    text=uniqueid,
                                    attrs={"id":"fanficfare-uid"}))

        if self.getMetadata('title'):
            metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title'),
                                        attrs={"id":"id"}))

        def creator_attrs(idnum):
            if epub3:
                return {"id":"id-%d"%idnum}
            else:
                return {"opf:role":"aut"}
        idnum = 1
        if self.getMetadata('author'):
            if self.story.isList('author'):
                for auth in self.story.getList('author'):
                    metadata.appendChild(newTag(contentdom,"dc:creator",
                                                attrs=creator_attrs(idnum),
                                                text=auth))
                    idnum += 1
            else:
                metadata.appendChild(newTag(contentdom,"dc:creator",
                                            attrs=creator_attrs(idnum),
                                            text=self.getMetadata('author')))
                idnum += 1

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="FanFicFare [https://github.com/JimmXinu/FanFicFare]",
                                    attrs={"id":"id-%d"%idnum}))
        idnum += 1
        # metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
        if self.story.getMetadata('langcode'):
            langcode=self.story.getMetadata('langcode')
        else:
            langcode='en'
        metadata.appendChild(newTag(contentdom,"dc:language",text=langcode))

        #  published, created, updated, calibre
        #  Leave calling self.story.getMetadataRaw directly in case date format changes.
        if epub3:
            ## epub3 requires an updated modified date on every change of
            ## any kind, not just *content* change.
            from datetime import datetime
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"property":"dcterms:modified"},
                                        text=datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")))
        else:
            if self.story.getMetadataRaw('datePublished'):
                metadata.appendChild(newTag(contentdom,"dc:date",
                                            attrs={"opf:event":"publication"},
                                            text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")))

            if self.story.getMetadataRaw('dateCreated'):
                metadata.appendChild(newTag(contentdom,"dc:date",
                                            attrs={"opf:event":"creation"},
                                            text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")))

            if self.story.getMetadataRaw('dateUpdated'):
                metadata.appendChild(newTag(contentdom,"dc:date",
                                            attrs={"opf:event":"modification"},
                                            text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d")))
                metadata.appendChild(newTag(contentdom,"meta",
                                            attrs={"name":"calibre:timestamp",
                                                   "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))

        series = self.story.getMetadata('series')
        if series and self.getConfig('calibre_series_meta'):
            series_index = "0.0"
            if '[' in series:
                # logger.debug(series)
                ## assumed "series [series_index]"
                series_index = series[series.rindex(' [')+2:-1]
                series = series[:series.rindex(' [')]

                ## calibre always outputs a series_index and it's
                ## always a float with 1 or 2 decimals.  FFF usually
                ## has either an integer or no index. (injected
                ## calibre series is the only float at this time)
                series_index = "%.2f" % float(series_index)

            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:series",
                                               "content":series}))
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:series_index",
                                               "content":series_index}))

        if self.getMetadata('description'):
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))

        if self.getMetadata('site'):
            metadata.appendChild(newTag(contentdom,"dc:publisher",
                                        text=self.getMetadata('site')))

        if self.getMetadata('storyUrl'):
            if epub3:
                metadata.appendChild(newTag(contentdom,"dc:identifier",
                                            text="URL:"+self.getMetadata('storyUrl')))
            else:
                metadata.appendChild(newTag(contentdom,"dc:identifier",
                                            attrs={"opf:scheme":"URL"},
                                            text=self.getMetadata('storyUrl')))
            metadata.appendChild(newTag(contentdom,"dc:source",
                                        text=self.getMetadata('storyUrl')))

        if epub3:
            # <meta refines="#id" property="title-type">main</meta>
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"property":"title-type",
                                               "refines":"#id",
                                               },
                                        text="main"))

            # epub3 removes attrs that identify dc:creator and
            # dc:contributor types and instead put them here.
            # 'aut' for 1-(idnum-1)
            for j in range(1,idnum-1):
                #<meta property="role" refines="#id-1" scheme="marc:relators">aut</meta>
                metadata.appendChild(newTag(contentdom,"meta",
                                            attrs={"property":"role",
                                                   "refines":"#id-%d"%j,
                                                   "scheme":"marc:relators",
                                                   },
                                            text="aut"))

            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"property":"role",
                                               "refines":"#id-%d"%(idnum-1),
                                               "scheme":"marc:relators",
                                               },
                                        text="bkp"))

        ## end of metadata, create manifest.
        items = [] # list of (id, href, type, title) tuples(all strings)
        itemrefs = [] # list of strings -- idrefs from .opfs' spines
        items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                   ## but it needs to be in the items manifest.

        guide = None
        coverIO = None

        coverimgid = "image0000"
        if self.use_oldcover:
            logger.debug("using old cover")
            (oldcoverhtmlhref,
             oldcoverhtmltype,
             oldcoverhtmldata,
             oldcoverimghref,
             oldcoverimgtype,
             oldcoverimgdata) = self.story.oldcover
            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
            outputepub.writestr(oldcoverimghref,oldcoverimgdata)

            coverimgid = "image0"
            items.append((coverimgid,
                          oldcoverimghref,
                          oldcoverimgtype,
                          None))
            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
            itemrefs.append("cover")
            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
                                                           "name":"cover"}))
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                                   "title":"Cover",
                                                                   "href":oldcoverhtmlhref}))

        if self.getConfig('include_images'):
            imgcount=0
            for imgmap in self.story.getImgUrls():
                imgfile = "OEBPS/"+imgmap['newsrc']
                # don't overwrite old cover.
                if not self.use_oldcover or imgfile != oldcoverimghref:
                    outputepub.writestr(imgfile,imgmap['data'])
                    items.append(("image%04d"%imgcount,
                                  imgfile,
                                  imgmap['mime'],
                                  None))
                    imgcount+=1
                    if 'cover' in imgfile:
                        # make sure coverimgid is set to the cover, not
                        # just the first image.
                        coverimgid = items[-1][0]

        items.append(("style","OEBPS/stylesheet.css","text/css",None))

        if self.story.cover and not self.use_oldcover:
            # Note that the id of the cover xhmtl *must* be 'cover'
            # for it to work on Nook.
            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
            itemrefs.append("cover")
            #
            # <meta name="cover" content="cover.jpg"/>
            metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid,
                                                           "name":"cover"}))
            # cover stuff for later:
            # at end of <package>:
            # <guide>
            # <reference type="cover" title="Cover" href="Text/cover.xhtml"/>
            # </guide>
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                       "title":"Cover",
                                                       "href":"OEBPS/cover.xhtml"}))

            if self.hasConfig("cover_content"):
                COVER = string.Template(self.getConfig("cover_content"))
            else:
                COVER = self.EPUB_COVER
            coverIO = BytesIO()
            self._write(coverIO,COVER.substitute(dict(list(self.story.getAllMetadata().items())+list({'coverimg':self.story.cover}.items()))))

        if self.getConfig("include_titlepage"):
            items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
            itemrefs.append("title_page")
        if self.story.getChapterCount() > 1 and self.getConfig("include_tocpage") and not self.metaonly :
            items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
            itemrefs.append("toc_page")

        ## save where to insert logpage.
        logpage_indices = (len(items),len(itemrefs))

        dologpage = ( self.getConfig("include_logpage") == "smart" and \
                          (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") )  \
                     or self.getConfig("include_logpage") == "true"

        ## collect chapter urls and file names for internalize_text_links option.
        chapurlmap = {}
        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
            if chap['html']:
                i=index+1
                items.append(("file%s"%chap['index04'],
                              "OEBPS/file%s.xhtml"%chap['index04'],
                              "application/xhtml+xml",
                              chap['title']))
                itemrefs.append("file%s"%chap['index04'])
                chapurlmap[chap['url']]="file%s.xhtml"%chap['index04'] # url -> relative epub file name.

        if dologpage:
            if self.getConfig("logpage_at_end") == "true":
                ## insert logpage after chapters.
                logpage_indices = (len(items),len(itemrefs))
            items.insert(logpage_indices[0],("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
            itemrefs.insert(logpage_indices[1],"log_page")

        manifest = contentdom.createElement("manifest")
        package.appendChild(manifest)
        for item in items:
            (id,href,type,title)=item
            manifest.appendChild(newTag(contentdom,"item",
                                        attrs={'id':id,
                                               'href':href,
                                               'media-type':type}))
        if epub3:
            # epub3 nav
            # <item href="nav.xhtml" id="nav" media-type="application/xhtml+xml" properties="nav"/>
            manifest.appendChild(newTag(contentdom,"item",
                                        attrs={'href':'nav.xhtml',
                                               'id':'nav',
                                               'media-type':'application/xhtml+xml',
                                               'properties':'nav'
                                               }))

        spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
        package.appendChild(spine)
        for itemref in itemrefs:
            spine.appendChild(newTag(contentdom,"itemref",
                                     attrs={"idref":itemref,
                                            "linear":"yes"}))
        # guide only exists if there's a cover.
        if guide:
            package.appendChild(guide)

        # write content.opf to zip.
        contentxml = contentdom.toxml(encoding='utf-8')
        # tweak for brain damaged Nook STR.  Nook insists on name before content.
        contentxml = contentxml.replace(ensure_binary('<meta content="%s" name="cover"/>'%coverimgid),
                                        ensure_binary('<meta name="cover" content="%s"/>'%coverimgid))

        outputepub.writestr("content.opf",contentxml)

        contentdom.unlink()
        del contentdom

        ## create toc.ncx file
        tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
        ncx = tocncxdom.documentElement
        ncx.setAttribute("version","2005-1")
        ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
        head = tocncxdom.createElement("head")
        ncx.appendChild(head)
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:uid", "content":uniqueid}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:depth", "content":"1"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:totalPageCount", "content":"0"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:maxPageNumber", "content":"0"}))

        docTitle = tocncxdom.createElement("docTitle")
        docTitle.appendChild(newTag(tocncxdom,"text",text=self.getMetadata('title')))
        ncx.appendChild(docTitle)

        tocnavMap = tocncxdom.createElement("navMap")
        ncx.appendChild(tocnavMap)

        # <navPoint id="<id>" playOrder="<risingnumberfrom0>">
        #   <navLabel>
        #     <text><chapter title></text>
        #   </navLabel>
        #   <content src="<chapterfile>"/>
        # </navPoint>
        index=0
        for item in items:
            (id,href,type,title)=item
            # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title.
            if title :
                navPoint = newTag(tocncxdom,"navPoint",
                                  attrs={'id':id,
                                         'playOrder':unicode(index)})
                tocnavMap.appendChild(navPoint)
                navLabel = newTag(tocncxdom,"navLabel")
                navPoint.appendChild(navLabel)
                ## the xml library will re-escape as needed.
                navLabel.appendChild(newTag(tocncxdom,"text",text=stripHTML(title)))
                navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                index=index+1

        # write toc.ncx to zip file
        outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
        tocncxdom.unlink()
        del tocncxdom

        if epub3:
            ##############################################################################################################
            ## create nav.xhtml file
            tocnavdom = getDOMImplementation().createDocument(None, "html", None)
            navxhtml = tocnavdom.documentElement
            navxhtml.setAttribute("xmlns","http://www.w3.org/1999/xhtml")
            navxhtml.setAttribute("xmlns:epub","http://www.idpf.org/2007/ops")
            navxhtml.setAttribute("lang",langcode)
            navxhtml.setAttribute("xml:lang",langcode)
            head = tocnavdom.createElement("head")
            navxhtml.appendChild(head)
            head.appendChild(newTag(tocnavdom,"title",text="Navigation"))
            # <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
            head.appendChild(newTag(tocnavdom,"meta",
                                    attrs={"http-equiv":"Content-Type",
                                           "content":"text/html; charset=utf-8"}))

            body = tocnavdom.createElement("body")
            navxhtml.appendChild(body)
            nav = newTag(tocnavdom,"nav",
                         attrs={"epub:type":"toc"})
            body.appendChild(nav)
            ol = newTag(tocnavdom,"ol")
            nav.appendChild(ol)

            for item in items:
                (id,href,type,title)=item
                # only items to be skipped, cover.xhtml, images, toc.nav,
                # stylesheet.css, should have no title.
                if title:
                    li = newTag(tocnavdom,"li")
                    ol.appendChild(li)
                    atag = newTag(tocnavdom,"a",
                                  attrs={"href":href},
                                  text=stripHTML(title))
                    li.appendChild(atag)

            if self.story.cover and not self.use_oldcover:
                #
                #

                #
Cover

                #

                #
                nav = newTag(tocnavdom,"nav",
                             attrs={"epub:type":"landmarks",
                                    "hidden":""})
                body.appendChild(nav)
                ol = newTag(tocnavdom,"ol")
                nav.appendChild(ol)
                li = newTag(tocnavdom,"li")
                ol.appendChild(li)
                atag = newTag(tocnavdom,"a",
                              attrs={"href":"OEBPS/cover.xhtml",
                                     "epub:type":"cover"},
                              text="Cover")
                li.appendChild(atag)

            # write nav.xhtml to zip file
            outputepub.writestr("nav.xhtml",tocnavdom.toxml(encoding='utf-8'))
            tocnavdom.unlink()
            del tocnavdom
            ##############################################################################################################

        # write stylesheet.css file.
        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata()))

        # write title page.
        if self.getConfig("titlepage_use_table"):
            TITLE_PAGE_START  = self.EPUB_TABLE_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TABLE_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TABLE_TITLE_WIDE_ENTRY
            NO_TITLE_ENTRY    = self.EPUB_TABLE_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TABLE_TITLE_PAGE_END
        else:
            TITLE_PAGE_START  = self.EPUB_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TITLE_ENTRY # same, only wide in tables.
            NO_TITLE_ENTRY    = self.EPUB_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TITLE_PAGE_END

        if coverIO:
            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
            coverIO.close()

        titlepageIO = BytesIO()
        self.writeTitlePage(out=titlepageIO,
                            START=TITLE_PAGE_START,
                            ENTRY=TITLE_ENTRY,
                            WIDE_ENTRY=WIDE_TITLE_ENTRY,
                            END=TITLE_PAGE_END,
                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
        if titlepageIO.getvalue(): # will be false if no title page.
            outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
        titlepageIO.close()

        # write toc page.
        tocpageIO = BytesIO()
        self.writeTOCPage(tocpageIO,
                          self.EPUB_TOC_PAGE_START,
                          self.EPUB_TOC_ENTRY,
                          self.EPUB_TOC_PAGE_END)
        if tocpageIO.getvalue(): # will be false if no toc page.
            outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
        tocpageIO.close()

        if dologpage:
            # write log page.
            logpageIO = BytesIO()
            self.writeLogPage(logpageIO)
            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
            logpageIO.close()

        if self.hasConfig('chapter_start'):
            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
        else:
            CHAPTER_START = self.EPUB_CHAPTER_START

        if self.hasConfig('chapter_end'):
            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
        else:
            CHAPTER_END = self.EPUB_CHAPTER_END

        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
            # logger.debug("chapter:%s %s %s"%(len(chap['html']), chap['title'],chap['url']))
            if chap['html']:
                chap_data = chap['html']
                if self.getConfig('internalize_text_links'):
                    soup = bs4.BeautifulSoup(chap['html'],'html5lib')
                    changed=False
                    for alink in soup.find_all('a'):
                        ## Chapters can be inserted in the middle
                        ## which can break existing internal links.
                        ## So let's save the original href and update.
                        # logger.debug("found %s"%alink)
                        if alink.has_attr('data-orighref') and alink['data-orighref'] in chapurlmap:
                            alink['href']=chapurlmap[alink['data-orighref']]
                            # logger.debug("set1  %s"%alink)
                            changed=True
                        elif alink.has_attr('href') and alink['href'] in chapurlmap:
                            if not alink['href'].startswith('file'):
                                # only save orig href if not already internal.
                                alink['data-orighref']=alink['href']
                            alink['href']=chapurlmap[alink['href']]
                            # logger.debug("set2  %s"%alink)
                            changed=True
                    if changed:
                        chap_data = unicode(soup)
                        # Don't want html, head or body tags in
                        # chapter html--bs4 insists on adding them.
                        chap_data = re.sub(r"</?(html|head|body)[^>]*>\r?\n?","",chap_data)

                # logger.debug('Writing chapter text for: %s' % chap.title)
                chap['url']=removeEntities(chap['url'])
                chap['chapter']=removeEntities(chap['chapter'])
                chap['title']=removeEntities(chap['title'])
                chap['origchapter']=removeEntities(chap['origtitle'])
                chap['tocchapter']=removeEntities(chap['toctitle'])
                # escape double quotes in all vals.
                for k,v in chap.items():
                    if isinstance(v,basestring): chap[k]=v.replace('"','"')
                fullhtml = CHAPTER_START.substitute(chap) + \
                    chap_data.strip() + \
                    CHAPTER_END.substitute(chap)
                # strip to avoid ever growning numbers of newlines.
                # ffnet(& maybe others) gives the whole chapter text
                # as one line.  This causes problems for nook(at
                # least) when the chapter size starts getting big
                # (200k+)
                fullhtml = re.sub(r'(
|
)\n*',r'\1\n',fullhtml)

                # logger.debug("write OEBPS/file%s.xhtml"%chap['index04'])
                outputepub.writestr("OEBPS/file%s.xhtml"%chap['index04'],fullhtml.encode('utf-8'))
                del fullhtml

        if self.story.calibrebookmark:
            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)

        # declares all the files created by Windows.  otherwise, when
        # it runs in appengine, windows unzips the files as 000 perms.
        for zf in outputepub.filelist:
            zf.create_system = 0
        outputepub.close()
        out.write(zipio.getvalue())
        zipio.close()

## Utility method for creating new tags.
def newTag(dom,name,attrs=None,text=None):
    tag = dom.createElement(name)
    if( attrs is not None ):
        for attr in attrs.keys():
            tag.setAttribute(attr,attrs[attr])
    if( text is not None ):
        tag.appendChild(dom.createTextNode(text))
    return tag


"""
Definitions of the GA4GH protocol types.
"""
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import datetime
import json
import inspect
from sys import modules

import google.protobuf.json_format as json_format
import google.protobuf.message as message
import google.protobuf.struct_pb2 as struct_pb2

import ga4gh.pb as pb

from ga4gh._protocol_version import version  # noqa
from ga4gh.common_pb2 import *  # noqa
from ga4gh.assay_metadata_pb2 import *  # noqa
from ga4gh.metadata_pb2 import *  # noqa
from ga4gh.metadata_service_pb2 import *  # noqa
from ga4gh.read_service_pb2 import *  # noqa
from ga4gh.reads_pb2 import *  # noqa
from ga4gh.reference_service_pb2 import *  # noqa
from ga4gh.references_pb2 import *  # noqa
from ga4gh.variant_service_pb2 import *  # noqa
from ga4gh.variants_pb2 import *  # noqa
from ga4gh.allele_annotations_pb2 import *  # noqa
from ga4gh.allele_annotation_service_pb2 import *  # noqa
from ga4gh.sequence_annotations_pb2 import *  # noqa
from ga4gh.sequence_annotation_service_pb2 import *  # noqa
from ga4gh.bio_metadata_pb2 import *  # noqa
from ga4gh.bio_metadata_service_pb2 import *  # noqa
from ga4gh.genotype_phenotype_pb2 import *  # noqa
from ga4gh.genotype_phenotype_service_pb2 import *  # noqa
from ga4gh.rna_quantification_pb2 import *  # noqa
from ga4gh.rna_quantification_service_pb2 import *  # noqa

# A map of response objects to the name of the attribute used to
# store the values returned.
_valueListNameMap = {
    SearchVariantSetsResponse: "variant_sets",  # noqa
    SearchVariantsResponse: "variants",  # noqa
    SearchDatasetsResponse: "datasets",  # noqa
    SearchReferenceSetsResponse: "reference_sets",  # noqa
    SearchReferencesResponse: "references",  # noqa
    SearchReadGroupSetsResponse: "read_group_sets",  # noqa
    SearchReadsResponse: "alignments",  # noqa
    SearchCallSetsResponse: "call_sets",  # noqa
    SearchVariantAnnotationSetsResponse: "variant_annotation_sets",  # noqa
    SearchVariantAnnotationsResponse: "variant_annotations",  # noqa
    SearchFeatureSetsResponse: "feature_sets",  # noqa
    SearchFeaturesResponse: "features",  # noqa
    SearchBioSamplesResponse: "biosamples",  # noqa
    SearchIndividualsResponse: "individuals",  # noqa
    SearchPhenotypeAssociationSetsResponse: "phenotype_association_sets",  # noqa
    SearchPhenotypesResponse: "phenotypes",  # noqa
    SearchGenotypePhenotypeResponse: "associations",  # noqa
    SearchRnaQuantificationSetsResponse: "rna_quantification_sets",  # noqa
    SearchRnaQuantificationsResponse: "rna_quantifications",  # noqa
    SearchExpressionLevelsResponse: "expression_levels",  # noqa
}

def getValueListName(protocolResponseClass):
    """
    Returns the name of the attribute in the specified protocol class
    that is used to hold the values in a search response.
    """
    return _valueListNameMap[protocolResponseClass]

def convertDatetime(t):
    """
    Converts the specified datetime object into its appropriate protocol
    value. This is the number of milliseconds from the epoch.
    """
    epoch = datetime.datetime.utcfromtimestamp(0)
    delta = t - epoch
    millis = delta.total_seconds() * 1000
    return int(millis)

def getValueFromValue(value):
    """
    Extract the currently set field from a Value structure
    """
    if type(value) != struct_pb2.Value:
        raise TypeError("Expected a Value, but got {}".format(type(value)))
    if value.WhichOneof("kind") is None:
        raise AttributeError("Nothing set for {}".format(value))
    return getattr(value, value.WhichOneof("kind"))

def toJson(protoObject, indent=None):
    """
    Serialises a protobuf object as json
    """
    # Using the internal method because this way we can reformat the JSON
    js = json_format._MessageToJsonObject(protoObject, True)
    return json.dumps(js, indent=indent)

def toJsonDict(protoObject):
    """
    Converts a protobuf object to the raw attributes
    i.e. a key/value dictionary
    """
    return json.loads(toJson(protoObject))

def fromJson(json, protoClass):
    """
    Deserialise json into an instance of protobuf class
    """
    return json_format.Parse(json, protoClass())

def validate(json, protoClass):
    """
    Check that json represents data that could be used to make
    a given protobuf class
    """
    try:
        fromJson(json, protoClass)
        # The json conversion automatically validates
        return True
    except Exception:
        return False

class SearchResponseBuilder(object):
    """
    A class to allow sequential building of SearchResponse objects.
    """
    def __init__(self, responseClass, pageSize, maxBufferSize):
        """
        Allocates a new SearchResponseBuilder for the specified
        responseClass, user-requested pageSize and the system mandated
        maxBufferSize (in bytes). The maxBufferSize is an
        approximate limit on the overall length of the serialised
        response.
        """
        self._responseClass = responseClass
        self._pageSize = pageSize
        self._maxBufferSize = maxBufferSize
        self._numElements = 0
        self._nextPageToken = None
        self._protoObject = responseClass()
        self._valueListName = getValueListName(responseClass)
        self._bufferSize = self._protoObject.ByteSize()

    def getPageSize(self):
        """
        Returns the page size for this SearchResponseBuilder. This is the
        user-requested maximum size for the number of elements in the
        value list.
        """
        return self._pageSize

    def getMaxBufferSize(self):
        """
        Returns the maximum internal buffer size for responses, which
        corresponds to total length (in bytes) of the serialised protobuf
        objects. This will always be less than the size of JSON output.
        """
        return self._maxBufferSize

    def getNextPageToken(self):
        """
        Returns the value of the nextPageToken for this
        SearchResponseBuilder.
        """
        return self._nextPageToken

    def setNextPageToken(self, nextPageToken):
        """
        Sets the nextPageToken to the specified value.
        """
        self._nextPageToken = nextPageToken

    def addValue(self, protocolElement):
        """
        Appends the specified protocolElement to the value list for this
        response.
        """
        self._numElements += 1
        self._bufferSize += protocolElement.ByteSize()
        attr = getattr(self._protoObject, self._valueListName)
        obj = attr.add()
        obj.CopyFrom(protocolElement)

    def isFull(self):
        """
        Returns True if the response buffer is full, and False otherwise.
        The buffer is full if either (1) the number of items in the value
        list is >= pageSize or (2) the total length of the serialised
        elements in the page is >= maxBufferSize.

        If page_size or max_response_length were not set in the request
        then they're not checked.
        """
        return (
            (self._pageSize > 0 and self._numElements >= self._pageSize) or
            (self._bufferSize >= self._maxBufferSize)
        )

    def getSerializedResponse(self):
        """
        Returns a string version of the SearchResponse that has
        been built by this SearchResponseBuilder.
        """
        self._protoObject.next_page_token = pb.string(self._nextPageToken)
        s = toJson(self._protoObject)
        return s

def getProtocolClasses(superclass=message.Message):
    """
    Returns all the protocol classes that are subclasses of the
    specified superclass. Only 'leaf' classes are returned,
    corresponding directly to the classes defined in the protocol.
    """
    # We keep a manual list of the superclasses that we define here
    # so we can filter them out when we're getting the protocol
    # classes.
    superclasses = set([message.Message])
    thisModule = modules[__name__]
    subclasses = []
    for name, class_ in inspect.getmembers(thisModule):
        if ((inspect.isclass(class_) and
                issubclass(class_, superclass) and
                class_ not in superclasses)):
            subclasses.append(class_)
    return subclasses

postMethods = \
    [('/callsets/search',
      SearchCallSetsRequest,  # noqa
      SearchCallSetsResponse),  # noqa
     ('/datasets/search',
      SearchDatasetsRequest,  # noqa
      SearchDatasetsResponse),  # noqa
     ('/readgroupsets/search',
      SearchReadGroupSetsRequest,  # noqa
      SearchReadGroupSetsResponse),  # noqa
     ('/reads/search',
      SearchReadsRequest,  # noqa
      SearchReadsResponse),  # noqa
     ('/references/search',
      SearchReferencesRequest,  # noqa
      SearchReferencesResponse),  # noqa
     ('/referencesets/search',
      SearchReferenceSetsRequest,  # noqa
      SearchReferenceSetsResponse),  # noqa
     ('/variants/search',
      SearchVariantsRequest,  # noqa
      SearchVariantsResponse),  # noqa
     ('/datasets/search',
      SearchDatasetsRequest,  # noqa
      SearchDatasetsResponse),  # noqa
     ('/callsets/search',
      SearchCallSetsRequest,  # noqa
      SearchCallSetsResponse),  # noqa
     ('/featuresets/search',
      SearchFeatureSetsRequest,  # noqa
      SearchFeatureSetsResponse),  # noqa
     ('/features/search',
      SearchFeaturesRequest,  # noqa
      SearchFeaturesResponse),  # noqa
     ('/variantsets/search',
      SearchVariantSetsRequest,  # noqa
      SearchVariantSetsResponse),  # noqa
     ('/variantannotations/search',
      SearchVariantAnnotationsRequest,  # noqa
      SearchVariantAnnotationSetsResponse),  # noqa
     ('/variantannotationsets/search',
      SearchVariantAnnotationSetsRequest,  # noqa
      SearchVariantAnnotationSetsResponse),  # noqa
     ('/rnaquantificationsets/search',
      SearchRnaQuantificationSetsRequest,  # noqa
      SearchRnaQuantificationSetsResponse),  # noqa
     ('/rnaquantifications/search',
      SearchRnaQuantificationsRequest,  # noqa
      SearchRnaQuantificationsResponse),  # noqa
     ('/expressionlevels/search',
      SearchExpressionLevelsRequest,  # noqa
      SearchExpressionLevelsResponse)]  # noqa


from core_functions import Chain, Reel
from tabulate import tabulate
from colorama import init as colorama_init, Fore

class ChainOutput():

    # constant: colorama colours for output
    COLORS = {
        "element": Fore.LIGHTWHITE_EX,
        "element_loop": Fore.LIGHTYELLOW_EX,
        "edge": Fore.LIGHTBLACK_EX,
    }

    # constant: symbols for output
    SYMBOLS = {
        "edge": " ━ "
    }

    def __init__(self, chain: Chain.Chain):
        """
        constructor
        @param chain: Chain to output

        OUT:
        a string of ints representing the chain
        - each value is separated by an edge symbol
        - loop values are highlighted in yellow
        """

        # initialise colorama
        colorama_init()

        self.out = ""

        # calculate chain and loop start index
        array, loop_start = chain

        # create list to store if idx in array is a loop value
        self.idx_is_loop = []

        for idx, item in enumerate(array):

            # add item
            self.out += self.COLORS["element_loop"] if idx >= loop_start else self.COLORS["element"]
            self.out += str(item)

            # add edge symbol if not last item
            self.out += self.COLORS["edge"] + self.SYMBOLS["edge"] if idx != len(array) - 1 else ""

            # reset highlight; it's the right thing to do :)
            self.out += Fore.RESET

    def __str__(self):
        """
        string representation of ChainOutput
        @return: out
        """

        return self.out

class ReelOutput():

    # constant: colorama colours for output
    COLORS = {
        "separator": Fore.LIGHTBLACK_EX
    }

    # constant: symbols for output
    SYMBOLS = {
        "separator": "┆"
    }

    # constant: table styles for tabulate
    TABLE_STYLE = "plain"

    # constant: custom middle separator for tabulate
    MIDDLE_SEPARATOR = True

    def __init__(self, reel: Reel.Reel):
        """
        constructor
        @param reel: Reel to output
        """

        # initialise colorama
        colorama_init()

        # prepare table
        self.array = [["Index", self.COLORS["separator"] + self.SYMBOLS["separator"] + Fore.RESET, "Chain"]]

        # calculate chain and loop start index
        for item in reel:
            self.array.append([item.starting_value, self.COLORS["separator"] + self.SYMBOLS["separator"] + Fore.RESET, ChainOutput(item.chain)])

        # remove middle column if MIDDLE_SEPARATOR is False
        if not self.MIDDLE_SEPARATOR:
            for col in self.array:
                col.pop(1)

        # use tabulate to format table
        self.out = tabulate(self.array, tablefmt=self.TABLE_STYLE, headers="firstrow")

    def __str__(self):
        """
        string representation of ReelOutput
        @return: out
        """

        return self.out


import argparse
import os
from time import sleep

# === subroutines ===

def collect_files_for_removal(root: str) -> tuple[list[str], list[str]]:
    if not os.path.exists(root):
        return ([], [])

    res_files = list()
    res_folders = list()

    for (dir_path, dirs, files) in os.walk(root, topdown=False):
        files = [os.path.join(dir_path, file) for file in files if file.endswith('.class')]
        res_files.extend(files)
        res_folders.append(dir_path)
    res_folders.remove(root)

    return (res_folders, res_files)

# === CLI and Environment ===

workdir = os.path.abspath(os.curdir)

parser = argparse.ArgumentParser()
parser.add_argument(
    '--output-dir', metavar='DIR', default='out',
    help='a directory with *.CLASS files')

args = parser.parse_args()
output_dir = args.output_dir

# === main ===

output_dir = os.path.abspath(output_dir)
print('[i] Cleaning output directory:', output_dir)

dirs, files = collect_files_for_removal(output_dir)

# files first
for path in files:
    os.remove(path)

# then folders
for path in dirs:
    os.rmdir(path)

print('[i] Done!')
sleep(1)


from PySide2 import QtWidgets
from PySide2.QtCore import Signal

# widget to get input for vector 3 types
class Vector3Widget(QtWidgets.QWidget):

    # Signals
    on_value_changed = Signal(tuple)

    _main_layout = None

    def __init__(self, value=(0, 0, 0)):
        QtWidgets.QWidget.__init__(self)

        self._value = value

        # main layout
        self._main_layout = QtWidgets.QGridLayout()
        self._main_layout.setSpacing(0)
        self.setLayout(self._main_layout)

        # input fields
        self._unit_1_field = None
        self._unit_2_field = None
        self._unit_3_field = None

        self._create_ui()

    def _create_ui(self):
        # field for value x
        self._unit_1_field = QtWidgets.QSpinBox()
        self._unit_1_field.setValue(self._value[0])
        self._unit_1_field.editingFinished.connect(self._on_field_value_changed)

        # field for value y
        self._unit_2_field = QtWidgets.QSpinBox()
        self._unit_2_field.setValue(self._value[1])
        self._unit_2_field.editingFinished.connect(self._on_field_value_changed)

        # field for value z
        self._unit_3_field = QtWidgets.QSpinBox()
        self._unit_3_field.setValue(self._value[2])
        self._unit_3_field.editingFinished.connect(self._on_field_value_changed)

        # add to layout
        self._main_layout.addWidget(self._unit_1_field, 1, 2)
        self._main_layout.addWidget(self._unit_2_field, 1, 3)
        self._main_layout.addWidget(self._unit_3_field, 1, 4)

    def _on_field_value_changed(self, value=0):
        self.on_value_changed.emit(self.get_value())

    def get_value(self):
        return (self._unit_1_field.value(), self._unit_2_field.value(), self._unit_3_field.value())


import os
import json
from typing import List
from datetime import datetime

import pandas as pd

#############Load config.json and get input and output paths
with open('config.json','r') as f:
    config = json.load(f)

input_folder_path = config['input_folder_path']
output_folder_path = config['output_folder_path']
ingestion_record_file_name = config['ingestion_record_file_name']
ingestion_file_name = config['ingestion_file_name']

#############Function for data ingestion
def read_tabular_file(dir_name: str, file_name: str):
    return pd.read_csv(os.path.join(dir_name, file_name), sep=',', encoding='utf-8')

def clean_dataframe(df: pd.DataFrame):
    # filter out duplicates
    df = df.drop_duplicates()
    return df

def save_dataframe(df, output_dir_name: str, file_name: str):
    df.to_csv(os.path.join(output_dir_name, file_name), sep=',', encoding='utf-8', index=False)

def write_ingested_file_record(file_name: str, file_dir: str, ingested_file_loc: str, ingested_file_name: str, ingested_file_length: int):
    with open(os.path.join(file_dir, file_name), 'a') as f:
        f.write("{datetime}\t{location}\t{filename}\t{length}\n".format(
            datetime=datetime.now(),
            location=ingested_file_loc,
            filename=ingested_file_name,
            length=ingested_file_length
        ))

def merge_multiple_dataframe(input_folder_dir: str,
                             output_folder_dir: str,
                             output_file_name: str,
                             record_file_name: str,
                             data_cols: List[str] = ["corporation", "lastmonth_activity", "lastyear_activity", "number_of_employees", "exited"]):
    # check for datasets, compile them together, and write to an output file

    file_name_ls = os.listdir(input_folder_dir)

    df_list = pd.DataFrame(columns=data_cols)
    for file_name in file_name_ls:
        df = read_tabular_file(input_folder_dir, file_name)
        df_list = df_list.append(df)
        write_ingested_file_record(record_file_name, output_folder_dir,
                                   input_folder_dir, file_name, len(df))

    df_list = clean_dataframe(df_list)
    save_dataframe(df_list, output_folder_path, output_file_name)

if __name__ == '__main__':
    merge_multiple_dataframe(input_folder_path, output_folder_path, ingestion_file_name, ingestion_record_file_name)


import numpy as np
import pandas as pd
import os

import cv2
import re
import torch
import torchvision
from torchvision import transforms
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection import FasterRCNN
from torchvision.models.detection.rpn import AnchorGenerator
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.sampler import SequentialSampler
from matplotlib import pyplot as plt
from sklearn.model_selection import KFold
from tqdm import tqdm

class TurbineDataset(Dataset):
    """
    Custom PyTorch dataset for turbine image data.

    This dataset is designed for object detection tasks where each image contains
    annotations of turbine components. It can be used for both training and inference.

    Args:
        dataframe (pandas.DataFrame): The DataFrame containing image and annotation
            information.
        transforms (callable, optional): A function/transform to apply
            to the image data.
        train (bool, optional): Specify if the dataset is for training (True)
            or inference (False).

    Attributes:
        image_ids (numpy.ndarray): Unique image IDs extracted from the DataFrame.
        df (pandas.DataFrame): The input DataFrame containing image and annotation
            information.
        transforms (callable, optional): A function/transform for image data
            augmentation.
        train (bool): Indicates whether the dataset is for training (True) or
            inference (False).

    Methods:
        __len__(): Returns the number of unique images in the dataset.
        __getitem__(index): Retrieves an image and its associated annotations.

    For training:
    - Images are loaded and transformed.
    - Annotations are retrieved and organized into a dictionary.

    For inference:
    - Only images are loaded and returned.

    Returns:
        If 'train' is True:
            Tuple containing:
                - image (torch.Tensor): The preprocessed image.
                - target (dict): A dictionary containing annotations
                    (boxes, labels, etc.).
                - image_id (str): ID of the image.
        If 'train' is False:
            Tuple containing:
                - image (torch.Tensor): The preprocessed image.
                - image_id (str): ID of the image.
    """
    def __init__(self, dataframe, transforms=None, train=True):
        super().__init__()

        self.image_ids = dataframe['image'].unique()
        self.df = dataframe
        self.transforms = transforms
        self.train = train

    def __len__(self) -> int:
        return self.image_ids.shape[0]

    def __getitem__(self, index: int):
        image_id = self.image_ids[index]
        image = cv2.imread(image_id, cv2.IMREAD_COLOR)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)
        image /= 255.0
        if self.transforms is not None:
            image = self.transforms(image)
        if self.train is False:
            return image, image_id

        records = self.df[self.df['image'] == image_id]
        boxes = records[['minx', 'miny', 'maxx', 'maxy']].values
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        area = torch.as_tensor(area, dtype=torch.float32)
        labels = torch.ones((records.shape[0],), dtype=torch.int64)
        iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)
        target = {}
        target['boxes'] = boxes
        target['labels'] = labels
        target['image_id'] = torch.tensor([index])
        target['area'] = area
        target['iscrowd'] = iscrowd
        return image, target, image_id

class Averager:
    """
    A utility class for calculating and maintaining the average of a series of values.

    This class is typically used for computing the average loss during training
    iterations.

    Attributes:
        current_total (float): The running total of values to be averaged.
        iterations (float): The number of values added to the running total.

    Methods:
        send(value): Add a new value to the running total and update the number of
            iterations.
        value: Property that returns the average of the added values.
        reset(): Reset the running total and number of iterations to zero.

    Example Usage:
    ```
    avg_loss = Averager()
    avg_loss.send(2.0)
    avg_loss.send(3.0)
    average = avg_loss.value  # Returns 2.5
    avg_loss.reset()  # Resets the total and iterations to zero.
    ```

    Note:
        If no values are added (iterations = 0), the `value` property returns 0 to
        prevent division by zero.

    """
    def __init__(self):
        self.current_total = 0.0
        self.iterations = 0.0

    def send(self, value):
        """
        Add a new value to the running total and update the number of iterations.

        Args:
            value (float): The value to be added to the running total.
        """
        self.current_total += value
        self.iterations += 1

    @property
    def value(self):
        """
        Get the average value of the added values.

        Returns:
            float: The average value, or 0 if no values have been added
                (iterations = 0).
        """
        if self.iterations == 0:
            return 0
        else:
            return 1.0 * self.current_total / self.iterations

    def reset(self):
        """
        Reset the running total and number of iterations to zero.
        """
        self.current_total = 0.0
        self.iterations = 0.0

def collate_fn(batch):
    """
    Collates a batch of data elements into a structured format.

    This function is typically used in data loading pipelines, such as when working with
    PyTorch's DataLoader. It takes a batch of individual data elements and arranges them
    into a structured format, often as a tuple or a dictionary, making it suitable for
    further processing.

    Args:
        batch (list): A list of individual data elements to be collated.

    Returns:
        tuple: A tuple containing the collated data elements. The specific structure
        of the returned tuple may vary depending on the data and the application.

    Example Usage:
    ```
    batch = [(image1, label1), (image2, label2), (image3, label3)]
    collated_batch = collate_fn(batch)
    # Example collated_batch: ((image1, image2, image3), (label1, label2, label3))
    ```

    Note:
        The structure of the returned tuple should match the requirements of the
        downstream processing steps, such as model input.

    """
    return tuple(zip(*batch))

def prepare_batches_for_training(folds: dict, selected_data: gdp.GeoDataFrame, number_of_fold: int):
    trans = transforms.Compose([transforms.ToTensor()])
    train_df = selected_data[selected_data['image'].isin(folds[number_of_fold]['train'])]
    test_df = selected_data[selected_data['image'].isin(folds[number_of_fold]['test'])]
    train_dataset = TurbineDataset(train_df, trans,True)
    test_dataset = TurbineDataset(test_df, trans,True)

    indices = torch.randperm(len(train_dataset)).tolist()
    train_data_loader = DataLoader(
        train_dataset,
        batch_size=16,
        shuffle=False,
        num_workers=4,
        collate_fn=collate_fn
    )

    test_data_loader = DataLoader(
        test_dataset,
        batch_size=8,
        shuffle=False,
        num_workers=4,
        collate_fn=collate_fn
    )


import sys
import time
import random

import pygame

from event_listener import event_listener
from functions import render_all, one_dimensional_list, update_frames

sys.path.insert(1, 'player')
from yoshi import Yoshi
from movement import move_all, set_direction, move

sys.path.insert(1, 'eggs')
from egg import Egg

sys.path.insert(1, 'end_game')
from end_game import  end_game
from death_collided import collided_walls

pygame.init()

# Setting app name.
pygame.display.set_caption("Yoshi Snake Game.")

# It should be 17x15.
SCREEN_WIDTH = 680
SCREEN_HEIGHT = 600
GREEN = (31, 134, 31)
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))

yoshis = []
eggs = []

objects = []
objects.append(yoshis)
objects.append(eggs)

# Creating player character.
player = Yoshi()
yoshis.append(player)

# Testing egg.
eggy = Egg("green")
eggs.append(eggy)

# Allows player to choose first direction.
direction = ""

score = 1
frame = 0
while True:
    # Rendering frame
        # Reseting screen for every frame.
    screen.fill(GREEN)

        # Rendering all objects
    render_all( one_dimensional_list(objects), screen )

        # Flip the display
    pygame.display.flip()

        # Verifying if player lost:
    if collided_walls(player, SCREEN_WIDTH, SCREEN_HEIGHT):
       end_game(screen)

    time.sleep(0.5)

    # Taking input.
    last_relevant_event = event_listener()
    # Closing game if clicked to quit:
    if last_relevant_event == "quit":
        sys.exit(0)

    # Preparing for the next rendering:

    move_all(yoshis)

    direction = set_direction(direction, last_relevant_event)
    move(player, direction, 40, 40)

        # Checking egg collision.
    if eggy.check_collision(player):
        new_yoshi = Yoshi(x = player.x, y = player.y, current_frame = frame)
        yoshis.append(new_yoshi)

        # Changing egg location.
        eggy.x = random.randint(0, 16) * 40
        eggy.y = random.randint(0, 14) * 40

        score += 1
        pygame.display.set_caption("Yoshi's Snake Game." + " " * 3 + "Score: " + str(score))

        # Updating gif frames.
    update_frames(one_dimensional_list(objects))

    frame += 1


# from django.shortcuts import render
from django.views.generic import ListView, DetailView, UpdateView, CreateView, DeleteView  # импортируем класс, который говорит нам о том, что в этом представлении мы будем выводить список объектов из БД
from .models import Post
from datetime import datetime
from .filters import PostFilter
from .forms import PostForm # импортируем нашу форму
from django.contrib.auth.mixins import LoginRequiredMixin
from django.core.paginator import Paginator
from django.contrib.auth.mixins import PermissionRequiredMixin

class PostList(ListView):
    model = Post  # указываем модель, объекты которой мы будем выводить
    template_name = 'news.html'  # указываем имя шаблона, в котором будет лежать html,
    # в котором будут все инструкции о том, как именно пользователю должны вывестись наши объекты
    context_object_name = 'news'  # это имя списка, в котором будут лежать все объекты,
    # его надо указать, чтобы обратиться к самому списку объектов через html-шаблон
    queryset = Post.objects.order_by('-id')
    form_class = PostForm  # добавляем форм класс, чтобы получать доступ к форме через метод POST
    #paginate_by = 1

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['time_now'] = datetime.utcnow()  # добавим переменную текущей даты time_now
        context['value1'] = None  # добавим ещё одну пустую переменную, чтобы на её примере посмотреть работу
        # другого фильтра
        return context

# создаём представление в котором будет детали конкретного отдельного товара
class PostDetail(DetailView):
    model = Post # модель всё та же, но мы хотим получать детали конкретно отдельного товара
    template_name = 'post.html' # название шаблона будет post.html
    context_object_name = 'post' # название объекта. в нём будет

class Search(ListView):
    model = Post
    template_name = 'search.html'
    context_object_name = 'search'
    ordering = ['-time_in']
    paginate_by = 1  # поставим постраничный вывод в один элемент

    def get_context_data(self, **kwargs):  # забираем отфильтрованные объекты переопределяя метод
        # get_context_data у наследуемого класса (привет полиморфизм, мы скучали!!!)
        context = super().get_context_data(**kwargs)
        context['filter'] = PostFilter(self.request.GET, queryset=self.get_queryset())  # вписываем наш фильтр
        # в контекст
        return context

class PostCreateView(PermissionRequiredMixin,CreateView):
    template_name = 'add.html'
    form_class = PostForm
    permission_required = ('news.add_post',)

# дженерик для редактирования объекта
class PostEditView(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    template_name = 'edit.html'
    form_class = PostForm
    permission_required = ('news.change_post',)

    # метод get_object мы используем вместо queryset, чтобы получить информацию об объекте
    # который мы собираемся редактировать
    def get_object(self, **kwargs):
        id_1 = self.kwargs.get('pk')
        return Post.objects.get(id=id_1)

# дженерик для удаления товара
class PostDeleteView(PermissionRequiredMixin, DeleteView):
    template_name = 'delete.html'
    queryset = Post.objects.all()
    permission_required = ('news.delete_post',)
    success_url = '/news/'


from collections import deque
from threading import Thread

class Sequencer:
    def __init__(self, name):
        self.name = name
        self.file = open(name, "wb")
        self.queue = deque()
        self.running = False
        self.byte_sequence = []
        self.thread = Thread(target=self._writer_thread)

    def add(self, byte):
        index = byte['index']
        self.queue.append(byte)
        # self.byte_sequence.append(byte)
        # self.arrange()
        pass

    def close(self):
        """
        Wait for the writer thread to finish and close the open file.
        """
        self.running = False
        self.thread.join()
        self.file.close()

    def _writer_thread(self):
        """
        Runs until a stop is requested and the queue is exhausted.
        """
        self.running = True
        while self.running or len(self.queue):
            if len(self.queue):
                byte = self.queue.popleft()
                chunk = byte['index']
                data = byte['data']
                chunk_size = len(data)
                self.file.seek(chunk*chunk_size)
                self.file.write(data)

    def arrange(self):
        self.byte_sequence.sort(key=lambda x: x['index'])
        i = 0
        while i != len(self.byte_sequence):
            current = self.byte_sequence[i]
            previous = self.byte_sequence[i - 1]
            current_sequence = current.get('index')
            previous_sequence = previous.get('last', previous.get('index'))
            if (current_sequence - 1) == previous_sequence and (current_sequence - 1) > -1:
                current = self.byte_sequence.pop(i)
                previous['data'] = previous['data'] + current.get('data', b'')
                last_sequence = current.get('last', current.get('index'))
                previous['last'] = last_sequence
                continue
            i += 1
        print(self.byte_sequence)

if __name__ == '__main__':
    obj = Sequencer('test')
    obj.add({'index': 0, 'data': b'0'})
    obj.add({'index': 1, 'data': b'1'})
    obj.add({'index': 2, 'data': b'2'})
    obj.add({'index': 3, 'data': b'3'})
    obj.add({'index': 4, 'data': b'4'})
    obj.add({'index': 5, 'data': b'5'})
    obj.add({'index': 6, 'data': b'6'})
    obj.add({'index': 7, 'data': b'7'})
    obj.add({'index': 8, 'data': b'8'})
    obj.add({'index': 9, 'data': b'9'})
    obj.add({'index': 10, 'data': b'10'})
    obj.add({'index': 11, 'data': b'11'})