import argparse
import os
import torch

from data.loader import data_loader
from models import TrajectoryPrediction
import utils
from utils import int_tuple

parser = argparse.ArgumentParser()

parser.add_argument("--log_dir", default="./")
parser.add_argument("--dataset_name", default="zara2", type=str)
parser.add_argument("--delim", default="\t")
parser.add_argument("--loader_num_workers", default=4, type=int)
parser.add_argument("--obs_len", default=8, type=int)
parser.add_argument("--pred_len", default=12, type=int)
parser.add_argument("--skip", default=1, type=int)
parser.add_argument("--batch_size", default=32, type=int)
parser.add_argument("--noise_dim", default=(16,), type=int_tuple)
parser.add_argument("--noise_type", default="gaussian")

parser.add_argument("--dropout", default=0, type=float) # dropout rate
parser.add_argument("--alpha", default=0.2, type=float)   # alpha for the leaky relu
parser.add_argument("--heads", default="4, 1", type=str)
parser.add_argument("--hidden_units", default="16", type=str)

parser.add_argument("--action_input_dim", default=16, type=int)
parser.add_argument("--action_encoder_hidden_dim", default=32, type=int)
parser.add_argument("--goal_input_dim", default=16, type=int)
parser.add_argument("--goal_encoder_hidden_dim", default=32, type=int)
parser.add_argument("--distance_embedding_dim", default=128, type=int)

parser.add_argument("--dset_type", default="test", type=str)
parser.add_argument(
    "--resume", default="./model_best.pth.tar", type=str,
    metavar="PATH", help="path to latest checkpoint (default: none)"
)
parser.add_argument("--num_samples", default=20, type=int)      # 暂时调整为1,与 best_k 一致
parser.add_argument("--gpu_num", default="0", type=str)

def evaluate_helper(error, seq_start_end):
    sum_ = 0
    error = torch.stack(error, dim=1)
    for (start, end) in seq_start_end:
        start = start.item()
        end = end.item()
        _error = error[start:end]
        _error = torch.sum(_error, dim=0)
        _error = torch.min(_error)
        sum_ += _error
    return sum_

def get_model(checkpoint):
    n_units = [
        [args.action_encoder_hidden_dim]
        + [int(x) for x in args.hidden_units.strip().split(",")]
        + [args.action_encoder_hidden_dim]
    ]
    n_heads = [int(x) for x in args.heads.strip().split(",")]

    model = TrajectoryPrediction(
        obs_len=args.obs_len,
        pred_len=args.pred_len,
        action_input_dim=args.action_input_dim,
        action_encoder_hidden_dim=args.action_encoder_hidden_dim,
        goal_input_dim=args.goal_input_dim,
        goal_encoder_hidden_dim=args.goal_encoder_hidden_dim,
        distance_embedding_dim=args.distance_embedding_dim,
        n_units=n_units,
        n_heads=n_heads,
        dropout=args.dropout,
        alpha=args.alpha,
        noise_dim=args.noise_dim,
        noise_type=args.noise_type,
    )
    model.load_state_dict(checkpoint["state_dict"])
    model.cuda()
    model.eval()
    return model

def evaluate(args, loader, model):
    ADE_outer, FDE_outer = [], []
    traj_sum = 0

    with torch.no_grad():
        for batch in loader:
            batch = [tensor.cuda() for tensor in batch]
            (
                obs_traj,  # obs_traj, [8,1413,2]
                pred_traj_gt,  # pred_traj, [12,1413,2]
                obs_traj_rel,  # obs_traj_rel, [8,1413,2]
                pred_traj_gt_rel,  # pred_traj_rel, [12, 1413,2]
                non_linear_ped,
                loss_mask,
                seq_start_end,
            ) = batch

            ADE, FDE = [], []
            traj_sum += pred_traj_gt.size(1)

            for _ in range(args.num_samples):
                # traj_rel_gt = torch.cat((obs_traj_rel, pred_traj_gt_rel), dim=0)

                pred_traj_fake_rel = model(obs_traj, obs_traj_rel, seq_start_end, training_step=3)

                # pred_traj_fake = pred_action_fake
                # pred_traj_fake_predpart = pred_traj_fake[-args.pred_len:]
                pred_traj_fake_abs = utils.relative_to_abs(pred_traj_fake_rel, obs_traj[-1])

                ADE_, FDE_ = utils.cal_ADE_FDE(pred_traj_gt, pred_traj_fake_abs, mode="raw")

                temp = utils.loss_test(pred_traj_fake_abs, pred_traj_gt)
                print(temp)

                ADE.append(ADE_)
                FDE.append(FDE_)

            ADE_sum = evaluate_helper(ADE, seq_start_end)
            FDE_sum = evaluate_helper(FDE, seq_start_end)
            ADE_outer.append(ADE_sum)
            FDE_outer.append(FDE_sum)

        ADE_output = sum(ADE_outer) / (traj_sum * args.pred_len)
        FDE_output = sum(FDE_outer) / traj_sum

        return ADE_output, FDE_output

def main(args):
    checkpoint = torch.load(args.resume)
    model = get_model(checkpoint)
    path = utils.get_dset_path(args.dataset_name, args.dset_type)
    _, loader = data_loader(args, path)
    ADE, FDE = evaluate(args, loader, model)
    print(
        "Dataset: {}, Pred Len: {}, ADE: {:.12f}, FDE: {:.12f}".format(
            args.dataset_name, args.pred_len, ADE, FDE
        )
    )

if __name__ == "__main__":
    args = parser.parse_args()
    torch.manual_seed(72)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_num
    main(args)


# Make the SNP-fragment list from a multi-sample barcoded 10x-Genomics bam file and its corresponding VCF file with input options: mmq, mbq and qoffset.
# Written by Ehsan Motazedi, Wageningen UR, 04-11-2016.
# Last mofified: 18-07-2018.

import copy
import difflib
import os
import pysam
import re
import subprocess
import sys
from sys import argv
from genotypes import getAllelesPop, getGenotypesPop
from math import log
from reads import Read
# from memory_profiler import profile

def adjust_seq(read):
	""" Adjust the read sequence according to the mapped positions, i.e. get rid of the insertions and clipped bases."""
	cig = list(_x for _x in re.split('[0-9]{1,}', read.cigarstring) if _x)
	cign = list(_x for _x in re.split('[^0-9]', read.cigarstring) if _x)
	cig = list(cig[_n]*int(cign[_n]) for _n in range(0, len(cig)))
	cig = ''.join(_x for _x in cig if 'D' not in _x) # deleted nucleotides from the reference are not present in the read sequence
	adj_seq = []
	adj_qual = []
	for _n, _x in enumerate(read.seq):
		if cig[_n]=='M': # Allow only match/mismatch of the read nucleotides, i.e. no clipping, no insertion.
			adj_qual.append(read.qual[_n])
			adj_seq.append(_x)
	return ''.join(adj_seq), adj_qual

def frag_gen(varpos, allelelst, genolst, coordinates, nucleotides, qscores):
	""" Generate SNP-fragments from (linked) reads."""
	if (not coordinates) or (not nucleotides):
		return Read(), {}
	var_codes = []
	var_num = []
	var_q = []
	for _n, _x in enumerate(varpos):
		if _x < coordinates[0]:
			continue
		if _x > coordinates[-1]:
			break
		if _x in coordinates:
			if set(genolst[_n].GetGenes()).intersection(set(['.','-'])): # Throw away missing genotypes or genotypes with one or more missing allele(s)
				continue
			if len(set(genolst[_n].GetGenes()))<2: # Do not include in the SNP-fragments belonging to a population member its homozygous alleles
				continue
			try:
				var_codes.append(allelelst[_n][2][nucleotides[coordinates.index(_x)]])
				var_num.append(allelelst[_n][0])
				var_q.append(qscores[coordinates.index(_x)])
			except KeyError: # if the called nucleotide is wrong, i.e. does not exist in VCF alleles
				pass
	try: # return the reads {SNP number: allele} and the associated quality scores {SNP number: Qscore}
		return Read({_x:str(_y) for _x, _y in zip([var_num[0], var_num[1]]+var_num[2:], var_codes)}), {_x:str(_y) for _x, _y in zip(var_num, var_q)}
	except IndexError: # throw away reads with less than 2 SNPs
		return Read(), {}

class InputError(Exception):
	""" Handle invalid input specifications."""
	def __init__(self, msg, *args):
		super(InputError, self).__init__(args)
		self.msg = msg
	def __str__(self):
		return "InputError: {}\n".format(self.msg)
	def __repr__(self):
		return (self.msg,)+self.args+('\n',)

# @profile
def get_frags(bamfile, vcffile, mbq=13, mmq=20, qoffset=33):
	"""
mmq : minimum read mapping quality to consider a read for phasing, default 20\n
qvoffset <33/64> : quality value offset, 33/64 depending on how quality values were encoded, default is 33\n
mbq  : minimum base quality to consider a base for haplotype fragment, default 13\n
	"""
	try:
		all_reads = pysam.Samfile(bamfile, 'rb')
	except IOError:
		raise InputError('The input BAM file was not found!')
	ReadHeader = subprocess.Popen(["samtools","view","-H", bamfile], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
	header, err_header = ReadHeader.communicate()
	if ReadHeader.returncode!=0:
		raise InputError('Failed to read the header from the bam file! Original error message:\n'+err_header)
	if isinstance(header, bytes):
		header=bytes.decode(header)
	else:
		pass
	RGIDs, SMnames = [], []
	for _headerline in header.splitlines(): # pasre the header of the bam file to extract the ID and SM fields of each Read Group
		if _headerline[0:3]=='@RG':
			RGID_added, SM_added = False, False
			for _n, _field in enumerate(_headerline.split()):
				if 'ID' in _field:
					if not RGID_added:
						RGIDs.append(''.join(_headerline.split()[_n].split(':')[1:])) # add read group ID
						RGID_added = True
					else:
						raise InputError('Double ID fields detected in @RG header line!')
				elif 'SM' in _field:
					if not SM_added:
						SMnames.append(''.join(_headerline.split()[_n].split(':')[1:])) # add the sample name associated with the read group ID
						SM_added = True
					else:
						raise InputError('Double SM fields detected in @RG header line!')
			if SM_added and RGID_added:
				pass
			elif SM_added:
				raise InputError('ID field missing in @RG header line!')
			elif RGID_added:
				raise InputError('SM field missing in @RG header line!')
			else:
				raise InputError('ID and SM fields missing in @RG header line!')
	if len(RGIDs)!=len(set(RGIDs)):
		raise InputError('Duplicate read group IDs detected in the bam header!')
	GroupedReadsWithID = [[] for _id in RGIDs] # separate reads belonging to each Read Group
	for _read in all_reads:
		try:
			GroupedReadsWithID[RGIDs.index(dict(_read.get_tags())['RG'])].append(_read)
		except ValueError: # this error occurs if a read group in the read tags is not present in the header. This is NOT expected to happen though!
			SMnames.append(SMnames[RGIDs.index(difflib.get_close_matches(dict(_read.get_tags())['RG'],RGIDs)[0])]) # Choose a sample name for the new read group from the header read groups the most close to it
			RGIDs.append(dict(_read.get_tags())['RG'])
			GroupedReadsWithID.append([_read])
	GroupedReads, GroupedSM = [], [] # combine the reads with different RGID but the same SM as they are assumed to belong to the same sample
	for _SMname, _ReadGroup in zip(SMnames, GroupedReadsWithID):
		if _SMname not in GroupedSM:
			GroupedReads.append(_ReadGroup)
			GroupedSM.append(_SMname)
		else:
			GroupedReads[GroupedSM.index(_SMname)]+=_ReadGroup
	del GroupedReadsWithID
	try:
		genolst = getGenotypesPop(vcffile, GroupedSM)
		allelelst = getAllelesPop(vcffile, GroupedSM)
	except IOError:
		raise InputError('The VCF file was not found!')
	except:
		raise
	varpos = list(_x[1] for _x in allelelst)
	for _group, reads in enumerate(GroupedReads):
		out=open('HapCUT_frags_{}.matrix'.format(GroupedSM[_group]),'w')
		BX_reads = []
		Chromium_barcode = []
		for _r in reads:
			try:
				Chromium_barcode.append(_r.get_tag('BX'))
				BX_reads.append(_r)
			except KeyError:
				pass
		garbage = sys.stderr.write('WARNING: {0:d} reads were not considered as they lacked an adjusted chromium barcode!\n'.format(len(reads)-len(BX_reads)))
		reads = BX_reads
		linked_reads = {_barcode:[] for _barcode in Chromium_barcode}
		frag_lst = {_barcode:[] for _barcode in Chromium_barcode}
		q_lst = {_barcode:[] for _barcode in Chromium_barcode}
		HapCUT =  {_barcode:[] for _barcode in Chromium_barcode}
		for _rnum, _barcode in enumerate(Chromium_barcode): # scan through the alignments to find the linked reads)
			read = copy.deepcopy(reads[_rnum])
			if read.is_unmapped or read.is_duplicate or (read.flag & 0x900): # throw away unmapped reads, duplicates and secondary/supplementary alignments
				continue
			if read.mapping_quality >= mmq:
				try:
					adj_seq, adj_qual =  adjust_seq(read)
					coordinates, nucleotides, quals = list(zip(*[(int(_x), _y, _z) for _x, _y, _z in zip(read.positions, adj_seq.upper(), list(ord(_x)-qoffset for _x in adj_qual)) if _z>=mbq]))
				except ValueError as e:
					if e.args[0][0:len("need more than 0 values to unpack")]=="need more than 0 values to unpack" or e.args[0][0:len("not enough values to unpack")]=="not enough values to unpack":
						coordinates, nucleotides, quals= [(), (), ()]
					else:
						raise
				linked_reads[_barcode].append((coordinates, nucleotides, quals))
		for _barcode, _rec in linked_reads.items():
			try:
				_rec = [tuple(zip(*sorted(zip(*_r), key = lambda _x: int(_x[0])))) for _r in _rec if len(_r[0])>0]
				_rec = sorted(_rec, key = lambda x: x[0][0])
			except IndexError as e:
				if "tuple index out of range" in e.args[0]:
					_rec = [(), (), ()]
				else:
					raise
			else: # Take care of overlaps between paired-end reads or between reads in barcoded molecules
				_rn = 0
				while _rn < len(_rec)-1:
					if _rec[_rn][0][-1]<_rec[_rn+1][0][0]:
						_rn+=1
					else:
						coordinates = [_c for _c in _rec[_rn][0]]+[_c for _c in _rec[_rn+1][0]]
						nucleotides = [_n for _n in _rec[_rn][1]]+[_n for _n in _rec[_rn+1][1]]
						quals = [_q for _q in _rec[_rn][2]]+[_q for _q in _rec[_rn+1][2]]
						coordinates, nucleotides, quals = [_t for _t in zip(*sorted(zip(coordinates,nucleotides,quals), key=lambda x: int(x[0])))]
						unique_q = []
						unique_c = []
						unique_n = []
						for _n, _c in enumerate(coordinates): # remove the duplicates from overlapping positions
							try:
								if unique_c[-1]!=_c:
									#print("\nN:\n",nucleotides, "\nQ:\n", quals)
									unique_c.append(_c)
									unique_n.append(nucleotides[_n])
									unique_q.append(quals[_n])
								elif unique_n[-1]==nucleotides[_n]:
									unique_q[-1] = min(126-qoffset, unique_q[-1]+quals[_n])
								else: # if the called nucleotides differ at overlapping sites, use the one with the highest Phred score and adjust the Phred score.
									if quals[_n]>unique_q[-1]:
										_new_q_score = round(-10*log(1-10**(-unique_q[-1]/10)*(1-10**(-quals[_n]/10)), 10), 5) # Q=-10log(p,10)
										if _new_q_score >= mbq:
											unique_n[-1] = nucleotides[_n]
											unique_q[-1] = _new_q_score
										else:
											del(unique_c[-1], unique_n[-1], unique_q[-1])
									else:
										_new_q_score = round(-10*log(1-(1-10**(-unique_q[-1]/10))*10**(-quals[_n]/10), 10), 5)
										if _new_q_score >= mbq:
											unique_q[-1] = _new_q_score
										else:
											del(unique_c[-1], unique_n[-1], unique_q[-1])
							except IndexError:
								unique_c.append(_c)
								unique_n.append(nucleotides[_n])
								unique_q.append(quals[_n])
						_rec[_rn+1] = (tuple(unique_c), tuple(unique_n), tuple(unique_q))
						_rec[_rn] = ((),(),())
						_rn+=1
				_rec = [tuple(zip(*sorted(zip(*_r), key = lambda _x: int(_x[0])))) for _r in _rec if len(_r[0])>0]
				coordinates = [int(_c)+1 for _r in _rec for _c in _r[0]] # Convert the zero-based BAM coordinates to 1-based, as the coordinates are 1-based in the VCF (like the SAM format).
				nucleotides = [_n for _r in _rec for _n in _r[1]]
				quals = [_q for _r in _rec for _q in _r[2]]
				frag_lst[_barcode], q_lst[_barcode] = frag_gen(varpos, allelelst, genolst[_group], coordinates, nucleotides, quals)
				_HapCUT_alleles = [[]]
				_HapCUT_starts = []
				_snp_alleles = frag_lst[_barcode].GetalleleSet(frag_lst[_barcode].GetBegin(), frag_lst[_barcode].GetEnd())
				if len([_x for _x in _snp_alleles if _x!='-'])>1: #keep only SNP-fragments that have a length of at least 2
					for _n, _allele in enumerate(_snp_alleles): # prepare records from the fragment list and the quality list to be written in HapCUT format
						if _allele == '-' and _HapCUT_alleles[-1]!=[]:
							_HapCUT_alleles.append([])
						elif _allele != '-' and _HapCUT_alleles[-1]==[]:
							_HapCUT_alleles[-1].append(_allele)
							_HapCUT_starts.append(str(_n+frag_lst[_barcode].GetBegin()+1))
						elif _allele != '-':
							_HapCUT_alleles[-1].append(_allele)
					_HapCUT_alleles = _HapCUT_alleles[0:len(_HapCUT_starts)] # eliminate empty allele sets
					for _n in range(0, len(_HapCUT_starts)):
						_HapCUT_alleles[_n] = ''.join(_HapCUT_alleles[_n])
					HapCUT[_barcode] = str(len(_HapCUT_starts))+'\t'+'BX::'+_barcode+'\t'+'\t'.join('\t'.join([_start, _alleles]) for _start, _alleles in zip(_HapCUT_starts, _HapCUT_alleles))+'\t'+''.join(chr(int(_q)+qoffset) for _q in q_lst[_barcode].values())
					out.write(HapCUT[_barcode]+os.linesep)
		out.close()

if __name__ == "__main__":

	bamfile=argv[1]
	vcffile=argv[2]
	#get_frags('../20180327/cultivar1_C1R5_BXsorted.bam','../20180327/cultivar1_C1R5_recoded.vcf', mmq=9, mbq=12)
	get_frags(bamfile,vcffile)#, mmq=9, mbq=12)


from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from rest_framework.exceptions import ParseError
from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer
from yaas_app.models import Auction
from yaas_app.serializers import AuctionSerializer, BidSerializer
from yaas_app.views import send_new_bid_email

def json_response(data, status=200):
    content = JSONRenderer().render(data)
    return HttpResponse(content=content,
                        status=status,
                        content_type='application/json')

@csrf_exempt
def rest_browse_auctions(request):
    if request.method != 'GET':
        return HttpResponse(status=404)

    auctions = Auction.all()
    serializer = AuctionSerializer(auctions, many=True)
    return json_response(serializer.data)

@csrf_exempt
def rest_search_auctions(request):
    if request.method != 'GET':
        return HttpResponse(status=404)

    title = request.GET['title']
    auctions = Auction.search(title)
    serializer = AuctionSerializer(auctions, many=True)
    return json_response(serializer.data)

@csrf_exempt
@login_required
def rest_bid_auction(request):
    if request.method != "POST":
        return HttpResponse(status=404)

    try:
        bid = JSONParser().parse(request)
        bid['bidder'] = request.user.id
        bid_serializer = BidSerializer(data=bid)
        if bid_serializer.is_valid():
            new_bid = bid_serializer.save()
            data = bid_serializer.data
            send_new_bid_email(new_bid)
        else:
            data = bid_serializer.errors
    except ParseError as error:
        return HttpResponse(content='{"error":"' + error.detail + '"}',
                            status=200,
                            content_type='application/json')

    return json_response(data)


from django.urls import path
from django.conf.urls import url
from post.views import *

app_name = 'post'

urlpatterns = [
    path('', index, name='index'),
    path('contact/', contact, name='contact'),
    path('about/', about, name='about'),
    path('posts/', posts, name='posts'),
    path('posts/<int:id>/', post_detail, name='post_detail'),
    path('post_create_view/', post_create_view, name = 'post_create_view'),
    path('posts/<int:id>/delete/', post_delete ,name = 'delete'),
    path('posts/<int:id>/update/', post_update, name = 'update'),
]


from random import randint
from random import seed
import networkx as nx
import numpy as np
from matplotlib import pyplot as plt

from prog.utils import graph_directory

def draw_graph(graph, n, p):
    nx.draw_circular(graph)
    plt.savefig(graph_directory + 'watts_strogatz_' + str(n) + '_' + str(p) + '.pdf')
    plt.clf()

def generate_watts_strogatz_graph(n: int, m: int, p: float) -> nx.Graph:
    seed()

    graph = nx.Graph()
    graph.add_nodes_from(range(0, n))

    for node in graph.nodes_iter():
        for i in range(1, m + 1):
            graph.add_edge(node, np.remainder(node - i, n))
            graph.add_edge(node, np.remainder(node + i, n))

    edges = graph.edges()
    for (u, v) in edges:
        if np.random.rand() < p:
            graph.remove_edge(u, v)
            graph.add_edge(v, randint(0, n - 1))

    return graph

def frange_mul(mul: float, n: float):
    p = 0.001
    while p <= n:
        yield p
        p *= mul

def question_a():
    for (n, m, p) in [(15, 2, 0.0), (15, 2, 0.5)]:
        draw_graph(generate_watts_strogatz_graph(n, m, p), n, p)

def question_b():
    N = 1000
    m = 5

    avg_c_zero = nx.average_clustering(generate_watts_strogatz_graph(N, m, 0))
    avg_l_zero = nx.average_shortest_path_length(generate_watts_strogatz_graph(N, m, 0))

    p_values, ravg_c_values, ravg_l_values = [], [], []
    for p in frange_mul(2, 0.512):
        g = generate_watts_strogatz_graph(N, m, p)

        try:
            p_values.append(p)
            ravg_c_values.append(nx.average_clustering(g) / avg_c_zero)
            ravg_l_values.append(nx.average_shortest_path_length(g) / avg_l_zero)

        except Exception as e:
            print(e)
            print("err for p={p}".format(p=p))

    fig = plt.figure()
    fig.clf()
    ax = fig.add_subplot(111)

    ax.semilogx()

    ax.scatter(x=p_values, y=ravg_c_values, color="b", label='relative average clustering')
    ax.scatter(x=p_values, y=ravg_l_values, color="r", label='relative average shortest path')

    ax.set_xlabel("probability of a link")
    ax.set_ylabel("average values")
    ax.set_title('Averages against p')
    ax.legend(loc='best')

    fig.savefig(graph_directory + "fig4.pdf")

def watts_strogatz():
    question_a()
    question_b()

if __name__ == '__main__':
    watts_strogatz()


from .enums import Namespace, Method

from homeassistant.const import STATE_ON, STATE_OFF

class ToggleXMixin(object):
    def togglex_turn_on(self):
        payload_options = {
            'togglex': {
                'channel': 0,
                'onoff': 1
            }
        }
        return self._get_mqtt_payload(Method.SET, Namespace.CONTROL_TOGGLEX, payload_options)

    def togglex_turn_off(self):
        payload_options = {
            'togglex': {
                'channel': 0,
                'onoff': 0
            }
        }
        return self._get_mqtt_payload(Method.SET, Namespace.CONTROL_TOGGLEX, payload_options)

    def togglex_get_state(self, payload):
        if payload.get('all'):
            """ deal with digest payload """
            togglex = payload.get('all').get('digest').get('togglex')
        else:
            togglex = payload.get('togglex')
        return STATE_ON if togglex[0]['onoff'] == 1 else STATE_OFF


#!/usr/bin/python
# -*- coding: utf-8  -*-
"""Common functions not specifically related to batchuploads or wiki."""
from __future__ import unicode_literals
from builtins import dict, open
import json
import os
import operator  # needed by sorted_dict
import sys  # needed by convert_from_commandline()
import locale  # needed by convert_from_commandline()
from pywikibot.tools import deprecated

# avoid having to use from past.builtins import basestring
try:
    basestring  # attempt to evaluate basestring
except NameError:
    def is_str(s):
        return isinstance(s, str)
else:
    def is_str(s):
        return isinstance(s, basestring)

def is_int(value):
    """Check if the given value is an integer.

    @param value: The value to check
    @type value: str, or int
    @return bool
    """
    try:
        int(value)
        return True
    except (ValueError, TypeError):
        return False

def is_pos_int(value):
    """Check if the given value is a positive integer.

    @param value: The value to check
    @type value: str, or int
    @return bool
    """
    if is_int(value) and int(value) > 0:
        return True
    return False

def open_and_read_file(filename, codec='utf-8', as_json=False):
    """
    Open and read a file using the provided codec.

    Automatically closes the file on return.

    @param filename: the file to open
    @param codec: the used encoding (defaults to "utf-8")
    @param json: load as json instead of reading
    """
    with open(filename, 'r', encoding=codec) as f:
        if as_json:
            return json.load(f)
        return f.read()

def open_and_write_file(filename, text, codec='utf-8', as_json=False):
    """
    Open and write to a file using the provided codec.

    Automatically closes the file on return.

    @param filename: the file to open
    @param text: the text to output to the file
    @param codec: the used encoding (defaults to "utf-8")
    @param json: if text is an object which should be dumped as json
    """
    with open(filename, 'w', encoding=codec) as f:
        if as_json:
            f.write(json.dumps(text, indent=4, ensure_ascii=False))
        else:
            f.write(text)

def sorted_dict(ddict):
    """
    Turn a dict into a sorted list.

    @param ddict: dict to sort
    @type ddict: dict
    @return: list of tuples
    """
    sorted_ddict = sorted(ddict.items(),
                          key=operator.itemgetter(1),
                          reverse=True)
    return sorted_ddict

def strip_dict_entries(dict_in):
    """Strip whitespace from all keys and (string) values in a dictionary."""
    dict_out = dict()
    if not isinstance(dict_in, dict):
        raise MyError('strip_dict_entries() expects a dictionary object'
                      'as input but found "%s"' % type(dict_in).__name__)
    for k, v in dict_in.items():
        if is_str(v):
            v = v.strip()
        dict_out[k.strip()] = v
    return dict_out

def strip_list_entries(list_in):
    """Strip whitespace from all entries in a list."""
    list_out = []
    if not isinstance(list_in, list):
        raise MyError('strip_list_entries() expects a list object'
                      'as input but found "%s"' % type(list_in).__name__)
    for l in list_in:
        if is_str(l):
            l = l.strip()
        list_out.append(l)
    return list_out

def trim_list(old_list):
    """Given a list remove any empty entries."""
    # return any empty or None
    if not old_list:
        return old_list

    old_list = strip_list_entries(old_list)
    new_list = []
    for l in old_list:
        if l:
            new_list.append(l)
    return new_list

def listify(value):
    """Turn the given value, which might or might not be a list, into a list.

    @param value: The value to listify
    @type value: any
    @rtype: list, or None
    """
    if value is None:
        return None
    elif isinstance(value, list):
        return value
    else:
        return [value, ]

def deep_sort(obj):
    """Recursively sort list or dict of nested lists."""
    if isinstance(obj, dict):
        _sorted = dict()
        for key in sorted(obj):
            _sorted[key] = deep_sort(obj[key])
    elif isinstance(obj, list):
        new_list = []
        for val in obj:
            new_list.append(deep_sort(val))
        _sorted = sorted(new_list)
    else:
        _sorted = obj
    return _sorted

def add_or_increment(dictionary, val, key=None):
    """
    Add a value to the dictionary or increments the counter for the value.

    @param dictionary: the dictionary to update
    @param val: the value to look for in the dictionary
    @param key: the key holding the counter
    """
    if val not in dictionary:
        if key:
            dictionary[val] = {key: 0}
        else:
            dictionary[val] = 0
    if key:
        dictionary[val][key] += 1
    else:
        dictionary[val] += 1

@deprecated('helpers.get_all_template_entries')
def get_all_template_entries(wikitext, template_name):
    """Return a list of all arguments for instances of a given template."""
    import batchupload.helpers as helpers
    return helpers.get_all_template_entries(wikitext, template_name)

@deprecated('helpers.get_all_template_entries_from_page')
def get_all_template_entries_from_page(page, template_name):
    """Return a list of all arguments for instances of a given template."""
    import batchupload.helpers as helpers
    return helpers.get_all_template_entries_from_page(page, template_name)

def modify_path(base_path, out_path):
    """
    Modify a path by appending it to a base path, if one is given.

    @param base_path: The prefixing path
    @param out_path: The path to modify, prefix
    @return: string, the modified base_path
    """
    if base_path:
        if not os.path.isdir(base_path):
            raise MyError('"%s" is not a directory.' % base_path)
        out_path = os.path.join(base_path, out_path)
    return out_path

def create_dir(out_path):
    """
    Create a directory if it doesn't exist.

    @param out_path: directory to create
    """
    if not out_path:
        raise MyError('Cannot create directory without a name.')
    if not os.path.exists(out_path):
        os.mkdir(out_path)
    elif os.path.isfile(out_path):
        raise MyError('Cannot create the directory "%s" as a file with that '
                      'name already exists.' % out_path)

def to_unicode(text):
    """
    Converts a str to unicode (if python2).

    @param text: text to convert
    @return: unicode string
    """
    try:
        unicode
    except NameError:
        # python 3 so don't worry
        pass
    else:
        if isinstance(text, str):
            text = unicode(text)
    return text

def convert_from_commandline(string):
    """
    Convert a string read from the commandline to a standard unicode format.

    @param string: string to convert
    @return: str
    """
    try:
        unicode
    except NameError:
        # python 3 so don't worry (I think)
        return string
    else:
        return string.decode(sys.stdin.encoding or
                             locale.getpreferredencoding(True))

class MyError(Exception):

    """Home made errors"""

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


import os
from ffmpy import FFmpeg

src='/home/opekta/Downloads/y2mate.com - Furious Red Cards Moments In Football 2021_1080pFHR.mp4'
os.makedirs('/home/opekta/copaeurope/mmaction2/data/soccernet/fromYouTube', exist_ok=True)
os.makedirs('/home/opekta/copaeurope/mmaction2/data/soccernet/fromYouTube/RedCard',exist_ok=True)
finalPath='/home/opekta/copaeurope/mmaction2/data/soccernet/fromYouTube/RedCard/Furious Red Cards Moments In Football 2021_'

nb=0
finalPath+=str(nb)+'.mp4'
minuteStart=7
secondStart=44
start=60*minuteStart+secondStart
minuteEnd=7
secondEnd=47
end=60*minuteEnd+secondEnd

ff = FFmpeg(
    inputs={src: ['-fflags', 'genpts', '-y', '-an', '-ss', str(start),
            '-to', str(end)]},
    outputs={finalPath: ['-avoid_negative_ts', 'make_zero']}
)
ff.run()


#!/usr/bin/env python3
import gzip
import hashlib
import io
import json
import sys
import urllib3

CA_CERTS_LOCATION = '/etc/ssl/certs/ca-certificates.crt'
disable_strict_security = True

def download_cve_file(http, url_root, url_filename):
    r_meta = http.request('GET', '%s%s.meta' % (url_root, url_filename))
    print(r_meta.status)
    meta_lines = list(map(lambda bytes: bytes.decode("utf-8"), r_meta.data.split(b'\r\n')))
    print(meta_lines)
    metas = {}
    for line in meta_lines:
        if line:
            [k, v] = line.split(':', maxsplit=1)
            metas[k] = v
    r_meta.close()
    r_zip = http.request('GET', '%s%s.json.gz' % (url_root, url_filename))
    print(r_zip.status)
    print(r_zip.headers)
    if r_zip.getheader('Content-Length') != metas['gzSize']:
        print('bad gzip file length')
    else:
        print('good gzip file length')

    data = gzip.GzipFile(fileobj=io.BytesIO(r_zip.data)).read()
    r_zip.close()

    o = json.loads(data)
    print(json.dumps(o, sort_keys=True, indent=4))

    # calc hash of file
    sha256_hash = hashlib.sha256()
    sha256_hash.update(data)
    ze_hash = sha256_hash.hexdigest().upper()
    # print('Calc:', ze_hash)
    if metas['sha256'] != ze_hash:
        print('bad json file hash')
    else:
        print('good json file hash')

    return o

def main():
    urllib3.util.make_headers(accept_encoding=True)
    # temporary to disable until the HTTPS cert authority issue can be figued out
    if disable_strict_security:
        urllib3.disable_warnings()
        http = urllib3.PoolManager(
            cert_reqs='CERT_NONE',
            assert_hostname=False)
    else:
        http = urllib3.PoolManager(
            cert_reqs='CERT_REQUIRED',
            ca_certs=CA_CERTS_LOCATION)

    download_cve_file(http, 'https://nvd.nist.gov/feeds/json/cve/1.1/', 'nvdcve-1.1-modified')

if __name__ == '__main__':
    main()


from collections import defaultdict
from helpers import round_array
from ..config import Config

class RunInfo:
    """
    Stores metrics for the runs.
    """

    def __init__(self, run_id, seed):
        self.run_id = run_id
        self.seed = seed
        self.elapsed_time = None
        self.best_team = None
        self.teams_in_last_generation = []
        self.pareto_front_in_last_generation = []
        self.train_score_per_validation = []
        self.test_score_per_validation = []
        self.recall_per_validation = []
        self.actions_distribution_per_validation = []
        self.inputs_distribution_per_instruction_per_validation = []
        self.inputs_distribution_per_team_per_validation = []
        self.global_diversity_per_validation = defaultdict(list)
        self.global_mean_fitness_score_per_validation = []
        self.global_max_fitness_score_per_validation = []
        self.global_mean_validation_score_per_validation = []
        self.global_max_validation_score_per_validation = []
        self.info_per_team_per_generation = []
        self.global_mean_fitness_per_generation = []
        self.global_max_fitness_per_generation = []
        self.global_fitness_per_diversity_per_generation = defaultdict(list)
        self.global_diversity_per_generation = defaultdict(list)
        self.novelty_type_per_generation = []
        self.mean_team_size_per_validation = []
        self.mean_program_size_with_introns_per_validation = []
        self.mean_program_size_without_introns_per_validation = []

    def __str__(self):
        msg = "RUN "+str(self.run_id)+"\n"
        msg += "seed: "+str(self.seed)

        msg += "\n\n\n##### BEST TEAM METRICS PER VALIDATION"
        msg += "\n\nBest Team Fitness per Validation: "+str(round_array(self.train_score_per_validation))
        msg += "\n\nBest Team Validation Score per Validation (champion): "+str(round_array(self.test_score_per_validation))
        msg += "\n\nBest Team Recall per Action per Validation: "+str(self.recall_per_validation)

        msg += "\n\n\n##### GLOBAL METRICS PER VALIDATION"
        msg += "\n\nGlobal Mean Fitness Score per Validation: "+str(self.global_mean_fitness_score_per_validation)
        msg += "\n\nGlobal Max. Fitness Score per Validation: "+str(self.global_max_fitness_score_per_validation)
        msg += "\n\nGlobal Mean Validation Score per Validation: "+str(self.global_mean_validation_score_per_validation)
        msg += "\n\nGlobal Max. Validation Score per Validation: "+str(self.global_max_validation_score_per_validation)
        if len(Config.RESTRICTIONS['used_diversities']) > 0:
            msg += "\n\nGlobal Diversities per Validation"
            for key in self.global_diversity_per_validation:
                msg += "\n"+str(key)+": "+str(self.global_diversity_per_validation[key])

        msg += "\n\n\n##### DISTRIBUTION METRICS PER VALIDATION"
        msg += "\n\nDistribution of Actions per Validation: "+str(self.actions_distribution_per_validation)
        msg += "\n\nDistribution of Inputs per Validation (per instruction): "+str(self.inputs_distribution_per_instruction_per_validation)
        msg += "\n\nDistribution of Inputs per Validation (per team): "+str(self.inputs_distribution_per_team_per_validation)

        msg += "\n\n\n##### METRICS FOR SIZES PER VALIDATION"
        msg += "\n\nMean Team Sizes: "+str(self.mean_team_size_per_validation)
        msg += "\n\nMean Program Sizes (with introns): "+str(self.mean_program_size_with_introns_per_validation)
        msg += "\n\nMean Program Sizes (without introns): "+str(self.mean_program_size_without_introns_per_validation)

        msg += "\n\n\n##### GLOBAL METRICS PER TRAINING"
        msg += "\n\nGlobal Mean Fitness Score per Training: "+str(self.global_mean_fitness_per_generation)
        msg += "\n\nGlobal Max. Fitness Score per Training: "+str(self.global_max_fitness_per_generation)
        msg += "\n\nGlobal Fitness Score per Training (per diversity):"
        if len(Config.RESTRICTIONS['used_diversities']) > 1:
            for key in self.global_fitness_per_diversity_per_generation:
                msg += "\n"+str(key)+": "+str(self.global_fitness_per_diversity_per_generation[key])
        if len(Config.RESTRICTIONS['used_diversities']) > 0:
            msg += "\n\nGlobal Diversities per Training"
            for key in self.global_diversity_per_generation:
                msg += "\n"+str(key)+": "+str(self.global_diversity_per_generation[key])
            if len(Config.RESTRICTIONS['used_diversities']) > 1:
                msg += "\n\nDiversity Type per Training: "+str(self.novelty_type_per_generation)

        return msg


"""
Experiments on the Robust DE problems
"""

import numpy as np
import pandas as pd
from test_suite.robust_de_problems import RobustDEProblem
from methods.reckless import Reckless
from utils.latex_tbl import df_2_tex
from utils.plot_curves import plot_curves
import json
import os

RESULTS_DIR = os.path.join(os.path.dirname(__file__), 'results')

def run_one_fun(fun_num):
    if not os.path.exists(RESULTS_DIR):
        os.mkdir(RESULTS_DIR)

    print("Running ES experiments for fun_num:{}".format(fun_num))
    np.random.seed(0)
    num_runs = 60
    num_fevals = np.logspace(2, 5, 4)

    funs_dim = [3] * 2 + [1] * 2 + [2] * 2

    alg_variants = {
        "p=0.5": {
            "minimizer": "NES",
            "is_restart": False,
            "p": 0.5,
            "minimizer_opts": {
                # "is_restart": False,
                "is_mirror": False
            }
        },
        "p=0.4": {
            "minimizer": "NES",
            "is_restart": False,
            "p": 0.4,
            "minimizer_opts": {
                # "is_restart": False,
                "is_mirror": False
            }
        },
        "p=0.3": {
            "minimizer": "NES",
            "is_restart": False,
            "p": 0.3,
            "minimizer_opts": {
                # "is_restart": False,
                "is_mirror": False
            }
        },
        "p=0.2": {
            "minimizer": "NES",
            "is_restart": False,
            "p": 0.2,
            "minimizer_opts": {
                # "is_restart": False,
                "is_mirror": False
            }
        },
        "p=0.1": {
            "minimizer": "NES",
            "is_restart": False,
            "p": 0.1,
            "minimizer_opts": {
                # "is_restart": False,
                "is_mirror": False
            }
        }
    }

    regret_curves = {
        'metadata': {
            'xlabel': 'FEs',
            'ylabel': '$r(\mathbf{x}_*)$',
            'title': '',
            'filepath': None,  # 'plot.pdf'
            'plt_type': ''
        },
        'data': []
    }

    regret_file = os.path.join(RESULTS_DIR, "p_regret_curves_%d" % fun_num)
    regret_curves['data'] = []
    regret_curves['metadata']['filepath'] = regret_file + ".pdf"
    regret_curves['metadata']['title'] = '$\mathcal{L}_%d$' % fun_num
    for alg_variant, alg_opts in alg_variants.items():
        xs = []
        regret_fevals = []
        regret_errs_fevals = []
        regrets = []
        dim = funs_dim[fun_num - 1]
        for max_fevals in num_fevals:
            regret_runs = []
            for run in range(num_runs):
                np.random.seed(run)
                prob = RobustDEProblem(D_x=dim, D_y=dim, fun_num=fun_num)
                x_opt, y_opt, _ = Reckless(prob.evaluate, D_x=dim, D_y=dim, max_fevals=int(max_fevals), seed=run,
                                           minimizer=alg_opts["minimizer"],
                                           minimizer_opts=alg_opts["minimizer_opts"],
                                           is_restart=alg_opts["is_restart"],
                                           p=alg_opts["p"]).run()

                if run == 0:
                    xs.append(prob.get_num_fevals())
                else:
                    xs[-1] = max(xs[-1], prob.get_num_fevals())

                regret = prob.regret(x_opt, y_opt)

                regret_runs.append(regret)

            regret_fevals.append(np.mean(regret_runs))
            regret_errs_fevals.append(np.std(regret_runs))
            regrets.append(regret_runs)

        regret_curves['data'].append(
            {
                'name': alg_variant,
                'ys': regrets,
                'm_ys': regret_fevals,
                'std_ys': regret_errs_fevals,
                'xs': xs
            })

    assert len(regret_curves['data']) == len(alg_variants)

    # plot_curves(regret_curves)

    with open(regret_file + ".json", "w") as f:
        json.dump(regret_curves, f)

def main():
    from multiprocessing import Pool
    p = Pool(8)
    fun_nums = [1, 2, 3, 4, 5, 6]
    # for fun_num in fun_nums:
    #     run_one_fun(fun_num)
    p.map(run_one_fun, fun_nums)

if __name__ == "__main__":
    main()


import numpy as np
import cv2
import os
import random
import pickle
dir_dataset=r'C:\Users\Soumalya\Desktop\ML_H_v_H\dataset'
CATEGORIES=['horse','human']

img_size=100

data=[]

for catagory in CATEGORIES:
	folder=os.path.join(dir_dataset,catagory)
	label=CATEGORIES.index(catagory)
	for img in os.listdir(folder):
		img_path=os.path.join(folder,img)
		img_arr=cv2.imread(img_path)
		img_arr=cv2.resize(img_arr,(img_size,img_size))
		data.append([img_arr,label])
random.shuffle(data)
X=[]
y=[]
for features,label in data:
	X.append(features)
	y.append(label)
X=np.array(X)
y=np.array(y)
pickle.dump(X, open('X.pkl', 'wb'))
pickle.dump(y, open('y.pkl', 'wb'))


# -*- coding: utf-8 -*-
"""
Created on Tue May 22 12:19:49 2018

Script for boundary treatment accuracy test - Smooth case.

@author: Sven
"""

import csv
import matplotlib.pyplot as plt
import numpy as np
import os
from numerical_helpers import convergence_rate
from uniform_CWENO_1D import CWENO5_1D

logging = False

H = 0.25*np.power(2.,range(-1,-14,-1))   # cell sizes

errors = [[],[],[],[],[]]

# smooth function u(x) = sin(2*pi*x)
def smooth_cell_averages(x,h):
    return (np.cos(2*np.pi*(x-h/2))-np.cos(2*np.pi*(x+h/2)))/(2*np.pi*h)

# parameter sets
q = [2,1,2,1,1]
p = [3,4,3,3,4]
gamma_1 = [4,3,3,3,3]
gamma_2 = [2,1,1,0,2]

for j in range(len(q)):
  # reconstruction for different cell sizes
  step_size_error = [[],[]]
  for h in H:
      X = np.linspace(0, 4*h, 5)
      u_avg = smooth_cell_averages(X,h)
      # parameters of reconstruction
      if gamma_2[j]==0:
          params = ((0.5-h**gamma_1[j], h**gamma_1[j], 0.25, 0.25),
                    (0.75-h**gamma_1[j], h**gamma_1[j], 0.25))
      else:
          params = ((1-h**gamma_1[j]-2*h**gamma_2[j], h**gamma_1[j], h**gamma_2[j], h**gamma_2[j]),
                    (1-h**gamma_1[j]-h**gamma_2[j], h**gamma_1[j], h**gamma_2[j]))

      operator = CWENO5_1D(u_avg, h, h**q[j], p[j], 0.5, params)
      rec_coeffs = operator.compute_reconstruction_polynomial()

      # reconstruction errors from smooth cell averages
      step_size_error[0].append(np.abs(np.dot(rec_coeffs[0,:],np.power(-5/2*h,range(5)))
                                        - np.sin(2*np.pi*(X[0]-h/2))))
      step_size_error[1].append(np.abs(np.dot(rec_coeffs[1,:],np.power(-5/2*h,range(5)))
                                        - np.sin(2*np.pi*(X[0]-h/2))))

  errors[j] = step_size_error

convergence_rates = [[],[],[],[],[]]
for i in range(len(q)):
    rates = [convergence_rate(value, H) for value in errors[i]]
    convergence_rates[i] = rates

if logging:
    path = 'C:\\Users\\Sven\\Documents\\Studium\\Master\\Masterarbeit_Seminar\\Masterarbeit\\log\\'
    with open(os.path.join(path,'testData_CwenoFiveBoundariesSmooth.csv'),
              'w', newline='') as csvfile:
        wr = csv.writer(csvfile,delimiter='&',lineterminator='\\\ \n',
                        quoting=csv.QUOTE_NONE,escapechar=' ')
        for i in range(len(errors[0])):
            wr.writerow('----- {} -----'.format(i))
            for j in range(len(errors[0][0])):
                wr.writerow(['%i &%.2e &%.1f &&%.2e &%.1f &&%.2e &%.1f &&%.2e &%.1f &&%.2e &%.1f ' %
                    (j+1,errors[0][i][j],convergence_rates[0][i][j],
                     errors[1][i][j],convergence_rates[1][i][j],
                     errors[2][i][j],convergence_rates[2][i][j],
                     errors[3][i][j],convergence_rates[3][i][j],
                     errors[4][i][j],convergence_rates[4][i][j])])

fig, axs = plt.subplots(2, 3)
fig.set_size_inches(13,9)
axs = axs.flatten()
for set_idx in range(len(q)):
    plt.subplot(axs[set_idx])
    N = 1./H
    axs[set_idx].set_xlabel(xlabel='N',labelpad=-0.5)
    plt.ylabel('|$u(x)-P_{rec}(x)$|')
    plt.title('$\gamma_1={}$, $\gamma_2={}$, $p={}$, $q={}$'.format(
        gamma_1[set_idx],gamma_2[set_idx],p[set_idx],q[set_idx]))
    slope_one = plt.loglog(N[4:10],np.exp(-5*np.log(N[4:10])+10),color='gray',linestyle='--')
    plt.loglog(N,errors[set_idx][0],color='red',marker='*',label='$P_{rec}^{0.5}$')
    plt.loglog(N,errors[set_idx][1],color='blue',marker='o',label='$P_{rec}^{1.5}$')

    props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
leg = plt.legend(title='Error in $x_{0}$',loc=9, bbox_to_anchor=(-0.1,2.5), ncol=2)
leg_lines = leg.get_lines()
leg_texts = leg.get_texts()
plt.setp(leg_lines, linewidth=2)
plt.show()


import os
from os.path import join, getsize
import re
import nltk.data

text_dir = 'data/multi-document/DUC 2003/text'
def rewriteDocsIntoSingleHTMLFreeTex():
    for root, dirs, files in os.walk(text_dir):
        if dirs != []:
            for dir in dirs:
                print(dir)
                curr_dir = text_dir + '/' + dir
                new_file = join(curr_dir, 'merge.txt')
                f = open(new_file,"w+")
                for sub_root, sub_dirs, sub_files in os.walk(curr_dir):
                    for file in sub_files:
                        if file != 'merge.txt':
                            print(file)
                            with open(join(curr_dir, file), 'r') as reading_file:
                                text = reading_file.read()
                                result = re.search('<TEXT>([\s\S]*)<\/TEXT>', text)
                                if result is None:
                                    print(text)
                                text = result.group(1)
                                TAG_RE = re.compile(r'<[^>]+>')
                                text = TAG_RE.sub('', text)
                                f.write("%s" % text)
                f.close()

def splitTextIntoSentences():
    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    for root, dirs, files in os.walk(text_dir):
        if dirs != []:
            for dir in dirs:
                print(dir)
                curr_dir = text_dir + '/' + dir
                summaryFile = join(curr_dir, 'merge.txt')
                fp = open(summaryFile,"r")
                data = fp.read()
                TAG_RE = re.compile(r'\n')
                data = TAG_RE.sub(' ', data)
                sentences = tokenizer.tokenize(data)
                fp.close()
                summaryFile2 = join(curr_dir, 'merge.txt')
                f = open(summaryFile2,"w+")
                for sentence in sentences:
                    f.write("%s\n" % sentence)
                f.close()

rewriteDocsIntoSingleHTMLFreeTex()
splitTextIntoSentences()


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
u"""
Created by ygidtu@gmail.com at 2018.12.16

Main function to plot sashimi plot
"""
from multiprocessing import cpu_count

import click
from conf.plot_settings import parse_settings
from ioutils.reading_input import (index_gtf, read_reads_depth_from_bam,
                                   read_reads_depth_from_count_table,
                                   read_transcripts)
from ioutils.utils import *

from plot.sashimi_plot_utils import draw_sashimi_plot

__dir__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

def darw_without_bam(
    bam,
    count_table,
    splice_region,
    colors,
    color_factor,
    threshold,
    output,
    sashimi_plot_settings,
    no_gene,
    dpi,
    distance_ratio,
    title
):
    color_index = 0
    tmp_color = {}
    required_cols = {}
    if bam:
        with open(bam) as r:
            for line in r:
                lines = re.split(r"\t| {2,}", line.strip())

                if not lines:
                    continue

                if len(lines) > 1:
                    required_cols[lines[0]] = lines[1]
                else:
                    required_cols[lines[0]] = clean_star_filename(lines[0])

                if lines[color_factor - 1] not in tmp_color.keys():
                    tmp_color[required_cols[lines[0]]
                              ] = colors[color_index % len(colors)]
                    color_index += 1

    reads_depth = read_reads_depth_from_count_table(
        count_table=count_table,
        splice_region=splice_region,
        required=colors,
        threshold=threshold,
        colors=colors
    )

    draw_sashimi_plot(
        output_file_path=output,
        settings=sashimi_plot_settings,
        average_depths_dict=reads_depth,
        splice_region=splice_region,
        no_bam=True,
        show_gene=not no_gene,
        dpi=dpi,
        distance_ratio=distance_ratio,
        title=title,
    )

def draw_default(
    bam, splice_region,
    color_factor, colors,
    share_y, share_y_by,
    barcode,
    sort_by_color,
    threshold, threshold_of_reads,
    log, process, reads,
    barcode_tag, save_depth,
    output,
    customized_junction,
    sashimi_plot_settings,
    no_gene, dpi,
    distance_ratio,
    title, show_side,
    stack, side_strand,
):
    bam_list, shared_y = prepare_bam_list(
        bam, color_factor, colors, share_y_by, barcodes=barcode)

    if sort_by_color:
        bam_list = sorted(bam_list, key=lambda x: x.color)

    reads_depth = read_reads_depth_from_bam(
        bam_list=bam_list,
        splice_region=splice_region.copy(),
        threshold=threshold,
        threshold_of_reads=threshold_of_reads,
        log=log,
        n_jobs=process,
        reads=reads,
        barcode_tag=barcode_tag
    )

    if save_depth:
        with open(output, "w+") as w:
            for key, value in reads_depth.items():
                for val in value:
                    w.write("{},{}\n".format(key.to_csv(), val))
        exit(0)

    # set shared y
    if share_y:
        assign_max_y(shared_y.values(), reads_depth)

    # read customized junctions
    if customized_junction and os.path.exists(customized_junction):
        customized_junction = read_reads_depth_from_count_table(
            customized_junction,
            splice_region=splice_region,
            required=None,
            colors=colors
        )

        temp_customized_junction = {
            k.alias: v for k, v in customized_junction.items()}

        for key, value in reads_depth.items():
            temp = temp_customized_junction.get(
                key.alias,
                customized_junction.get(os.path.basename(key.path), None)
            )

            if temp:
                value.add_customized_junctions(temp)

    draw_sashimi_plot(
        output_file_path=output,
        settings=sashimi_plot_settings,
        average_depths_dict=reads_depth,
        splice_region=splice_region.copy(),
        no_bam=False,
        show_gene=not no_gene,
        dpi=dpi,
        logtrans=log,
        distance_ratio=distance_ratio,
        title=title,
        stack=stack,
        show_side=show_side,
        side_strand_choice={"+": "plus", "-": "minus"}.get(side_strand)
    )

@click.command()
@click.option(
    "-e",
    "--event",
    type=click.STRING,
    required=True,
    help="Event range eg: chr1:100-200:+"
)
@click.option(
    "-b",
    "--bam",
    type=click.Path(exists=True),
    required=True,
    help="""
    Path to input BAM file. \b

    Or a tab separated text file, \b
    - first column is path to BAM file, \b
    - second column is BAM file alias(optional) \b
    \b
    Path to tab separated list file\b
    1. the column use to plot sashimi, identical with count table column names \b
    2. optional, the alias of 1st column \b
    3. additional columns \b
    """
)
@click.option(
    "-c",
    "--count-table",
    type=click.Path(),
    help="""
    Path to input count table file. \b
    To make sashimi plot without bam
    """
)
@click.option(
    "-g",
    "--gtf",
    type=click.Path(exists=True),
    help="Path to gtf file, both transcript and exon tags are necessary"
)
@click.option(
    "-o",
    "--output",
    type=click.Path(),
    help="Path to output graph file",
    show_default=True
)
@click.option(
    "--config",
    default=os.path.join(__dir__, "settings.ini"),
    type=click.Path(),
    help="Path to config file, contains graph settings of sashimi plot",
    show_default=True
)
@click.option(
    "-t",
    "--threshold",
    default=0,
    type=click.IntRange(min=0, clamp=True),
    help="Threshold to filter low abundance junctions",
    show_default=True
)
@click.option(
    "-T",
    "--threshold-of-reads",
    default=0,
    type=click.IntRange(min=0, clamp=True),
    help="Threshold to filter low abundance reads for stacked plot",
    show_default=True
)
@click.option(
    "-d",
    "--dpi",
    default=300,
    type=click.IntRange(min=1, clamp=True),
    help="The resolution of output file",
    show_default=True
)
@click.option(
    "--indicator-lines",
    default=None,
    type=click.STRING,
    help="""
    Where to plot additional indicator lines, comma separated int, sites occured multiple times will highlight in red \b
    Or \b
    Path to file contains indicator lines, \b
    1st column is the line site
    2nd column is transcript id
    3rd column is the weights
    """
)
@click.option(
    "--share-y",
    default=False,
    is_flag=True,
    type=click.BOOL,
    help="""
    Whether different sashimi plots shared same y axis
    """
)
@click.option(
    "--no-gene",
    is_flag=True,
    type=click.BOOL,
    help="Do not show gene id next to transcript id"
)
@click.option(
    "--color-factor",
    type=str,
    help="""
    The index of specific column in --bam or path to color settings, 2 columns are required, first if key of bam or cell group, second column is color
    """,
    show_default=True
)
@click.option(
    '--log',
    type=click.Choice(["0", "2", "10", "zscore"]),
    default="0",
    help="y axis log transformed, 0 -> not log transform; 2 -> log2; 10 -> log10"
)
@click.option(
    "--customized-junction",
    type=click.STRING,
    default=None,
    help="""
    Path to junction table column name needs to be bam name or bam alias. \b
    """
)
@click.option(
    "-p",
    "--process",
    type=click.IntRange(min=1, max=cpu_count()),
    default=1,
    help="""
    How many cpu to use \b
    """
)
@click.option(
    "-f",
    "--genome",
    type=click.Path(),
    default=None,
    help="""
    Path to genome fasta \b
    """
)
@click.option(
    "--sort-by-color",
    is_flag=True,
    type=click.BOOL,
    help="""
    Whether sort input bam order, for better looking \b
    """
)
@click.option(
    "--stack",
    default=False,
    is_flag=True,
    type=click.BOOL,
    help="Whether to draw stacked reads"
)
@click.option(
    "--share-y-by",
    type=click.INT,
    default=-1,
    help="""
    Index of column with share y axis (1-based), Need --share-y\.
    For example, first 3 bam files use same y axis, and the rest use another
    """,
    show_default=True
)
@click.option(
    "--remove-empty-gene",
    is_flag=True,
    type=click.BOOL,
    help="""
    Whether to plot empty transcript \b
    """
)
@click.option(
    "--distance-ratio",
    type=click.FLOAT,
    default=0.3,
    help="distance between transcript label and transcript line",
    show_default=True
)
@click.option(
    "--title",
    type=click.STRING,
    default=None,
    help="Title",
    show_default=True
)
@click.option(
    "--save-depth",
    is_flag=True,
    type=click.BOOL,
    help="""
    Whether to save reads depth to file,
    the last 3 columns are chrom, position and depth,
    The same pos will repeated multiple times for joyplot in R
     \b
    """
)
@click.option(
    "--barcode",
    type=click.Path(),
    help="""
    Path to barcode list file,
    At list  three columns were required,
    1st The name of bam file; 2nd the barcode;
    3rd The group label
     \b
    """
)
@click.option(
    "--barcode-tag",
    type=click.STRING,
    default="CB",
    help="""
    The default cell barcode tag label
     \b
    """
)
@click.option(
    "--reads",
    type=click.Choice(["All", "R1", "R2"]),
    default="All",
    help="""
    Whether filter R1 or R2
     \b
    """
)
@click.option(
    "--show-side",
    is_flag=True,
    type=click.BOOL,
    help="""
    Whether to draw additional side plot,
     \b
    """
)
@click.option(
    "--side-strand",
    type=click.Choice(["All", "+", "-"]),
    default="All",
    help="""
    which strand kept for side plot, default use all
     \b
    """
)
@click.option(
    "--show-id",
    is_flag=True,
    help="""
    which show gene id or gene name
    \b
    """
)
def plot(
        bam: str, event: str, gtf: str, output: str,
        config: str, threshold: int, indicator_lines: str,
        share_y: bool, no_gene: bool, color_factor: str,
        dpi: int, log: str, customized_junction: str,
        process: int, sort_by_color: bool, share_y_by: int,
        remove_empty_gene: bool, distance_ratio: float,
        title: str, genome: str, save_depth: str, stack: bool,
        threshold_of_reads: int, barcode: str, barcode_tag: str,
        reads: str, show_side: bool, side_strand: str, count_table: str,
        show_id: bool = False
):
    u"""
    This function is used to plot single sashimi plotting
    \f
    Created by ygidtu@gmail.com at 2018.12.19
    :param ctx: passed parameters from main
    :param bam: list of input BAM files
    :param event: event id, chr:100-200-100-200:+ etc
    :param bam: list of input BAM files
    :param gtf: path to gtf file
    :param output: path to output file
    :param event: event id, chr:100-200-100-200:+ etc
    :param config: path to config file, default using settings.ini file under this suite of scripts
    :param threshold: filter out low abundance junctions
    :param indicator_lines: draw vertical lines in sashimi to show the spliced sites
    :param share_y: make different plots use same y axis boundary
    :param no_gene: do not show gene id
    :param color_factor: 1-based index, only work with bam list
    :param dpi: output file resolution
    :param log: whether to perform y axis log transform
    :param customized_junction: add customized junction to plot
    :param process:
    :param sort_by_color:
    :param share_y_by:
    :param remove_empty_gene:
    :param distance_ratio:
    :param title
    :return:
    """
    try:
        log = int(log)
    except ValueError:
        pass

    out_dir = os.path.dirname(os.path.abspath(output))

    # check reads to kept
    reads = None if reads == "All" else reads == "R1"

    try:
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
    except IOError as err:
        print(err)
        print("Create output directory failed, please check %s" % out_dir)
        exit(err)

    sashimi_plot_settings = parse_settings(config)

    colors = sashimi_plot_settings["colors"]

    splice_region = get_sites_from_splice_id(
        event, indicator_lines=indicator_lines)

    splice_region = read_transcripts(
        gtf_file=index_gtf(input_gtf=gtf),
        genome=genome,
        region=splice_region,
        show_id=show_id
    )

    if remove_empty_gene:
        splice_region.remove_empty_transcripts()

    if count_table:
        darw_without_bam(
            bam,
            count_table,
            splice_region,
            colors,
            color_factor,
            threshold,
            output,
            sashimi_plot_settings,
            no_gene,
            dpi,
            distance_ratio,
            title
        )
    else:
        draw_default(
            bam, splice_region,
            color_factor, colors,
            share_y, share_y_by,
            barcode,
            sort_by_color,
            threshold, threshold_of_reads,
            log, process, reads,
            barcode_tag, save_depth,
            output,
            customized_junction,
            sashimi_plot_settings,
            no_gene, dpi,
            distance_ratio,
            title, show_side,
            stack, side_strand
        )

if __name__ == '__main__':
    pass


import csv
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np

data_folder = 'C:\\Users\\fahad\\Downloads\\object-detection-crowdai.tar\\object-detection-crowdai'
output_folder = 'D:\\carnd\\CarND-Vehicle-Detection\\data\\crowd_ai'

with open('{}/labels.csv'.format(data_folder), 'r') as f:
    reader = csv.reader(f)
    next(reader, None)

    index = 1
    for row in reader:
        output_file = '{}/{}-{}.jpeg'.format(output_folder, row[5], index)
        #print(row)
        filename = row[4]

        img = mpimg.imread('{}/{}'.format(data_folder, filename))

        x = [int(row[0]), int(row[2])]
        y = [int(row[1]), int(row[3])]

        if x[0] < x[1]:
            x1, x2 = x[0], x[1]
        else:
            x2, x1 = x[0], x[1]

        if y[0] < y[1]:
            y1, y2 = y[0], y[1]
        else:
            y2, y1 = y[0], y[1]

        if y2-y1 > 25 and x2-x1 > 25:
            img_car = img[y1:y2,x1:x2]
            img_car_64 = cv2.resize(img_car, (64,64))

            img_car_64bgr = cv2.cvtColor(img_car_64, cv2.COLOR_RGB2BGR)
            print(output_file)
            cv2.imwrite(output_file, img_car_64bgr)
        else:
            print('Discarding...small size')

        index += 1

        #cv2.rectangle(img, (x1,y1), (x2,y2), (255, 0, 0), 6)

        #f, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5))
        #ax1.imshow(img)
        #ax1.set_title('Row #: {}'.format(index))
        #ax2.imshow(img_car_64)
        #ax2.set_title('{},{},{},{}'.format(x1,y1, x2, y2))


import random
import string
import datetime
from random import choice
from string import ascii_lowercase, digits

from django import forms
from django.contrib.auth import get_user_model
from django.contrib.auth.forms import (
    AuthenticationForm, UserCreationForm, PasswordChangeForm,
    PasswordResetForm, SetPasswordForm
)
from django.utils import timezone

from .models import Salon, Stylist, Reservation

User = get_user_model()

class EmailChangeForm(forms.ModelForm):
    """メールアドレス変更フォーム"""

    class Meta:
        model = User
        fields = ('email',)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

    def clean_email(self):
        email = self.cleaned_data['email']
        User.objects.filter(email=email, is_active=False).delete()
        return email

class LoginForm(AuthenticationForm):
    """ログインフォーム"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'
            field.widget.attrs['placeholder'] = field.label  # placeholderにフィールドのラベルを入れる

class UserCreateForm(UserCreationForm):
    """ユーザー登録用フォーム"""

    class Meta:
        model = User
        fields = ('user_name', 'email', 'first_name', 'last_name', 'select_plan',)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # self.fields['username'].widget.attrs['class'] = 'form-control'
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

        # randusername = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(6)])
        # randusername = generate_random_username
        # print(randusername)
        # self.fields['randusername'].queryset = randusername

    def clean_email(self):
        email = self.cleaned_data['email']
        User.objects.filter(email=email, is_active=False).delete()
        return email

# ********* ランダムなusernameを生成するメソッド **************

# def generate_random_username(length=6, chars=ascii_lowercase + digits, split=4, delimiter='-'):
#     username = ''.join([choice(chars) for i in range(length)])
#
#     if split:
#         username = delimiter.join([username[start:start + split] for start in range(0, len(username), split)])
#         print(username)
#
#     try:
#         print(username)
#         User.objects.get(username=username)
#         return generate_random_username(length=length, chars=chars, split=split, delimiter=delimiter)
#     except User.DoesNotExist:
#         return username

# ******************************************************

class UserUpdateForm(forms.ModelForm):
    """ユーザー情報更新フォーム"""

    class Meta:
        model = User
        fields = ('last_name', 'first_name',)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

class MyPasswordChangeForm(PasswordChangeForm):
    """パスワード変更フォーム"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

class MyPasswordResetForm(PasswordResetForm):
    """パスワード忘れたときのフォーム"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

class MySetPasswordForm(SetPasswordForm):
    """パスワード再設定用フォーム(パスワード忘れて再設定)"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

class SalonUpdateForm(forms.ModelForm):
    class Meta:
        model = Salon
        exclude = ('salon', 'cut_prise', 'color_prise', 'mainte_cut_prise', 'mainte_color_prise',)

class StylistCreateForm(forms.ModelForm):
    class Meta:
        model = Stylist
        fields = ('stylist_name',)

# class ReservationDateTimeForm(forms.ModelForm):
#     class Meta:
#         model = Reservation
#         fields = ('date', 'start', 'end')
#         widgets = {
#             'date': forms.SelectDateWidget,
#             'start': forms.TimeInput(format='%H:%M'),
#             'end': forms.TimeInput(format='%H:%M'),
#         }

class ReservationCreateForm(forms.ModelForm):
    class Meta:
        model = Reservation
        fields = '__all__'
        exclude = ('status',)
        widgets = {
            'member': forms.TextInput(attrs={'readonly': 'readonly'}),
            'plan': forms.Select(attrs={'readonly': 'readonly'}),
            'date': forms.TextInput(attrs={'readonly': 'readonly'}),
            'start': forms.TextInput(attrs={'readonly': 'readonly'}),
            'end': forms.TextInput(attrs={'readonly': 'readonly'}),
        }

    def __init__(self, *args, **kwargs):
        self.salon = kwargs.pop('salon_pk')
        self.member = kwargs.pop('user_name')

        reserve_date = kwargs.pop('reserve_date')
        reserve = datetime.datetime.strptime(reserve_date, '%Y-%m-%d')
        date = datetime.datetime.date(reserve)
        self.date = date

        self.start = kwargs.pop('start')
        self.end = kwargs.pop('end')
        super(ReservationCreateForm, self).__init__(*args, **kwargs)

        self.fields['member'].initial = self.member
        self.fields['member'].required = True
        # ログインサロンの初期値を設定
        salon = Salon.objects.get(id=self.salon)
        self.fields['salon'].choices = ((salon.pk, salon.salon_name),)
        self.fields['stylist'].queryset = Stylist.objects.filter(affiliation_salon=self.salon)

        # ユーザー選択プランの初期値設定
        plan = User.objects.filter(user_name=self.member).values('select_plan')
        if plan[0]['select_plan'] == 0:
            select_plan = (('0', 'カットし放題'),)
        elif plan[0]['select_plan'] == 1:
            select_plan = (('1', 'カラーし放題'),)
        elif plan[0]['select_plan'] == 2:
            select_plan = (('2', 'カット＆カラーし放題'),)
        else:
            select_plan = (('9', '利用不可'),)

        self.fields['plan'].choices = select_plan
        self.fields['date'].initial = self.date
        self.fields['start'].initial = self.start
        self.fields['end'].initial = self.end

        # ユーザーの利用可能施術を設定
        full_cut = (('0', 'フルカット'),)
        full_color = (('1', 'フルカラー'),)
        mainte_cut = (('2', 'メンテカット'),)
        mainte_color = (('3', 'メンテカラー'),)
        unusable = (('9', '利用不可'),)

        # 最新の予約情報取得
        try:
            treatment_info = Reservation.objects.filter(member=self.member).order_by('-date')
            self.fields['treatment'].choices = recursive_choice_setting(treatment_info, 0, date)

        # 予約した記録がなかったら
        except IndexError:
            self.fields['treatment'].choices = full_cut

    def clean_treatment(self):
        treatment = self.cleaned_data['treatment']
        if treatment == 9:
            raise forms.ValidationError('利用できません')
        return treatment

########################################
# 施術可能選択肢を再起処理で実現(カットのみ) #
########################################
def recursive_choice_setting(obj_treatment_info, int_list_no, date_reserve_date):
    # ユーザーの利用可能施術を設定
    full_cut = (('0', 'フルカット'),)
    full_color = (('1', 'フルカラー'),)
    mainte_cut = (('2', 'メンテカット'),)
    mainte_color = (('3', 'メンテカラー'),)
    unusable = (('9', '利用不可'),)

    # 最新の予約が「予約済み」状態だった場合
    if obj_treatment_info.values('status')[int_list_no]['status'] == 0:
        return unusable
    # 最新の予約が「済」だった場合
    elif obj_treatment_info.values('status')[int_list_no]['status'] == 1:
        reserved_date = obj_treatment_info.values('date')[int_list_no]['date']
        # now = datetime.datetime.now()
        # interval_date = now.date() - reserved_date
        interval_date = date_reserve_date - reserved_date

        # 当日に2回行こうとした場合、利用不可
        if interval_date == datetime.timedelta(days=0):
            return unusable

        # 最新の予約から30日経過していたらフルカット
        elif interval_date > datetime.timedelta(days=30):
            return full_cut

        # 最新の予約から30日経過していなかったらメンテカット
        elif interval_date <= datetime.timedelta(days=30):
            return mainte_cut

    # 最新の予約が「キャンセル」だった場合
    elif obj_treatment_info.values('status')[int_list_no]['status'] == 2:
        try:
            return recursive_choice_setting(obj_treatment_info, int_list_no + 1, date_reserve_date)
        except IndexError:
            return full_cut

    # 最新の予約が「ドタキャン」だった場合
    elif obj_treatment_info.values('status')[int_list_no]['status'] == 3:
        reserved_date = obj_treatment_info.values('date')[int_list_no]['date']
        # now = datetime.datetime.now()
        # interval_date = now.date() - reserved_date
        interval_date = date_reserve_date - reserved_date

        # ドタキャンから20日経過していた場合
        if interval_date > datetime.timedelta(days=20):
            try:
                reserved_date = obj_treatment_info.values('date')[int_list_no + 1]['date']
                # now = datetime.datetime.now()
                # interval_date = now.date() - reserved_date
                interval_date = date_reserve_date - reserved_date

                # ドタキャンより前の予約から30日経過していた場合
                if interval_date > datetime.timedelta(days=30):
                    return full_cut

                # ドタキャンより前の予約から30日経過していなかった場合
                elif interval_date <= datetime.timedelta(days=30):
                    return mainte_cut

            # ドタキャンより前の予約がなかったら
            except IndexError:
                return full_cut

        # ドタキャンから20日経過していなかったら利用不可
        elif interval_date <= datetime.timedelta(days=20):
            return unusable


import pandas as pd
import numpy as np

from sklearn.cluster import KMeans

from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.metrics.cluster import homogeneity_completeness_v_measure

data = pd.read_pickle('/home/mluser/master8_projects/clustering_vacancies/data/corpus/df_vacancies_full_ru_42K_w2v.pkl')
vectors_name = 'w2v_300'

co = data[data.is_prog]

co_train = co[co.is_train].sample(4000)
co_test = co[co.is_test]

X_train = np.array(co_train[vectors_name])
X_train = X_train.tolist()

X_test = np.array(co_test[vectors_name])
X_test = X_test.tolist()

count = []
ars = []
hcv = []

for i in range(10, 101, 1):
    print('k = ' + str(i))
    model = KMeans(n_clusters=i, n_jobs=-1)
    model.fit(X_train)
    labels = model.predict(X_test)
    co_test['label_test'] = labels

    count.append(co_test.label_test.nunique())
    ars.append(adjusted_rand_score(co_test.label_true, co_test.label_test))
    hcv.append(homogeneity_completeness_v_measure(co_test.label_true, co_test.label_test))

hcv = np.array(hcv)
count = np.array(count)
ars = np.array(ars)

df = pd.DataFrame(hcv)
df['count'] = count
df['ars'] = ars

df.columns = ['homogeneity', 'completeness', 'v_measure', 'count', 'ars']
df['k'] = range(10, 101, 1)

co = df[['k', 'count', 'ars', 'homogeneity', 'completeness', 'v_measure']]
co.to_csv('/home/mluser/master8_projects/clustering_vacancies/results/df_vacancies_full_clusters_results_ru_prog_k.csv', index=False)

import seaborn as sns
import matplotlib.pyplot as plt

plt.clf()
sns.lineplot(x=co.k, y=co['count'])
plt.savefig('/home/mluser/master8_projects/clustering_vacancies/results/plots/clustering_KMeans_k_count.png', format='png', dpi=300)

plt.clf()
sns.lineplot(x=co.k, y=co.homogeneity, label='homogeneity')
sns.lineplot(x=co.k, y=co.completeness, label='completeness')
sns.lineplot(x=co.k, y=co.v_measure, label='v_measure')
sns.lineplot(x=co.k, y=co.ars, label='ars')
plt.savefig('/home/mluser/master8_projects/clustering_vacancies/results/plots/clustering_KMeans_k_metrics.png', format='png', dpi=300)


"""Rename some files. For example:

* PXL_YYYYMMDD_nnn.jpg becomes IMG_YYYYMMDD_nnn.jpg
* PXL_YYYYMMDD_nnn.NIGHT.jpg becomes IMG_YYYYMMDD_nnn.jpg
* PXL_YYYYMMDD_nnn.MP.jpg becomes IMG_YYYYMMDD_nnn_MV.jpg
* PXL_YYYYMMDD_nnn.mp4 becomes VID_YYYYMMDD_nnn.mp4

Usage::

    python rename_img_vid.py "D:\\My\\Windows\\Path" -v

"""
import glob
import os

__all__ = []

def undo_pxl(path, verbose=False):
    pxl_prefix = 'PXL_'
    for filetype, new_prefix in (('*.jpg', 'IMG_'), ('*.mp4', 'VID_')):
        for filepath in glob.iglob(os.path.join(path, filetype)):
            filename = os.path.basename(filepath)
            if filename.startswith(pxl_prefix):
                newname = os.path.join(
                    path, filename.replace(pxl_prefix, new_prefix))
                if not os.path.exists(newname):
                    os.rename(filepath, newname)
                    if verbose:
                        print(f'{filepath} -> {newname}')

def rename_suffix(path, from_suffix='.MP.jpg', to_suffix='_MV.jpg',
                  verbose=False):
    for filepath in glob.iglob(os.path.join(path, f'*{from_suffix}')):
        newname = filepath.replace(from_suffix, to_suffix)
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def append(path, ext, suffix, verbose=False):
    for filepath in glob.iglob(os.path.join(path, '*.' + ext)):
        s = filepath.split(f'.{ext}')
        newname = f'{s[0]}_{suffix}.{ext}'
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def prepend(path, suffix, prefix, verbose=False):
    for filepath in glob.iglob(os.path.join(path, '*.' + suffix)):
        filename = os.path.basename(filepath)
        if filename.startswith(prefix):
            continue
        newname = os.path.join(path, prefix + '_' + filename)
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def main(path, verbose=False):
    undo_pxl(path, verbose=verbose)
    rename_suffix(path, verbose=verbose)
    rename_suffix(path, from_suffix='.NIGHT.jpg', to_suffix='.jpg',
                  verbose=verbose)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Rename some files.')
    parser.add_argument('path', type=str, help='Path name')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Verbose output')

    args = parser.parse_args()
    main(args.path, verbose=args.verbose)


import twython
import logging

class twytbot:
    def __init__(self, key, secret, acctok, sectok):
        self.KEY = key
        self.SECRET = secret
        self.ACCESS_TOKEN = acctok
        self.SECRET_TOKEN = sectok
        self.twitter = None

    def authentificate(self):
        self.twitter = twython.Twython(self.KEY, self.SECRET, self.ACCESS_TOKEN, self.SECRET_TOKEN)
        try:
            self.twitter.verify_credentials()
        except Exception as e:
            logging.warn("Twitter log failed %s" % e)


# -*- coding: utf-8 -*-

from __future__ import print_function, division

import sys
import os
import time
import json
import IPython
import datetime
import argparse
import numpy as np
#import matplotlib
#matplotlib.use('agg')
import os.path as osp
from PIL import Image
#import matplotlib.pyplot as plt

import torch
import torchvision
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import torch.backends.cudnn as cudnn
from torchvision import datasets, models, transforms

from eval_metrics import evaluate_deep
from random_erasing import RandomErasing
from model import ft_net, ft_net_dense, PCB, MUB
from utils import AverageMeter, Logger, save_checkpoint

version = torch.__version__

if int(version[2]) <= 3:
    from torch.autograd import Variable
    flag = True
    print("Now using pytoch version {}".format(version))
else:
    flag = False
    print("Now using pytoch version {}".format(version))

######################################################################
# Options
# --------
parser = argparse.ArgumentParser(description='Training')
#parser.add_argument('--gpu_ids',default=None, nargs='+', type=int, help='gpu_ids: e.g. 0  0,1,2  0,2')
parser.add_argument('--gpu-ids',default='0', type=str, help='gpu_ids: e.g. 0  0,1,2  0,2')
parser.add_argument('--name',default='ft_ResNet50', type=str, help='output model name')
parser.add_argument('--data-dir',default='/home/zzd/Market/pytorch',type=str, help='training dir path')
parser.add_argument('--train-all', action='store_true', help='use all training data' )
parser.add_argument('--use-clean-imgs', action='store_true', help='use cleaned images' )
parser.add_argument('--color-jitter', action='store_true', help='use color jitter in training' )
parser.add_argument('--erasing-p', default=0, type=float, help='Random Erasing probability, in [0,1]')
parser.add_argument('--use-dense', action='store_true', help='use densenet121' )
parser.add_argument('--MUB', action='store_true', help='use MUB' )
parser.add_argument('--print-freq', type=int, default=10, help='print frequency')
parser.add_argument('--max-epoch', default=60, type=int,
                    help="maximum epochs to run")
parser.add_argument('--start-epoch', default=0, type=int,
                    help="manual epoch number (useful on restarts)")
parser.add_argument('--train-batch', default=32, type=int,
                    help="train batch size")
parser.add_argument('--test-batch', default=32, type=int, help="test batch size")
parser.add_argument('--stepsize', default=20, type=int,
                    help="stepsize to decay learning rate (>0 means this is enabled)")
parser.add_argument('--resume', type=str, default='', metavar='PATH')
parser.add_argument('--evaluate', action='store_true', help="evaluation only")
parser.add_argument('--eval-step', type=int, default=-1,
                    help="run evaluation for every N epochs (set to -1 to test after training)")
parser.add_argument('--start-eval', type=int, default=0, help="start to evaluate after specific epoch")
parser.add_argument('--save-dir', type=str, default='logs')
parser.add_argument('--train-log', type=str, default='log_train.txt')
parser.add_argument('--test-log', type=str, default='log_test.txt')
parser.add_argument('--use-cpu', action='store_true', help="use cpu")
opt = parser.parse_args()

def main():
    print("==========\nArgs:{}\n==========".format(opt))
    data_dir = opt.data_dir
    name = opt.name
    os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_ids
    use_gpu = torch.cuda.is_available()
    if opt.use_cpu: use_gpu = False

    if not opt.evaluate:
        sys.stdout = Logger(osp.join(opt.save_dir, opt.train_log))
    else:
        sys.stdout = Logger(osp.join(opt.save_dir, opt.test_log))

    if use_gpu:
        print("Currently using GPU {}".format(opt.gpu_ids))
        cudnn.benchmark = True
    else:
        print("Currently using CPU (GPU is highly recommended)")
    #str_ids = opt.gpu_ids.split(',')
    #gpu_ids = []
    #for str_id in str_ids:
    #    gid = int(str_id)
    #    if gid >=0:
    #        gpu_ids.append(gid)
    #
    ## set gpu ids
    #if len(gpu_ids)>0:
    #    torch.cuda.set_device(gpu_ids[0])
    #print(gpu_ids[0])

    # Load train Data
    # ---------
    print("==========Preparing trian dataset========")
    transform_train_list = [
        transforms.Resize((384,192), interpolation=3),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]
    if opt.erasing_p>0:
        transform_train_list = transform_train_list +  [RandomErasing(probability = opt.erasing_p, mean=[0.0, 0.0, 0.0])]

    if opt.color_jitter:
        transform_train_list = [transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0)] + transform_train_list

    train_data_transforms = transforms.Compose( transform_train_list )
    train_all = ''
    if opt.train_all:
        if opt.use_clean_imgs:
            train_all = '_all_clean'
        else:
            train_all = '_all'
            print("Using all the train images")

    train_image_datasets = {}
    train_image_datasets['train'] = datasets.ImageFolder(os.path.join(data_dir, 'train' + train_all),
                                              train_data_transforms)
    train_dataloaders = {x: torch.utils.data.DataLoader(train_image_datasets[x], batch_size=opt.train_batch,
                                                 shuffle=True, num_workers=0)
                  for x in ['train']}
    dataset_sizes = {x: len(train_image_datasets[x]) for x in ['train']}
    class_names = train_image_datasets['train'].classes
    inputs, classes = next(iter(train_dataloaders['train']))

    ######################################################################
    #Prepare test data
    print("========Preparing test dataset========")
    transform_test_list = transforms.Compose([
            transforms.Resize((384,192), interpolation=3),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])
    test_image_datasets = {x: datasets.ImageFolder( os.path.join(data_dir,x) ,transform_test_list) for x in ['gallery','query']}
    test_dataloaders = {x: torch.utils.data.DataLoader(test_image_datasets[x], batch_size=opt.test_batch,
                                                 shuffle=False, num_workers=0) for x in ['gallery','query']}

    print("Initializing model...")
    if opt.MUB:
        model = MUB(len(class_names))
    print("Model size: {:.5f}M".format(sum(p.numel() for p in model.parameters())/1000000.0))
    start_epoch = opt.start_epoch

    if opt.resume:
        print("Loading checkpoint from '{}'".format(opt.resume))
        checkpoint = torch.load(opt.resume)
        model.load_state_dict(checkpoint['state_dict'])
        #model.load_state_dict(checkpoint)
        start_epoch = checkpoint['epoch']
    if use_gpu:
        model = nn.DataParallel(model).cuda()
    if opt.evaluate:
        print("Evaluate only")
        test(model, test_image_datasets,test_dataloaders,  use_gpu)
        return
    criterion = nn.CrossEntropyLoss().cuda()

    ignored_params = list(map(id, model.module.resnet50.fc.parameters() ))
    ignored_params += (list(map(id, model.module.classifier0.parameters() ))
                     +list(map(id, model.module.classifier1.parameters() ))
                     +list(map(id, model.module.classifier2.parameters() ))
                     +list(map(id, model.module.classifier3.parameters() ))
                     +list(map(id, model.module.classifier4.parameters() ))
                     +list(map(id, model.module.classifier5.parameters() ))
                     +list(map(id, model.module.classifier.parameters() ))
                     #+list(map(id, model.classifier7.parameters() ))
                      )
    base_params = filter(lambda p: id(p) not in ignored_params, model.parameters())
    optimizer = optim.SGD([
             {'params': base_params, 'lr': 0.01},
             {'params': model.module.resnet50.fc.parameters(), 'lr': 0.1},
             {'params': model.module.classifier0.parameters(), 'lr': 0.1},
             {'params': model.module.classifier1.parameters(), 'lr': 0.1},
             {'params': model.module.classifier2.parameters(), 'lr': 0.1},
             {'params': model.module.classifier3.parameters(), 'lr': 0.1},
             {'params': model.module.classifier4.parameters(), 'lr': 0.1},
             {'params': model.module.classifier5.parameters(), 'lr': 0.1},
             {'params': model.module.classifier.parameters(), 'lr': 0.01},
             #{'params': model.classifier7.parameters(), 'lr': 0.01}
         ], weight_decay=5e-4, momentum=0.9, nesterov=True)
    # Decay LR by a factor of 0.1 every 40 epochs
    if opt.stepsize > 0:
        scheduler = lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)

    start_time = time.time()
    train_time = 0
    best_rank1 = -np.inf
    best_epoch = 0
    print("==> Start training")
######################################################################
# Training the model
# ------------------
#
# Now, let's write a general function to train a model. Here, we will
# illustrate:
#
# -  Scheduling the learning rate
# -  Saving the best model
#
# In the following, parameter ``scheduler`` is an LR scheduler object from
# ``torch.optim.lr_scheduler``.

    for epoch in range(start_epoch, opt.max_epoch):
        start_train_time = time.time()
        train(epoch, model, criterion, optimizer, train_dataloaders, use_gpu)
        train_time += round(time.time() - start_train_time)

        if opt.stepsize > 0: scheduler.step()

        if (epoch+1) > opt.start_eval and opt.eval_step > 0 and (epoch+1) % opt.eval_step == 0 or (epoch+1) == opt.max_epoch:
            print("==> Test")
            rank1 = test(model, test_image_datasets, test_dataloaders, use_gpu)
            is_best = rank1 > best_rank1
            if is_best:
                best_rank1 = rank1
                best_epoch = epoch + 1
            if use_gpu:
                state_dict = model.module.state_dict()
            else:
                state_dict = model.state_dict()
            save_checkpoint({
                'state_dict': state_dict,
                'rank1': rank1,
                'epoch': epoch,
            }, is_best, osp.join(opt.save_dir, 'checkpoint_ep' + str(epoch+1) + '.pth.tar'))

    print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(best_rank1, best_epoch))

    elapsed = round(time.time() - start_time)
    elapsed = str(datetime.timedelta(seconds=elapsed))
    train_time = str(datetime.timedelta(seconds=train_time))
    print("Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(elapsed, train_time))

def train(epoch, model, criterion, optimizer, train_dataloaders, use_gpu):
    since = time.time()
    losses = AverageMeter()
    # Each epoch has a training and validation phase
    for phase in ['train']:
        model.train(True)  # Set model to training mode
        # Iterate over data.
        for batch_index, data in enumerate(train_dataloaders[phase]):
            # get the inputs
            inputs, labels = data
            #print(inputs.shape)
            # wrap them in Variable
            if flag:
                if use_gpu:
                    inputs = Variable(inputs.cuda())
                    labels = Variable(labels.cuda())
                else:
                    inputs, labels = Variable(inputs), Variable(labels)
            else:
                if use_gpu:
                    inputs = inputs.cuda()
                    labels = labels.cuda()
                else:
                    inputs, labels = inputs, labels

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward
            y_g, y_cp, y_p = model(inputs)

            # global loss
            loss_g = criterion(y_g, labels)

            # cropped loss
            loss_cp = criterion(y_cp, labels)

            # part loss
            part = {}
            sm = nn.Softmax(dim=1)
            num_part = 6
            for i in range(num_part):
                part[i] = y_p[i]
            loss_p = criterion(part[0], labels)
            for i in range(num_part-1):
                loss_p += criterion(part[i+1], labels)
            loss = loss_g + loss_cp + loss_p/num_part
            # backward + optimize only if in training phase
            if phase == 'train':
                loss.backward()
                optimizer.step()
            if flag:
                losses.update(loss.data[0], labels.size(0))
            else:
                losses.update(loss.item(), labels.size(0))

            if (batch_index+1) % opt.print_freq == 0:
                print('Epoch: [{0}][{1}/{2}]\t'
                'Loss: {loss.val:.4f} ({loss.avg:.4f})\t'.format(epoch+1, batch_index+1,len(train_dataloaders[phase]),loss=losses))

######################################################################
# Draw Curve
#---------------------------
#x_epoch = []
#fig = plt.figure()
#ax0 = fig.add_subplot(121, title="loss")
#ax1 = fig.add_subplot(122, title="top1err")
#def draw_curve(current_epoch):
#    x_epoch.append(current_epoch)
#    ax0.plot(x_epoch, y_loss['train'], 'bo-', label='train')
#    ax0.plot(x_epoch, y_loss['val'], 'ro-', label='val')
#    ax1.plot(x_epoch, y_err['train'], 'bo-', label='train')
#    ax1.plot(x_epoch, y_err['val'], 'ro-', label='val')
#    if current_epoch == 0:
#        ax0.legend()
#        ax1.legend()
#    fig.savefig( os.path.join('./model',name,'train.jpg'))

######################################################################
# Save model
#---------------------------
def save_network(network, epoch_label):
    save_filename = 'net_%s.pth'% epoch_label
    save_path = os.path.join('./model',name,save_filename)
    torch.save(network.cpu().state_dict(), save_path)
    if torch.cuda.is_available:
        network.cuda()

#dir_name = os.path.join('./model',name)
#if not os.path.isdir(dir_name):
#    os.makedirs(dir_name)
#
## save opts
#with open('%s/opts.json'%dir_name,'w') as fp:
#    json.dump(vars(opt), fp, indent=1)

def test(model, test_image_datasets, test_dataloaders, use_gpu, ranks=[1, 5, 10, 20]):
    batch_time = AverageMeter()

    model.eval()
    queryloader = test_dataloaders['query']
    galleryloader = test_dataloaders['gallery']

    #with torch.no_grad():
    qf, q_pids, q_camids = [], [], []
    q_path = test_image_datasets['query'].imgs
    q_camids,q_pids = get_id(q_path)
    for batch_idx, q_data in enumerate(queryloader):
        imgs, _ = q_data
        if flag:
            imgs = Variable(imgs.cuda())
        else:
            imgs = imgs.cuda()

        end = time.time()
        f_g, f_cp, y_p = model(imgs)
        y_p = torch.squeeze(y_p)
        f_p = y_p.view(y_p.size(0), -1)
        features = torch.cat((f_g, f_cp, f_p), 1)
        batch_time.update(time.time() - end)

        features = features.data.cpu()
        qf.append(features)
        #q_pids.extend(pids)
        #q_camids.extend(camids)
    qf = torch.cat(qf, 0)
    q_pids = np.asarray(q_pids)
    q_camids = np.asarray(q_camids)

    print("Extracted features for query set, obtained {}-by-{} matrix".format(qf.size(0), qf.size(1)))

    gf, g_pids, g_camids = [], [], []
    g_path = test_image_datasets['gallery'].imgs
    g_camids,g_pids = get_id(g_path)
    end = time.time()
    for batch_idx, g_data in enumerate(galleryloader):
        imgs, _ = g_data
        if flag:
            imgs = Variable(imgs.cuda())
        else:
            imgs = imgs.cuda()

        end = time.time()
        f_g, f_cp, y_p = model(imgs)
        y_p = torch.squeeze(y_p)
        f_p = y_p.view(y_p.size(0), -1)
        features = torch.cat((f_g, f_cp, f_p), 1)
        batch_time.update(time.time() - end)

        features = features.data.cpu()
        gf.append(features)
    gf = torch.cat(gf, 0)
    g_pids = np.asarray(g_pids)
    g_camids = np.asarray(g_camids)

    print("Extracted features for gallery set, obtained {}-by-{} matrix".format(gf.size(0), gf.size(1)))

    print("==> BatchTime(s)/BatchSize(img): {:.3f}/{}".format(batch_time.avg, opt.test_batch))
    m, n = qf.size(0), gf.size(0)
    distmat = torch.pow(qf, 2).sum(dim=1, keepdim=True).expand(m, n) + \
              torch.pow(gf, 2).sum(dim=1, keepdim=True).expand(n, m).t()
    distmat.addmm_(1, -2, qf, gf.t())
    distmat = distmat.numpy()

    print("Computing CMC and mAP")
    cmc, mAP = evaluate_deep(distmat, q_pids, g_pids, q_camids, g_camids, use_metric_cuhk03=False)

    print("Results ----------")
    print("mAP: {:.1%}".format(mAP))
    print("CMC curve")
    for r in ranks:
        print("Rank-{:<3}: {:.1%}".format(r, cmc[r-1]))
    print("------------------")

    return cmc[0]

def get_id(img_path):
    camera_id = []
    labels = []
    for path, v in img_path:
        filename = path.split('/')[-1]
        label = filename[0:4]
        camera = filename.split('c')[1]
        if label[0:2]=='-1':
            labels.append(-1)
        else:
            labels.append(int(label))
        camera_id.append(int(camera[0]))
    return camera_id, labels

if __name__ == '__main__':
    main()


#!python 3.6
"""Model class for lane scoring table view."""

from PyQt5 import QtGui
from PyQt5.QtCore import Qt, QAbstractTableModel, QModelIndex

class LaneScoringModel(QAbstractTableModel):
    def __init__(self, spevent_model, game):
        super().__init__()
        self.spevent = spevent_model
        self.game = game
        self.font = QtGui.QFont()
        self.font.setPointSize(12)
        spevent_model.modelAboutToBeReset.connect(self.beginResetModel)
        spevent_model.modelReset.connect(self.endResetModel)
        spevent_model.layoutChanged.connect(self.layoutChanged)

    def rowCount(self, parent=None):
        return self.spevent.teamSize

    def columnCount(self, parent=None):
        return 1 + len(self.spevent.teamInfo.ids())

    def headerData(self, section, orientation, role):
        if role == Qt.DisplayRole:
            if section == 0:
                return None
            ids = self.spevent.teamInfo.ids()
            lanes = [self.spevent.teamLane(id, self.game) for id in ids]
            return sorted(lanes)[section-1]


import matplotlib.pyplot as plt
import numpy as np
import sys
import os
import zipfile
def recursive_path(mypath):
    for f in listdir(mypath):
        if isfile(join(mypath, f)):
            #print(mypath,' ', f)
            if(f.find("part")==-1):
                onlyfiles.append(join(mypath,f))
        else:
            recursive_path(join(mypath,f))

def zipcompress(fileName):
    import bz2
    compressionLevel=9
    zip = bz2.compress(open(fileName, 'rb').read(), compressionLevel)
    return sys.getsizeof(zip)
if __name__ == "__main__":

    output_file2 = open('bzip_supergene.txt','w')

    mypath = 'pep-unfiltered-alignemtns-original'
    from os import listdir
    from os.path import isfile, join
    onlyfiles = []
    recursive_path(mypath)
    array=[]

    #m = compressDataExpectation('MSA/23S.E/input.fasta_clustalo')
    n=0
    for f in onlyfiles:
        #print(f)
        #m=compressData("MSA/"+f)
        #myfile.write(f+"\n")
        if 'tar' in f :
            continue
        print(n)
        n+=1
        locfile = f
        loczip = f+".zip"
        zip = zipfile.ZipFile(loczip, "w", zipfile.ZIP_DEFLATED)
        zip.write (locfile)
        zip.close()
        m= os.path.getsize(f+'.zip')
        output_file2.write(f + '    ' + str(m) + '\n')
        os.remove(f+'.zip')

    print('DONE')

    output_file2.close()
'''tar = tarfile.open(f+'.tar.gz',"w:gz")
        tar.add(f)
        tar.close()
        m= os.path.getsize(f+'.tar.gz')
        output_file2.write(f + '    ' + str(m) + '\n')
        os.remove(f+'.tar.gz')'''