import torch
import torch.nn as nn
from cpu_offload import CPUOffload
import torchvision
class TorchCPUOffload(nn.Module):
    def __init__(self, module):
        super().__init__()
        self.module = module

    def forward(self, *args, **kwargs):
        with torch.autograd.graph.save_on_cpu(pin_memory=True):
            return self.module(*args, **kwargs)
# 定义 VGG16 模型
class VGG16(nn.Module):
    def __init__(self, num_classes=1000,num_channels = 64):
        super(VGG16, self).__init__()
        self.num_channels = num_channels
        # 第一段卷积层
        self.conv1_1 = nn.Conv2d(3, self.num_channels, kernel_size=3, padding=1)
        self.ReLU1_1 = nn.ReLU(inplace=True)
        self.conv1_2 = nn.Conv2d(self.num_channels, self.num_channels, kernel_size=3, padding=1)
        self.ReLU1_2 = nn.ReLU(inplace=True)
        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第二段卷积层
        self.conv2_1 = nn.Conv2d(self.num_channels, self.num_channels * 2, kernel_size=3, padding=1)
        self.ReLU2_1 = nn.ReLU(inplace=True)
        self.conv2_2 = nn.Conv2d(self.num_channels * 2, self.num_channels * 2, kernel_size=3, padding=1)
        self.ReLU2_2 = nn.ReLU(inplace=True)
        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第三段卷积层
        self.conv3_1 = nn.Conv2d(self.num_channels * 2,self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_1 = nn.ReLU(inplace=True)
        self.conv3_2 = nn.Conv2d(self.num_channels * 4, self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_2 = nn.ReLU(inplace=True)
        self.conv3_3 = nn.Conv2d(self.num_channels * 4, self.num_channels * 4, kernel_size=3, padding=1)
        self.ReLU3_3 = nn.ReLU(inplace=True)
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第四段卷积层
        self.conv4_1 = nn.Conv2d(self.num_channels * 4, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_1 = nn.ReLU(inplace=True)
        self.conv4_2 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_2 = nn.ReLU(inplace=True)
        self.conv4_3 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU4_3 = nn.ReLU(inplace=True)
        self.pool4 = nn.MaxPool2d(kernel_size=2, stride=2)

        # 第五段卷积层
        self.conv5_1 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_1 = nn.ReLU(inplace=True)
        self.conv5_2 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_2 = nn.ReLU(inplace=True)
        self.conv5_3 = nn.Conv2d(self.num_channels * 8, self.num_channels * 8, kernel_size=3, padding=1)
        self.ReLU5_3 = nn.ReLU(inplace=True)
        self.pool5 = nn.AdaptiveAvgPool2d((7,7))
        self.classifier = nn.Sequential(
            nn.Linear(self.num_channels * 8 * 7 * 7, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, num_classes)
        )

    def forward(self, x):
        # with torch.autograd.graph.save_on_cpu(pin_memory=True):
        x = self.conv1_1(x) # 122

        x =self.ReLU1_1(x) #121
        # with torch.autograd.graph.save_on_cpu(pin_memory=True):
        x = self.conv1_2(x) #121

        x = self.ReLU1_2(x) #120
        x = self.pool1(x) #60

        x = self.conv2_1(x) # 60

        x =self.ReLU2_1(x) #59

        x = self.conv2_2(x) #59

        x = self.ReLU2_2(x) #58
        x = self.pool2(x) #29

        x = self.conv3_1(x) # 29

        x =self.ReLU3_1(x) #28

        x = self.conv3_2(x) #28

        x = self.ReLU3_2(x) #27

        x = self.conv3_3(x) #27

        x = self.ReLU3_3(x) #26
        x = self.pool3(x) #13

        x = self.conv4_1(x) # 13

        x =self.ReLU4_1(x) #12

        x = self.conv4_2(x) #12

        x = self.ReLU4_2(x) #11

        x = self.conv4_3(x) #11

        x = self.ReLU4_3(x) #10
        x = self.pool4(x) #5

        x = self.conv5_1(x) # 5

        x =self.ReLU5_1(x) #4

        x = self.conv5_2(x) #4

        x = self.ReLU5_2(x) #3

        x = self.conv5_3(x) #3

        x = self.ReLU5_3(x) #2
        x = self.pool5(x) #1
        # x = self.features(x)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

class Net(nn.Module):
    def __init__(self,  offload_ratio= 0,num_channels=64):
        super(Net, self).__init__()
        self.model = VGG16(num_classes=10,num_channels= num_channels)
        self.offload_ratio = offload_ratio
    def forward(self, x):
        with CPUOffload(offload_ratio=self.offload_ratio, tag="vgg16"):
            x = self.model(x)
        return x


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 11 15:59:35 2016

@author: RJovelin
"""

# use this script to make a box plot figure comparing read depth betweem
# heteroplasmic and invariant positions either for WGS or RNAseq for a given cancer

# usage: PlotreadDepthVariablePositions.py [options]
# - inputfile: json file with dict {participant: [[read_depth_snps], [read_depth_not_variable]]}
# - [WGS/RNAseq]: datatype
# - outputfile: figure file

import os
import sys
import numpy as np
from scipy import stats
import json
import matplotlib.pyplot as plt

# get json file with read depth
datafile = sys.argv[1]
# get data type
data_type = sys.argv[2]
# get the outputfile name
outputfile = sys.argv[3]

# get the tumor type from the datafile
tumor_type = datafile[:datafile.index('_')]

# open file for reading
infile = open(datafile, 'r')
# load json dict {participant: [[read_depth_snps], [read_depth_not_variable]]}
ReadDepth = json.load(infile)
# close file
infile.close()

# create  a data list [[read_depth_snps], [read_depth_not_variable]]
AllData = [[], []]
# loop over dict
for participant in ReadDepth:
    # dump read depth in each inner list
    AllData[0].extend(ReadDepth[participant][0])
    AllData[1].extend(ReadDepth[participant][1])

# make a box plot figure comparing read depth betweem variable and non-variable positions

# make a list of position types
names = ['Variable', 'Conserved']

# create figure
#fig = plt.figure(1, figsize = (4.3,2.56))
fig = plt.figure(1, figsize = (2.5, 3))

# add a plot to figure (1 row, 1 column, 1 plot)
ax = fig.add_subplot(1, 1, 1)

# write label for y axis
ytext = ax.set_ylabel('Read depth', color = 'black', size = 10, ha = 'center', fontname = 'Arial', family = 'sans-serif')
xtext = ax.set_xlabel('MT sites', color = 'black', size = 10, ha = 'center', fontname = 'Arial', family = 'sans-serif')

# add labels to x-ticks, rotate and align right, set size to 14
ax.set_xticklabels(names, ha = 'center', size = 10, fontname = 'Arial', family = 'sans-serif')

# add a light grey horizontal grid to the plot, semi-transparent,
ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
# hide these grids behind plot objects
ax.set_axisbelow(True)

# remove top axes and right axes ticks
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()

# use a boxplot
bp = ax.boxplot(AllData, showmeans = True, showfliers = False, widths = 0.7, labels = names, patch_artist = True)

# color WGS boxes in grey
i = 0
# change box, whisker color to black
for box in bp['boxes']:
    # change line color
    box.set(color = 'black')
    if i % 2 == 0:
        # SNPs data, color box in grey
        box.set(facecolor = [0.518, 0.439, 0.804])
    else:
        box.set(facecolor = [0.808, 0.765, 0.9])
    i += 1

# change whisker color ro black
for wk in bp['whiskers']:
    wk.set(color = 'black', linestyle = '-')

# change color of the caps
for cap in bp['caps']:
    cap.set(color = 'black')

# change the color and line width of the medians
for median in bp['medians']:
    median.set(color = 'black')

# change the mean marker and marker
for mean in bp['means']:
    mean.set(marker = 'o', markeredgecolor = 'black', markerfacecolor = 'black', markersize = 5)

# do not show lines around figure
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)

# create a list with range of x-axis values
xvals = [i + 0.5 for i in range(len(names) + 1)]
# Set a buffer around the edge of the x-axis
plt.xlim([min(xvals)- 0.5, max(xvals)+ 0.5])

# do not show ticks
plt.tick_params(
    axis='y',       # changes apply to the x-axis and y-axis (other option : x, y)
    which='both',      # both major and minor ticks are affected
    bottom='off',      # ticks along the bottom edge are off
    top='off',         # ticks along the top edge are off
    right = 'off',
    left = 'off',
    labelbottom='off', # labels along the bottom edge are off
    colors = 'black',
    labelsize = 10)

# do not show ticks
plt.tick_params(
    axis='x',       # changes apply to the x-axis and y-axis (other option : x, y)
    which='both',      # both major and minor ticks are affected
    bottom='off',      # ticks along the bottom edge are off
    top='off',         # ticks along the top edge are off
    right = 'off',
    left = 'off',
    labelbottom='on', # labels along the bottom edge are on
    colors = 'black',
    labelsize = 10,
    direction = 'out') # ticks are outside the frame when bottom = 'on'

# Set the tick labels font name
for label in ax.get_yticklabels():
    label.set_fontname('Arial')

## note some ways to modify label font size and color of the ticks
#for label in ax.get_yticklabels():
#    label.set_color('black')
#    label.set_fontsize(10)

# add title
plt.title('Read depth in {0} for {1} data\n'.format(tumor_type, data_type), size = 12, fontname = 'Arial')

# annotate figure to add significance

# get the P value of Wilcoxon rank sum test
Pval = stats.ranksums(AllData[0], AllData[1])[1]
# get stars for significance
if Pval > 0.05:
    P = 'N.S.'
elif Pval < 0.05 and Pval > 0.01:
    P = '*'
elif Pval < 0.01 and Pval > 0.001:
    P = '**'
elif Pval < 0.001:
    P = '***'

# get the x and y coordinates
yvalues = []
yvalues.extend(AllData[0])
yvalues.extend(AllData[1])
# get max and min y values
y_min, y_max = min(yvalues), max(yvalues)

# add bracket
ax.annotate("", xy=(1, y_max), xycoords='data', xytext=(2, y_max),
            textcoords='data', arrowprops=dict(arrowstyle="-", ec='#aaaaaa',
            connectionstyle="bar,fraction=0.1"))
# add stars for significance
ax.text(1.5, y_max + abs(y_max - y_min)*0.05, P, horizontalalignment='center',
       verticalalignment='center', color = 'grey', fontname = 'Arial')

# add jittered data points to box plots (when data points are not too numerous)
# set x up coordinates
#a = np.random.uniform(-0.2, 0.2, len(AllData[0])) + 1
#b = np.random.uniform(-0.2, 0.2, len(AllData[1])) + 2
#plt.plot(a, AllData[0], 'o', markersize = 4, markeredgecolor='blue', markeredgewidth = 1.5, color = 'blue', alpha = 0.3)
#plt.plot(b, AllData[1], 'o', markersize = 4, markeredgecolor='blue', markeredgewidth = 1.5, color = 'blue', alpha = 0.3)

# save figure
fig.savefig(outputfile, bbox_inches = 'tight')


from django.shortcuts import render, redirect, reverse
from django.http import HttpResponse, JsonResponse
from django.template.loader import render_to_string
from django.views import View
from chess import gamerules
from chess.models import Tournament, Game, Move, AccountPage
from chess.forms import AccountPageForm, TournamentForm, ChooseTournamentsForm
from django.contrib.auth.decorators import login_required
from django.contrib.admin.views.decorators import staff_member_required
from django.utils import timezone
from django.utils.decorators import method_decorator
from datetime import timedelta
from copy import deepcopy

class IndexView(View):
    def get(self, request):
        upcoming_tournaments = Tournament.objects.filter(date__gte=timezone.now())
        recent_games = Game.objects.filter(result__isnull=False).order_by('-time')[:5]

        context_dict = {'upcoming_tournaments':upcoming_tournaments, 'recent_games':recent_games}
        return render(request, 'chess/index.html', context_dict)

class PlayView(View):
    def get(self, request, game_id):
        try:
            game = Game.objects.get(id=game_id)

            # if spectate_mode is on, the user will have no control over the game
            spectate_mode = 'spectate' in request.path.split('/')

            if timezone.now() < game.time: # dissalow users from entering games which are scheduled in the future
                return redirect(reverse('chess:index'))
            elif game.result is not None: # if the game has been played already, redirect user to its history page
                return redirect(reverse('chess:gamehistory', kwargs={'game_id':game_id}))

            # perspective decides if the board is shown normal or upside down
            perspective = 0
            if not spectate_mode:
                if request.user.is_authenticated and request.user == game.player_white:
                    perspective = 0
                elif request.user.is_authenticated and request.user == game.player_black:
                    perspective = 1
                else:
                    # user does not participate in this game
                    return redirect(reverse('chess:spectate', kwargs={'game_id':game_id}))

            # execute moves on the board in order
            board_state = deepcopy(gamerules.starting_board_state)
            moves = Move.objects.filter(game=game).order_by('move_id')
            for move in moves:
                gamerules.make_move(board_state, move.square_from, move.square_to)

            context_dict = {'perspective': perspective,
                            'board_state': board_state,
                            'moves': [move.square_from + move.square_to for move in moves],
                            'spectate_mode': spectate_mode,
                            'player_white_name': game.player_white.username,
                            'player_black_name': game.player_black.username}

            return render(request, 'chess/play.html', context_dict)
        except Game.DoesNotExist:
            return redirect(reverse('chess:index'))

class PlayLobbyView(View):
    def get(self, request):
        games = Game.objects.filter(time__lte=timezone.now()).filter(result__isnull=True)
        if request.user.is_authenticated:
            ongoing_games = games.exclude(player_black=request.user).exclude(player_white=request.user)
            games_white = games.filter(player_white=request.user)
            games_black = games.filter(player_black=request.user)
            playable_games = (games_white | games_black)
        else:
            ongoing_games = games
            playable_games = Game.objects.none()

        # playable games are the currently held games which the user participates in. ongoing games are all other currently held games.
        context_dict = {'playable_games': playable_games.order_by('time'), 'ongoing_games': ongoing_games.order_by('time') }

        # gamestatus is present if the user has just finished their game. It informs them if they won, lost or drawn the game
        if 'gamestatus' in request.GET:
            gamestatus = request.GET['gamestatus']
            if gamestatus == 'draw':
                context_dict['message'] = "Your game ended in a draw."
            elif gamestatus == 'win':
                context_dict['message'] = "Congratulations! You have won the game."
            elif gamestatus == 'loss':
                context_dict['message'] = "You have lost the game."

        return render(request, 'chess/playlobby.html', context_dict)

class MoveListView(View):
    def get(self, request, game_id):
        try:
            game = Game.objects.get(id=game_id)

            # execute moves on the board in order
            board_state = deepcopy(gamerules.starting_board_state)
            moves = Move.objects.filter(game=game).order_by('move_id')
            for move in moves:
                gamerules.make_move(board_state, move.square_from, move.square_to)

            # board_state['turn'] is false when it's white player's turn, true otherwise
            turn = board_state['turn']==(request.user==game.player_black)

            move_str = request.GET['move']
            if move_str and (move_str in gamerules.legal_moves(board_state)): # double checks if the received move is valid
                # create a new move in db
                move = Move()
                move.game = game
                move.square_from = move_str[:2]
                move.square_to = move_str[2:]
                move.save()
                # update the board
                gamerules.make_move(board_state, move.square_from, move.square_to)
                # now is the turn of the opponent
                turn = False

                # check if the current position is winning/losing/tied
                if gamerules.mate_white(board_state):
                    game.result = 0.0 # black won (0-1)
                    game.save()
                elif gamerules.mate_black(board_state):
                    game.result = 1.0 # white won (1-0)
                    game.save()
                elif gamerules.draw(board_state):
                    game.result = 0.5 # draw (1/2-1/2)
                    game.save()

            # if the game is already decided, redirect to playlobby with a message
            if game.result is not None:
                if game.result == 0.5:
                    gamestatus = 'draw'
                elif ((game.result == 0.0 and request.user == game.player_black) or
                        (game.result == 1.0 and request.user == game.player_white)):
                    gamestatus = 'win'
                else:
                    gamestatus = 'loss'
                return JsonResponse({'redirect':reverse('chess:playlobby')+'?gamestatus='+gamestatus})

            return JsonResponse({'turn':turn, 'legal_moves':gamerules.legal_moves(board_state)})
        except Game.DoesNotExist:
            return HttpResponse('')

class MoveUpdateView(View):
    def get(self, request, game_id):
        try:
            game = Game.objects.get(id=game_id)
            moves = Move.objects.filter(game=game).order_by('-move_id')
            if moves:
                # returns the latest move in the game
                return HttpResponse(moves[0].square_from + moves[0].square_to)
            else:
                return HttpResponse('')
        except Game.DoesNotExist:
            return HttpResponse('')

class GameHistoryView(View):
    def get(self, request, game_id):
        try:
            game = Game.objects.get(id=game_id)

            if timezone.now() < game.time:
                return redirect(reverse('chess:index'))
            elif game.result is None:
                redirect(reverse('chess:play', kwargs={'game_id':game_id}))

            moves = Move.objects.filter(game=game).order_by('move_id')

            context_dict = { 'moves': [move.square_from + move.square_to for move in moves]}
            return render(request, 'chess/gamehistory.html', context_dict)
        except Game.DoesNotExist:
            return redirect(reverse('chess:index'))

class BoardHistoryView(View):
    def get(self, request, game_id):
        try:
            game = Game.objects.get(id=game_id)
            moves = Move.objects.filter(game=game).order_by('move_id')

            if 'move_index' in request.GET:
                n_moves = int(request.GET['move_index'])
            else:
                n_moves = moves.count()

            board_state = deepcopy(gamerules.starting_board_state)
            for move in moves[:n_moves]:
                gamerules.make_move(board_state, move.square_from, move.square_to)

            context_dict = {'perspective': 0,
                            'board_state': board_state,
                            'player_white_name': game.player_white.username,
                            'player_black_name': game.player_black.username}
            return HttpResponse(render_to_string('chess/board.html', context_dict))
        except Game.DoesNotExist:
            return HttpResponse('')

def show_tournament_history(request):
    context_dict = {}

    try:
        tournaments = Tournament.objects.filter(winner__isnull=False)

        context_dict['tournaments'] = tournaments
    except Tournament.DoesNotExist:
        context_dict['tournaments'] = None

    return render(request, 'chess/tournament_history.html', context=context_dict)

def show_tournament(request, tournament_name):
    context_dict = {}

    try:
        tournament = Tournament.objects.get(name=tournament_name)
        games = Game.objects.filter(tournament=tournament)

        context_dict['games'] = games
        context_dict['tournament'] = tournament
    except Tournament.DoesNotExist:
        context_dict['games'] = None
        context_dict['tournament'] = None

    return render(request, 'chess/tournament.html', context=context_dict)

def show_calendar(request):
    return render(request, 'chess/calendar.html')

class CalendarUpdateView(View):
    def get(self, request):
        tournaments = Tournament.objects.all()

        time = timezone.now()
        if 'tournament' in request.GET:
            try:
                selected_tournament = Tournament.objects.get(id=request.GET['tournament'])
                time = selected_tournament.date
            except:
                pass

        event_list = []

        for tournament in tournaments:
            event = {'title':tournament.name,
                     'start':tournament.date.strftime('%Y-%m-%dT%H:%M:%S'),
                     'url':reverse('chess:show_tournament', kwargs={'tournament_name':tournament.name})}

            if tournament.participants.filter(id=request.user.id).exists():
                event['backgroundColor'] = '#0f0'

            event_list.append(event)

        return JsonResponse({'events':event_list, 'time':time})

@login_required
def create_accountpage(request):
    if request.method == 'POST':
        accountpage = AccountPage.objects.get_or_create(user=request.user)[0]
        accountpage_form = AccountPageForm(request.POST, request.FILES, instance=accountpage)

        if accountpage_form.is_valid():
            accountpage_form.save(commit=True)
            return redirect(reverse('chess:accountpage'))
        else:
            print(accountpage_form.errors)
    else:
        accountpage_form = AccountPageForm()

    return render(request, 'chess/create_accountpage.html', {'accountpage_form': accountpage_form})

def show_accountpage(request):
    context_dict = {}

    try:
        accountpage = AccountPage.objects.get(user=request.user)
        context_dict['accountpage'] = accountpage
    except AccountPage.DoesNotExist:
        context_dict['accountpage'] = None

    return render(request, 'chess/accountpage.html', context=context_dict)

@login_required
def add_tournament(request):
    tournament_form = TournamentForm()

    if request.method == 'POST':
        tournament_form = TournamentForm(request.POST)
        if tournament_form.is_valid():
            tournament = tournament_form.save(commit=True)
            return redirect(reverse('chess:calendar')+'?tournament='+str(tournament.id))
        else:
            print(tournament_form.errors)
    return render(request, 'chess/add_tournament.html', {'tournament_form': tournament_form})

class DeleteTournamentView(View):
    @method_decorator(staff_member_required)
    def get(self, request):
        form = ChooseTournamentsForm()
        return render(request, 'chess/delete_tournament.html', {'form': form})

    @method_decorator(staff_member_required)
    def post(self, request):
        form = ChooseTournamentsForm(request.POST)
        if form.is_valid():
            for tournament in form.cleaned_data.get('tournaments'):
                print('success')
                Tournament.objects.get(id=tournament).delete()
            return redirect(reverse('chess:calendar'))
        else:
            print(form.errors)
        return render(request, 'chess/sign_up_for_tournaments.html', {'form': form})

class TournamentSignupView(View):
    @method_decorator(login_required)
    def get(self, request):
        form = ChooseTournamentsForm(user=request.user)
        return render(request, 'chess/sign_up_for_tournaments.html', {'form': form})

    @method_decorator(login_required)
    def post(self, request):
        form = ChooseTournamentsForm(request.POST, user=request.user)
        if form.is_valid():
            for tournament in form.cleaned_data.get('tournaments'):
                Tournament.objects.get(id=tournament).participants.add(request.user)
            return redirect(reverse('chess:calendar'))
        else:
            print(form.errors)
        return render(request, 'chess/sign_up_for_tournaments.html', {'form': form})


from itertools import accumulate

def dice_results(dices, n):
    a1, b1, a2, b2 = dices
    results = [0] * (n + 1)
    for d1 in range(a1, b1 + 1):
        for d2 in range(a2, b2 + 1):
            results[d1 + d2] += 1
    return results

if __name__ == '__main__':
    dices_g = list(map(int, input().split()))
    dices_e = list(map(int, input().split()))
    number_max = max([dices_g[1] + dices_g[3], dices_e[1] + dices_e[3]])
    results_g = dice_results(dices_g, number_max)
    results_e = dice_results(dices_e, number_max)

    wins_g = 0
    cumulative_e = list(accumulate(results_e))
    for result, frequency in enumerate(results_g):
        wins_g += frequency * cumulative_e[result - 1]

    wins_e = 0
    cumulative_g = list(accumulate(results_g))
    for result, frequency in enumerate(results_e):
        wins_e += frequency * cumulative_g[result - 1]

    if wins_g > wins_e:
        print('Gunnar')
    elif wins_g < wins_e:
        print('Emma')
    else:
        print('Tie')


import json
import os
from datetime import datetime, timedelta
import hashlib
import shutil
from logging import Logger
from glob import glob
import time

import pytz
import piexif
from pymediainfo import MediaInfo
import ffmpeg
from PIL import Image, UnidentifiedImageError
import pyheif

MEDIA_TYPES = {
    'image': ['.jpg', '.jpeg', '.png', '.heic'],
    'video': ['.mp4', '.mpg', '.mov', '.m4v', '.mts', '.mkv'],
}

class OrganizePictures:
    FILENAME_DATE_FORMAT = "%Y-%m-%d_%H'%M'%S"
    ENCODED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
    PIEXIF_DATE_FORMAT = '%Y:%m:%d %H:%M:%S'
    VIDEO_DATE_FORMATS = {
        "default": "%Z %Y-%m-%d %H:%M:%S",
        ".mkv": "%Z %Y-%m-%dT%H:%M:%SZ",
    }
    IMG_CONVERT_EXTS = ['.heic']
    IMG_CHANGE_EXTS = ['.jpeg']
    VID_CONVERT_EXTS = ['.mpg', '.mov', '.m4v', '.mts', '.mkv']
    PREFERRED_IMAGE_EXT = '.jpg'
    PREFERRED_VIDEO_EXT = '.mp4'

    # pylint: disable=too-many-arguments
    def __init__(
            self,
            logger: Logger,
            source_directory: str,
            destination_directory: str,
            extensions: list = None,
            media_type: str = None,
            dry_run: bool = False,
            cleanup: bool = False,
            sub_dirs: bool = True,
            verbose: bool = False,
    ):
        self.logger = logger
        self.source_dir = source_directory
        self.dest_dir = destination_directory
        self.media_type = media_type
        self.dry_run = dry_run
        self.cleanup = cleanup
        self.sub_dirs = sub_dirs
        self.verbose = verbose

        self.results = {"moved": 0, "duplicate": 0, "failed": 0, "deleted": 0}

        self.extensions = extensions
        if self.extensions is None:
            if media_type is not None:
                self.extensions = MEDIA_TYPES.get(media_type)
            else:
                self.extensions = []
                for exts in MEDIA_TYPES.values():
                    self.extensions += exts

    @staticmethod
    def _get_file_ext(file):
        _, ext = os.path.splitext(os.path.basename(file))
        return ext

    def _get_json_file(self, _file):
        """Get the json file name for the given file"""
        if "(" in _file and ")" in _file:
            ext = self._get_file_ext(_file)
            start = _file.find("(")
            end = _file.find(")")
            base_file = _file[:start]
            file_num = _file[start + 1:end]
            _file = f"{base_file}{ext}({file_num})"
        return f"{_file}.json"

    @staticmethod
    def _load_json_file(json_file: str):
        parsed_json = None
        if os.path.isfile(json_file):
            with open(json_file, encoding="utf-8") as user_file:
                parsed_json = json.load(user_file)
        return parsed_json

    @staticmethod
    def _md5(fname):
        hash_md5 = hashlib.md5()
        with open(fname, "rb") as file_handle:
            for chunk in iter(lambda: file_handle.read(4096), b""):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()

    def _get_files(self, path: str):
        files = []
        paths = glob(f"{path}/*")
        for file in paths:
            if os.path.isfile(file):
                ext = self._get_file_ext(file)
                ext_lower = ext.lower()
                if ext_lower in self.extensions:
                    add = True
                    if self.media_type is not None and ext_lower not in MEDIA_TYPES.get(self.media_type):
                        add = False
                    if add:
                        files.append(file)
            elif os.path.isdir(file):
                files += self._get_files(file)
        return sorted(files)

    def _get_date_taken(self, _file: str) -> datetime:
        date_time_obj = None
        ext = self._get_file_ext(_file).lower()
        if ext in MEDIA_TYPES.get('video'):
            media_info = MediaInfo.parse(_file)
            for track in media_info.tracks:
                if track.track_type in ['Video', 'General']:
                    if track.encoded_date is not None:
                        date_time_obj = datetime.strptime(
                            track.encoded_date, self.VIDEO_DATE_FORMATS.get(
                                ext,
                                self.VIDEO_DATE_FORMATS.get("default")
                            )
                        )
                        _fromtz = pytz.timezone(track.encoded_date[0:track.encoded_date.find(" ")])
                        _totz = pytz.timezone('US/Mountain')
                        date_time_obj = datetime.astimezone(date_time_obj.replace(tzinfo=_fromtz), _totz)
                        break
                    if track.recorded_date is not None:
                        date_time_obj = datetime.strptime(track.recorded_date, "%Y-%m-%d %H:%M:%S%z")
                        break
        elif ext in MEDIA_TYPES.get('image'):
            json_file = self._get_json_file(_file)
            if os.path.isfile(json_file):
                date_time_obj = datetime.fromtimestamp(
                    int(self._load_json_file(json_file).get('photoTakenTime', {}).get('timestamp')))
            else:
                try:
                    exif_dict = piexif.load(_file)
                    for tag, value in exif_dict['Exif'].items():
                        if "DateTimeDigitized" in piexif.TAGS['Exif'][tag]["name"]:
                            date_time_obj = datetime.strptime(value.decode(), self.PIEXIF_DATE_FORMAT)
                            break
                except piexif._exceptions.InvalidImageDataError:
                    self.logger.error(f'Unable to get exif data for file: {_file}')
        # if other dates are not found, use the file modified date
        if date_time_obj is None:
            date_time_obj = datetime.strptime(time.ctime(os.path.getmtime(_file)), '%c')

        return date_time_obj

    def _convert_video(self, _file: str, _new_file: str):
        converted = False
        if os.path.isfile(_new_file):
            self.logger.info(f"Skipping conversion of \"{_file}\" to \"{_new_file}\" as it already exists")
            converted = True
        self.logger.info(f"Converting \"{_file}\" to \"{_new_file}\"")
        stream = ffmpeg.input(_file)
        stream = ffmpeg.output(
            stream,
            _new_file,
            acodec="aac",
            vcodec="h264",
            map_metadata=0,
            metadata=f"comment=Converted {_file} to {_new_file}",
            loglevel="verbose" if self.verbose else "quiet"
        )
        _, err = ffmpeg.run(stream)
        if err is None:
            self.results['moved'] += 1
            self.logger.info(f"Successfully converted \"{_file}\" to \"{_new_file}\"")
            converted = True
        else:
            self.logger.error(f"Failed to convert \"{_file}\" to \"{_new_file}\"")
        return converted

    def _convert_image(self, source_file: str, dest_file: str):
        self.logger.debug(f"Converting file:\n\tSource: {source_file}\n\tDestination: {dest_file}")
        image_ext = self._get_file_ext(source_file).lower()
        method = "pillow"
        try:
            image = Image.open(source_file)
            image.convert('RGB').save(dest_file)
        except UnidentifiedImageError as pilexc:
            if image_ext != ".heic":
                return False
            method = "pyheif"
            self.logger.error(f"Failed first conversion attempt via {method}: {source_file}\n{pilexc}")
            heif_file = pyheif.read(source_file)
            image = Image.frombytes(
                heif_file.mode,
                heif_file.size,
                heif_file.data,
                "raw",
                heif_file.mode,
                heif_file.stride,
            )
            image.save(dest_file)
        except Exception as exc:
            self.logger.error(f"Failed second conversion attempt via {method}: {source_file}\n{exc}")
            return False
        self.results['moved'] += 1
        self.logger.debug(
            f"Successfully converted file via {method}:\n\tSource: {source_file}\n\tDestination: {dest_file}"
        )
        return True

    def _media_file_matches(self, source_file: str, dest_file: str):
        matches = False
        # assume source file exists, ensure dest file exists
        if os.path.isfile(dest_file):
            if self._md5(source_file) == self._md5(dest_file):
                self.results['duplicate'] += 1
                matches = True
                self.logger.debug(f"""Source file matches existing destination file:
                            Source: {source_file}
                            Destination: {dest_file}""")
            else:
                self.logger.debug(f"""Source file does not match existing destination file:
                            Source: {source_file}
                            Destination: {dest_file}""")
                source_file_ext = self._get_file_ext(source_file).lower()
                if source_file_ext in MEDIA_TYPES.get('video'):
                    self.logger.debug(f"Checking if video file has already been converted")
                    _media_info = MediaInfo.parse(dest_file)
                    for _track in _media_info.tracks:
                        if hasattr(_track, 'comment') and _track.comment is not None:
                            if "Converted" in _track.comment:
                                self.results['duplicate'] += 1
                                self.logger.info(f"Video file already converted: {source_file}")
                                self.logger.debug(f"\t{_track.comment}")
                                matches = True
                                break
        return matches

    @staticmethod
    def _find_image_animation(_file: str, _ext: str):
        for ext in MEDIA_TYPES.get('video'):
            image_animation = _file.replace(_ext, ext)
            if os.path.isfile(image_animation):
                return image_animation
            image_animation = _file.replace(_ext, ext.upper())
            if os.path.isfile(image_animation):
                return image_animation
        return None

    def _update_file_date(self, _file, _date: datetime):
        try:
            new_date = _date.strftime(self.PIEXIF_DATE_FORMAT)
            image = Image.open(_file)

            exif_dict = piexif.load(_file)
            if exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] != new_date:
                self.logger.debug(f"Updating date digitized for {_file}")
                exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] = new_date
                exif_bytes = piexif.dump(exif_dict)
                image.save(_file, exif=exif_bytes)
            else:
                self.logger.debug(f"Exif date already matches for {_file}")
        except Exception as exc:
            self.logger.error(f"Failed to update file date for {_file}: {exc}")

    def _get_new_fileinfo(self, _file: str, _date: datetime):
        _ext = self._get_file_ext(_file)
        _ext_lower = _ext.lower()
        _year = _date.strftime("%Y")
        _month = _date.strftime("%b")
        _dir = self.dest_dir
        if self.sub_dirs:
            _dir += f"/{_year}/{_month}"

        _filename = f"{_date.strftime(self.FILENAME_DATE_FORMAT)}{_ext_lower}"
        _new_file_info = {
            'ext': _ext_lower,
            'dir': _dir,
            'filename': _filename,
            'path': f"{_dir}/{_filename}",
            'date_encoded': _date
        }
        json_file = self._get_json_file(_file)
        if os.path.isfile(json_file):
            _new_file_info['json_file'] = json_file
            _new_file_info['new_json_file'] = f"{_dir}/{self._get_json_file(_filename)}"

        image_animation = self._find_image_animation(_file, _ext)
        if self.media_type == 'image' and image_animation is not None:
            _new_file_info['animation_source'] = image_animation
            _new_file_info['animation_dest'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_VIDEO_EXT)}"

        if _ext_lower in self.IMG_CONVERT_EXTS:
            _new_file_info['convert_path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_IMAGE_EXT)}"
        if _ext_lower in self.VID_CONVERT_EXTS:
            _new_file_info['path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_VIDEO_EXT)}"
        if _ext_lower in self.IMG_CHANGE_EXTS:
            _new_file_info['path'] = f"{_dir}/{_filename.replace(_ext_lower, self.PREFERRED_IMAGE_EXT)}"

        if not os.path.isdir(_new_file_info['dir']):
            self.logger.debug(f"Destination path does not exist, creating: {_new_file_info['dir']}")
            os.makedirs(_new_file_info['dir'])
        if not os.path.exists(_new_file_info['path']):
            return _new_file_info

        self.logger.debug(f"Destination file already exists: {_new_file_info['path']}")
        if self._media_file_matches(_file, _new_file_info['path']):
            _new_file_info['duplicate'] = True
            return _new_file_info
        else:
            # increment 1 second and try again
            new_dt = _date + timedelta(seconds=1)
            # Need to work out issues with this before enabling
            # self._update_file_date(_file, new_dt)
            return self._get_new_fileinfo(_file, new_dt)

    def run(self):
        files = self._get_files(self.source_dir)
        for index, media_file in enumerate(files, start=1):
            self.logger.info(
                f"Processing file {index} / {len(files)}:\n\t{media_file}"
            )
            cleanup_files = []
            date_taken = self._get_date_taken(media_file)
            if date_taken is not None:
                new_file_info = self._get_new_fileinfo(media_file, date_taken)
                if not new_file_info.get('duplicate'):
                    if new_file_info['ext'] in MEDIA_TYPES.get('video'):
                        if self._convert_video(media_file, new_file_info['path']):
                            cleanup_files.append(media_file)
                    else:
                        try:
                            self.logger.info(
                                f"Moving file:\n\tSource: {media_file}\n\tDestination: {new_file_info['path']}"
                            )
                            if new_file_info.get('convert_path') is not None:
                                if not self._convert_image(media_file, new_file_info['convert_path']):
                                    self.logger.error(f"Failed to convert image: {media_file}")
                                    continue
                            # Need to work out issues with this before enabling
                            # self._update_file_date(media_file, new_file_info['date_encoded'])
                            shutil.copyfile(media_file, new_file_info['path'])
                            self.results['moved'] += 1
                            cleanup_files.append(media_file)
                            if "animation_source" in new_file_info:
                                if self._media_file_matches(
                                        new_file_info['animation_source'],
                                        new_file_info['animation_dest']
                                ) or self._convert_video(
                                        new_file_info['animation_source'],
                                        new_file_info['animation_dest']
                                ):
                                    cleanup_files.append(new_file_info['animation_source'])
                        except shutil.Error as exc:
                            self.results['failed'] += 1
                            self.logger.error(f"Failed to move file: {media_file}\n{exc}")

                    if new_file_info.get('json_file') is not None:
                        self.logger.info(
                            f"Moving file:\n\tSource: {new_file_info.get('json_file')}\n\t"
                            f"Destination: {new_file_info.get('new_json_file')}")
                        shutil.copyfile(new_file_info['json_file'], new_file_info['new_json_file'])
                        self.results['moved'] += 1
                        cleanup_files.append(new_file_info['json_file'])
                else:
                    # file is already moved
                    self.logger.info(f"File already moved: {media_file}")
                    cleanup_files.append(media_file)
                    if new_file_info.get('json_file'):
                        cleanup_files.append(new_file_info.get('json_file'))
                    if new_file_info.get('animation_source'):
                        cleanup_files.append(new_file_info.get('animation_source'))

            if cleanup_files and self.cleanup:
                for cleanup_file in cleanup_files:
                    self.results['deleted'] += 1
                    self.logger.info(f"Deleting file: {cleanup_file}")
                    os.remove(cleanup_file)

        return self.results


import re

from django import forms
from django.contrib import admin
from django.contrib.flatpages.admin import FlatPageAdmin
from django.contrib.flatpages.models import FlatPage
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from django.db.models import Count, Sum
from django.db.models import F
from django.utils.html import mark_safe
from django.urls import reverse

from filebrowser.settings import ADMIN_VERSIONS, VERSIONS
from letters.admin import LetterInline
from payment.admin import CommissionInline, InvoiceInline
from republisher.admin import RepublisherInline
from socialmedia.admin import TweetInline
from socialmedia.common import SCHEDULE_RESULTS

from . import models, utils

# Used to select sizes of images
IMAGE_SIZE_CHOICES = [(item, VERSIONS[item]['verbose_name'],)
                      for item in ADMIN_VERSIONS]
IMAGE_SIZE_CHOICES.append(('LEAVE', 'LEAVE',))

# Regular expression to replace TinyMCE's pesky insertion of width
# and height attributes into images.

blankpara_regex = re.compile(r'
]*?>\s*?
|
]*?>\s*? \s*?
')

img_regex = re.compile(r'()')

figure_regex = re.compile(r'(
)(.*?)()(.*?)(
)(.*?)\r\n(
(.*?)(
)')

class CorrectionInline(admin.StackedInline):
    model = models.Correction
    extra = 0

class ArticleForm(forms.ModelForm):
    summary_image_size = forms.ChoiceField(choices=IMAGE_SIZE_CHOICES,
                                           initial="medium")
    # primary_image_size = forms.ChoiceField(choices=IMAGE_SIZE_CHOICES,
    #                                       initial="extra_large")

    '''Remove height and width from TinyMCE image insertions.
    '''

    def clean_main_topic(self):
        if self.cleaned_data["main_topic"] == "(None)":
            self.cleaned_data["main_topic"] = None
        return self.cleaned_data["main_topic"]

    def clean(self, *args, **kwargs):

        if self.instance.pk:
            if self.instance.version > self.cleaned_data["version"]:
                raise ValidationError(utils.get_edit_lock_msg(
                    self.instance.user))

        if self.cleaned_data["main_topic"]:
            self.cleaned_data["topics"] = self.cleaned_data["topics"] | \
                    models.Topic.objects.filter(
                        name=self.cleaned_data["main_topic"])

        # Transform the body to fix Wysiwyg editor limitations

        if self.cleaned_data["use_editor"]:
            body = self.cleaned_data["body"]
            self.cleaned_data["body"] = utils.replaceBadHtmlWithGood(body)

        super(ArticleForm, self).clean(*args, **kwargs)

class ArticleAdmin(admin.ModelAdmin):
    form = ArticleForm
    list_display = ('new_edit', 'old_edit', 'created',
                    'modified', 'published',
                    'is_published', 'cached_byline_no_links',
                    'category', )
    prepopulated_fields = {"slug": ("title", )}
    search_fields = ['title', 'cached_byline_no_links', ]
    date_hierarchy = 'modified'
    ordering = ['-modified', ]
    list_filter = ['published', 'category', 'region', 'topics']
    raw_id_fields = ('author_01', 'author_02', 'author_03',
                     'author_04', 'author_05',
                     'topics', 'main_topic', )
    autocomplete_lookup_fields = {
        'fk': ['author_01', 'author_02', 'author_03',
               'author_04', 'author_05', 'main_topic', ],
        'm2m': ['topics', ]
    }

    readonly_fields = ('cached_byline_no_links',
                       'cached_summary_text',
                       'user', 'modified', )

    fieldsets = (
        ('Identifying Information', {
            'classes': ('wide',),
            'fields': ('title', 'subtitle',
                       'cached_byline_no_links',
                       'author_01',)
        }),
        ('Additional authors', {
            'classes': ('wide grp-collapse grp-closed',),
            'fields': ('author_02', 'author_03', 'author_04',
                       'author_05',
                       ('byline', 'byline_style', ),
                       'editor_feedback')
        }),
        ('Content', {
            'classes': ('wide',),
            'fields': ('body', )
        }),
        ('Publish', {
            'fields': ('category', 'topics', 'main_topic',
                       'region', ('recommended',  'undistracted_layout', ),
                       'slug', 'published', ),
        }),
        ('Summary Image and text', {
            'classes': ('grp-collapse grp-closed',),
            'fields': (
                'summary_image', 'summary_image_size',
                'summary_image_alt', 'cached_summary_text',
                'summary_text', 'summary_template',),
        }),
        ('Advanced', {
            'classes': ('grp-collapse grp-closed',),
            'fields': ('copyright', 'include_in_rss',
                       ('letters_on', 'comments_on', 'collapse_comments', ),
                       'stickiness', 'exclude_from_list_views',
                       'use_editor',
                       ('template', 'template_process', ),
                       'additional_head_scripts', 'additional_body_scripts',
                       'activate_slideshow',
                       ('last_tweeted', 'notified_authors',),
                       ('author_payment', 'override_commissions_system',
                        'commissions_processed',),
                       ('suppress_ads', 'promote_article',
                        'encourage_republish',),
                       ('user', 'modified', 'version'),
                       ('secret_link', 'secret_link_view',))
        })
    )

    inlines = [
        TweetInline, RepublisherInline, CorrectionInline,
        LetterInline, # CommissionInline,
    ]

    @admin.display(description='new_edit')
    def new_edit(self, obj):
        return mark_safe('%s' %
                         (obj.get_absolute_url(),
                          obj.title))
    new_edit.allow_tags = True
    new_edit.short_description = "New editor"

    @admin.display(description='old_edit')
    def old_edit(self, obj):
        return mark_safe(
            '%s' %
            (reverse('admin:newsroom_article_change',
                     args=[obj.pk]),
            'Old edit'))
    old_edit.allow_tags = True
    old_edit.short_description = "Old editor"

    def __init__(self,*args,**kwargs):
        super().__init__(*args, **kwargs)
        # self.list_display_links = (None, )

    def changelist_view(self, request, extra_context=None):
        extra_context = extra_context or {}
        return super(ArticleAdmin, self).\
            changelist_view(request, extra_context=extra_context)

    def save_model(self, request, obj, form, change):
        obj.user = request.user
        obj.save()

    class Media:
        css = {'all': ('/static/newsroom/css/admin_enhance.css', )}
        js = [
            '//cdn.ckeditor.com/4.14.0/standard-all/ckeditor.js',
            '/static/newsroom/js/ck_styles.js?v=20220203',
            '/static/newsroom/js/ck_init_admin.js?v=20220203',
            '/static/newsroom/js/statistics.js?v=20220203',
            '/static/newsroom/js/admin_enhance.js?v=20220203',
            '/static/socialmedia/js/tweets.js?v=20220203',
        ]

class CategoryAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name", )}

class RegionAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name", )}

class TopicAdmin(admin.ModelAdmin):
    search_fields = ['name', 'slug', ]
    list_display = ('name', 'num_articles', 'num_qanda' )
    prepopulated_fields = {"slug": ("name", )}

    def get_queryset(self, request):
        queryset = super().get_queryset(request)
        queryset = queryset.annotate(
            num_articles=Count('article', distinct=True),
            num_qanda=Count('qanda', distinct=True)
        )
        return queryset

    def num_articles(self, obj):
        return obj.num_articles

    num_articles.admin_order_field = 'num_articles'

    def num_qanda(self, obj):
        return obj.num_qanda

    num_qanda.admin_order_field = 'num_qanda'

    def total(self, obj):
        return obj.total

    total.admin_order_field = 'total'

admin.site.register(models.Article, ArticleAdmin)
admin.site.register(models.UserEdit)
admin.site.register(models.Category, CategoryAdmin)
admin.site.register(models.Region, RegionAdmin)
admin.site.register(models.Topic, TopicAdmin)

class AuthorForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(AuthorForm, self).__init__(*args, **kwargs)
        self.fields['email'].required = True

class AuthorAdmin(admin.ModelAdmin):
    form = AuthorForm
    list_display = ('last_name', 'first_names', 'created',
                    'email', 'freelancer', 'level', )
    list_editable = ('email', 'freelancer','level', )
    search_fields = ['last_name', 'first_names', ]
    # inlines = [
    #    InvoiceInline,
    # ]
    raw_id_fields = ('user', )
    autocomplete_lookup_fields = {
        'fk': ['user',],
    }

class CorrectionAdmin(admin.ModelAdmin):
    list_display = ('article', 'update_type', 'created', 'modified',)
    list_editable = ('update_type', )
    list_filter = ['update_type', ]
    ordering = ['-modified', ]
    search_fields = ['text', 'article__title', ]
    raw_id_fields = ('article',)
    autocomplete_lookup_fields = {
        'fk': ['article',],
    }

class WetellAdmin(admin.ModelAdmin):
    list_display = ['pk', 'service', 'published',
                    'created', 'modified']
    ordering = ['-modified']
    search_fields = ['data']

admin.site.register(models.Author, AuthorAdmin)
admin.site.register(models.MostPopular)
admin.site.register(models.Correction, CorrectionAdmin)
admin.site.register(models.WetellBulletin, WetellAdmin)

# Define a new FlatPageAdmin
class FlatPageAdmin(FlatPageAdmin):
    fieldsets = (
        (None, {'fields': ('url', 'title', 'content', 'sites')}),
        (_('Advanced options'), {
            'classes': ('collapse', ),
            'fields': (
                'registration_required',
                'template_name',
            ),
        }),
    )

    class Media:
        css = {'all': ('/static/newsroom/css/admin_enhance.css', )}
        js = [
            '//cdn.ckeditor.com/4.14.0/standard-all/ckeditor.js',
            '/static/newsroom/js/ck_styles.js',
            '/static/newsroom/js/ck_init_admin.js',
        ]

# Re-register FlatPageAdmin
admin.site.unregister(FlatPage)
admin.site.register(FlatPage, FlatPageAdmin)


import netCDF4 as nc
import numpy as np
f_w = nc.Dataset('Y_distribution.nc', 'w', format='NETCDF4')

# define dimensions
longs = f_w.createDimension('longitude', size=len(lon))
lats = f_w.createDimension('latitude', size=len(lat))

# create variables
lat_w = f_w.createVariable('lat', np.float32, 'latitude')
lon_w = f_w.createVariable('lon', np.float32, 'longitude')
output_w = f_w.createVariable('2d_data', np.float32, ('latitude', 'longitude'))

# lon/lat/output are np.array, 1d, 1d, 2d
lon_w[:] = lon
lat_w[:] = lat
output_w[:] = output
f_w.close()


import sys
import json
import pysam
import truvari
from collections import Counter
from truvari.annotations.lcr import sequence_entropy
def filter_TR_entry(entry):
    """
    Returns True if this should be filtered
    """
    if "TRFdiff" not in entry.info or entry.info["TRFdiff"] == None:
        return True
    if entry.info["TRFperiod"] == 1:
        return True
    if len(entry.alts[0]) > len(entry.ref):
        seq = entry.alts[0][1:]
    else:
        seq = entry.ref[1:]
    if sequence_entropy(seq) < 0.25:
        return True
    return False

if __name__ == '__main__':
    v = pysam.VariantFile(sys.argv[1])
    #bed = sys.argv[2]
    header = v.header.copy()
    header.add_line('##FILTER=<ID=NONTR,Description="NonTR variant">')
    o = pysam.VariantFile('/dev/stdout', 'w', header=header)
    for entry in v:
        if truvari.entry_size(entry) < 5:
            o.write(entry)
            continue
        entry.translate(header)
        if filter_TR_entry(entry):
            entry.filter.clear()
            entry.filter.add('NONTR')
        o.write(entry)


from PIL import Image, ImageTk, ImageDraw

class Point:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __str__(self):
        return '(' + str(self.x) + ', ' + str(self.y) + ')'

def cropping(startPoint, endPoint, bc, img):
    pixelList = []

    x0 = startPoint[0]
    y0 = startPoint[1]
    x1 = endPoint[0]
    y1 = endPoint[1]

    if y0 > y1:
        y0, y1 = y1, y0
    if x0 > x1:
        x0, x1 = x1, x0

    for j in range(y0, y1):
        for i in range(x0, x1):
            if img.getpixel((i, j)) != bc:
                color = img.getpixel((i, j))
                pixelObj = ((i, j), color)
                pixelList.append(pixelObj)
                img.putpixel((i, j), bc)

    return pixelList

def curve(p0, p1, p2, color, img):
    t = 0
    while t < 1:
        x = int(p0[0] * (1 - t) ** 2 + 2 * (1 - t) * t * p1[0] + p2[0] * t ** 2)
        y = int(p0[1] * (1 - t) ** 2 + 2 * (1 - t) * t * p1[1] + p2[1] * t ** 2)
        img.putpixel((x, y), color)
        t = t + 0.001
    curveImg = ImageTk.PhotoImage(img)
    return curveImg

def draw_eclipse(centerPoint, x, y, color, img):
    img.putpixel((centerPoint[0] + x, centerPoint[1] + y), color);
    img.putpixel((centerPoint[0] - x, centerPoint[1] + y), color);
    img.putpixel((centerPoint[0] + x, centerPoint[1] - y), color);
    img.putpixel((centerPoint[0] - x, centerPoint[1] - y), color);

def eclipse_mid_point(centerPoint, rx, ry, color, img):
    rxSq = rx ** 2
    rySq = ry ** 2
    x = 0
    y = ry
    px = 0
    py = 2 * rxSq * y
    draw_eclipse(centerPoint, x, y, color, img)
    p = rySq - (rxSq * ry) + (0.25 * rxSq)
    while px < py:
        x = x + 1
        px = px + 2 * rySq
        if p < 0:
            p = p + rySq + px
        else:
            y = y - 1
            py = py - 2 * rxSq
            p = p + rySq + px - py
        draw_eclipse(centerPoint, x, y, color, img)

    p = rySq * (x + 0.5) * (x + 0.5) + rxSq * (y - 1) * (y - 1) - rxSq * rySq
    while y > 0:
        y = y - 1
        py = py - 2 * rxSq;
        if p > 0:
            p = p + rxSq - py;
        else:
            x = x + 1
            px = px + 2 * rySq;
            p = p + rxSq - py + px;
        draw_eclipse(centerPoint, x, y, color, img);

    eclipseImg = ImageTk.PhotoImage(img)
    return eclipseImg

def move_transition(pixelList, newPoint, bc, img):
    deltaX = newPoint[0] - pixelList[0][0][0]
    deltaY = newPoint[1] - pixelList[0][0][1]

    for i in range(0, len(pixelList) - 1):
        pixel = pixelList[i]
        img.putpixel((pixel[0][0] + deltaX, pixel[0][1] + deltaY), pixel[1])

    transitImg = ImageTk.PhotoImage(img)
    return transitImg

def pencil(previousPoint, pointNow, color, img):
    draw = ImageDraw.Draw(img)
    draw.line((previousPoint, pointNow), color)

    pencilImg = ImageTk.PhotoImage(img)
    return pencilImg

def eraser(previousPoint, pointNow, color, img):
    draw = ImageDraw.Draw(img)
    draw.line((previousPoint, pointNow), fill=color, width=10)

    eraserImg = ImageTk.PhotoImage(img)
    return eraserImg

def triangle(startPoint, endPoint, color, img, defaultState):
    a = (endPoint[1] - startPoint[1]) / float(2)
    b = (endPoint[0] - startPoint[0]) / float(2)

    A = (int(startPoint[0]), int(endPoint[1]))
    B = (int(endPoint[0]), int(endPoint[1]))
    C = (int(startPoint[0] + b), int(startPoint[1]))

    draw = ImageDraw.Draw(img)

    draw.line((A, B), color)
    draw.line((B, C), color)
    draw.line((C, A), color)

    triangleImg = ImageTk.PhotoImage(img)
    return triangleImg

def rectangle(pointA, pointB, color, img, defaultState):
    draw = ImageDraw.Draw(img)
    if defaultState:  # draw square
        edge = abs(pointB[0] - pointA[0])
        if pointB[1] > pointA[1]:
            pointB = (pointB[0], edge + pointA[1])
        else:
            pointB = (pointB[0], abs(pointA[1] - edge))
        draw.rectangle([pointA, pointB], None, color)
    else:
        draw.rectangle([pointA, pointB], None, color)

    rectangleImg = ImageTk.PhotoImage(img)
    return rectangleImg

def fillColor(img, center, bc, newColor, paperWidth, paperHeight):
    print('center ', center)

    oldColor = img.getpixel(center)
    if oldColor == newColor:
        return

    listSeed = []
    listSeed.append(center)
    while listSeed:
        seed = listSeed.pop(0)
        try:
            seedColor = img.getpixel(seed)
        except IndexError:
            seedColor = None
        if seedColor == oldColor:
            img.putpixel(seed, newColor)
            x, y = seed[0], seed[1]
            listSeed.append((x + 1, y))
            listSeed.append((x - 1, y))
            listSeed.append((x, y + 1))
            listSeed.append((x, y - 1))

    filledImg = ImageTk.PhotoImage(img)
    return filledImg


import argparse

import torch.optim.lr_scheduler
import torchvision.transforms as transforms
import wandb

from torch.optim import SGD
from torch.utils.data import Subset
from tqdm import tqdm
import torch.nn as nn

from datasets.dataloader_animal10n import animal_dataset
from torchvision.models import vgg19_bn
from utils import *

parser = argparse.ArgumentParser('Train with ANIMAL-10N dataset')
parser.add_argument('--dataset_path', default='~/ANIMAL-10N', help='dataset path')

# model settings
parser.add_argument('--lambda_fc', default=1.0, type=float, help='weight of feature consistency loss (default: 1.0)')
parser.add_argument('--theta_s', default=1.0, type=float, help='threshold for voted correct samples (default: 1.0)')
parser.add_argument('--theta_r', default=0.95, type=float, help='threshold for relabel samples (default: 0.95)')
parser.add_argument('--k', default=200, type=int, help='neighbors for soft-voting (default: 200)')

# train settings
parser.add_argument('--epochs', default=150, type=int, metavar='N', help='number of total epochs to run (default: 200)')
parser.add_argument('--batch_size', default=128, type=int, help='mini-batch size (default: 128)')
parser.add_argument('--lr', default=0.02, type=float, help='initial learning rate (default: 0.02)')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M', help='momentum of SGD solver (default: 0.9)')
parser.add_argument('--weight_decay', default=5e-4, type=float, help='weight decay (default: 5e-4)')
parser.add_argument('--seed', default=3047, type=int, help='seed for initializing training. (default: 3047)')
parser.add_argument('--gpuid', default='0', type=str, help='Selected GPU (default: "0")')
parser.add_argument('--entity', type=str, help='Wandb user entity')
parser.add_argument('--run_path', type=str, help='run path containing all results')

def train(labeled_trainloader, modified_label, all_trainloader, encoder, classifier, proj_head, pred_head, optimizer, epoch, args):
    encoder.train()
    classifier.train()
    proj_head.train()
    pred_head.train()
    xlosses = AverageMeter('xloss')
    ulosses = AverageMeter('uloss')
    labeled_train_iter = iter(labeled_trainloader)
    all_bar = tqdm(all_trainloader)
    for batch_idx, ([inputs_u1, inputs_u2],  _, _) in enumerate(all_bar):
        try:
            [inputs_x1, inputs_x2], labels_x, index = labeled_train_iter.next()
        except:
            labeled_train_iter = iter(labeled_trainloader)
            [inputs_x1, inputs_x2], labels_x, index = labeled_train_iter.next()

        # cross-entropy training with mixup
        batch_size = inputs_x1.size(0)
        inputs_x1, inputs_x2 = inputs_x1.cuda(), inputs_x2.cuda()
        labels_x = modified_label[index]
        targets_x = torch.zeros(batch_size, args.num_classes, device=inputs_x1.device).scatter_(1, labels_x.view(-1, 1), 1)
        l = np.random.beta(0.5, 0.5)
        l = max(l, 1 - l)
        all_inputs_x = torch.cat([inputs_x1, inputs_x2], dim=0)
        all_targets_x = torch.cat([targets_x, targets_x], dim=0)
        idx = torch.randperm(all_inputs_x.size()[0])
        input_a, input_b = all_inputs_x, all_inputs_x[idx]
        target_a, target_b = all_targets_x, all_targets_x[idx]
        mixed_input = l * input_a + (1 - l) * input_b
        mixed_target = l * target_a + (1 - l) * target_b

        logits = classifier(encoder(mixed_input))
        Lce = -torch.mean(torch.sum(F.log_softmax(logits, dim=1) * mixed_target, dim=1))

        # optional feature-consistency
        inputs_u1, inputs_u2 = inputs_u1.cuda(), inputs_u2.cuda()

        feats_u1 = encoder(inputs_u1)
        feats_u2 = encoder(inputs_u2)
        f, h = proj_head, pred_head

        z1, z2 = f(feats_u1), f(feats_u2)
        p1, p2 = h(z1), h(z2)
        Lfc = D(p2, z1)
        loss = Lce + args.lambda_fc * Lfc
        xlosses.update(Lce.item())
        ulosses.update(Lfc.item())
        all_bar.set_description(
            f'Train epoch {epoch} LR:{optimizer.param_groups[0]["lr"]} Labeled loss: {xlosses.avg:.4f} Unlabeled loss: {ulosses.avg:.4f}')

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    logger.log({'ce loss': xlosses.avg, 'fc loss': ulosses.avg})

def test(testloader, encoder, classifier, epoch):
    encoder.eval()
    classifier.eval()
    accuracy = AverageMeter('accuracy')
    data_bar = tqdm(testloader)
    with torch.no_grad():
        for i, (data, label, _) in enumerate(data_bar):
            data, label = data.cuda(), label.cuda()
            feat = encoder(data)
            res = classifier(feat)
            pred = torch.argmax(res, dim=1)
            acc = torch.sum(pred == label) / float(data.size(0))
            accuracy.update(acc.item(), data.size(0))
            data_bar.set_description(f'Test epoch {epoch}: Accuracy#{accuracy.avg:.4f}')
    logger.log({'acc': accuracy.avg})
    return accuracy.avg

def evaluate(dataloader, encoder, classifier, args, noisy_label):
    encoder.eval()
    classifier.eval()
    feature_bank = []
    prediction = []

    ################################### feature extraction ###################################
    with torch.no_grad():
        # generate feature bank
        for (data, target, index) in tqdm(dataloader, desc='Feature extracting'):
            data = data.cuda()
            feature = encoder(data)
            feature_bank.append(feature)
            res = classifier(feature)
            prediction.append(res)
        feature_bank = F.normalize(torch.cat(feature_bank, dim=0), dim=1)

        ################################### sample relabelling ###################################
        prediction_cls = torch.softmax(torch.cat(prediction, dim=0), dim=1)
        his_score, his_label = prediction_cls.max(1)

        print(f'Prediction track: mean: {his_score.mean()} max: {his_score.max()} min: {his_score.min()}')
        conf_id = torch.where(his_score > args.theta_r)[0]
        modified_label = torch.clone(noisy_label).detach()
        modified_label[conf_id] = his_label[conf_id]

        ################################### sample selection ###################################
        prediction_knn = weighted_knn(feature_bank, feature_bank, modified_label, args.num_classes, args.k, 10)  # temperature in weighted KNN
        vote_y = torch.gather(prediction_knn, 1, modified_label.view(-1, 1)).squeeze()
        vote_max = prediction_knn.max(dim=1)[0]
        right_score = vote_y / vote_max
        clean_id = torch.where(right_score >= args.theta_s)[0]
        noisy_id = torch.where(right_score < args.theta_s)[0]

    return clean_id, noisy_id, modified_label

def main():
    args = parser.parse_args()
    seed_everything(args.seed)
    if args.run_path is None:
        args.run_path = f'Dataset(animal10n_Model({args.theta_r}_{args.theta_s})'

    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpuid
    global logger
    logger = wandb.init(project='animal10n', entity=args.entity, name=args.run_path)
    logger.config.update(args)

    if not os.path.isdir(f'animal10n'):
        os.mkdir(f'animal10n')
    if not os.path.isdir(f'animal10n/{args.run_path}'):
        os.mkdir(f'animal10n/{args.run_path}')

    ############################# Dataset initialization ##############################################
    args.num_classes = 10
    args.image_size = 64

    # data loading
    weak_transform = transforms.Compose([
        transforms.RandomCrop(args.image_size, padding=8),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor()])
    none_transform = transforms.Compose([transforms.ToTensor()])  # no augmentation
    strong_transform = transforms.Compose([transforms.RandomCrop(args.image_size, padding=8),
                                           transforms.RandomHorizontalFlip(),
                                           RandAugment(),
                                           transforms.ToTensor()])

    # eval data served as soft-voting pool
    train_data = animal_dataset(root=args.dataset_path, transform=KCropsTransform(strong_transform, 2), mode='train')
    eval_data = animal_dataset(root=args.dataset_path, transform=weak_transform, mode='train')
    test_data = animal_dataset(root=args.dataset_path, transform=none_transform, mode='test')
    all_data = animal_dataset(root=args.dataset_path, transform=MixTransform(strong_transform, weak_transform, 1), mode='train')

    # noisy labels
    noisy_label = torch.tensor(eval_data.targets).cuda()

    test_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True)
    eval_loader = torch.utils.data.DataLoader(eval_data, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True)
    all_loader = torch.utils.data.DataLoader(all_data, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True)

    ################################ Model initialization ###########################################
    encoder = vgg19_bn(pretrained=False)
    encoder.classifier = nn.Sequential(
        nn.Linear(512 * 7 * 7, 4096),
        nn.ReLU(True),
        nn.Dropout(p=0.2),
        nn.Linear(4096, 4096),
        nn.ReLU(True),
        nn.Dropout(p=0.2),
    )

    def init_weights(m):
        if type(m) == nn.Linear:
            nn.init.normal_(m.weight, 0, 0.005)
            nn.init.constant_(m.bias, 0)

    encoder.classifier.apply(init_weights)
    classifier = nn.Linear(4096, args.num_classes)
    proj_head = torch.nn.Sequential(torch.nn.Linear(4096, 256),
                                    torch.nn.BatchNorm1d(256),
                                    torch.nn.ReLU(),
                                    torch.nn.Linear(256, 128))
    pred_head = torch.nn.Sequential(torch.nn.Linear(128, 256),
                                    torch.nn.BatchNorm1d(256),
                                    torch.nn.ReLU(),
                                    torch.nn.Linear(256, 128))
    classifier.apply(init_weights)

    encoder.cuda()
    classifier.cuda()
    proj_head.cuda()
    pred_head.cuda()

    #################################### Training initialization #######################################
    optimizer = SGD([{'params': encoder.parameters()}, {'params': classifier.parameters()}, {'params': proj_head.parameters()}, {'params': pred_head.parameters()}],
                    lr=args.lr, weight_decay=args.weight_decay, momentum=args.momentum)
    scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[50, 100], gamma=0.1)

    acc_logs = open(f'animal10n/{args.run_path}/acc.txt', 'w')
    save_config(args, f'animal10n/{args.run_path}')
    print('Train args: \n', args)
    best_acc = 0

    ################################ Training loop ###########################################
    for i in range(args.epochs):
        clean_id, noisy_id, modified_label = evaluate(eval_loader, encoder, classifier, args, noisy_label)

        clean_subset = Subset(train_data, clean_id.cpu())
        sampler = ClassBalancedSampler(labels=modified_label[clean_id], num_classes=args.num_classes)
        labeled_loader = torch.utils.data.DataLoader(clean_subset, batch_size=args.batch_size, sampler=sampler, num_workers=4, drop_last=True)

        train(labeled_loader, modified_label, all_loader, encoder, classifier, proj_head, pred_head, optimizer, i, args)

        cur_acc = test(test_loader, encoder, classifier, i)
        scheduler.step()
        if cur_acc > best_acc:
            best_acc = cur_acc
            save_checkpoint({
                'cur_epoch': i,
                'classifier': classifier.state_dict(),
                'encoder': encoder.state_dict(),
                'proj_head': proj_head.state_dict(),
                'pred_head': pred_head.state_dict(),
                'optimizer': optimizer.state_dict(),
            }, filename=f'animal10n/{args.run_path}/best_acc.pth.tar')
        acc_logs.write(f'Epoch [{i}/{args.epochs}]: Best accuracy@{best_acc}! Current accuracy@{cur_acc} \n')
        acc_logs.flush()
        print(f'Epoch [{i}/{args.epochs}]: Best accuracy@{best_acc}! Current accuracy@{cur_acc} \n')
    save_checkpoint({
        'cur_epoch': i,
        'classifier': classifier.state_dict(),
        'encoder': encoder.state_dict(),
        'proj_head': proj_head.state_dict(),
        'pred_head': pred_head.state_dict(),
        'optimizer': optimizer.state_dict(),
    }, filename=f'animal10n/{args.run_path}/last.pth.tar')

if __name__ == '__main__':
    main()


import os
import json
from dotenv import load_dotenv

load_dotenv()

FILEPATH = os.getenv('FILEPATH')

def readJsonFile(file):
     with open(file) as json_file:
        return json.load(json_file)

def calcTotal(itens):
    sum = 0
    for item in itens:
        sum += item['total']
    return sum

def main():
    print(calcTotal(readJsonFile(FILEPATH)))

if __name__ == '__main__':
    main()


import _data_read as dr
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

def power_law(name):
   eta,gamma,tau,t=dr.Data_Read(name)
   func1 = lambda x,kappa,n: kappa*np.power(x,n-1)
   func=np.vectorize(func1)
   Xdata=gamma
   Ydata=eta
   plt.figure( num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k' )
   plt.plot( Xdata, Ydata, 'o', label='rownanie potegowe' )
   popt, pcov = curve_fit(func, Xdata, Ydata, p0=[1,1])
   print('Standardowe odchylenie dopasowania fit (eta) = ',dr.stdev(Xdata,Ydata,func,popt))
   plt.plot( Xdata, func(Xdata,*popt), 'b',
             label='dopasowanie: kappa = %5.3e, n = %5.3e' % tuple( popt ) )
   plt.xlabel('gamma')
   plt.ylabel('eta')
   plt.legend()
   plt.show()

power_law("wyniki.xls")


# download the casecase file from the CV2 git hub :
# https://github.com/opencv/opencv/tree/master/data/haarcascades

import cv2

face_Cascade = cv2.CascadeClassifier('c:\demo\haarcascade_frontalface_default.xml')

# you can use any video to capure the faces
cap = cv2.VideoCapture('C:\Python Code\FacialDetection\people.mp4')

while True:
    ret,img = cap.read()

    # convert to gray
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # grab faces from the gray image

    faces = face_Cascade.detectMultiScale(gray,1.1,4)

    # draw a rectangle for each face (for each frame)
    for (x,y,w,h) in faces :
        cv2.rectangle(img,(x,y), (x+w,y+h),(255,0,0),3)

    # show each image
    cv2.imshow('img',img)

    # break if 'q' pressed

    if cv2.waitKey(1) & 0xff == ord('q'):
        break

cap.release()


import findspark
from pyspark import SparkContext
from pyspark.streaming import StreamingContext

# print(findspark.find())
findspark.init('/home/rachneet/spark-3.0.1-bin-hadoop2.7')
sc = SparkContext('local[2]', 'NetworkWordCount')
ssc = StreamingContext(sc, 1)  # get data at interval of 1 sec

lines = ssc.socketTextStream('localhost', 9999)
words = lines.flatMap(lambda line: line.split(' '))
pairs = words.map(lambda word: (word,1))
word_counts = pairs.reduceByKey(lambda num1, num2: num1+num2)
word_counts.pprint()
ssc.start()


###########
# IMPORTS #
###########

# STANDARD LIBRARY
import os
import sys

# THIRD PARTY DEPENDENCIES
from flask_restplus import Resource
from flask_restplus import Api
from flask import Flask

# INTERNAL DEPENDENCIES
sys.path.append(os.path.dirname(__file__))
import analyzer
import database

###########
# GLOBALS #
###########

# META FLAGS
__author__ = ['Cosmia', 'Bryce']
__version__ = '0.1'

# CONSTANTS

##################
# INITIALIZATION #
##################

# FLASK INITIALIZATION
app = Flask(__name__)
api = Api(app, title='Arrow API', version=__version__)

# DATABASE INITIALIZATION
db = database.Mongo('arrow', 'test')

#############
# FUNCTIONS #
#############

def http_response(code, message=None, **kwargs):
    response = {'code': code}
    if message:
        response['messsage'] = message
    if 200 <= code < 300:
        response['status'] = 'success'
    elif 300 <= code < 400:
        response['status'] = 'error'
    elif 500 <= code < 600:
        response['status'] = 'failure'
    response.update(kwargs)
    return response

#########
# USERS #
#########

users = api.namespace('Users', path='/user')

@users.route('/<string:username>')
class User(Resource):

    def get(self, username):
        return db.get_user(username)

    def put(self, username):
        db.put_user(username)
        return http_response(200)

############
# ANALYSIS #
############

analysis = api.namespace('Analysis', path='/analyze')

@analysis.route('/<string:sentence>')
class Analysis(Resource):

    def get(self, sentence):
        score = analyzer.get_sentiment_scores(sentence)
        return http_response(200, score=score)

if __name__ == '__main__':
    app.run(debug=True)


""" Unit tests for service functions of consumer app. """
#pylint: disable=C0103
from django.test import TestCase

from consumer.factories.consumer_factory import CONSUMER_FACTORY
from consumer.models import Consumer, ConsumerHistoryEvent, EmailSubscription
from consumer.service import (build_consumer_count_list,
    create_subscriber_for_consumer, get_consumer_instance_type, get_site_rep,
    process_consumer_opt_out, update_default_site_consumers,
    update_subscriber_of_consumer, qry_qualified_consumers)
from geolocation.models import USZip
from market.models import Site
from subscriber.models import MobilePhone, Subscriber

class FakeRequest(object):
    """ Fake request object. """
    pass

class TestService(TestCase):
    """ Test case for service functions of consumer app. """

    fixtures = ['test_consumer', 'test_sales_rep']

    @classmethod
    def setUpClass(cls):
        """ Build fake request for testing process_consumer_opt_out. """
        super(TestService, cls).setUpClass()
        cls.request = FakeRequest()
        cls.request.META = {'REMOTE_ADDR': '127.0.0.1'}

    def common_assertions_history(self, consumer):
        """ Assert ConsumerHistoryEvent log record. """
        log_history = ConsumerHistoryEvent.objects.latest('id')
        self.assertEqual(int(log_history.event_type), 1)
        self.assertEqual(log_history.ip, '127.0.0.1')
        self.assertEqual(log_history.consumer_id, consumer.id)
        data_dict = eval(log_history.data)
        return data_dict

    def test_update_default_site_cons(self):
        """ Assert consumers on site 1 having a zip code for site 2 are moved.
        """
        self.assertEqual(Consumer.objects.filter(site=1).count(), 3)
        update_default_site_consumers()
        self.assertEqual(Consumer.objects.filter(site=1).count(), 0)

    def test_get_site_rep_default(self):
        """ Assert that if site rep does not exist for this site, it will use
        default (site 1's).
        """
        site = Site.objects.get(id=2)
        rep = get_site_rep(site)
        self.assertEqual(rep.email_domain, '10Coupons.com')
        self.assertEqual(rep.consumer_id, 300)

    def test_get_site_rep_28(self):
        """ Assert that if site rep does exist, it will be selected. This test
        is for the North Jersey market.
        """
        site = Site.objects.get(id=28)
        rep = get_site_rep(site)
        self.assertEqual(rep.email_domain, '10NorthJerseyCoupons.com')
        self.assertEqual(rep.consumer_id, 301)

    def test_single_opt_out_current(self):
        """ Assert a single consumer opt out works with subscription_list
        param in current supported format.
        """
        subscription = [5]
        consumer = CONSUMER_FACTORY.create_consumer()
        consumer.email_subscription.add(subscription[0])
        self.assertTrue(consumer.email_subscription.get(id=1))
        self.assertTrue(consumer.email_subscription.get(id=subscription[0]))
        process_consumer_opt_out(self.request, consumer, subscription)
        self.assertTrue(consumer.email_subscription.get(id=1))
        with self.assertRaises(EmailSubscription.DoesNotExist):
            subscription = consumer.email_subscription.get(id=subscription[0])
        log_history_data = self.common_assertions_history(consumer)
        self.assertEqual(log_history_data['requested_list'], subscription)
        self.assertEqual(log_history_data['unsubscribed_list'], [5, 6])

    def test_flyer_opt_out_current(self):
        """ Assert consumer opt out of flye remail works with subscription_list
        param in current supported format.
        """
        subscription = [1]
        consumer = CONSUMER_FACTORY.create_consumer()
        consumer.email_subscription.add(2, 3, 4, 5, 6)
        self.assertTrue(consumer.email_subscription.get(id=1))
        process_consumer_opt_out(self.request, consumer, subscription)
        self.assertEqual(consumer.email_subscription.count(), 0)
        log_history_data = self.common_assertions_history(consumer)
        self.assertEqual(log_history_data['requested_list'], subscription)
        self.assertEqual(log_history_data['unsubscribed_list'], ['ALL'])

    def test_multi_opt_out_current(self):
        """ Assert consumer opt out works with multiple items in the
        subscription_list param in current supported format.
        """
        subscription = [2, 5]
        consumer = CONSUMER_FACTORY.create_consumer()
        consumer.email_subscription.add(2, 5, 6)
        self.assertTrue(consumer.email_subscription.get(id=1))
        self.assertTrue(consumer.email_subscription.get(id=2))
        self.assertTrue(consumer.email_subscription.get(id=5))
        self.assertTrue(consumer.email_subscription.get(id=6))
        process_consumer_opt_out(self.request, consumer, subscription)
        self.assertTrue(consumer.email_subscription.get(id=1))
        with self.assertRaises(EmailSubscription.DoesNotExist):
            subscription = consumer.email_subscription.get(id=2)
        with self.assertRaises(EmailSubscription.DoesNotExist):
            subscription = consumer.email_subscription.get(id=5)
        with self.assertRaises(EmailSubscription.DoesNotExist):
            subscription = consumer.email_subscription.get(id=6)
        log_history_data = self.common_assertions_history(consumer)
        self.assertEqual(log_history_data['requested_list'], subscription)
        self.assertEqual(log_history_data['unsubscribed_list'], [2, 5, 6])

    def test_flyer_opt_out_deprecated(self):
        """ Assert consumer opt out of flye remail works with subscription_list
        param in current supported format.
        """
        subscription = '01343'
        consumer = CONSUMER_FACTORY.create_consumer()
        self.assertTrue(consumer.email_subscription.get(id=1))
        consumer.email_subscription.add(2)
        process_consumer_opt_out(self.request, consumer, subscription)
        self.assertEqual(consumer.email_subscription.count(), 0)
        log_history_data = self.common_assertions_history(consumer)
        self.assertEqual(log_history_data['requested_list'], [1])
        self.assertEqual(log_history_data['unsubscribed_list'], ['ALL'])

    def test_opt_out_deprecated(self):
        """ Assert a single consumer opt out works with subscription_list
        param in deprecated format (as 5 character string).
        """
        subscription = '03343'
        consumer = CONSUMER_FACTORY.create_consumer()
        consumer.email_subscription.add(3)
        self.assertTrue(consumer.email_subscription.get(id=1))
        self.assertTrue(consumer.email_subscription.get(id=3))
        process_consumer_opt_out(self.request, consumer, subscription)
        self.assertTrue(consumer.email_subscription.get(id=1))
        with self.assertRaises(EmailSubscription.DoesNotExist):
            subscription = consumer.email_subscription.get(id=3)
        log_history_data = self.common_assertions_history(consumer)
        self.assertEqual(log_history_data['requested_list'], [3])
        self.assertEqual(log_history_data['unsubscribed_list'], [3])

class TestConsumerInstance(TestCase):
    """ Test identification of Consumer instances. """
    fixtures = ['test_consumer', 'test_advertiser', 'test_ad_rep',
        'test_media_partner']

    def test_unidentifiable(self):
        """ Test when no Consumer found with email. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('unidentifiable@10coupons.com')
        self.assertEqual(user_type, None)
        self.assertEqual(is_ad_rep, False)

    def test_identify_consumer(self):
        """ Test when only Consumer instance. """
        user_type, is_ad_rep = get_consumer_instance_type('101@example.com')
        self.assertEqual(user_type, 'consumer')
        self.assertEqual(is_ad_rep, False)

    def test_identify_advertiser(self):
        """ Test when Advertiser instance. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('113_user@example.com')
        self.assertEqual(user_type, 'advertiser')
        self.assertEqual(is_ad_rep, False)

    def test_identify_affiliate(self):
        """ Test when Affiliate instance. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('test_affiliate@example.com')
        self.assertEqual(user_type, 'affiliate_partner')
        self.assertEqual(is_ad_rep, False)

    def test_identify_media_partner(self):
        """ Test when MediaPartner instance. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('test_media_group@example.com')
        self.assertEqual(user_type, 'media_group_partner')
        self.assertEqual(is_ad_rep, False)

    def test_identify_consumer_ad_rep(self):
        """ Test when AdRep Consumer instance. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('test-ad-rep@example.com')
        self.assertEqual(user_type, 'consumer')
        self.assertEqual(is_ad_rep, True)

    def test_identify_adv_ad_rep(self):
        """ Test when AdRep Advertiser instance. """
        user_type, is_ad_rep = \
            get_consumer_instance_type('test-ad-rep2@example.com')
        self.assertEqual(user_type, 'advertiser')
        self.assertEqual(is_ad_rep, True)

class TestConsumerSiteCountList(TestCase):
    """ Test case for service function build_consumer_count_list. """
    fixtures = ['test_advertiser', 'test_consumer', 'test_coupon',
        'test_coupon_views', 'test_flyer']

    def setUp(self):
        """ Build comparison of current before adding a consumer. """
        super(TestConsumerSiteCountList, self).setUp()
        self.hv_list = build_consumer_count_list(2)
        self.hv_count = self.hv_list[0]['county_count']
        self.hv_zip = self.hv_list[0]['cities'][0]['zips'][0]['zip']
        self.hv_site = Site.objects.get(id=2)
        self.consumer = Consumer.objects.create_consumer(
            username='illegit_consumer@site.com',
            email='illegit_consumer@site.com',
            consumer_zip_postal=self.hv_zip,
            site = self.hv_site
            )
        self.consumer.is_email_verified = True
        self.consumer.save()

    def test_orange_county_count(self):
        """ Assert the consumer count site list is built accurately. """
        oc_count = Consumer.objects.filter(site__id=2, is_email_verified=True,
            is_emailable=True,
            email_subscription=1, consumer_zip_postal__in=
            USZip.objects.filter(us_county__name='Orange'
            ).values_list('code', flat=True)).count()
        test_list = build_consumer_count_list(2)
        # Only one county is currently configured for site 2 (Orange).
        self.assertEqual(oc_count, test_list[0]['county_count'])
        self.assertEqual('Orange', test_list[0]['county'])
        self.assertEqual('12550', test_list[0]['cities'][0]['zips'][0]['zip'])
        self.assertEqual(oc_count,
            test_list[0]['cities'][0]['zips'][0]['zip_count'])

    def test_valid_count_site(self):
        """ Assert the consumers included in these counts are on the right site.
        """
        site = Site.objects.get(id=3)
        self.consumer.site = site
        self.consumer.save()
        test_list1 = build_consumer_count_list(2)
        self.assertEqual(self.hv_list[0]['county_count'],
            test_list1[0]['county_count'])
        self.consumer.site = self.hv_site
        self.consumer.save()
        test_list2 = build_consumer_count_list(2)
        self.assertEqual(self.hv_list[0]['county_count'] + 1,
            test_list2[0]['county_count'])

    def test_valid_count_subscription(self):
        """ Assert the consumers included in these counts have an email
        subscription.
        """
        self.consumer.email_subscription = []
        self.consumer.save()
        test_list6 = build_consumer_count_list(2)
        self.assertEqual(self.hv_list[0]['county_count'],
            test_list6[0]['county_count'])
        self.consumer.email_subscription = [1]
        self.consumer.save()
        test_list7 = build_consumer_count_list(2)
        self.assertEqual(self.hv_list[0]['county_count'] + 1,
            test_list7[0]['county_count'])

class TestQualifiedConsumers(TestCase):
    """ Assert result of queryset returned by qry_qualified_consumers abides by
    rules that exclude eligilbity.
    """
    fixtures = ['test_consumer', 'test_advertiser', 'test_coupon_views',
        'test_subscriber']

    def prep_consumer(self):
        """ Prepare consumer for tests. """
        mobile_phone = MobilePhone.objects.get(mobile_phone_number='8455550000')
        mobile_phone.is_verified = True
        mobile_phone.save()
        self.good_subscriber = Subscriber.objects.get(id=2)
        self.good_consumer = Consumer.objects.get(id=115)
        self.good_consumer.is_email_verified = True
        self.good_consumer.save()

    def common_asserts(self, consumer, test_name, test_msg):
        """ Common shared assertions for this test case. """
        result = qry_qualified_consumers()
        if result.filter(id=consumer.id).exists():
            self.fail('%s: %s' % (test_name, test_msg))

    def test_consumer_is_verified(self):
        """ Assert consumer has to be verified to be qualified. """
        self.prep_consumer()
        unverified_consumer = self.good_consumer
        unverified_consumer.is_email_verified = False
        unverified_consumer.subscriber = self.good_subscriber
        unverified_consumer.save()
        self.common_asserts(unverified_consumer, 'test_consumer_is_verified',
            'Unverified consumer email is identified as qualifying')

    def test_consumer_w_subscription(self):
        """ Assert consumer has to have Email Flyer subscription to qualify. """
        self.prep_consumer()
        unsubscribed_consumer = Consumer.objects.get(id=122)
        unsubscribed_consumer.subscriber = self.good_subscriber
        unsubscribed_consumer.save()
        self.common_asserts(unsubscribed_consumer, 'test_consumer_w_subscription',
            'Consumer without subscription is identified as qualifying')

    def test_consumer_no_subscriber(self):
        """ Assert consumer must have subscriber. """
        self.prep_consumer()
        self.common_asserts(self.good_consumer, 'test_consumer_no_subscriber',
            'Consumer without subscriber is identified as qualifying')

    def test_subscriber_no_phone(self):
        """ Assert consumer must have subscriber with mobile. """
        self.prep_consumer()
        subscriber = Subscriber.objects.get(id=3)
        self.good_consumer.subscriber = subscriber
        self.good_consumer.save()
        self.common_asserts(self.good_consumer, 'test_consumer_no_subscriber',
            'Consumer with subscriber with no mobile is qualifying.')

    def test_unverified_phone(self):
        """ Assert consumer must have subscriber with verified mobile. """
        self.prep_consumer()
        subscriber = Subscriber.objects.get(id=7)
        self.good_consumer.subscriber = subscriber
        self.good_consumer.save()
        self.common_asserts(self.good_consumer, 'test_consumer_no_subscriber',
            'Consumer with subscriber with no mobile is qualifying.')

    def test_qualified_consumer(self):
        """ Assert qualified consumer is selectable. """
        self.prep_consumer()
        self.good_consumer.subscriber = self.good_subscriber
        self.good_consumer.save()
        result = qry_qualified_consumers()
        self.assertEqual(result.filter(id=self.good_consumer.id).count(), 1)

class TestCreateSubForConsumer(TestCase):
    """ Test case for service function create_subscriber_for_consumer. """
    fixtures = ['test_consumer']

    def test_consumer_has_sub_no_phone(self):
        """ Assert a consumer having a subscriber who has no phone, a phone is
        created.
        """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        self.assertEqual(create_subscriber_for_consumer(consumer, 1,
            '2225550000', '12550', site), 0)
        try:
            mobile_phone = MobilePhone.objects.get(
                mobile_phone_number='2225550000')
        except MobilePhone.DoesNotExist:
            self.fail('Mobile phone not created.')
        self.assertEqual(mobile_phone.subscriber.consumer(), consumer)

    def test_consumer_has_subscriber(self):
        """ Assert when a consumer already has a subscriber who has a phone,
        0 is returned.
        """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        MobilePhone.objects.create(subscriber_id=30, carrier_id='2',
            mobile_phone_number='2225550001')
        self.assertEqual(create_subscriber_for_consumer(consumer, 1,
            '2225550001', '12550', site), 0)

    def test_con_has_sub_diff_number(self):
        """ Assert when this number belongs to a different subscriber, 0 is
        returned.
        """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        MobilePhone.objects.create(subscriber_id=31, carrier_id='2',
            mobile_phone_number='2225550002')
        self.assertEqual(create_subscriber_for_consumer(consumer, 1,
            '2225550002', '12550', site), 1)

class TestUpdateSubscriberOfConsumer(TestCase):
    """ Test case for update_subscriber_of_consumer. """
    fixtures = ['test_consumer']

    def test_mobile_phone_in_use(self):
        """ Assert if this mobile phone is already used, 1 is returned. """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        MobilePhone.objects.create(subscriber_id=31, carrier_id='2',
            mobile_phone_number='2225550003')
        self.assertEqual(update_subscriber_of_consumer(consumer, '2',
            '2225550003', '12550', site), 1)

    def test_subscriber_no_phone(self):
        """ Assert a subscriber without a phone gets a phone. """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        self.assertEqual(update_subscriber_of_consumer(consumer, '2',
            '2225550004', '12550', site), 0)
        try:
            mobile_phone = MobilePhone.objects.get(
                mobile_phone_number='2225550004')
        except MobilePhone.DoesNotExist:
            self.fail("Mobile phone not created.")
        self.assertEqual(list(consumer.subscriber.mobile_phones.all()),
            [mobile_phone])

    def test_con_has_sub_diff_number(self):
        """ Assert a consumer updates her phone number. """
        site = Site.objects.get(id=2)
        consumer = Consumer.objects.get(id=105)
        MobilePhone.objects.create(subscriber_id=30, carrier_id='2',
            mobile_phone_number='2225550005')
        # A different phone number:
        self.assertEqual(update_subscriber_of_consumer(consumer, '2',
            '2225550006', '12550', site), 0)
        try:
            mobile_phone = MobilePhone.objects.get(
                mobile_phone_number='2225550006')
        except MobilePhone.DoesNotExist:
            self.fail('Mobile phone not created.')
        self.assertEqual(mobile_phone.subscriber.consumer(), consumer)