import cv2
import numpy as np
import argparse
import time
import find_boundary as bz
import os
import glob
def kmeans_seg(img, K):
    size = img.shape
    height = size[0]
    width = size[1]
    channel = size[2]

    imgPos = np.zeros(shape=(height, width, channel + 2))

    for i in range(height):
        for j in range(width):
            imgPos[i][j] = np.append(img[i][j], [i, j])

    Z = imgPos.reshape((-1, 5))

    # convert to np.float32
    Z = np.float32(Z)

    # define criteria, number of clusters(K) and apply kmeans()
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 1000, 0.1)
    # criteria = (10, 100, 1.0)

    ret, label, center = cv2.kmeans(Z, K, None, criteria, 20, cv2.KMEANS_RANDOM_CENTERS)

    # Now convert back into uint8, and make original image
    center = np.uint8(center)
    res = center[label.flatten(), 0:3]
    res2 = res.reshape((img.shape))

    mask = bz.find_bound(label, size)

    # mask.dtype = "uint8"

    # cv2.imshow('res2', res2)
    # cv2.imshow('mask',mask)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()
    return res2, mask

def seg(input_path,output_path, output_path2,K):
    start = time.time()
    img = cv2.imread(input_path)
    img_seg, img_binary = kmeans_seg(img, K)
    cv2.imwrite(output_path, img_seg)
    cv2.imwrite(output_path2, img_binary)
    end = time.time()
    print("Spend time: " + str(end - start))

def main():
    # Dir = '/home/tone/workspace/jupyter_lab/lab/Hanh/data/images/image_test'
    Desresult = '/home/buiduchanh/WorkSpace/Javis/Kmeans/result_Kmeans'
    Desresult_bound = '/home/buiduchanh/WorkSpace/Javis/Kmeans/result_bound'
    # imagelist = sorted(glob.glob('{}/*'.format(Dir)))
    listimage = [['/home/buiduchanh/Downloads/画像データ/61/6162/15100040/610000119/610005146.jpg', 'shadow'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/10500115/610000690/610032722.jpg', 'small_rust'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/10500115/610000690/610032743.jpg', 'pipe'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100090/610000143/610006427.jpg', 'large_rust'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100090/610000143/610006454.jpg', 'shadow'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100040/610000119/610005120.jpg', 'cable'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100190/610000688/610032632.jpg', 'tree'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100420/610000299/610014266.jpg', 'large_rust'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/10500115/610000004/610000203.jpg', 'high_contrast'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/10500180/610000033/610001218.jpg', 'medium_contrast'],
                 ['/home/buiduchanh/Downloads/画像データ/61/6162/15100390/610000263/610012394.jpg', 'low_contrast']]

    for input_path in listimage:
        print(input_path[0])
        basename = os.path.splitext(os.path.basename(input_path[0]))[0]
        # newname = basename + '_result' + '.jpg'
        newname = '{}_{}.jpg'.format(basename,input_path[1])
        output_path = os.path.join(Desresult,newname)
        output_path2 = os.path.join(Desresult_bound,newname)
        numberK = 3
        seg(input_path[0],output_path,output_path2,numberK)

if __name__ == "__main__":
    main()


from django.contrib.auth import get_user_model
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.shortcuts import get_object_or_404, redirect, render

from .forms import CommentForm, PostForm
from .models import Follow, Group, Post

COUNT_POSTS = 10
User = get_user_model()

def index(request):
    posts = Post.objects.all()
    paginator = Paginator(posts, COUNT_POSTS)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, 'posts/index.html', {'page': page})

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    posts = group.posts.all()
    paginator = Paginator(posts, COUNT_POSTS)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, "group.html", {"group": group, "page": page})

@login_required
def new_post(request):
    form = PostForm(request.POST or None, files=request.FILES or None)
    if form.is_valid():
        comment = form.save(commit=False)
        comment.author = request.user
        comment.save()
        return redirect("posts:index")
    return render(
        request, "posts/new_post.html",
        {"form": form, "is_new": True})

def profile(request, username):
    user_profile = get_object_or_404(User, username=username)
    paginator = Paginator(user_profile.posts.all(), COUNT_POSTS)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    following = (request.user.is_authenticated and Follow.objects.filter(
        user__username=request.user,
        author=user_profile).exists())
    return render(
        request, 'posts/profile.html',
        {'page': page, 'user_profile': user_profile,
         'following': following})

def post_view(request, username, post_id):
    post = get_object_or_404(Post, id=post_id, author__username=username)
    user_profile = post.author
    comments = post.comments.all()
    form = CommentForm()
    return render(
        request, 'posts/post.html',
        {'post': post, 'user_profile': user_profile, 'form': form,
         'comments': comments})

@login_required
def post_edit(request, username, post_id):
    post = get_object_or_404(Post, id=post_id, author__username=username)
    if request.user != post.author:
        return redirect('posts:post_view', username, post_id)
    form = PostForm(
        request.POST or None, files=request.FILES or None, instance=post)
    if form.is_valid():
        form.save()
        return redirect('posts:post_view', username, post_id)
    return render(
        request, 'posts/new_post.html',
        {"form": form, "post": post, "is_new": False})

def page_not_found(request, exception):
    return render(
        request,
        "misc/404.html",
        {"path": request.path},
        status=404
    )

def server_error(request):
    return render(request, "misc/500.html", status=500)

@login_required
def add_comment(request, username, post_id):
    post = get_object_or_404(Post, author__username=username, id=post_id)
    form = CommentForm(request.POST or None)
    if form.is_valid():
        comment = form.save(commit=False)
        comment.author = request.user
        comment.post = post
        comment.save()
    return redirect('posts:post_view', username, post_id)

@login_required
def follow_index(request):
    post_list = Post.objects.filter(author__following__user=request.user)
    paginator = Paginator(post_list, COUNT_POSTS)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(
        request,
        "posts/follow.html",
        {'page': page, 'paginator': paginator}
    )

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username=username)
    if author != request.user:
        Follow.objects.get_or_create(user=request.user, author=author)
    return redirect('posts:profile', username=username)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username=username)
    Follow.objects.filter(author=author, user=request.user).delete()
    return redirect('posts:profile', username=username)


from datetime import date, timedelta
from django.http import HttpResponse
from django.http import FileResponse

from .models import *

from proyecto.models import ActividadesCronograma, Equipo, RegistroPerfil
from mensaje.funciones import isVisto

from statistics import mean, pstdev

def isActividad(equipo, nombre):
    estudiante = equipo.datosestudiante_set.first()
    if estudiante.actividad.filter(nombre=nombre):
        return True
    else:
        return False

def mensajesAvisosLista(equipo, usuario):
    aviso = AvisoActividad.objects.filter(usuario=usuario, equipo=equipo)
    if aviso.exists():
        lista_actividades = aviso[0].actividades.all()
        mensajes = [f"{n.nombre}" for n in lista_actividades]
    else:
        mensajes = []
    return mensajes

def avisosEstudiantes(datos_est, usuario):
    avisos = AvisoActividad.objects.filter(usuario=usuario)
    grupo = usuario.groups.get().name
    datos_estudiantes = {}
    for dato_est in datos_est:
        aviso_estudiante = avisos.filter(equipo=dato_est.equipo)
        if aviso_estudiante.exists():
            lista_actividades = aviso_estudiante[0].actividades.all()
            if lista_actividades.count() != 0:
                nombre_actividades = [f"{n}: {a.nombre}" for n, a in enumerate(lista_actividades, 1)]
                mensaje = "\n".join(nombre_actividades)
            else:
                mensaje = "No realizó actividad nueva"
            if grupo == 'tutor' or grupo == 'docente':
                datos_estudiantes[dato_est] = [aviso_estudiante[0].actividades.all().count(), mensaje,
                    isVisto(dato_est.usuario, usuario)]
            else:
                datos_estudiantes[dato_est] = [aviso_estudiante[0].actividades.all().count(), mensaje]
        else:
            if grupo == 'tutor' or grupo == 'docente':
                datos_estudiantes[dato_est] = [0, "No tiene ninguna actividad", isVisto(dato_est.usuario, usuario)]
            else:
                datos_estudiantes[dato_est] = [0, "No tiene ninguna actividad"]
    orden_datos_estudiantes = dict(sorted(datos_estudiantes.items(), key=lambda cantidad: cantidad[1][0], reverse=True))
    return orden_datos_estudiantes

def avisosEquipos(equipos_multiple, usuario):
    avisos = AvisoActividad.objects.filter(usuario=usuario)
    datos_equipos= {}
    for equipo in equipos_multiple:
        aviso_equipo = avisos.filter(equipo=equipo)
        if aviso_equipo.exists():
            lista_actividades = aviso_equipo[0].actividades.all()
            if lista_actividades.count() != 0:
                nombre_actividades = [f"{n}: {a.nombre}" for n, a in enumerate(lista_actividades, 1)]
                mensaje = "\n".join(nombre_actividades)
            else:
                mensaje = "No realizó actividad nueva"
            datos_equipos[equipo] = [aviso_equipo[0].actividades.all().count(), mensaje]
        else:
            datos_equipos[equipo] = [0, "No tiene ninguna actividad"]
    orden_datos_equipos = dict(sorted(datos_equipos.items(), key=lambda cantidad: cantidad[1][0], reverse=True))
    return orden_datos_equipos

def agregarAviso(texto_actividad, equipo, usuario):
    aviso, created = AvisoActividad.objects.get_or_create(
        usuario=usuario,
        equipo=equipo,
    )
    actividad = Actividad.objects.get(nombre=texto_actividad)
    aviso.actividades.add(actividad)
    aviso.save()

def marcarAvisosVistos(equipo, usuario):
    aviso = AvisoActividad.objects.filter(usuario=usuario, equipo=equipo)
    if aviso.exists():
        aviso[0].actividades.clear()

def progress(estudiante):
    actividades = Actividad.objects.all()
    actividades_estudiante = estudiante.actividad.all()
    suma_valores = 0
    for actividad in actividades:
        suma_valores += actividad.valor
    porcentaje_por_unidad = 100/suma_valores
    suma_valores_estudiante = 0
    for actividad in actividades_estudiante:
        suma_valores_estudiante += actividad.valor
    progreso_sobre_100 = int(suma_valores_estudiante * porcentaje_por_unidad)
    return progreso_sobre_100

def agregarActividadEstudiante(texto_actividad, estudiante):
    actividad = Actividad.objects.get(nombre=texto_actividad)
    estudiante.actividad.add(actividad)
    estudiante.save()

def agregarActividadEquipo(texto_actividad, equipo):
    estudiantes = equipo.datosestudiante_set.all()
    for estudiante in estudiantes:
        actividad = Actividad.objects.get(nombre=texto_actividad)
        estudiante.actividad.add(actividad)
        estudiante.save()

def actividadRealizadaEstudiante(texto_actividad, estudiante):
    hecho = estudiante.actividad.filter(nombre=texto_actividad).exists()
    return hecho

def pasosRealizados(estudiante):
    pasos = {1:2, 2:3, 3:7, 4:13, 5:18, 6:26}
    cantidad_actividades = estudiante.actividad.all().count()
    pasos_realizados = []
    # print(cantidad_actividades)

    for paso, actividad in pasos.items():
        if cantidad_actividades >= actividad:
            pasos_realizados.append(paso)
    # print(pasos_realizados)

    return pasos_realizados

def informarCronograma(pk):
    equipo = Equipo.objects.get(id=pk)
    cronograma_existe = ActividadesCronograma.objects.filter(equipo=equipo).exists()
    estudiante = equipo.datosestudiante_set.first()
    progreso = progress(estudiante)
    # mensaje_limite = 'Aún tienes tiempo para elaborar el sistema'
    mensaje_limite = ''
    if cronograma_existe:
        cronograma = ActividadesCronograma.objects.filter(equipo=estudiante.equipo)
            # fecha de registro del cronograma o fecha de registro del proyecto
        fecha = RegistroPerfil.objects.get(equipo=estudiante.equipo).fecha_creacion
            # fecha limite sistema 2 años y medio
        # prueba modificar el 0 del delta para eliminar al usuario
        fecha = fecha.astimezone().date()#-timedelta(0)
        fecha_limite_sistema = fecha+ timedelta(365*2.5)
        dia_restante_sistema = fecha_limite_sistema - date.today()
        dia_restante_sistema = dia_restante_sistema.days
        # fecha transcurrida desde el inicio
        dias_transcurridos = date.today() - fecha
        dias_transcurridos = dias_transcurridos + timedelta(0)
        # dias a semanas:
        semanas = dias_transcurridos.days // 7# - 1
        num_semana = dias_transcurridos.days // 7 + 1
        dias = dias_transcurridos.days % 7
        dias_transcurridos = dias_transcurridos.days# - 7
        # duracion del proyecto
        max_semana = range(1,1+max([n.semana_final for n in cronograma]))
        semana_total = len(max_semana)
        dia_total = 7*semana_total
        # fecha limite cronograma
        fecha_limite_crono = fecha + timedelta(dia_total)
        dia_restante_crono = fecha_limite_crono - date.today()
        dia_restante_crono = dia_restante_crono.days
        # fecha limite sistema 2 años y medio
        fecha_limite_sistema = fecha + timedelta(365*2.5)
        dia_restante_sistema = fecha_limite_sistema - date.today()
        dia_restante_sistema= dia_restante_sistema.days
        # porcentaje
        por_dia_crono = (dia_restante_crono* 100) / dia_total
        por_dia_sistema = dia_restante_sistema* 100 / (365*2.5)
        por_dia_crono = str(por_dia_crono)
        por_dia_sistema = str(por_dia_sistema)

        dia_retrazo = dia_restante_crono * -1
        por_dia_retrazo = ( dia_restante_crono *-1* 100)/(365*2.5-dia_total)
        por_dia_retrazo= str(por_dia_retrazo)

        if num_semana <= semana_total:
            limite_cronograma = False
        else:
            actividades = []
            limite_cronograma = True
        # ********** casos de eliminacion del estudiante
        # pasa 2 años y medio
        if dia_restante_sistema <= -1 and progreso.nivel < 100:
            # estudiante.usuario.delete()
            print('Se jodio')
            mensaje_limite = 'El estudiante fue eliminado del sistema por pasar los 2 años sin concluir el proyecto'
            return (mensaje_limite)
        # En caso de conclusion de proyecto
        if progress(estudiante) >= 100:
            fecha_100 = equipo.fecha_conclusion
            fecha_eliminar = fecha_100 + timedelta(180)
            if fecha_eliminar.date() < date.today():
                # estudiante.usuario.delete()
                print('cuenta eliminada')
            mensaje_limite = 'Concluiste con éxito el Proyecto de Grado, en 6 meses se eliminará tu cuenta'
            dia_restante_crono = ''
            dia_restante_sistema = ''
            dia_retrazo = ''
            semana_total = ''
            por_dia_crono = ''
            por_dia_sistema = ''
            por_dia_retrazo = ''
            limite_cronograma = ''
        # caso de reglamento sanabria, no conclusion de perfil
        if not estudiante.equipo.registroperfil:
            fecha_ingreso = estudiante.fecha_inscripcion.date()
        # se establese fecha limite del semestre de fin de septiembre y fin de marzo
            if fecha_ingreso.month < 6:
                fecha_limite = date(fecha_ingreso.year,9,30)
            else:
                fecha_limite = date(fecha_ingreso.year+1,3,30)
            if fecha_limite < date.today():
                # estudiante.usuario.delete()
                print('cuenta eliminada')
                print('Se jodio')
                mensaje_limite = 'El estudiante fue eliminado del sistema por no aprobar el perfil en el semestre inscrito'
                return (mensaje_limite)
    else:
        dia_restante_crono = ''
        dia_restante_sistema = ''
        dia_retrazo = ''
        semana_total = ''
        por_dia_crono = ''
        por_dia_sistema = ''
        por_dia_retrazo = ''
        limite_cronograma = ''
        mensaje_limite = ''
    context = {
        'dia_restante_crono':dia_restante_crono,
        'dia_restante_sistema':dia_restante_sistema,
        'dia_retrazo':dia_retrazo,
        'semana_total':semana_total,
        'por_dia_crono':por_dia_crono,
        'por_dia_sistema':por_dia_sistema,
        'por_dia_retrazo':por_dia_retrazo,
        'limite_cronograma':limite_cronograma,
        'cronograma_existe':cronograma_existe,
        'estudiante':estudiante,
        'dia_restante_sistema':dia_restante_sistema,
        'mensaje_limite':mensaje_limite}
    return context

def diasRestantes(pk):
    equipo = Equipo.objects.get(id=pk)
    cronograma_existe = ActividadesCronograma.objects.filter(equipo=equipo).exists()
    estudiante = equipo.datosestudiante_set.first()
    progreso = progress(estudiante)
    mensaje_limite = ''
    if cronograma_existe:
        cronograma = ActividadesCronograma.objects.filter(equipo=estudiante.equipo)
        # fecha de registro del cronograma o fecha de registro del proyecto
        fecha = RegistroPerfil.objects.get(equipo=estudiante.equipo).fecha_creacion
        # fecha limite sistema 2 años y medio
        # prueba modificar el 0 del delta para eliminar al usuario
        fecha = fecha.astimezone().date()#-timedelta(0)
        # fecha limite sistema 2 años y medio
        fecha_limite_sistema = fecha+ timedelta(365*2.5)
        dias_restantes_sistema = fecha_limite_sistema - date.today()
        dias_restantes_sistema = dias_restantes_sistema.days
        # fecha transcurrida desde el inicio
        dias_transcurridos = date.today() - fecha
        dias_transcurridos = dias_transcurridos + timedelta(0)
        dias_transcurridos = dias_transcurridos.days
        # duracion del proyecto
        max_semana = range(1,1+max([n.semana_final for n in cronograma]))
        semana_total = len(max_semana)
        dia_total = 7*semana_total
        # fecha limite cronograma
        fecha_limite_crono = fecha + timedelta(dia_total)
        dias_restantes_cronograma = fecha_limite_crono - date.today()
        dias_restantes_cronograma = dias_restantes_cronograma.days

    else:
        dias_restantes_sistema = ''
        dias_restantes_cronograma = ''
        dias_transcurridos = ''

    return dias_restantes_cronograma, dias_restantes_sistema, dias_transcurridos

def distanciaEntreActividades(lista_actividades, equipo):
    """Extrae un numero que ayudará para ordenar por esfuerzo del estudiante.
    Se realiza un análisis entre las diferencias de tiempo entre cada
    actividad.
    A menor el nivel, mas esfuerzo el estudiante tiene en completar las
    actividades.
    Se nombrea nivel_ie"""

    actividades = lista_actividades.order_by('fecha_creacion')
    fecha_inicio = equipo.fecha_creacion
    fechas_actividades = [n.fecha_creacion for n in actividades]
    fechas_actividades.insert(0, fecha_inicio)

    # diff_tiempo = []
    diff_tiempo_dias = []
    for x, y in zip(fechas_actividades[0::], fechas_actividades[1::]):
        diff = y- x
        diff_dias = diff.total_seconds()/(3600*24)
        # diff_dias = diff.days
        diff_tiempo_dias.append(diff_dias)

    # Por media y desviasion estandar, datos la diferencia de tiempos
    # print(diff_tiempo_dias)
    media = mean(diff_tiempo_dias)
    desviacion = pstdev(diff_tiempo_dias)
    nivel_ie = media + desviacion
    return nivel_ie


from marshmallow import ValidationError
from marshmallow.fields import String
from pyramid.httpexceptions import HTTPNotFound
from pyramid.request import Request
from pyramid.response import Response
from sqlalchemy.exc import IntegrityError

from tildes.enums import LogEventType
from tildes.models.group import Group
from tildes.models.log import LogTopic
from tildes.models.topic import Topic, TopicBookmark, TopicIgnore, TopicVote
from tildes.schemas.group import GroupSchema
from tildes.schemas.topic import TopicSchema
from tildes.views import IC_NOOP
from tildes.views.decorators import ic_view_config, use_kwargs

@ic_view_config(
    route_name="topic",
    request_method="GET",
    request_param="ic-trigger-name=edit",
    renderer="topic_edit.jinja2",
    permission="edit",
)
def get_topic_edit(request: Request) -> dict:
    """Get the edit form for a topic with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic",
    request_method="GET",
    renderer="topic_contents.jinja2",
    permission="view",
)
def get_topic_contents(request: Request) -> dict:
    """Get a topic's body with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic",
    request_method="PATCH",
    renderer="topic_contents.jinja2",
    permission="edit",
)
@use_kwargs(TopicSchema(only=("markdown",)), location="form")
def patch_topic(request: Request, markdown: str) -> dict:
    """Update a topic with Intercooler."""
    topic = request.context

    topic.markdown = markdown

    return {"topic": topic}

@ic_view_config(route_name="topic", request_method="DELETE", permission="delete")
def delete_topic(request: Request) -> Response:
    """Delete a topic with Intercooler and redirect to its group."""
    topic = request.context
    topic.is_deleted = True

    response = Response()
    response.headers["X-IC-Redirect"] = request.route_url(
        "group", path=topic.group.path
    )

    return response

@ic_view_config(
    route_name="topic_vote",
    request_method="PUT",
    renderer="topic_voting.jinja2",
    permission="vote",
)
def put_topic_vote(request: Request) -> Response:
    """Vote on a topic with Intercooler."""
    topic = request.context

    savepoint = request.tm.savepoint()

    new_vote = TopicVote(request.user, topic)
    request.db_session.add(new_vote)

    request.db_session.add(LogTopic(LogEventType.TOPIC_VOTE, request, topic))

    try:
        # manually flush before attempting to commit, to avoid having all objects
        # detached from the session in case of an error
        request.db_session.flush()
        request.tm.commit()
    except IntegrityError:
        # the user has already voted on this topic
        savepoint.rollback()

    # re-query the topic to get complete data
    topic = (
        request.query(Topic)
        .join_all_relationships()
        .filter_by(topic_id=topic.topic_id)
        .one()
    )

    return {"topic": topic}

@ic_view_config(
    route_name="topic_vote",
    request_method="DELETE",
    renderer="topic_voting.jinja2",
    permission="vote",
)
def delete_topic_vote(request: Request) -> Response:
    """Remove the user's vote from a topic with Intercooler."""
    topic = request.context

    request.query(TopicVote).filter(
        TopicVote.topic == topic, TopicVote.user == request.user
    ).delete(synchronize_session=False)

    request.db_session.add(LogTopic(LogEventType.TOPIC_UNVOTE, request, topic))

    # manually commit the transaction so triggers will execute
    request.tm.commit()

    # re-query the topic to get complete data
    topic = (
        request.query(Topic)
        .join_all_relationships()
        .filter_by(topic_id=topic.topic_id)
        .one()
    )

    return {"topic": topic}

@ic_view_config(
    route_name="topic_tags",
    request_method="GET",
    renderer="topic_tags_edit.jinja2",
    permission="tag",
)
def get_topic_tags(request: Request) -> dict:
    """Get the tagging form for a topic with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic_tags",
    request_method="PUT",
    renderer="topic_tags.jinja2",
    permission="tag",
)
@use_kwargs(
    {"tags": String(missing=""), "conflict_check": String(missing="")}, location="form"
)
def put_tag_topic(request: Request, tags: str, conflict_check: str) -> dict:
    """Apply tags to a topic with Intercooler."""
    topic = request.context

    # check for edit conflict by verifying tags didn't change after they loaded the form
    if conflict_check:
        conflict_check_tags = conflict_check.split(",")
    else:
        conflict_check_tags = []

    if conflict_check_tags != topic.tags:
        raise ValidationError(
            {"tags": ["Someone else edited simultaneously, please cancel and retry"]}
        )

    if tags:
        # split the tag string on commas
        new_tags = tags.split(",")
    else:
        new_tags = []

    old_tags = topic.tags

    try:
        topic.tags = new_tags
    except ValidationError as exc:
        raise ValidationError({"tags": ["Invalid tags"]}) from exc

    # if tags weren't changed, don't add a log entry or update page
    if set(topic.tags) == set(old_tags):
        return IC_NOOP

    request.db_session.add(
        LogTopic(
            LogEventType.TOPIC_TAG,
            request,
            topic,
            info={"old": old_tags, "new": topic.tags},
        )
    )

    # show the tag field by default in the future when the tagger is posting a new topic
    if not request.user.show_tags_on_new_topic:
        request.user.show_tags_on_new_topic = True
        request.db_session.add(request.user)

    return {"topic": topic}

@ic_view_config(
    route_name="topic_group",
    request_method="GET",
    renderer="topic_group_edit.jinja2",
    permission="move",
)
def get_topic_group(request: Request) -> dict:
    """Get the form for moving a topic with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic",
    request_param="ic-trigger-name=topic-move",
    request_method="PATCH",
    permission="move",
)
@use_kwargs(GroupSchema(only=("path",)), location="form")
def patch_move_topic(request: Request, path: str) -> dict:
    """Move a topic to a different group with Intercooler."""
    topic = request.context

    new_group = request.query(Group).filter(Group.path == path).one_or_none()
    if not new_group:
        raise HTTPNotFound("Group not found")

    old_group = topic.group

    if new_group == old_group:
        return IC_NOOP

    topic.group = new_group

    request.db_session.add(
        LogTopic(
            LogEventType.TOPIC_MOVE,
            request,
            topic,
            info={"old": str(old_group.path), "new": str(topic.group.path)},
        )
    )

    return Response("Moved")

@ic_view_config(
    route_name="topic_remove",
    request_method="PUT",
    permission="remove",
    renderer="post_action_toggle_button.jinja2",
)
def put_topic_remove(request: Request) -> dict:
    """Remove a topic with Intercooler."""
    topic = request.context

    topic.is_removed = True
    request.db_session.add(LogTopic(LogEventType.TOPIC_REMOVE, request, topic))

    return {"name": "remove", "subject": topic, "is_toggled": True}

@ic_view_config(
    route_name="topic_remove",
    request_method="DELETE",
    permission="remove",
    renderer="post_action_toggle_button.jinja2",
)
def delete_topic_remove(request: Request) -> dict:
    """Un-remove a topic with Intercooler."""
    topic = request.context

    topic.is_removed = False
    request.db_session.add(LogTopic(LogEventType.TOPIC_UNREMOVE, request, topic))

    return {"name": "remove", "subject": topic, "is_toggled": False}

@ic_view_config(
    route_name="topic_lock",
    request_method="PUT",
    permission="lock",
    renderer="post_action_toggle_button.jinja2",
)
def put_topic_lock(request: Request) -> dict:
    """Lock a topic with Intercooler."""
    topic = request.context

    topic.is_locked = True
    request.db_session.add(LogTopic(LogEventType.TOPIC_LOCK, request, topic))

    return {"name": "lock", "subject": topic, "is_toggled": True}

@ic_view_config(
    route_name="topic_lock",
    request_method="DELETE",
    permission="lock",
    renderer="post_action_toggle_button.jinja2",
)
def delete_topic_lock(request: Request) -> dict:
    """Unlock a topic with Intercooler."""
    topic = request.context

    topic.is_locked = False
    request.db_session.add(LogTopic(LogEventType.TOPIC_UNLOCK, request, topic))

    return {"name": "lock", "subject": topic, "is_toggled": False}

@ic_view_config(
    route_name="topic_title",
    request_method="GET",
    renderer="topic_title_edit.jinja2",
    permission="edit_title",
)
def get_topic_title(request: Request) -> dict:
    """Get the form for editing a topic's title with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic",
    request_param="ic-trigger-name=topic-title-edit",
    request_method="PATCH",
    permission="edit_title",
)
@use_kwargs(TopicSchema(only=("title",)), location="form")
def patch_topic_title(request: Request, title: str) -> dict:
    """Edit a topic's title with Intercooler."""
    topic = request.context

    if title == topic.title:
        return IC_NOOP

    request.db_session.add(
        LogTopic(
            LogEventType.TOPIC_TITLE_EDIT,
            request,
            topic,
            info={"old": topic.title, "new": title},
        )
    )

    topic.title = title

    return Response(topic.title)

@ic_view_config(
    route_name="topic_link",
    request_method="GET",
    renderer="topic_link_edit.jinja2",
    permission="edit_link",
)
def get_topic_link(request: Request) -> dict:
    """Get the form for editing a topic's link with Intercooler."""
    return {"topic": request.context}

@ic_view_config(
    route_name="topic",
    request_param="ic-trigger-name=topic-link-edit",
    request_method="PATCH",
    permission="edit_link",
)
@use_kwargs(TopicSchema(only=("link",)), location="form")
def patch_topic_link(request: Request, link: str) -> dict:
    """Edit a topic's link with Intercooler."""
    topic = request.context

    if link == topic.link:
        return IC_NOOP

    request.db_session.add(
        LogTopic(
            LogEventType.TOPIC_LINK_EDIT,
            request,
            topic,
            info={"old": topic.link, "new": link},
        )
    )

    topic.link = link
    topic.content_metadata = None

    return Response(f'{topic.link}')

@ic_view_config(
    route_name="topic_bookmark",
    request_method="PUT",
    permission="bookmark",
    renderer="post_action_toggle_button.jinja2",
)
def put_topic_bookmark(request: Request) -> dict:
    """Bookmark a topic with Intercooler."""
    topic = request.context

    savepoint = request.tm.savepoint()

    bookmark = TopicBookmark(request.user, topic)
    request.db_session.add(bookmark)

    try:
        # manually flush before attempting to commit, to avoid having all
        # objects detached from the session in case of an error
        request.db_session.flush()
        request.tm.commit()
    except IntegrityError:
        # the user has already bookmarked this topic
        savepoint.rollback()

    return {"name": "bookmark", "subject": topic, "is_toggled": True}

@ic_view_config(
    route_name="topic_bookmark",
    request_method="DELETE",
    permission="bookmark",
    renderer="post_action_toggle_button.jinja2",
)
def delete_topic_bookmark(request: Request) -> dict:
    """Unbookmark a topic with Intercooler."""
    topic = request.context

    request.query(TopicBookmark).filter(
        TopicBookmark.user == request.user, TopicBookmark.topic == topic
    ).delete(synchronize_session=False)

    return {"name": "bookmark", "subject": topic, "is_toggled": False}

@ic_view_config(
    route_name="topic_ignore",
    request_method="PUT",
    permission="ignore",
    renderer="post_action_toggle_button.jinja2",
)
def put_topic_ignore(request: Request) -> dict:
    """Ignore a topic with Intercooler."""
    topic = request.context

    savepoint = request.tm.savepoint()

    ignore = TopicIgnore(request.user, topic)
    request.db_session.add(ignore)

    try:
        # manually flush before attempting to commit, to avoid having all
        # objects detached from the session in case of an error
        request.db_session.flush()
        request.tm.commit()
    except IntegrityError:
        # the user has already ignored this topic
        savepoint.rollback()

    return {"name": "ignore", "subject": topic, "is_toggled": True}

@ic_view_config(
    route_name="topic_ignore",
    request_method="DELETE",
    permission="ignore",
    renderer="post_action_toggle_button.jinja2",
)
def delete_topic_ignore(request: Request) -> dict:
    """Unignore a topic with Intercooler."""
    topic = request.context

    request.query(TopicIgnore).filter(
        TopicIgnore.user == request.user, TopicIgnore.topic == topic
    ).delete(synchronize_session=False)

    return {"name": "ignore", "subject": topic, "is_toggled": False}

@ic_view_config(
    route_name="topic",
    request_method="GET",
    request_param="ic-trigger-name=markdown-source",
    renderer="markdown_source.jinja2",
    permission="view_content",
)
def get_topic_markdown_source(request: Request) -> dict:
    """Get the Markdown source for a topic with Intercooler."""
    return {"post": request.context}


#%% Import packages
import os
import pickle
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from scipy.signal import argrelextrema
import squarify as sq
import time
import numpy.ma as ma

#%% Define classes

class election(object):
    def __init__(self, hh_dict):
        self.households = hh_dict

    def utility(self, index):
        u = dict( [ (hh.id, hh.eu_df.eu.loc[index]) for hh in self.households.values() ] )
        return(u)

    def count_unhappy(self, index):
        utilities = self.utility(index)
        baseline = dict( [ (hh.id, hh.eu_df.eu.iloc[0]) for hh in self.households.values() ] )
        n = sum([ baseline[hh_id] > utilities[hh_id] for hh_id in utilities.keys() ])
        return n

    def vote(self):
        ballots = np.array( [ np.array(hh.vote(), dtype = np.integer) \
                  for hh in self.households.values() ] )
        winner = election.instant_runoff(ballots)
        return (winner, self.utility(winner))

    @staticmethod
    def instant_runoff(ballots):
        majority = ballots.shape[0] * 0.5
        n_choices = ballots.shape[1]
        ballots = pd.DataFrame(ballots)
        vc = ballots[0].value_counts()
        while vc.iloc[0] <= majority:
            vc = ballots[n_choices - 1].value_counts()
            eliminate = vc.index[0]
            blist = [ list(x) for x in list(np.array(ballots)) ]
            for b in blist: b.remove(eliminate)
            ballots = pd.DataFrame(np.array(blist, dtype = np.integer))
            n_choices = ballots.shape[1]
            vc = ballots[0].value_counts()
        return vc.index[0]

class transaction(object):
    def __init__(self, buyer, seller, year, price):
        self.buyer = buyer
        self.seller = seller
        self.year = year
        self.price = price

    def buyer_id(self):
        return self.buyer.id

    def seller_id(self):
        return self.seller.id

class auction(object):
    def __init__(self, hh_dict):
        self.households = hh_dict
        self.transactions = []
        self.ballots = None
        self.initialize_votes()

    def utility(self, index):
        u = dict( [ (hh.id, hh.eu_df.eu.loc[index]) for hh in self.households.values() ] )
        trans = [t for t in self.transactions if t.year == index]
        for t in trans:
            # print "Transaction: buyer = ", t.buyer_id(), ", seller = ", t.seller_id(), ", year = ", t.year[0], ", price = ", t.price
            u[t.buyer_id()] -= t.price
            u[t.seller_id()] += t.price
        return(u)

    def unhappy(self, index):
        potentially_unhappy = dict([(hh.id, hh.eu_df) for hh in self.households.values() if hh.eu_df.index[0] != index])
        u = self.utility(index)
        utilities = dict([(k, u[k]) for k in potentially_unhappy.keys()])
        baseline = dict( [ (k, potentially_unhappy[k].iloc[0]) for k in potentially_unhappy.keys()])
        unhappy = dict([(k, (utilities[k], potentially_unhappy[k])) \
                        for k in potentially_unhappy.keys() \
                        if utilities[k] < potentially_unhappy[k].iloc[0].eu])
        return unhappy

    def count_unhappy(self, index):
        return len(self.unhappy(index))

    def vote(self, force = False):
        if self.ballots is None:
            return None
        majority = 0.5 * self.ballots.shape[0]
        votes = pd.DataFrame(self.ballots)[0].value_counts()
        print(votes)
        if votes.iloc[0] > majority:
            return votes.index[0]
        elif force:
            return votes.index[0]
        return None

    def initialize_votes(self):
        self.ballots = np.array( [ hh.vote() for hh in self.households.values() ] )
        self.transactions = []

    def auction(self, max_rounds = 1000):
        global tt, bbids
        self.initialize_votes()
        for round in range(max_rounds):
            print("Round ", round)
            winner = self.vote()
            if winner is not None:
                return (winner, self.utility(winner))
            target = self.vote(force = True)
            print("Target = ", target)
            buyers  = [trans.buyer for trans in self.transactions]
            sellers = [trans.seller for trans in self.transactions]
            buyer_ids = [ b.id for b in buyers ]
            seller_ids = [ s.id for s in sellers ]
            neutral_ids = list(set(self.households.keys()) - set(buyer_ids) - set(seller_ids))
            neutral = [ self.households[hh_id] for hh_id in neutral_ids ]
            for hh in neutral:
                hh.construct_bids(target)
            for hh in buyers:
                purchases = [ t for t in self.transactions if t.buyer_id() == hh.id ]
                hh.construct_bids(target, purchases)
            bids = pd.concat([hh.bids for hh in neutral + buyers])
            bbids = bids.copy()
            transactions = self.bidding_round(bids)
            tt = transactions
            for trans in transactions:
                trans.buyer.wealth -= trans.price
                trans.seller.wealth += trans.price
                self.ballots[trans.seller_id()] = trans.year
            if len(transactions) > 0:
                print(len(transactions), " Transactions")
                self.transactions += transactions
            else:
                print("No transactions")
                break
        winner = self.vote(force = True)
        return (winner, self.utility(winner))

    def bidding_round(self,bids):
        global pp, wtpp, wtaa, p0, bcc, scc
        transactions = []
        wta = bids[np.logical_not(bids.is_offer.values)][['id', 'year','amount']]
        wtp = bids[bids.is_offer.values][['id', 'year','amount']]
        wtpp = wtp.copy()
        wtaa = wta.copy()
        wta = wta[np.logical_not(wta.duplicated())]
        wtp = wtp[np.logical_not(wtp.duplicated())]
        # wta = wta.drop_duplicates(inplace = True)
        # wtp = wtp.drop_duplicates(inplace = True)
        if wta is None or wta.shape[0] < 2:
            # print "Empty wta"
            return transactions
        if wtp is None or wtp.shape[0] < 2:
            # print "Empty wtp"
            return transactions
        wta = wta.pivot(index = 'id', columns = 'year', values = 'amount')
        wtp = wtp.pivot(index = 'id', columns = 'year', values = 'amount')
        wtpp = wtp.copy()
        wtaa = wta.copy()
        wta_min = wta.min()
        wtp_max = wtp.max()
        for buyer_index in np.random.choice(wtp.index, wtp.shape[0], replace = False):
            buyer = wtp.loc[buyer_index]
            buyer_year = buyer[np.logical_not(buyer.isnull())]
            offer = buyer_year.values[0]
            # print "Buyer_year.index - ", buyer_year.index
            # print "WTA.columns = ", wta.columns
            if buyer_year.index.values in wta.columns.values:
                seller_candidates = wta[buyer_year.index]
            else:
                continue
            seller_candidates = seller_candidates[np.logical_not(pd.isnull(seller_candidates.values))[:,0]]
            seller_candidates = seller_candidates[(seller_candidates.values <= offer)[:,0]]
            if seller_candidates.shape[0] > 0:
                # print "Year ", buyer_year.index[0], ": ", buyer.index[0], " offers ", offer
                # print "     seller_candidates has shape ", seller_candidates.shape
                seller_index = np.random.choice(seller_candidates.index, 1)[0]
                seller = seller_candidates.loc[seller_index]
                accept = seller.values[0]
                # print "Year ", buyer_year.index[0], ": ", buyer.index[0], " offers ", offer, " and ", \
                #        seller.index[0], " will accept ", accept
                if  offer >= accept:
                    price = (offer + accept) / 2.0
                    # print "Offer accepted: price = ", price
                    if True: # self.households[p.buyer].wealth >= price:
                        # print buyer_index, seller_index
                        bh = self.households[buyer_index]
                        sh = self.households[seller_index]
                        tx = transaction(bh, sh, buyer_year.index, price)
                        transactions.append(tx)
                        wtp.drop(buyer_index, inplace = True)
                        wta.drop(seller_index, inplace = True)
                else:
                    # print "Offer rejected."
                    pass
            if wta.shape[0] == 0 or wtp.shape[0] == 0:
                break
        # print len(transactions), " transactions."
        return transactions

class household(object):
    def __init__(self, id, wealth = 0, plots = None, discount = 0.03, eu_df = None):
        self.id = id
        self.wealth = wealth
        self.discount = discount
        self.eu_df = eu_df
        self.bids = pd.DataFrame(index = [], columns = ['id', 'year', 'is_offer', 'amount'])
        if plots is None:
            self.plots = np.zeros((0,5), dtype=np.integer)
        else:
            self.plots = np.array(plots, dtype=np.integer)

    def utility(self, profit_dc):
        own_patches_profit = np.concatenate([ household.extract_and_collapse(profit_dc, p) \
                                             for p in self.plots ],
                                      axis = 0)
        profit = np.sum(own_patches_profit, axis = 0)
        eu = np.sum(profit * np.exp(- self.discount * np.arange(len(profit))))
        return eu

    def set_eu(self, eu_df):
        self.eu_df = eu_df.sort_values('eu', ascending = False)

    def construct_bids(self, target, purchases = []):
        bid_scale = 1.1
        assert(self.eu_df is not None)
        favorite = self.eu_df.iloc[0]
        others = self.eu_df.iloc[1:]
        # print "Target = ", target
        if target == favorite.year:
            target = self.eu_df.index[1]
        target_eu = self.eu_df.eu.loc[target]

        sell_bids = pd.DataFrame({ \
                  'id': self.id,
                  'year':others.year,
                  'is_offer':False,
                  'amount':(favorite.eu - others.eu) * np.random.uniform(1.0, bid_scale),
                  })
        buy_bids = pd.DataFrame({ \
                  'id': (self.id,),
                  'year': (self.eu_df.index[0],),
                  'is_offer': (True,),
                  'amount':((favorite.eu - target_eu) / np.random.uniform(1.0, bid_scale),)\
                  })
        if False:
            buy_bids.amount = buy_bids.amount.clip(None, self.wealth)
        if (len(purchases) > 0):
            self.bids = buy_bids[ buy_bids.year == purchases[0].year]
        else:
            self.bids = pd.concat([buy_bids, sell_bids])
        self.bids = self.bids[self.bids.amount > 0.0]

    def vote(self):
        return self.eu_df.year.copy()

    #==========================================================================
    # EXTRACT A RECTANGULAR SECTION THROUGH A CUBE AND COLLAPSE
    #==========================================================================

    # Given a cube dc[z,y,x], extract a rectangular prism in (x,y), that extends
    # through all z-values, then ravel the x and y dimensions to produce a
    # 2D array with rows = z and columns = raveled x,y.

    @staticmethod
    def extract_and_collapse(dc, p):
        x = dc[:,p[1]:(p[1] + p[3]),p[0]:(p[0] + p[2])].copy()
        x = x.reshape((x.shape[0], x.shape[1] * x.shape[2]))
        return x

class breach(object):
    def __init__(self, pldr, breach_x, breach_y, breach_z):
        self.pldr = pldr
        self.x = breach_x
        self.y = breach_y
        self.z_breach = breach_z
        xx,yy = np.meshgrid(np.arange(pldr.width), np.arange(pldr.height))
        delta_x = xx - breach_x
        delta_y = yy - breach_y
        self.dist = np.hypot(delta_x, delta_y)
        self.scaled_dist = self.dist / 1000. + 1.
        self.A = 0.0

class polder(object):
    def __init__(self, x, y,
                 time_horizon,
                 n_households = 0,
                 max_wealth = 1.0E4, max_profit = 100.,
                 gini = 0.3,
                 border_height = 1.0,
                 amplitude = 1.5,
                 noise = 0.05):
        self.width = x
        self.height = y
        self.border_height = border_height
        self.max_wealth = max_wealth
        self.max_profit = max_profit
        self.time_horizon = time_horizon
        self.breach_duration = 0
        self.current_period = 0
        self.plots = np.zeros(shape = (0,5), dtype = np.integer)
        self.breaches = []
        self.initialize_elevation(border_height = border_height,
                                  amplitude = amplitude, noise = noise)
        self.initialize_hh(n_households)

    def initialize_elevation(self, border_height = None, amplitude = 1.0, noise = 0.05):
        if border_height is None:
            border_height = self.border_height
        wx = np.pi / self.width
        wy = np.pi / self.height
        self.elevation = border_height - amplitude * \
              np.outer(np.sin(np.arange(self.height) * wy),
                        np.sin(np.arange(self.width) * wx)) + \
              noise * np.random.normal(0.0, 1.0, (self.height, self.width))
        self.elevation_cube = np.zeros((self.time_horizon + 1, self.height, self.width))
        self.elevation_cube[0] = self.elevation
        self.current_period = 0

    def set_elevation(self, elevation, plots, n_households = None):
        if n_households is None:
            n_households = len(self.households)
        self.elevation = elevation
        self.owners = np.zeros_like(self.elevation, dtype = np.integer)
        self.plots = plots
        self.initialize_hh_from_plots(n_households)
        self.elevation_cube = np.zeros((self.time_horizon + 1, self.height, self.width))
        self.elevation_cube[0] = self.elevation
        self.current_period = 0

    def initialize_hh(self, n_households):
        self.owners = np.zeros_like(self.elevation, dtype = np.integer)
        self.households = dict()
        if n_households > 0:
            self.build_households(n_households)
            self.owners.fill(-1)
            for hh in self.households.values():
                for p in hh.plots:
                    self.owners[p[1]:(p[1]+p[3]),p[0]:(p[0]+p[2])] = hh.id

    def initialize_hh_from_plots(self, n_households):
        assert max(self.owners) < n_households
        self.households = dict([(i, household(id = i)) for i in range(n_households)])
        self.set_hh_plots()

    def set_households(self, households):
        if isinstance(households, dict):
            self.households = households
        else:
            # list of households
            self.households = dict((hh.id, hh) for hh in households)
        self.owners = np.zeros_like(self.elevation, dtype = np.integer)
        self.set_owners_wealth()

    def set_hh_wealth(self, hh):
        z0 = self.elevation.min() - 0.5
        z = self.elevation[self.owners == hh.id]
        if z.size == 0:
            hh.wealth = 0
        else:
            hh.wealth = self.max_wealth * np.sqrt(z.size) * (z.mean() - z0) / (self.border_height - z0)

    def set_owners_wealth(self):
        self.owners.fill(-1.0)
        for hh in self.households.values():
            for p in hh.plots:
                self.owners[p[1]:(p[1]+p[3]),p[0]:(p[0]+p[2])] = hh.id
            self.set_hh_wealth(hh)

    def set_hh_plots(self):
        for hh in self.households.values():
            hh.plots = []
        for i in range(self.plots.shape[0]):
            hh = self.households[self.plots[i,4]]
            hh.plots.append(self.plots[i,:4])
        self.set_owners_wealth()

    @staticmethod
    def build_subplots(weights, x0, y0, dx, dy, ix0 = 0):
        plot_sizes = sq.normalize_sizes(weights, dx, dy)
        plots = sq.squarify(plot_sizes, x0, y0, dx, dy)
        plots = pd.DataFrame(plots, columns = ('x', 'y', 'dx', 'dy'))
        plots['dx'] = np.round(plots['x'] + plots['dx']) - np.round(plots['x'])
        plots['dy'] = np.round(plots['y'] + plots['dy']) - np.round(plots['y'])
        plots = plots[['x','y','dx','dy']]
        plots = np.array(np.round(plots), dtype = np.integer)
        plots = np.concatenate( \
                   ( \
                    plots, \
                    np.expand_dims( np.arange(plots.shape[0], dtype=np.integer),
                                   axis = 1)  + int(ix0) \
                   ), \
                 axis = 1)
        return plots

    def build_plots(self, weights, n_boxes = 10):
        n = int(weights.size / n_boxes)
        remainder = weights.size % n
        w = np.random.choice(weights, weights.size, False)
        wr = w[0:remainder]
        w = w[remainder:]
        w_list = np.random.choice(w, size = (n_boxes, n), replace = False)
        w_list = [ w_list[i] for i in range(w_list.shape[0])]
        if remainder > 0:
            i_dest = np.random.choice(n_boxes, remainder, replace = True)
            for i, j in enumerate(i_dest):
                w_list[j] = np.append(w_list[j], wr[i])
        grid_weights = [ np.sum(x) for x in w_list ]
        scaled_grid_weights = sq.normalize_sizes(grid_weights, self.width, self.height)
        grid = sq.squarify(scaled_grid_weights, 0, 0, self.width, self.height)
        grid = pd.DataFrame(grid, columns = ('x', 'y', 'dx', 'dy'))
        grid['dx'] = np.round(grid['x'] + grid['dx']) - np.round(grid['x'])
        grid['dy'] = np.round(grid['y'] + grid['dy']) - np.round(grid['y'])
        grid = np.array(np.round(grid), dtype = np.integer)
#        self.grid = grid.copy()
#        self.w_list = w_list
#        self.grid_weights = grid_weights

        cum_len = np.cumsum( np.concatenate( (np.zeros((1,)), [len(ww) for ww in w_list[:-1]]) ) )

        plot_list = [ self.build_subplots(w_list[i], \
                          grid[i,0], grid[i,1], grid[i,2], grid[i,3],
                          ix0 = cum_len[i]) \
                      for i in range(len(w_list)) ]
#        self.plot_list = plot_list
        plots = np.concatenate(plot_list, axis = 0)
        self.plots = plots

    def build_households(self, n = None, gini = 0.3):
        if n is not None and n != len(self.households):
            print("Initializing", n, "households")
            self.households = dict([(i, household(id = i)) for i in range(n)])
        else:
            print("n = ", type(n), ", ", n, ", length = ", len(self.households))
        if isinstance(gini, dict):
            gini_land = gini['land']
        elif isinstance(gini, (list,tuple)):
            if (len(gini) > 1):
                gini_land = gini[0]
            else:
                gini_land = gini[0]
        else:
            gini_land = gini

        alpha = (1.0 / gini_land + 1.0) / 2.0
        weights = np.random.pareto(alpha, size = len(self.households))

        self.build_plots(weights)
        self.set_hh_plots()

    def calc_profit(self, water_level, k, elevation_cube = None, save = True):
        if elevation_cube is None:
            elevation_cube = self.elevation_cube
        profit = self.max_profit * logit(elevation_cube, k, water_level / 2.0)
        if save:
            self.profit = profit.copy()
        return profit

    def calc_eu(self, profit_cube = None, save = True):
        if profit_cube is None:
            profit_cube = self.profit.copy()
        hh_eu = dict([(hh.id, hh.utility(profit_cube)) for hh in self.households.values()])
        eu = np.zeros_like(self.owners, np.double)
        for i in range(eu.shape[0]):
            for j in range(eu.shape[1]):
                eu[i,j] = hh_eu[self.owners[i,j]]
        if save:
            self.eu = eu.copy()
        return (eu, hh_eu)

    def set_hh_eu(self, ids, eu_array):
        n_years = eu_array.shape[0]
        for i, hh_id in enumerate(ids):
            df = pd.DataFrame({'year':range(n_years), 'eu':eu_array[:,i]})
            self.households[hh_id].set_eu(df)

    def calc_eu_slice(self, trm_water_level, trm_k, wl_water_level, wl_k, ec, horizon, duration):
            ec1 = ec.copy()
            profit = np.zeros_like(ec, np.double)
            for j in range(duration + 1, horizon + 1):
                ec1[j] = ec1[duration]
            profit[:duration] = self.calc_profit(trm_water_level, trm_k, ec1[:duration], False)
            profit[duration:] = self.calc_profit(wl_water_level, wl_k, ec1[duration:], False)
            eu, hh_eu = self.calc_eu(profit, False)
            return(eu, hh_eu)

    def calc_eu_series(self, trm_water_level, trm_k, wl_water_level, wl_k, horizon = None, elevation_cube = None, save = True):
        if horizon is None:
            horizon = self.time_horizon
        if elevation_cube is None:
            elevation_cube = self.elevation_cube
        eu_cube = np.zeros((horizon , self.elevation.shape[0], self.elevation.shape[1]), np.double)
        ec0 = elevation_cube[:horizon+1].copy()
        hh_eu_array = np.zeros((horizon, len(self.households)))
        hh_id_list = self.households.keys()
        for i in range(horizon):
            eu, hh_eu = self.calc_eu_slice(trm_water_level, trm_k, wl_water_level, wl_k, ec0, horizon, i)
            eu_cube[i] = eu.copy()
            hh_eu_array[i] = [hh_eu[hh_id].copy() for hh_id in hh_id_list]
        self.eu_cube = eu_cube.copy()
        self.set_hh_eu(hh_id_list, hh_eu_array)
        return eu_cube

    def add_breach(self, breach_x, breach_y, duration):
        self.breach_duration = duration,
        self.breaches.append(breach(self, breach_x, breach_y, self.border_height))

    def aggrade(self, heads, ws, rho, SSC, dP, dO, period = -1):
        if period < 0:
            period = self.current_period + 1
        assert(period > 0 and period <= self.time_horizon)
        sed_load = np.zeros_like(self.elevation)
        for b in self.breaches:
            sed_load += SSC * b.scaled_dist ** -2.3
        new_layer = self.elevation_cube[period - 1]
        new_layer = aggrade_patches(heads, heads.index, ws, rho, sed_load, dP, dO, new_layer, self.border_height)
        self.elevation_cube[period] = new_layer
        self.current_period = period

#%% Define functions

def load_tides(file,parser,start,end):
    df = pd.read_csv(file,parse_dates=['datetime'],date_parser=parser,index_col='datetime')
    df1 = df[(df.index >= start) & (df.index < end) & (df.index.minute == 0)]
    df2 = df1['pressure'] - np.mean(df1['pressure'])
    return df2

def aggrade_patches(heads,times,ws,rho,SSC,dP,dO,z0, z_breach):
    z = z0.copy()
    C_last = np.zeros_like(z0)
    dt = float((times[1]-times[0]).seconds)
    delta_h = (heads.values[1:] - heads.values[:-1])
    for h, dh in zip(heads[1:], delta_h):
        if h > z_breach:
            delta_z = ma.masked_less_equal(h-z, 0.0)
            delta_z.set_fill_value(0.0)
            if dh > 0:
                # C0 = 0.69 * SSC * delta_z
                C_next = ( delta_z * ( 0.69 * dh * SSC + C_last) ) / (delta_z + dh + ws/dt)
            else:
                C_next = ( C_last * delta_z ) / (delta_z + ws/dt)
        else:
            C_next = ma.array(np.zeros_like(z0), ma.nomask)
        C_last = C_next.filled()
        dz = C_last * ws * dt / rho
        z += dz + dO - dP
        # print "Sum(dz) = ", np.sum(dz), ", Sum(C_last) = ", np.sum(C_last)
    return (z)

def logit(z,k,mid):
    x = 1.0 / (1.0 + np.exp(-k*(z-mid)))
    return x

def calc_trm(pdr, discount, horizon, trm_k = 2.0, wl_k = 1.0):
    global euc
    for hh in pdr.households.values(): hh.discount = discount
    euc = pdr.calc_eu_series(MHW, trm_k, MW, wl_k, horizon)
    d_euc = euc[1:] - euc[0]
    dem = max(d_euc.max(), -d_euc.min())
    for i in range(d_euc.shape[0]):
        plt.figure();
        plt.imshow(d_euc[i], "seismic_r", vmin = -dem, vmax = dem)
        plt.axis("off")
        cbar = plt.colorbar()
        cbar.set_label("Net present value")

def test(ec = None):
    global pdr
    global tides
    global ws
    global rho
    global SSC
    global dP
    global dO
    global MW
    global HW
    global LW
    global MHW
    global MLW
    global elevation_cube
    global euc
    # global profit
    global trm_profit
    global wl_profit

    file = './data/p32_tides.dat'
    parser = lambda x: pd.datetime.strptime(x, '%d-%b-%Y %H:%M:%S')
    start = pd.datetime(2015,5,15,1)
    end = pd.datetime(2016,5,14,1)

    tides = load_tides(file,parser,start,end) + 0.25

    # Calculate Mean High Water
    pressure = tides.values
    MW = np.mean(tides)
    HW = pressure[argrelextrema(pressure, np.greater)[0]]
    LW = pressure[argrelextrema(pressure, np.less)[0]]
    MHW = np.mean(HW)
    MLW = np.mean(LW)

    X = 500 # X size of polder
    Y = 300 # Y size of polder
    year = 8759 # hours in a year
    N = 100 # number of households

    # Initialize dataframe of household paramters
    max_wealth = 10000 # initial max wealth in Taka
    max_profit = 100 # max profit per 1 m^2 land in Taka

    t = 10 # in years
    gs = 0.03 # grain size in m
    ws = ((gs/1000)**2*1650*9.8)/0.018 # settling velocity calculated using Stoke's Law
    rho = 1100 # dry bulk density in kg/m^2
    SSC = 0.2 # suspended sediment concentration in g/L
    dP = 0 # compaction
    dO = 0 # organic matter deposition

    # breach coordinates on polder
    breachX = 0
    breachY = Y/2

    pdr = polder(x = X, y = Y, time_horizon= t, n_households = N,
                 max_wealth=max_wealth, max_profit = max_profit,
                 border_height = 0.5, amplitude = 1.5, noise = 0.05)
    pdr.add_breach(breachX, breachY, t)

    if ec is None:
        t0 = time.time()
        t1 = t0
        for i in range(pdr.time_horizon):
            pdr.aggrade(tides, ws, rho, SSC, dP, dO, i + 1)
            t2 = time.time()
            print("%2d: %.02f, %.02f" % (i, float(t2 - t1), float(t2 - t0)))
            t1 = t2
        elevation_cube = pdr.elevation_cube.copy()
    else:
        pdr.elevation_cube = ec.copy()
        pdr.elevation = ec[0].copy()

    calc_trm(pdr, 0.15, 4, trm_k = 5.0)
    trm_profit = pdr.calc_profit(MHW, 5.0, elevation_cube, False)
    wl_profit = pdr.calc_profit(MW, 1.0, elevation_cube, False)

def save_images(folder = "figures", euc = None, ec = None, wl_profit = None, trm_profit = None, note = None):
    plt.ioff()
    if euc is not None:
        d_euc = euc[1:] - euc[0]
        dem = max(d_euc.max(), -d_euc.min())
        for i in range(d_euc.shape[0]):
            f = plt.figure();
            plt.imshow(d_euc[i], "seismic_r", vmin = -dem, vmax = dem)
            plt.axis("off")
            cbar = plt.colorbar()
            cbar.set_label("Net present value")
            plt.draw()
            if note is not None:
                caption = "polder_%02d_du_%s.png" % (i, note)
            else:
                caption = "polder_%02d_du.png" % i
            plt.savefig(os.path.join(folder, caption), dpi = 300)
            plt.close(f)
    if ec is not None:
        el_min = ec.min()
        el_max = ec.min()
        scale = max(el_max - MW, MW - el_min)
        for i in range(ec.shape[0]):
            f = plt.figure();
            plt.imshow(ec[i], "terrain", vmin = MW - scale, vmax = MW + scale)
            plt.axis("off")
            cbar = plt.colorbar()
            cbar.set_label("Elevation (m) rel. MW")
            plt.draw()
            if note is not None:
                caption = "polder_%02d_elev_%s.png" % (i, note)
            else:
                caption = "polder_%02d_elev.png" % i
            plt.savefig(os.path.join(folder, caption), dpi = 300)
            plt.close(f)
    if wl_profit is None:
        pmax = 0.0
    else:
        pmax = wl_profit.max()
    if trm_profit is not None:
        pmax = max(pmax, trm_profit.max())

    if wl_profit is not None:
        # pmax = max(wl_profit)
        for i in range(wl_profit.shape[0]):
            f = plt.figure();
            plt.imshow(wl_profit[i], "plasma", vmin = 0, vmax = pmax)
            plt.axis("off")
            cbar = plt.colorbar()
            cbar.set_label("Profit")
            plt.draw()
            if note is not None:
                caption = "polder_%02d_wl_profit_%s.png" % (i, note)
            else:
                caption = "polder_%02d_wl_profit.png" % i
            plt.savefig(os.path.join(folder, caption), dpi = 300)
            plt.close(f)
    if trm_profit is not None:
        # pmax = max(wl_profit)
        for i in range(trm_profit.shape[0]):
            f = plt.figure();
            plt.imshow(trm_profit[i], "plasma", vmin = 0, vmax = pmax)
            plt.axis("off")
            cbar = plt.colorbar()
            cbar.set_label("Profit")
            plt.draw()
            if note is not None:
                caption = "polder_%02d_trm_profit_%s.png" % (i, note)
            else:
                caption = "polder_%02d_trm_profit.png" % i
            plt.savefig(os.path.join(folder, caption), dpi = 300)
            plt.close(f)
    plt.ion()

def runit(force = False):
    global elevation_cube, a, v, a_res, v_res

    plt.ioff()
    if os.path.exists('elevation_cube.pickle') and not force:
        elevation_cube = pickle.load(open('elevation_cube.pickle', 'rb'))
        test(elevation_cube)
    else:
        test()
        pickle.dump(elevation_cube, open('elevation_cube.pickle', 'wb'))

    plt.draw()
    plt.close('all')
    plt.ion()

    a = auction(pdr.households)
    a_res = a.auction()

    v = election(pdr.households)
    v_res = v.vote()

def batch(force = False, trm_k = 5.0):
    global vres_list, ares_list
    vres_list = []
    ares_list = []
    if not os.path.exists('batch'):
        os.mkdir('batch')
    pickle.dump(elevation_cube, open(os.path.join('batch', 'elevation_cube.pickle'), 'wb'))
    pickle.dump(pdr.owners, open(os.path.join('batch', 'owners.pickle'), 'wb'))
    pickle.dump(pdr.plots, open(os.path.join('batch', 'plots.pickle'), 'wb'))
    # trm_profit = pdr.calc_profit(MHW, 5.0, elevation_cube, False)
    # wl_profit = pdr.calc_profit(MW, 1.0, elevation_cube, False)
    for horizon in range(3,7):
        calc_trm(pdr, 0.15, horizon, trm_k = trm_k)

        a = auction(pdr.households)
        v = election(pdr.households)
        ares = a.auction()
        vres = v.vote()
        vres_list.append(vres)
        ares_list.append(ares)
        print("Vote: winner = ", vres[0], " min utility = ", min(vres[1].values()), ", ", v.count_unhappy(vres[0]), " unhappy households")
        print("Auction: winner = ", ares[0], " min utility = ", min(ares[1].values()), ", ", a.count_unhappy(ares[0]), " unhappy households")

#%% Run program
runit()


"""
app/services/term_service.py

This module contains term-methods.
"""

from app.repositorys.term_repository import term_repository
from app.repositorys.lender_repository import lender_repository
from app.repositorys.debt_repository import debt_repository
from app.repositorys.debtor_repository import debtor_repository
from app.repositorys.address_repository import address_repository
from app.repositorys.parcels_repository import parcels_repository
from app.services.render_template_service import render
from app.models import Credor, Devedor, Divida, Endereco, TermoDivida, Parcela, Parcelamento
from datetime import date, timedelta

def adicionar_meses(data, n):
    # Adicionar n meses à data fornecida
    novo_ano = data.year + (data.month + n - 1) // 12
    novo_mes = (data.month + n - 1) % 12 + 1

    # Garantir que o dia não ultrapasse o último dia do novo mês
    ultimo_dia_do_mes = (data.replace(year=novo_ano, month=novo_mes, day=1) + timedelta(days=31)).replace(day=1) - timedelta(days=1)
    novo_dia = min(data.day, ultimo_dia_do_mes.day)

    nova_data = data.replace(year=novo_ano, month=novo_mes, day=novo_dia)
    return nova_data

class TermService:
    """
    Term Service
    """
    def get_term_by_id(self, id_termo:int):
        """
        Get data term by id
        """
        return term_repository.get_term_by_id(id_termo)

    def get_dados(self):
        """
        Get data term by id
        """
        return {'nome':'John Doe', 'cpf':'225.815.220-89', 'nacionalidade': 'Brasileira', 'estado_civil':'Solteiro', 'endereco': 'Avenida João Naves de Ávila, 2121, Santa Mônica, Uberlândia', 'num_contrato':'3127313921',
                'data_divida': '2023-09-16', 'data_renegociacao':'2023-09-17', 'produto':'Internet 5G', 'montante_valor':'500.00', 'montante_atrasado': '500.00'}

    def get_all_term_by_id(self, id_user: str):
        """
        Get data term by id
        """
        return term_repository.get_all_term_by_id(id_user)

    def post_term(self, termo:TermoDivida, parcela:Parcelamento):
        """
        Insert new data term
        """
        divida = debt_repository.get_debt_by_id(termo.id_divida)
        devedor = lender_repository.get_lerder_by_id(divida.id_devedor)
        credor = debtor_repository.get_debtor_by_id(divida.id_credor)
        end_cred = address_repository.get_address_by_id(credor.id_endereco)
        end_dev = address_repository.get_address_by_id(devedor.id_endereco)

        data = date(date.today().year,date.today().month,10)

        parcelasDb = []

        for i in range(parcela.qtd):
            parcelasDb.append(Parcela(id_parcela = parcels_repository.generate_id_parcela()+i,
                                id_termo = termo.id_termo,
                                valor_parcela = parcela.valor,
                                data_vencimento = adicionar_meses(data,i+1),
                                parcela_paga=False))

        render(devedor,credor, divida, end_dev, end_cred,termo)

        ret = term_repository.post_term(termo)

        for i in parcelasDb:
            parcels_repository.post_parcels(i)

        # return ret
        return True

    def update_term(self):
        """
        Update term data
        """
        return term_repository.update_term()

term_service = TermService()


#!/usr/bin/env python
import os
import sys
import gc
from math import *
import time
import random
import pysam

# Get position of read based on chr with sam or bam file
def get_read_pos_with_sam_bam_file(sam_bam_file, chr_len_db, bin_size, out_list):
	long_bin_size = bin_size.upper()
	long_bin_size = long_bin_size.replace('K', '000')
	long_bin_size = long_bin_size.replace('M', '000000')
	long_bin_size = long_bin_size.replace('G', '000000000')
	long_bin_size = int(long_bin_size)

	random.seed()
	read_on_chr = {}
	with pysam.AlignmentFile(sam_bam_file, 'rb') as fin:
		with open(bin_size.upper()+"_"+out_list, 'w') as fout:
			for line in fin:
				if 'WGS' in line.query_name:
					fout.write(line.strip().split()[0]+"\n")
					continue
				read_id = line.query_name

				if line.is_unmapped or line.mate_is_unmapped:
					continue

				read_pos1 = line.reference_start+1
				chr1 = line.reference_name
				chr2 = line.next_reference_name
				read_pos2 = line.next_reference_start +1

				if chr1 == chr2 and chr1 in chr_len_db:
					bin_count_of_chr = int(round((chr_len_db[chr1]*1.0/long_bin_size+0.5)))
					pos1_index = int(read_pos1/long_bin_size)
					pos2_index = int(read_pos2/long_bin_size)
					if (pos1_index+pos2_index) in range(bin_count_of_chr-11, bin_count_of_chr+10):
						border = abs(bin_count_of_chr-1-(pos1_index+pos2_index))

						if abs(pos1_index-pos2_index) < 4:
							border = 0
						elif abs(pos1_index-pos2_index) < 7:
							border = 1
						else:
							border = 4.0-int((border+1)/2)
						if random.random() > 1.0/(2**border):
							fout.write(read_id+'\n')
							if read_id in read_on_chr:
								read_on_chr.pop(read_id)
							continue
				read_on_chr[read_id] = [chr1, read_pos1, chr2, read_pos2]
	return read_on_chr

# Get chromosome length
def get_chr_len(chr_list):
	chr_len_db = {}
	chr_order = []
	with open(chr_list, 'r') as f_in:
		for line in f_in:
			if line.strip() == '':
				continue
			data = line.strip().split()
			chr_order.append(data[0])
			chr_len_db[data[0]] = int(data[1])
	return chr_len_db, chr_order

# Calc read counts on each bin
def calc_read_count_per_bin(chr_len_db, chr_order, read_on_chr, bin_size):
	long_bin_size = bin_size.upper()
	long_bin_size = long_bin_size.replace('K', '000')
	long_bin_size = long_bin_size.replace('M', '000000')
	long_bin_size = long_bin_size.replace('G', '000000000')
	long_bin_size = int(long_bin_size)

	read_count_per_chr = {}
	read_count_whole_genome = {}

	bin_offset = [0 for i in range(0, len(chr_order)+1)]
	bin_count = [0 for i in range(0, len(chr_order)+1)]
	total_bin_count = 0

	for chrn in chr_len_db:
		bin_count_of_chr = int(round((chr_len_db[chrn]*1.0/long_bin_size+0.5)))
		total_bin_count += bin_count_of_chr
		bin_count[chr_order.index(chrn)+1] = bin_count_of_chr
		read_count_per_chr[chrn] = [[0 for i in range(0, bin_count_of_chr)] for j in range(0, bin_count_of_chr)]

	for i in range(0, len(bin_count)):
		for j in range(0, i+1):
			bin_offset[i] += bin_count[j]

	read_count_whole_genome = [[0 for i in range(0, total_bin_count)] for j in range(0, total_bin_count)]

	for read in read_on_chr:
		chr1, pos1, chr2, pos2 = read_on_chr[read]
		if chr1 not in chr_len_db or chr2 not in chr_len_db:
			continue
		pos1_index = int(pos1/long_bin_size)
		pos2_index = int(pos2/long_bin_size)
		if chr1 == chr2 and chr1 in read_count_per_chr:
			read_count_per_chr[chr1][pos1_index][pos2_index] += 1
			read_count_per_chr[chr1][pos2_index][pos1_index] += 1

		chr1_index = chr_order.index(chr1)
		chr2_index = chr_order.index(chr2)

		whole_pos1 = bin_offset[chr1_index] + pos1_index
		whole_pos2 = bin_offset[chr2_index] + pos2_index
		read_count_whole_genome[whole_pos1][whole_pos2] += 1
		read_count_whole_genome[whole_pos2][whole_pos1] += 1

	for chrn in read_count_per_chr:
		for i in range(0, len(read_count_per_chr[chrn])):
			for j in range(0, len(read_count_per_chr[chrn][i])):
				if read_count_per_chr[chrn][i][j] != 0:
					read_count_per_chr[chrn][i][j] = log(read_count_per_chr[chrn][i][j], 2)
				else:
					read_count_per_chr[chrn][i][j] = -float('inf')

	for i in range(0, len(read_count_whole_genome)):
		for j in range(0, len(read_count_whole_genome[i])):
			if read_count_whole_genome[i][j] != 0:
				read_count_whole_genome[i][j] = log(read_count_whole_genome[i][j], 2)
			else:
				read_count_whole_genome[i][j] = -float('inf')

	return read_count_per_chr, read_count_whole_genome

# Draw heatmap of allhic result with matplotlib
def draw_heatmap(data, chrn, bin_size, ext):

	import matplotlib as mpl
	mpl.use('Agg')
	import matplotlib.pyplot as plt

	short_bin_size = bin_size.upper()
	if bin_size[-9:] == '000000000':
		short_bin_size = bin_size[:-9]+'G'
	elif bin_size[-6:] == '000000':
		short_bin_size = bin_size[:-6]+'M'
	elif bin_size[-3:] == '000':
		short_bin_size = bin_size[:-3]+'K'

	ax = plt.gca()

	if chrn != 'all':
		file_prefix = short_bin_size + "_" + chrn
	else:
		file_prefix = short_bin_size + '_Whole_genome'

	print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+' Draw '+file_prefix)
	with open('log.csv', 'w') as flog:
		for i in range(0, len(data)):
			for j in range(0, len(data[i])):
				flog.write("%s,"%(str(data[i][j])))
			flog.write('\n')
	# mpl.cm.YlOrRd
	cmap = plt.get_cmap('YlOrRd')
	cmap.set_over('black')
	if chrn != 'all':
		hmap = ax.imshow(data, interpolation='nearest', origin='lower', cmap=cmap, aspect='auto')
	else:
		hmap = ax.imshow(data, interpolation='nearest', cmap=cmap, aspect='auto')

	plt.colorbar(mappable=hmap,cax=None, ax=None, shrink=0.5)
	plt.tick_params(labelsize=6)
	for ticks in ax.get_xticklabels():
		ticks.set_rotation(90)
	for ticks in ax.get_yticklabels():
		ticks.set_rotation(0)

	if chrn != 'all':
		title = chrn+'_'+short_bin_size
	else:
		title = 'Whole_genome_'+short_bin_size

	plt.xlabel("Bins ("+short_bin_size.lower()+"b per bin)", fontsize=8)
	if chrn == 'all':
		plt.xticks([])
		plt.yticks([])
		plt.title(title, y=1.01, fontsize=12)
	else:
		plt.title(title, y=1.1, fontsize=12)

	plt.savefig(file_prefix+'.'+ext, bbox_inches='tight', dpi=200)
	plt.close('all')

if __name__ == "__main__":
	if len(sys.argv) < 5:
		print("Notice: This script is using for drawing heatmap of the all-hic reasult")
		print("Usage: python "+sys.argv[0]+" <sam/bam file> <chr_list> <pic_ext> <out_filter_list>")
		print("\t<sam/bam_file> is the sam or bam file filtered by allhic")
		print("\t<chr_list> is the file contain ordered chrs and length")
		print("\t<pic_ext> is the file type of picture")
		print("\t<out_filter_list> is the list of reads to be filtered")

	else:
		sam_bam_file = sys.argv[1]
		chr_list = sys.argv[2]
		ext = sys.argv[3]
		out_list = sys.argv[4]

		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Step 1: Get chromosome length")
		chr_len_db, chr_order = get_chr_len(chr_list)

		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Step 2: Calculating and Drawing heatmap")

		bin_size = "150k"
		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Get read position based on chromosome")
		read_on_chr = get_read_pos_with_sam_bam_file(sam_bam_file, chr_len_db, bin_size, out_list)
		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Calculating")
		read_count_per_chr, read_count_whole_genome = calc_read_count_per_bin(chr_len_db, chr_order, read_on_chr, bin_size)

		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Drawing heatmap")

		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Drawing with bin size "+str(bin_size))
		for chrn in read_count_per_chr:
			draw_heatmap(read_count_per_chr[chrn], chrn, bin_size, ext)

		draw_heatmap(read_count_whole_genome, 'all', bin_size, ext)
		del read_count_per_chr, read_count_whole_genome
		gc.collect()

		del read_on_chr
		gc.collect()
		print(time.strftime('[%H:%M:%S]',time.localtime(time.time()))+" Success")


import math
import random
from collections import defaultdict
from copy import copy

import matplotlib.pyplot as plt
import uuid
import jsonpickle

from helpers import get_filename
from service_handling.dynamo_handling.emotion_randomization import get_emotion_ids_by_valence
from service_handling.dynamo_handling.get_bucket_contents import list_all_objects_in_bucket
from metadata.file_metadata import Metadata
from config import video_ids, valences
from service_handling.dynamo_handling.user_creation.plots import plot_emotion_distribution, \
    plot_video_id_distribution
from service_handling.dynamo_handling.user_creation.plots import plot_file_distribution

def get_random_subset_emotion(my_list):
    my_list = copy(my_list)
    subset_length = 11
    return random.sample(my_list, subset_length)

def get_custom_replication(my_list, total_length):
    ret = []

    my_list = copy(my_list)

    replications = math.ceil(total_length / len(my_list))

    for i in range(replications):
        random.shuffle(my_list)
        ret.extend(my_list)

    return ret[:total_length]

def get_objects_as_metadata(valence):
    ret = []

    meta_list = []
    s3_objects = list_all_objects_in_bucket()
    for obj in s3_objects:
        filename = get_filename(obj["Key"])
        meta = Metadata(filename)
        if meta.emotion_1_valence == valence or meta.emotion_1_valence == "neu":
            meta_list.append(meta)

    for i in range(8):
        random.shuffle(meta_list)
        ret.extend(meta_list)

    return ret

def set_items():
    user_pool = []

    for valence in valences:

        # e.g. positive or negative type
        user_type = []

        objects_metadata = get_objects_as_metadata(valence)
        emotions = get_emotion_ids_by_valence(valence)

        # 125 instances of each user_type
        for idx in range(125):
            added_filenames = set()

            print(idx)

            user = []

            random.shuffle(objects_metadata)

            emotions_subset = get_random_subset_emotion(emotions)
            emotion_pool = get_custom_replication(emotions_subset, total_length=132)
            video_pool = get_custom_replication(video_ids, total_length=132)

            rejects = 0

            # add items to the user while emotion pool is not exhausted
            while emotion_pool:
                item = random.choice(objects_metadata)

                while item.filename in added_filenames:
                    item = random.choice(objects_metadata)

                if item.emotion_1_id in emotion_pool and item.video_id in video_pool:

                    user.append({"item": copy(item),
                                 "emotions_id_subset": copy(emotions_subset)})

                    added_filenames.add(copy(item.filename))
                    objects_metadata.remove(item)
                    emotion_pool.remove(item.emotion_1_id)
                    video_pool.remove(item.video_id)

                else:
                    rejects += 1
                    if rejects > len(objects_metadata):
                        video_pool = get_custom_replication(video_ids, total_length=len(video_pool))

            user_type.append(user)

        user_pool.append(user_type)

    return user_pool

def main():
    user_pool = set_items()

    frozen = jsonpickle.encode(user_pool)

    # Writing to sample.json
    with open("user_pool.json", "w") as outfile:
        outfile.write(frozen)

if __name__ == "__main__":
    main()


import os
import random
import asyncio
import logging
import uuid
import itertools
from io import BytesIO
import openai
import replicate
import aiohttp
from PIL import Image

from .cfg import MAX_OUTPUT_IMAGES

# openai token setup
openai.organization = os.getenv("OPENAI_ORG")
openai.api_key = os.getenv("OPENAI_API_KEY")

# replicate model setup
stable_diffusion_model = replicate.models.get("stability-ai/stable-diffusion")
clip_caption_model = replicate.models.get("rmokady/clip_prefix_caption")

async def generate_images(images: list[Image]) -> list[Image]:
    """
    Abstraction of image generation pipeline.

    :param images: list of images to use.
    :return: extension of image set.
    """
    # captioning list of images
    async def caption_image(image: Image) -> str:  # noqa: F
        file = BytesIO()
        image.save(file, "PNG")
        prediction = replicate.predictions.create(
            version=clip_caption_model.versions.list()[0],
            input={
                "image": file,
                "model": "conceptual-captions",
                "use_beam_search": True,
            },
        )
        while prediction.status not in ["succeeded", "failed", "canceled"]:
            await asyncio.sleep(0.5)
            prediction.reload()
        return prediction.output

    clip_outputs = await asyncio.gather(*map(caption_image, images))
    logging.debug(f"CLIP outputs: {clip_outputs}")
    captions = "".join(pred.strip() + "\n" for pred in clip_outputs)

    # pass captions through gpt3
    gpt3_response = openai.Completion.create(
        model="text-davinci-002", prompt=captions, temperature=0.8, max_tokens=128
    )
    logging.debug(f"GPT3 response: {gpt3_response}")
    # limit to 2 prompts for cost reasons
    diffusion_prompts = list(
        filter(
            None,
            itertools.chain(
                *[p["text"].splitlines() for p in gpt3_response["choices"]]
            ),
        )
    )[:MAX_OUTPUT_IMAGES]
    logging.debug(f"Diffusion prompts: {diffusion_prompts}")

    async def download_image(session: aiohttp.ClientSession, url: str) -> Image:
        async with session.get(url) as response:
            raw = await response.read()
        return Image.open(BytesIO(raw))

    # generate new images using stable diffusion
    async def create_image(prompt: str) -> list[Image]:
        init_image = random.choice(images)
        file = BytesIO()
        init_image.save(file, "PNG")
        try:
            prediction = replicate.predictions.create(
                version=stable_diffusion_model.versions.list()[0],
                input={
                    "prompt": prompt,
                    "width": 512,
                    "height": 512,
                    "init_image": file,
                    "prompt_strength": 0.65,
                },
            )
            while prediction.status not in ["succeeded", "failed", "canceled"]:
                await asyncio.sleep(0.5)
                prediction.reload()
            async with aiohttp.ClientSession() as session:
                image_list = await asyncio.gather(
                    *[download_image(session, url) for url in prediction.output]
                )
            logging.debug(f"Created {len(image_list)} new images")
            return image_list  # noqa: E
        except Exception as e:
            logging.warning(e)
            return []

    new_images = await asyncio.gather(*map(create_image, diffusion_prompts))
    return list(itertools.chain(*new_images))

# script to use for testing
if __name__ == "__main__":
    test_dir = "app/photos"
    save_dir = "app/results"
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    original_images = []
    for path in os.listdir(test_dir):
        image = Image.open(f"{test_dir}/{path}")
        if image.mode != "RGB":
            image = image.convert(mode="RGB")
        original_images.append(image)
    output_images = asyncio.run(generate_images(original_images))
    for image in output_images:
        image.save(f"{save_dir}/{uuid.uuid4().hex}.png")


import base64 # library for decoding base64
import pyperclip  # library for copying to clipboard

def decodeInput():
    encoded = input("please put text to decode: ")
    decoded = base64.b64decode(encoded)
    decoded = decoded.decode("utf-8")
    print("your text is: {0} \n".format(decoded))
    copy2clip(decoded)

def copy2clip(txt):
    pyperclip.copy(txt)

def main():

    exit = False

    while not exit:
        flag = input("Want to decode some text? (y/n): ")

        if flag.lower() == "y":
            decodeInput()
        elif flag.lower() == "n":
            exit = True
            print("Goodbye! \n")
        else:
            print("Please enter y or n \n")

if __name__ == '__main__':
    main()


from sqlalchemy import Column, Integer, String, Boolean, Date, Enum

from db.base_class import Base
from db.models.enumeration.service_type import ServiceType

class Service(Base):
    id = Column(Integer, primary_key=True, index=True)
    name = Column(String, nullable=False)
    date = Column(Date)
    is_done = Column(Boolean)
    service_type = Column(Enum(ServiceType))
    price = Column(Integer)


import os
import sys
from uuid import uuid1
import json
import click
import requests
import traceback

from flask import Response

from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
from flow_client.flow_cli.utils.cli_utils import check_abs_path

def prettify(response, verbose=True):
    if verbose:
        if isinstance(response, requests.models.Response):
            try:
                response_dict = response.json()
            except TypeError:
                response_dict = response.json
        else:
            response_dict = response
        try:
            click.echo(json.dumps(response_dict, indent=4, ensure_ascii=False))
        except TypeError:
            click.echo(json.dumps(response_dict.json, indent=4, ensure_ascii=False))
        click.echo('')
    return response

# def dataset_upload(url, config_data):
#     file = config_data['file']
#     config_temp_path = f"/tmp/{uuid1()}"
#     with open(config_temp_path, "w", encoding="utf-8") as fp:
#         json.dump(config_data, fp)
#     with open(file, 'rb') as fp:
#         data = MultipartEncoder(
#             fields={'file': (file, fp, 'application/octet-stream')}
#         )
#         tag = [0]

#         def read_callback(monitor):
#             if config_data.get('verbose') == 1:
#                 sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len)))
#                 sys.stdout.flush()
#                 if monitor.bytes_read / monitor.len == 1:
#                     tag[0] += 1
#                     if tag[0] == 2:
#                         sys.stdout.write('\n')

#         data = MultipartEncoderMonitor(data, read_callback)
#         return access_server('post', url, 'data/upload', json_data=None, data=data,
#                         params=config_data, headers={'Content-Type': data.content_type})
def dataset_upload(file_name, params, url):
    with open(f"/tmp/{file_name}", 'rb') as fp:
        data = MultipartEncoder(
            fields={'file': (file_name, fp, 'application/octet-stream')}
        )
        tag = [0]
        def read_callback(monitor):
            if params.get('verbose') == 1:
                sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len)))
                sys.stdout.flush()
                if monitor.bytes_read / monitor.len == 1:
                    tag[0] += 1
                    if tag[0] == 2:
                        sys.stdout.write('\n')
        data = MultipartEncoderMonitor(data, read_callback)
        resp = access_server(
            'post',
            url,
            'data/upload',
            data=data,
            params=params,
            headers={"Content-Type": data.content_type}
        )
        return resp

def access_server(method, url, postfix, json_data=None, echo=True, **kwargs):
    try:
        url = "/".join([url, postfix])
        response = {}
        if method == 'get':
            response = requests.get(url=url, json=json_data, **kwargs)
        elif method == 'post':
            response = requests.post(url=url, json=json_data, **kwargs)
        if echo:
            prettify(response)
            return response
        else:
            return response
    except Exception as e:
        exc_type, exc_value, exc_traceback_obj = sys.exc_info()
        response = {'retcode': 100, 'retmsg': str(e),
                    'traceback': traceback.format_exception(exc_type, exc_value, exc_traceback_obj)}
        if 'Connection refused' in str(e):
            response['retmsg'] = 'Connection refused. Please check if the fate flow service is started'
            del response['traceback']
        if 'Connection aborted' in str(e):
            response['retmsg'] = 'Connection aborted. Please make sure that the address of fate flow server ' \
                                    'is configured correctly. The configuration file path is: ' \
                                    '{}.'.format(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                                os.pardir, os.pardir, 'settings.yaml')))
            del response['traceback']
        if echo:
            prettify(response)
            return ""
        else:
            return Response(json.dumps(response), status=500, mimetype='application/json')


#import numpy as np
import cv2

cap = cv2.VideoCapture(0)

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # Our operations on the frame come here
    # display in gray
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #cv2.imshow('frame', gray)

    # Display the resulting frame
    #print(frame)
    #print(np.shape(frame))
    #print(type(frame))
    #print(ret)
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #print(gray)
    #print(np.shape(gray))
    #edges = np.zeros_like(gray)
    #e = np.zeros((10,2), int)
    #print(e)
    #b = []
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #edges = cv2.goodFeaturesToTrack(gray, max_corners, quality_lev, min_dist)
    ##edges = cv2.Canny(frame, edges)
    ##edges = cv2.canny(frame, edges, 10, 30)
    ##print(gray)
    #print(edges)
    #print(np.shape(gray))
    cv2.imshow('frame', frame)
    #cv2.imshow('frame', frame)
    #break
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break


import os
import xarray as xr
from py_wake.site.xrsite import XRSite
import matplotlib.pyplot as plt
import yaml
import numpy as np
from plant.examples.example_data.make_example_data import xr2yml

xr2yml

def yml2xr(file):
    pass

if __name__ == '__main__':

    def load(f):
        yml_site = XRSite(xr.open_dataset(example_data_path + f + ".nc"))
        with open(example_data_path + f + '.yml') as fid:
            xr_site = XRSite(xr.Dataset.from_dict(yaml.safe_load(fid)))
        assert yml_site.ds.equals(xr_site.ds)
        return yml_site

    # UniformSite
    site = load('UniformSite')
    plt.figure()
    site.plot_wd_distribution(ws_bins=4)
    plt.title('UniformSite')

    # UniformWeibullSite
    f = 'UniformWeibullSite'
    site = load(f)
    plt.figure()
    site.plot_wd_distribution(ws_bins=4)
    plt.title(f)
    plt.figure()
    site.plot_ws_distribution()
    plt.title(f)

    # NonGriddedSite
    f = 'NonGriddedSite'
    site = load(f)
    x = y = np.arange(16)  # dummy positions
    lw = site.local_wind(x, y, wd=0, ws=10)
    plt.figure()
    plt.title(f)
    lw.WS.plot()

    # GriddedSite
    f = 'GriddedSite'
    site = load(f)
    x = y = np.arange(-2000, 2500, 500)
    X, Y = np.meshgrid(x, y)
    lw = site.local_wind(X.flatten(), Y.flatten(), wd=0, ws=10)
    plt.figure()
    plt.title(f)
    plt.contourf(X, Y, np.reshape(lw.WS, X.shape))

    plt.show()


import argparse
import re
import subprocess

import pytest

from cotainr.cli import CotainrCLI, Info
from ..util.patches import patch_empty_system, patch_system_with_actual_file

class TestExecute:
    def test_report_structure(self, capsys):
        Info().execute()
        stdout = capsys.readouterr().out

        # Dependency report section
        assert stdout.startswith(
            "Dependency report\n"
            "-------------------------------------------------------------------------------\n"
        )
        # One line with python version check
        assert (
            sum(line.startswith("    - Running python ") for line in stdout.split("\n"))
            == 1
        )
        # One line with singularity / apptainer version check
        assert (
            sum(
                any(provider in line for provider in ["singularity", "apptainer"])
                for line in stdout.split("\n")
            )
            == 1
        )

        # Systems info section
        assert (
            "\n"
            "System info\n"
            "-------------------------------------------------------------------------------\n"
        ) in stdout

    def test_with_systems(self, capsys, patch_system_with_actual_file):
        Info().execute()
        stdout = capsys.readouterr().out
        assert stdout.endswith(
            "System info\n"
            "-------------------------------------------------------------------------------\n"
            "Available system configurations:\n"
            "    - some_system_6021\n"
            "    - another_system_6021\n"
        )

    def test_without_systems(self, capsys, patch_empty_system):
        Info().execute()
        stdout = capsys.readouterr().out
        assert stdout.endswith(
            "System info\n"
            "-------------------------------------------------------------------------------\n"
            "No system configurations available\n"
        )

class TestAddArguments:
    def test_no_arguments_added(self):
        parser = argparse.ArgumentParser()
        Info.add_arguments(parser=parser)
        args = parser.parse_args(args=[])
        assert not vars(args)

class TestHelpMessage:
    def test_CLI_subcommand_help_message(self, argparse_options_line, capsys):
        with pytest.raises(SystemExit):
            CotainrCLI(args=["info", "--help"])
        stdout = capsys.readouterr().out
        assert stdout == (
            # Capsys apparently assumes an 80 char terminal (?) - thus extra '\n'
            "usage: cotainr info [-h]\n\n"
            "Obtain info about the state of all required dependencies for building a\n"
            "container.\n\n"
            f"{argparse_options_line}"
            "  -h, --help  show this help message and exit\n"
        )

class Test_check_python_dependency:
    def test_formatting(self):
        # Check for formatting like "Running python 3.11.0 >= 3.8.0, OK"
        assert re.match(
            (
                r"^Running python \d+\.\d+\.\d+ (>=)|(<) \d+\.\d+\.\d+, "
                r"(\\033\[92mOK\\033\[0m)|(\\033\[91mERROR\\033\[0m)$"
            ),
            Info()._check_python_dependency(),
        )

class Test_check_singularity_dependency:
    def test_found_apptainer(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "apptainer version 1.0.0",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found apptainer 1.0.0 >= 1.0.0, \x1b[92mOK\x1b[0m"
        )

    def test_found_singularity(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "singularity version 3.7.4-1",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found singularity 3.7.4-1 >= 3.7.4, \x1b[92mOK\x1b[0m"
        )

    def test_found_unknown(self, monkeypatch):
        monkeypatch.setattr(
            subprocess,
            "check_output",
            lambda *args, **kwargs: "container_tool_6021 version 60.2.1",
        )
        assert (
            Info()._check_singularity_dependency()
            == "Found unknown singularity provider: container_tool_6021 60.2.1"
        )

    def test_not_found(self, monkeypatch):
        def not_found(*args, **kwargs):
            raise FileNotFoundError()

        monkeypatch.setattr(
            subprocess,
            "check_output",
            not_found,
        )
        assert (
            Info()._check_singularity_dependency()
            == "apptainer/singularity not found, \033[91mERROR\033[0m"
        )

class Test_check_version:
    @pytest.mark.parametrize(
        "version,min_version,cmp_result",
        [
            ((3, 8, 0), (3, 8, 0), ">= 3.8.0, \x1b[92mOK\x1b[0m"),
            ((0, 1, 12), (0, 0, 1), ">= 0.0.1, \x1b[92mOK\x1b[0m"),
            ((3, 7, 14), (3, 8, 0), "< 3.8.0, \033[91mERROR\033[0m"),
            ((1, 0, 0), (3, 8, 0), "< 3.8.0, \033[91mERROR\033[0m"),
        ],
    )
    def test_version_comparison(self, version, min_version, cmp_result):
        assert (
            Info()._check_version(version=version, min_version=min_version)
            == cmp_result
        )

    def test_wrong_input_len(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=(1, 0, 0, 0), min_version=(1, 0, 0))
        assert str(exc_info.value) == (
            "The 'version=(1, 0, 0, 0)' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )

    def test_wrong_input_type(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=(1, 0, 0), min_version="1.0.0")
        assert str(exc_info.value) == (
            "The 'min_version=1.0.0' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )

    def test_wrong_part_type(self):
        with pytest.raises(TypeError) as exc_info:
            Info()._check_version(version=("1", "0", "0"), min_version=(1, 0, 0))
        assert str(exc_info.value) == (
            "The 'version=('1', '0', '0')' argument must be "
            "a (major, minor, patchlevel) tuple of integers"
        )