from django.shortcuts import get_object_or_404, render, redirect
from land.models import Land
from harvest.models import Harvest
from .models import Applications, Labors, SowingPurchases, ProductsRows
from .forms import SowingPurchasesForm, ApplicationForm, LoteForm, LaborsForm, ProductRowForm, EditTipodeCambioForm
from land .models import Campaign, Lote
from payments.models import EndorsedChecks, SelfChecks, Payments, ThirdPartyChecks
from payments.forms import PaymentForm, SelfChecksForm, EndorsedChecksForm
from django.contrib.auth.decorators import login_required
from purchases.forms import SearchForm, DateForm
from django.contrib.postgres.search import SearchVector
from harvest.forms import HarvestForm
from payments.forms import ChangeStateForm, DestroyObjectForm
from .forms import ChooseCampoForm, LoteNumberForm
import itertools
from payments.forms import DestroyObjectForm

@login_required
def sowing_purchases_list(request):

    search_form = SearchForm()

    date_form = DateForm()

    query = None

    date_query_start = None
    date_query_end = None

    if 'campaign' in request.session:
        campana = Campaign.objects.get(nombre=request.session['campaign'])
    elif Campaign.objects.all():
        campana = Campaign.objects.all()[0]

    sowing_purchases = SowingPurchases.objects.filter(campaña = campana)

    total_purchases = sowing_purchases.count()

    unpayed_purchases = sowing_purchases.filter(status='por pagar')

    total_unpayed_purchases = unpayed_purchases.count()

    if 'query' in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            query = form.cleaned_data['query']
            sowing_purchases = sowing_purchases.annotate(search=SearchVector('proveedor'),).filter(search=query)

    if 'date_query_start' and 'date_query_end' in request.GET:
        form = DateForm(request.GET)
        if form.is_valid():
            date_query_start = form.cleaned_data['date_query_start'].strftime("%Y-%m-%d")
            date_query_end = form.cleaned_data['date_query_end'].strftime("%Y-%m-%d")
            sowing_purchases = sowing_purchases.filter(date__range=[date_query_start, date_query_end])

    total_amounts_to_pay = []
    for purchase in unpayed_purchases:
        unpayed_amount = float(purchase.calculate_amount_to_pay())
        total_amounts_to_pay.append(unpayed_amount)

    total_amount_to_pay = sum(total_amounts_to_pay)

    return render(request, 'sowing/sowing_purchases_list.html', {
                                                                'campaña':campana,
                                                                'sowing_purchases':sowing_purchases,
                                                                'total_purchases':total_purchases,
                                                                'total_unpayed_purchases':total_unpayed_purchases,
                                                                'total_amount_to_pay':total_amount_to_pay,
                                                                'search_form':search_form,
                                                                'query':query,
                                                                'date_form':date_form,
                                                                'date_query_start':date_query_start,
                                                                'date_query_end':date_query_end,
                                                                })

@login_required
def sowing_purchases_create(request):

    new_sowing_purchase = None

    if request.method == 'POST':
        sowing_p_form = SowingPurchasesForm(data=request.POST)

        if sowing_p_form.is_valid():
            factura = sowing_p_form.cleaned_data.get('factura')
            proveedor = sowing_p_form.cleaned_data.get('proveedor')
            tipo_cambio = sowing_p_form.cleaned_data.get('tipo_cambio')
            iva = sowing_p_form.cleaned_data.get('iva')
            total = sowing_p_form.cleaned_data.get('total')

            if 'campaign' in request.session:
                campana = Campaign.objects.get(nombre=request.session['campaign'])
            elif Campaign.objects.all():
                campana = Campaign.objects.all()[0]

            attrs = {'campaña':campana,'factura':factura,
                                        'proveedor':proveedor,
                                        'tipo_cambio':tipo_cambio,
                                        'iva':iva, 'total':total}

            new_sowing_purchase = SowingPurchases(**attrs)
            new_sowing_purchase.save()

        return redirect(new_sowing_purchase.get_absolute_url())

    else:
        sowing_p_form = SowingPurchasesForm()

    return render(request,'sowing/sowing_purchases_create.html',{
                                                                'sowing_p_form':sowing_p_form,
                                                                    })

@login_required
def sowing_purchase_detail(request, id):

    sowing_purchase = get_object_or_404(SowingPurchases, id=id)

    precio_lt_kg = sowing_purchase.precio_lt_kg_usd * sowing_purchase.tipo_cambio

    sub_total_usd = sowing_purchase.precio_lt_kg_usd * sowing_purchase.lt_kg

    total_usd = sub_total_usd + (sub_total_usd * (sowing_purchase.iva/100))

    payments = sowing_purchase.payments

    self_checks = sowing_purchase.self_checks

    self_checks = [check for check in self_checks if check.estado != 'anulado']

    endorsed_checks = sowing_purchase.endorsed_checks

    third_p_checks = ThirdPartyChecks.objects.filter(estado='a depositar')

    initial_payment_data = {
        'content_type': sowing_purchase.get_content_type,
        'object_id': sowing_purchase.id,
    }

    payment_form = PaymentForm(request.POST or None, initial= initial_payment_data)

    self_check_form =SelfChecksForm(request.POST or None, initial=initial_payment_data)

    endorsed_checks_form = EndorsedChecksForm(request.POST or None, initial=initial_payment_data)

    products_row_form = ProductRowForm(request.POST or None)

    destroy_object_form = DestroyObjectForm(request.POST or None)

    if sowing_purchase.calculate_amount_to_pay() <= 0:
        sowing_purchase.status = 'pagado'
        sowing_purchase.save()

    if payment_form.is_valid():
        content_type = payment_form.cleaned_data.get('content_type')
        obj_id = payment_form.cleaned_data.get('object_id')
        monto = payment_form.cleaned_data.get('monto')
        tipo = payment_form.cleaned_data.get('tipo')

        attrs = {'content_type':content_type, 'object_id':obj_id, 'monto':monto, 'tipo':tipo}

        new_payment = Payments(**attrs)
        new_payment.save()

        return redirect(sowing_purchase.get_absolute_url())

    if self_check_form.is_valid():
        content_type = self_check_form.cleaned_data.get('content_type')
        obj_id = self_check_form.cleaned_data.get('object_id')
        fecha_pago = self_check_form.cleaned_data.get('fecha_pago')
        banco_emision = self_check_form.cleaned_data.get('banco_emision')
        numero_cheque = self_check_form.cleaned_data.get('numero_cheque')
        titular_cheque = self_check_form.cleaned_data.get('titular_cheque')
        monto = self_check_form.cleaned_data.get('monto')

        cliente = sowing_purchase.proveedor
        descripcion = sowing_purchase

        attrs = {'content_type':content_type, 'object_id':obj_id,
                                                'cliente':cliente,
                                                'descripcion':descripcion,
                                                'fecha_pago':fecha_pago,
                                                'banco_emision':banco_emision,
                                                'numero_cheque':numero_cheque,
                                                'titular_cheque':titular_cheque,
                                                'monto':monto,
                                                    }

        new_self_check = SelfChecks(**attrs)
        new_self_check.save()

        return redirect(sowing_purchase.get_absolute_url())

    if endorsed_checks_form.is_valid() and request.POST.get("check_id"):
        third_p_check = ThirdPartyChecks.objects.get(pk=int(request.POST.get("check_id")))

        content_type = endorsed_checks_form.cleaned_data.get('content_type')
        obj_id = endorsed_checks_form.cleaned_data.get('object_id')
        fecha_deposito = third_p_check.fecha_deposito
        banco_emision = third_p_check.banco_emision
        numero_cheque = third_p_check.numero_cheque
        titular_cheque = third_p_check.titular_cheque
        monto = third_p_check.monto
        cliente = third_p_check.cliente
        descripcion = third_p_check.descripcion
        observacion = ''
        third_p_id = third_p_check.id

        attrs = {'content_type':content_type, 'object_id':obj_id,
                                    'cliente':cliente,
                                    'descripcion': descripcion,
                                    'fecha_deposito':fecha_deposito,
                                    'banco_emision':banco_emision,
                                    'numero_cheque':numero_cheque,
                                    'titular_cheque':titular_cheque,
                                    'monto':monto,
                                    'observacion':observacion,
                                    'third_p_id':third_p_id,
                                    }

        new_endorsed_check = EndorsedChecks(**attrs)
        new_endorsed_check.save()

        third_p_check.estado = 'endosado'
        third_p_check.save()

        return redirect(sowing_purchase.get_absolute_url())

    if request.method == 'POST':
        destroy_object_form = DestroyObjectForm(data=request.POST)
        if destroy_object_form.is_valid() and request.POST.get('delete_token'):

            for check in endorsed_checks:
                check_id = check.third_p_id
                third_p_check = ThirdPartyChecks.objects.get(id=check_id)
                third_p_check.estado = 'a depositar'
                third_p_check.save()
                check.delete()

            for check in self_checks:
                check.delete()

            for payment in payments:
                payment.delete()

            sowing_purchase.delete()
            return redirect('sowing:sowing_purchases_list')
    else:
        destroy_object_form = DestroyObjectForm()

    if products_row_form.is_valid():

        lt_kg = products_row_form.cleaned_data.get('lt_kg')
        product = products_row_form.cleaned_data.get('product')
        precio_lt_kg_usd = products_row_form.cleaned_data.get('precio_lt_kg_usd')

        attrs = {
            'lt_kg':lt_kg, 'product':product.lower(), 'precio_lt_kg_usd':precio_lt_kg_usd,
            'sowing_purchase':sowing_purchase,
        }

        new_product_row = ProductsRows(**attrs)
        new_product_row.save()

        return redirect(sowing_purchase.get_absolute_url())

    if destroy_object_form.is_valid and request.POST.get('row_id'):
        row_id = int(request.POST.get("row_id"))
        row = ProductsRows.objects.get(pk=row_id)
        row.delete()

        return redirect(sowing_purchase.get_absolute_url())

    product_rows = sowing_purchase.productsrows_set.all()

    tipo_cambio_form = EditTipodeCambioForm(request.POST or None)

    if tipo_cambio_form.is_valid() and request.POST.get('change_token'):
        sowing_purchase.tipo_cambio = tipo_cambio_form.cleaned_data.get('number')

        if sowing_purchase.calculate_amount_to_pay() > 0:
            sowing_purchase.status = 'por pagar'

        sowing_purchase.save()

    return render(request, 'sowing/sowing_purchase_detail.html', {
                                                                'sowing_purchase':sowing_purchase,
                                                                'precio_lt_kg':precio_lt_kg,
                                                                'sub_total_usd':sub_total_usd,
                                                                'total_usd':total_usd,
                                                                'payment_form':payment_form,
                                                                'payments':payments,
                                                                'self_checks':self_checks,
                                                                'self_check_form':self_check_form,
                                                                'third_p_checks':third_p_checks,
                                                                'endorsed_checks_form':endorsed_checks_form,
                                                                'endorsed_checks':endorsed_checks,
                                                                'destroy_object_form':destroy_object_form,
                                                                'product_rows':product_rows,
                                                                'products_row_form':products_row_form,
                                                                'tipo_cambio_form':tipo_cambio_form,
                                                                })

@login_required
def sowing_purchase_update(request, id):

    sowing_purchase = get_object_or_404(SowingPurchases, id=id)

    if request.method == 'POST':
        sowing_p_form = SowingPurchasesForm(data=request.POST)

        if sowing_p_form.is_valid():

            payments = sowing_purchase.payments

            self_checks = sowing_purchase.self_checks

            endorsed_checks = sowing_purchase.endorsed_checks

            for check in endorsed_checks:
                    check_id = check.third_p_id
                    third_p_check = ThirdPartyChecks.objects.get(id=check_id)
                    third_p_check.estado = 'a depositar'
                    third_p_check.save()
                    check.delete()

            for check in self_checks:
                check.delete()

            for payment in payments:
                payment.delete()

            factura = sowing_p_form.cleaned_data.get('factura')
            proveedor = sowing_p_form.cleaned_data.get('proveedor')
            producto = sowing_p_form.cleaned_data.get('producto')
            precio_lt_kg_usd = sowing_p_form.cleaned_data.get('precio_lt_kg_usd')
            lt_kg = sowing_p_form.cleaned_data.get('lt_kg')
            tipo_cambio = sowing_p_form.cleaned_data.get('tipo_cambio')
            iva = sowing_p_form.cleaned_data.get('iva')

            date = sowing_purchase.date
            if 'campaign' in request.session:
                campana = Campaign.objects.get(nombre=request.session['campaign'])
            elif Campaign.objects.all():
                campana = Campaign.objects.all()[0]

            attrs = {'campaña':campana,
                                        'factura':factura,
                                        'proveedor':proveedor,
                                        'producto':producto,
                                        'precio_lt_kg_usd':precio_lt_kg_usd,
                                        'lt_kg':lt_kg,
                                        'tipo_cambio':tipo_cambio,
                                        'iva':iva,
                                        'date':date}

            sowing_purchase = SowingPurchases(id=id,**attrs)
            sowing_purchase.save()

        return redirect('sowing:sowing_purchases_list')

    else:
        sowing_p_form = SowingPurchasesForm()

    return render(request,'sowing/sowing_purchase_update.html',{'sowing_p_form':sowing_p_form})

@login_required
def products_averages(request):

    if 'campaign' in request.session:
        campana = Campaign.objects.get(nombre=request.session['campaign'])
    elif Campaign.objects.all():
        campana = Campaign.objects.all()[0]

    product_dict = SowingPurchases.calculate_averages(campana)[0]

    return render(request, 'sowing/product_averages.html',{
                                                            'product_dict':product_dict,
                                                            'campaña':campana,
                                                            })

@login_required
def lotes_list(request):
    if 'campaign' in request.session:
        campana = Campaign.objects.get(nombre=request.session['campaign'])
    elif Campaign.objects.all():
        campana = Campaign.objects.all()[0]

    lotes = Lote.objects.filter(campaña=campana)

    search_form = SearchForm()

    campo_form = ChooseCampoForm(campana=campana)

    number_form = LoteNumberForm()

    query = None

    campo = None

    number_query = None

    if 'number_query' in request.GET:
        form = LoteNumberForm(request.GET)
        if form.is_valid():
            number_query = int(form.cleaned_data['number_query'])
            lotes = lotes.filter(numero = number_query)

    if 'campo' in request.GET:
        form = ChooseCampoForm(request.GET, campana=campana)
        if form.is_valid():
            campo = str(form.cleaned_data['campo'])
            campo = campo.capitalize()
            campo_query = Land.objects.filter(nombre=campo).first()
            lotes = lotes.filter(campo=campo_query)

    if 'query' in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            query = form.cleaned_data['query']
            lotes = lotes.annotate(search=SearchVector('tipo'),).filter(search=query)

    return render(request,'sowing/lotes_list.html',{
                                                    'lotes':lotes,
                                                    'campaña':campana,
                                                    'search_form':search_form,
                                                    'query':query,
                                                    'campo_form':campo_form,
                                                    'campo':campo,
                                                    'number_form':number_form,
                                                    'number_query':number_query,
                                                })

@login_required
def lote_create(request):

    if 'campaign' in request.session:
        campana = Campaign.objects.get(nombre=request.session['campaign'])
    elif Campaign.objects.all():
        campana = Campaign.objects.all()[0]

    campos = Land.objects.all()

    if request.method == 'POST':
        lote_form = LoteForm(request.POST, campana=campana)

        if lote_form.is_valid():

            campo = lote_form.cleaned_data.get('campo')
            numero = lote_form.cleaned_data.get('numero')
            hectareas= lote_form.cleaned_data.get('hectareas')
            tipo= lote_form.cleaned_data.get('tipo')

            attrs = {'campo':campo, 'numero':numero,
                    'hectareas':hectareas, 'tipo':tipo,
                    'campaña':campana }

            new_lote = Lote(**attrs)
            new_lote.save()

            return redirect('sowing:lotes_list')

    else:
        lote_form = LoteForm(campana=campana)

    return render(request, 'sowing/lote_create.html',{
                                                    'lote_form':lote_form,
                                                    'campos':campos,
                                                    })

@login_required
def lote_detail(request,  lote_id):

    if 'campaign' in request.session:
        campana = Campaign.objects.get(nombre=request.session['campaign'])
    elif Campaign.objects.all():
        campana = Campaign.objects.all()[0]

    lote = get_object_or_404(Lote, pk=lote_id)

    product_choices = SowingPurchases.calculate_averages(campana)[1]

    product_choices = tuple(tuple(product) for product in product_choices)

    product_averages = SowingPurchases.calculate_averages(campana)[0]

    lote_view = request.get_full_path()

    sowing_purchases = SowingPurchases.objects.all()

    sowing_purchases_products_rows = [sowing_purchase.productsrows_set.all() for sowing_purchase in sowing_purchases]

    products = [list(set(map(str,row.values_list('product',flat=True)))) for row in sowing_purchases_products_rows]

    products = list(itertools.chain(*products))

    products = list(dict.fromkeys(products))

    if request.method == 'POST':
        application_form = ApplicationForm(data=request.POST, products=products)
        labors_form = LaborsForm(data=request.POST)
        harvest_form = HarvestForm(data=request.POST)
        change_state_form = ChangeStateForm(data=request.POST)
        destroy_object_form = DestroyObjectForm(data=request.POST)

        if application_form.is_valid():

            numero = application_form.cleaned_data.get('numero')
            lt_kg = application_form.cleaned_data.get('lt_kg')
            producto = application_form.cleaned_data.get('producto')
            tipo = application_form.cleaned_data.get('tipo')

            attrs = {'numero':numero, 'lt_kg':lt_kg,
                        'producto':producto, 'lote':lote, 'tipo':tipo}

            new_application = Applications(**attrs)
            new_application.save()

            return redirect(lote_view)

        if labors_form.is_valid():
            numero = labors_form.cleaned_data.get('numero')
            costo_ha = labors_form.cleaned_data.get('costo_ha')
            nombre = labors_form.cleaned_data.get('nombre').lower()

            attrs = {'numero':numero, 'costo_ha':costo_ha, 'nombre':nombre,
                        'lote':lote}

            new_labor = Labors(**attrs)
            new_labor.save()

            return redirect(lote_view)

        if harvest_form.is_valid():
            kg_totales = harvest_form.cleaned_data.get('kg_totales')
            attrs = {'kg_totales':kg_totales, 'lote':lote}
            new_harvest = Harvest(**attrs)
            new_harvest.save()
            lote.estado = 'cosechado'
            lote.save()

            return redirect(lote_view)

        if change_state_form.is_valid() and request.POST.get('change_state_token'):
            cosecha = Harvest.objects.filter(lote=lote)
            cosecha.delete()
            lote.estado = 'no cosechado'
            lote.save()

            return redirect(lote_view)

        if destroy_object_form.is_valid() and request.POST.get('delete_token'):
            lote.delete()
            return redirect('sowing:lotes_list')

    else:
        application_form = ApplicationForm(products=products)
        labors_form = LaborsForm()
        harvest_form = HarvestForm()
        change_state_form = ChangeStateForm()
        destroy_object_form = DestroyObjectForm()

    applications = lote.applications_set.all()

    application_numbers = list(map(int,applications.values_list('numero', flat=True)))

    applications_dict = {}
    for number in application_numbers:
        applications_dict[number] = {}
        number_x_applications = applications.filter(numero=number)
        for application in number_x_applications:
            applications_dict[number][application] = []
            applications_dict[number][application].append(application.calculate_sub_total(product_averages)[0])
            applications_dict[number][application].append(application.calculate_sub_total(product_averages)[1])

    application_totals_by_type = {}
    applicaton_types = list(map(str,applications.values_list('tipo', flat=True)))
    for type in applicaton_types:
        type_applications = applications.filter(tipo=type)
        type_total = 0

        for application in type_applications:
            application_sub_total = float(application.calculate_sub_total(product_averages)[0])
            type_total = type_total + application_sub_total

        application_totals_by_type[type] = type_total

    applications_total = sum(application_totals_by_type.values())

    labors = lote.labors_set.all()
    labors_numbers = list(map(int,labors.values_list('numero', flat=True)))
    labors_dict = {}
    for number in labors_numbers:
        labors_dict[number] = {}
        number_x_labors = labors.filter(numero=number)
        for labor in number_x_labors:
            labors_dict[number][labor] = labor.calculate_sub_total()

    labors_totals_by_type = {}
    labors_types = list(map(str,labors.values_list('nombre', flat=True)))
    for type in labors_types:
        type_labors = labors.filter(nombre=type)
        type_total = 0

        for labor in type_labors:
            labor_sub_total = float(labor.calculate_sub_total())
            type_total = type_total + labor_sub_total

        labors_totals_by_type[type] = type_total

    labors_total = sum(labors_totals_by_type.values())

    lote_total = applications_total + labors_total

    if lote.estado == 'cosechado':
        cosecha = Harvest.objects.filter(lote=lote).first()
        kg_totales = cosecha.kg_totales
        quintales_ha = (kg_totales/lote.hectareas)/100
    else:
        kg_totales = None
        quintales_ha = None

    return render(request, 'sowing/lote_detail.html', {
                                                        'lote':lote,
                                                        'application_form':application_form,
                                                        'applications_dict':applications_dict,
                                                        'application_totals_by_type':application_totals_by_type,
                                                        'labors_form':labors_form,
                                                        'labors':labors,
                                                        'labors_dict':labors_dict,
                                                        'labors_totals_by_type':labors_totals_by_type,
                                                        'applications_total':applications_total,
                                                        'labors_total': labors_total,
                                                        'lote_total':lote_total,
                                                        'harvest_form': harvest_form,
                                                        'change_state_form':change_state_form,
                                                        'kg_totales':kg_totales,
                                                        'quintales_ha':quintales_ha,
                                                        'destroy_object_form':destroy_object_form,
                                                        'products':products,
                                                        })

@login_required
def lote_update(request, id):

    lote = get_object_or_404(Lote, id=id)

    if request.method == 'POST':
        lote_form = LoteForm(data=request.POST)

        if lote_form.is_valid():

            campo = lote_form.cleaned_data.get('campo')
            numero = lote_form.cleaned_data.get('numero')
            hectareas= lote_form.cleaned_data.get('hectareas')
            tipo= lote_form.cleaned_data.get('tipo')

            if 'campaign' in request.session:
                campana = Campaign.objects.get(nombre=request.session['campaign'])
            elif Campaign.objects.all():
                campana = Campaign.objects.all()[0]

            attrs = {'campo':campo, 'numero':numero,
                    'hectareas':hectareas, 'tipo':tipo,
                    'campaña':campana }

            lote = Lote(id=id, **attrs)
            lote.save()

            return redirect(lote.get_absolute_url())

    else:
        lote_form = LoteForm()

    return render(request, 'sowing/lote_update.html',{
                                                        'lote':lote,
                                                        'lote_form':lote_form,
                                                    })

@login_required
def application_detail(request,id):

    application = get_object_or_404(Applications, id=id)

    lote = application.lote

    if request.method == 'POST':
        destroy_object_form = DestroyObjectForm(data=request.POST)
        if destroy_object_form.is_valid() and request.POST.get('delete_token'):
            application.delete()
            return redirect(lote.get_absolute_url())

    else:
        destroy_object_form = DestroyObjectForm()

    return render(request, 'sowing/application_detail.html',{
                                                            'application':application,
                                                            'destroy_object_form':destroy_object_form,
                                                            })

@login_required
def application_update(request,id):

    application = get_object_or_404(Applications, id=id)

    lote = application.lote

    sowing_purchases = SowingPurchases.objects.all()

    sowing_purchases_products_rows = [sowing_purchase.productsrows_set.all() for sowing_purchase in sowing_purchases]

    products = [list(set(map(str,row.values_list('product',flat=True)))) for row in sowing_purchases_products_rows]

    products = list(itertools.chain(*products))

    products = list(dict.fromkeys(products))

    if request.method == 'POST':
        application_form = ApplicationForm(data=request.POST, products=products)

        if application_form.is_valid():

            numero = application_form.cleaned_data.get('numero')
            producto = application_form.cleaned_data.get('producto')
            lt_kg =  application_form.cleaned_data.get('lt_kg')
            tipo= application_form.cleaned_data.get('tipo')

            date = application.date

            attrs = {'numero':numero, 'producto':producto,
                    'lt_kg':lt_kg, 'tipo':tipo, 'date':date,
                    'lote':lote }

            application = Applications(id=id, **attrs)
            application.save()

            return redirect(lote.get_absolute_url())

    else:
        application_form = ApplicationForm(products=products)

    return render(request, 'sowing/application_update.html',{
                                                            'application':application,
                                                            'application_form':application_form,
                                                            })

@login_required
def labor_detail(request, id):

    labor = get_object_or_404(Labors, id=id)

    lote = labor.lote

    if request.method == 'POST':
        destroy_object_form = DestroyObjectForm(data=request.POST)
        if destroy_object_form.is_valid() and request.POST.get('delete_token'):
            labor.delete()
            return redirect(lote.get_absolute_url())

    else:
        destroy_object_form = DestroyObjectForm()

    return render(request, 'sowing/labor_detail.html',{
                                                        'labor':labor,
                                                        'destroy_object_form':destroy_object_form,
                                                        })

@login_required
def labor_update(request, id):

    labor = get_object_or_404(Labors, id=id)

    lote = labor.lote

    if request.method == 'POST':
        labor_form = LaborsForm(data=request.POST)

        if labor_form.is_valid():

            numero = labor_form.cleaned_data.get('numero')
            nombre = labor_form.cleaned_data.get('nombre')
            costo_ha =  labor_form.cleaned_data.get('costo_ha')

            date = labor.date

            attrs = {'numero':numero, 'nombre':nombre,
                    'costo_ha':costo_ha, 'date':date,
                    'lote':lote }

            labor = Labors(id=id, **attrs)
            labor.save()

            return redirect(lote.get_absolute_url())

    else:
        labor_form = LaborsForm()

    return render(request, 'sowing/labor_update.html', {
                                                        'labor':labor,
                                                        'labor_form':labor_form,
                                                        })


import uvicorn
from fastapi import Depends, FastAPI, HTTPException
from sqlalchemy.engine import CursorResult
from sqlalchemy.ext.asyncio.engine import AsyncConnection
from sqlalchemy.schema import CreateTable

import domains
import tables
from conf.settings import settings
from dependencies import engine_begin, engine_connect

app = FastAPI()

@app.on_event("startup")
async def startup():
    print("on startup")

@app.on_event("shutdown")
async def shutdown():
    print("on shutdown")

@app.get("/init")
async def create_database(conn: AsyncConnection = Depends(engine_begin)):
    # tables.metadata.create_all(engien) - does not work asynchronously!
    # Inspection on an AsyncEngine is currently not supported.
    for table in tables.all:
        await conn.execute(CreateTable(table))
    return {"Database": "Initialized"}

@app.get("/")
async def read_all(conn: AsyncConnection = Depends(engine_connect)):
    cr: CursorResult = await conn.execute(tables.todos.select())
    # Use CursorResult if you need iterate data.
    """
    for row in cr:
        print(row)
    """
    # Use fetchall() if you need to return all
    return cr.fetchall()

@app.post("/")
async def create_todo(
    todo: domains.Todo, conn: AsyncConnection = Depends(engine_begin)
):
    await conn.execute(
        tables.todos.insert().values(
            title=todo.title,
            description=todo.description,
            priority=todo.priority,
            complete=todo.complete,
        )
    )
    return successful_response(201)

@app.get("/todo/{todo_id}")
async def read_todo(todo_id: int, conn: AsyncConnection = Depends(engine_connect)):
    todo_model = await conn.execute(
        tables.todos.select().where(tables.todos.c.id == todo_id)
    ).first()

    if todo_model:
        return todo_model

    raise http_exception()

@app.put("/{todo_id}")
async def update_todo(
    todo_id: int, todo: domains.Todo, conn: AsyncConnection = Depends(engine_begin)
):
    todo_model = await conn.execute(
        tables.todos.select().where(tables.todos.c.id == todo_id)
    ).first()

    if todo_model:
        await conn.execute(
            tables.todos.update()
            .where(tables.todos.c.id == todo_id)
            .values(
                title=todo.title,
                description=todo.description,
                priority=todo.priority,
                complete=todo.complete,
            )
        )
        return successful_response(200)

    raise http_exception()

@app.delete("/{todo_id}")
async def delete_todo(todo_id: int, conn: AsyncConnection = Depends(engine_begin)):
    todo_model = await conn.execute(
        tables.todos.select().where(tables.todos.c.id == todo_id)
    ).first()

    if todo_model:
        await conn.execute(tables.todos.delete().where(tables.todos.c.id == todo_id))

        return successful_response(200)

    raise http_exception()

def successful_response(status_code: int):
    return {"status": status_code, "transaction": "Successful"}

def http_exception():
    return HTTPException(status_code=404, detail="Todo not found")

if __name__ == "__main__":
    uvicorn.run(
        "main:app",
        host=settings["HOST"],
        port=settings["PORT"],
        debug=settings["DEBUG"],
        reload=settings["RELOAD"],
    )


import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.validation import check_X_y, check_array, check_is_fitted
from sklearn.metrics import euclidean_distances
from sklearn.utils.multiclass import unique_labels
import math

class Node:
    def __init__(self,
                depth,
                label = None,
                description = None,
                best_feature_id = -1,
                best_feature_threshold = -1 * float('inf'),
                impurity = 0):
        self.depth = depth
        self.label = label
        self.description = description
        self.best_feature_id = best_feature_id
        self.best_feature_threshold = best_feature_threshold
        self.impurity = impurity
        self.counts = {}
        self.probabilities = {}

        self.children = {}

class C45Classifier(BaseEstimator, ClassifierMixin):
    """ Classifier which implements c45 algorithm

    For more information regarding how to build your own classifier, read more
    in the :ref:`User Guide <user_guide>`.

    Parameters
    ----------
    demo_param : str, default='demo'
        A parameter used for demonstation of how to pass and store paramters.
    algo : str, default='c45'
        A parameter used for selecting proper tree algorithm. Now only two supported - 'id3' and 'c45'
        The only difference between them is the eway how amount of additional info is measured - via information gain or gain ratio
    max_depth: int, default=10
        Max depth of the tree
    min_samples_split: int, default=2
        Min size of a node for further split

    Attributes
    ----------
    X_ : ndarray, shape (n_samples, n_features)
        The input passed during :meth:`fit`.
    y_ : ndarray, shape (n_samples,)
        The labels passed during :meth:`fit`.
    classes_ : ndarray, shape (n_classes,)
        The classes seen at :meth:`fit`.
    """
    def __init__(self,
                algo='c45',
                max_depth=10,
                min_samples_split=2):
        self.algo = algo
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split

    def fit(self, X, y,
            types_auto=True,
            force_nominal_indices=[],
            force_continuous_indices=[],
            feature_names = [],
            class_names = []):
        """A reference implementation of a fitting function for a classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The training input samples.
        y : array-like, shape (n_samples,)
            The target values. An array of int.
        types_auto: boolean, default=True
            If true, try to determine if each variable is categorical or numeric; if False, all categories are considered as categorical
        force_nominal_indices: array-like
            Any variable can be explicitly marked as categorical.
            If both parameters, force_nominal_indices and force_continuous_indices are set and have intersections, parameter force_nominal_indices has priority
        force_continuous_indices: array-like
            Any variable can be explicitly marked as numeric/continuous.
            If both parameters, force_nominal_indices and force_continuous_indices are set and have intersections, parameter force_nominal_indices has priority
        feature_names: array_like, shape(n_features,)
            Names of features; set to their numbers if no names are provided

        Returns
        -------
        self : object
            Returns self.
        """
        self.nominal_features_ = []
        self.feature_names_ = []
        self.attrs_for_indices = {}

        # Check that X and y have correct shape
        X, y = check_X_y(X, y)

        # Store the classes seen during fit
        self.classes_ = np.array(unique_labels(y))

        self.X_ = np.array(X)
        self.y_ = np.array(y).reshape(-1, 1)

        self.data_ = np.append(self.X_, self.y_, axis=1)

        #print('shapes', self.X_.shape, self.y_.shape, self.data_.shape)

        #self.data_ = np.hstack( (self.X_, self.y_) )
        #print('Joined data with classes', self.data_)

        self.total_attributes_no = self.X_.shape[1]
        if len(feature_names) > 0:
            if len(feature_names) != self.total_attributes_no:
                raise ValueError("Attribute labels shape doesnot fit data shape")
        else:
            feature_names = [i for i in range(self.total_attributes_no)]

        self.feature_names_ = feature_names
        self.attr_indices_ = [i for i in range(self.total_attributes_no)]
        #print('total attrs', self.total_attributes_no, self.feature_names_)

        # Infer types of columns
        self.nominal_features_ = np.full(self.total_attributes_no, True)
        if types_auto == True:
            for i in range(self.total_attributes_no):
                self.nominal_features_[i] = self._is_attr_discrete(i)

        for cont_i in force_continuous_indices:
            self.nominal_features_[cont_i] = False
        for nom_i in force_nominal_indices:
            self.nominal_features_[nom_i] = True

        for i in range(self.total_attributes_no):
            self.attrs_for_indices[i] = np.unique(X[:,i])

        #
        #build tree
        self._generate_tree()

        return self

    def predict(self, X):
        """ An implementation of a prediction for a classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The input samples.

        Returns
        -------
        y : ndarray, shape (n_samples,)
            The label for each sample is the label of the closest sample
            seen udring fit.
        """
        # Check is fit had been called
        check_is_fitted(self, ['X_', 'y_'])

        # Input validation
        X = check_array(X)

        dummy = np.zeros( X.shape[0] )

        default_class_threshold = 0.5
        index = 0
        for row in X:
            probas = self._predict_proba_one(row)
            if probas is None:
                continue
            for c, p in probas.items():
                if p >= default_class_threshold:
                    dummy[index] = c
                    break
            #print(row, probas, index, dummy[index])
            index += 1

        return dummy

    def predict_proba(self, data):
        """ An implementation of a probabilitis prediction for a classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The input samples.

        Returns
        -------
        y : ndarray, shape (n_samples, n_classes)
            Probability for each class for each incoming sample.
        """
        output = []
        for row in data:
            proba = self._predict_proba_one(row)
            o = []
            for _, v in proba.items():
                o.append(v)
            output.append(o)

        return np.array(output)

    def _predict_proba_one(self, row):
        #traverse tree
        node = self.tree

        if node is None:
            return None

        while True:
            feature_id = node.best_feature_id
            observed_feature_value = row[feature_id]

            next_node = None

            if self.nominal_features_[feature_id] == True:
                for link, child in node.children.items():
                    if observed_feature_value == link:
                        next_node = child
                        break
            else:
                threshold = node.best_feature_threshold
                if len(node.children) == 2:
                    if observed_feature_value <= threshold:
                        next_node = node.children['left']
                    else:
                        next_node = node.children['right']

            if next_node == None:
                break

            node = next_node

        probas = node.probabilities
        return probas

    def feature_importances(self):
        node = self.tree
        importances = {}

        node_0_size = 0
        for _, v in node.counts.items():
            node_0_size += v

        node_layers, total_nodes = self._nodes_to_array()

        for layer in node_layers:
            for node in layer:
                feature_id = node.best_feature_id
                if feature_id == -1:
                    continue
                impurity = node.impurity

                total_entries = 0
                for _, v in node.counts.items():
                    total_entries += v

                children_importances = 0
                #check children
                for _, child in node.children.items():
                    child_impurity = child.impurity
                    if child_impurity == -1*float('inf'):
                        continue

                    total_child_entries = 0
                    for _, v in child.counts.items():
                        total_child_entries += v
                    children_importances += child_impurity * total_child_entries
                #print(impurity)
                node_importance = impurity * total_entries - children_importances
                node_importance /= node_0_size
                #print('Importance for', self.feature_names_[feature_id], 'is', node_importance, '(total', total_entries, ', node impurity', impurity, 'children cumulative importances', children_importances, ')')
                #print(feature_id)
                if feature_id in importances:
                    importances[feature_id] = max(node_importance, importances[feature_id])
                else:
                    importances[feature_id] = node_importance

        #print(importances)
        return importances

    def _nodes_to_array(self):
        layers = []

        layer_id = 0
        node = self.tree
        layer = []
        layer.append(node)
        layers.append(layer)

        total_nodes = len(layer)

        while True:
            all_nodes_in_layer = layers[layer_id]
            print('Layer size', len(all_nodes_in_layer))
            new_layer = []
            for n in all_nodes_in_layer:
                for _, child in n.children.items():
                    new_layer.append(child)
            if len(new_layer) == 0:
                break

            layer_id += 1
            layers.append(new_layer)
            total_nodes += len(new_layer)

        return layers, total_nodes

    def _log(self, x):
        if x == 0:
            return 0
        else:
            return math.log(x,2)

    def _entropy(self, data):
        s = len(data)
        if s == 0:
            return 0

        num_classes = np.array([0 for u in self.classes_])

        for row in data:
            class_index = np.where(self.classes_ == row[-1])
            num_classes[class_index] += 1

        num_classes = [x/s for x in num_classes]
        ent = 0
        for num in num_classes:
            ent += num * self._log(num)
        return (-1) * ent

    def _information_gain(self, data, subsets):
        #input : data and disjoint subsets of it
        #output : information gain
        s = len(data)
        #calculate impurity before split
        impurity_before_split = self._entropy(data)
        #calculate impurity after split
        weights = [len(subset)/s for subset in subsets]
        impurity_after_split = 0
        intrinsic_value = 0
        for i in range(len(subsets)):
            impurity_after_split += weights[i] * self._entropy(subsets[i])
            intrinsic_value += weights[i] * self._log(weights[i])
        #calculate total gain
        total_gain = impurity_before_split - impurity_after_split

        if self.algo == 'c45':
            gain_ratio = total_gain / ( -1 * intrinsic_value)
            return gain_ratio
        elif self.algo == 'id3':
            return total_gain
        else:
            raise ValueError('Unsupported algo:' + self.algo)

    def _is_attr_discrete(self, attr_id):
        dtype = self.X_[:,attr_id].dtype
        if (dtype == np.float) or (dtype == np.float64) :
            return False
        return True

    def _ig_for_nominal_feature(self, data, feature_id):
        X_column = data[:, feature_id]

        unique_values_for_feature = np.unique(X_column)
        X_column_splits = [ [] for i in range(len(unique_values_for_feature)) ]

        X_samples_count = data.shape[0]
        for sample_id in range(X_samples_count):
            sample = data[sample_id]
            for unique_value_id in range(len(unique_values_for_feature)):
                unique_value = unique_values_for_feature[unique_value_id]
                if unique_value == X_column[sample_id]:
                    X_column_splits[unique_value_id].append(sample)

        ig = self._information_gain(data, X_column_splits)
        threshold = None
        return (ig, threshold, X_column_splits, unique_values_for_feature) #to return same type as for cont variable

    def _ig_for_cont_feature(self, data, feature_id):
        X_column_sorted = np.sort(data[:, feature_id])
        #X_column_sorted = data[ data[:,feature_id].argsort() ]

        feature_threshold = -1* float('inf')
        best_ig = -1*float('inf')

        X_column_splits = None
        unique_feature_values = []

        for j in range(len(X_column_sorted) - 1):
            if X_column_sorted[j] != X_column_sorted[j + 1]:
                threshold = (X_column_sorted[j] + X_column_sorted[j + 1]) / 2
                less = []
                greater = []
                for row in data:
                    if row[feature_id] > threshold:
                        greater.append(row)
                    else:
                        less.append(row)

                ig = self._information_gain(data, [less, greater])

                if ig > best_ig:
                    best_ig = ig
                    feature_threshold = threshold
                    X_column_splits = [less, greater]
                    unique_feature_values = ['<=' + str(threshold), '>' + str(threshold)]

        return (best_ig, feature_threshold, X_column_splits, unique_feature_values)

    def _calculate_classes(self, y):
        counts = {}
        for x in self.classes_:
            counts[x] = 0

        for value in y:
            counts[value] += 1

        return counts

    def _generate_tree(self):
        self.tree = self._recursive_generate_tree(data = self.data_,
                                                    feature_ids=self.attr_indices_,
                                                    level = 0, label = 'root', verbose=1)

    def _recursive_generate_tree(self, data, feature_ids, level, label, verbose):
        #strategy
        #1. Check level. If exceeds max_depth - exit
        #2. Check amount of data. If less than min_split or min_leaf - exit
        #3. Cycle through all features.
        #   3.1 If feature is continuous or ordinal - find best binary split
        #   3.2 If feature is nominal - split by categories. Order by categories frequencies and select max_split_categories;
        #       other categories join to 'other'
        #4. Select feature which produces best split. Create subsets of data according to the best split.
        #5. For each subset:
        #   - increase depth
        #6. Recursive call

        node = Node(level, label)

        offset = '\t'*level
        if verbose > 0:
            print(offset + 'Level', level, '; label', label)
            print(offset + 'Incoming data shape:', data.shape)
            print(offset + 'Incoming feature_ids:', feature_ids)

        if len(data) == 0:
            if verbose > 0:
                print(offset + 'Not enough data at all, terminate')
            node.description = 'Not enough data at all, terminate'
            return None

        node.counts = self._calculate_classes(data[:, -1])
        if verbose > 0:
            print(offset + 'Node counts',node.counts)
        for c, f in node.counts.items():
            node.probabilities[c] = f / len(data[:, -1])

        if len(data) <= self.min_samples_split:
            if verbose > 0:
                print(offset + 'Not enough data to go deeper, terminate')
            node.description = 'Not enough data to go deeper, terminate'
            return None

        if level > self.max_depth:
            if verbose > 0:
                print(offset + 'Max depth exceeded, terminate')
            node.description = 'Max depth exceeded, terminate'
            return None

        if self._check_one_class_remains(data):
            if verbose > 0:
                print(offset + 'One class remains, terminate')
            node.description = 'One class remains, terminate'
            return node

        new_level = level + 1

        best_feature_id = -1
        max_ig = -1 * float('inf')
        best_threshold = None
        data_splits = []
        description = None
        unique_feature_values = []

        for feature_id in feature_ids:
            ig = max_ig
            threshold = best_threshold
            splits = None
            unique_values = []
            if self.nominal_features_[feature_id] == True:
                #nominal feature
                (ig, threshold, splits, unique_values) = self._ig_for_nominal_feature(data, feature_id)
                description = 'nominal'
            else:
                #contunuous or ordinal feature
                (ig, threshold, splits, unique_values) = self._ig_for_cont_feature(data, feature_id)
                description = 'non-nominal'
            if ig > max_ig:
                best_feature_id = feature_id
                best_threshold = threshold
                max_ig = ig
                data_splits = splits
                unique_feature_values = unique_values
                if verbose > 0:
                    print(offset + 'Feature', self.feature_names_[feature_id], 'has better split')
            else:
                if verbose > 0:
                    print(offset + 'Feature', self.feature_names_[feature_id], 'has less IG, ignoring')

        node.best_feature_id = best_feature_id
        node.best_feature_threshold = best_threshold
        node.description = description
        node.impurity = max_ig

        if best_feature_id < 0:
            if verbose > 0:
                print(offset + 'No better split. Leaf node')
            return node

        if data_splits is None:
            if verbose > 0:
                print(offset + 'No splits found. Leaf node')
            return node

        if len(data_splits) != len(unique_feature_values):
            if verbose > 0:
                print(offset + 'sizes (splits, values):', len(data_splits), len(unique_feature_values))
                print(offset + 'Shapes between splits and unique features don\'t match. Leaf node?')
            return node

        new_feature_ids = feature_ids.copy()
        if best_feature_id in new_feature_ids:
            new_feature_ids.remove(best_feature_id)

        if self.nominal_features_[node.best_feature_id] == True:
            i = 0
            for split in data_splits:
                if verbose > 0:
                    print(offset + 'Data shape for child:', np.array(split).shape)
                op = ''
                if self.nominal_features_[best_feature_id] == True:
                    op = ' is '
                unique_value = unique_feature_values[i]
                new_label = str(self.feature_names_[best_feature_id]) + op + str(unique_value)

                child = self._recursive_generate_tree(np.array(split), new_feature_ids, new_level, new_label, verbose)
                if child != None:
                    #Check if our child split improves our knowledge; if probs remain the same, child is useless
                    if node.counts != child.counts:
                        node.children[unique_value] = child
                i += 1
        else:
            #Expected data splits should be 2

            for i in range(2):
                unique_value = unique_feature_values[i]
                new_label = str(self.feature_names_[best_feature_id]) + str(unique_value)
                child = self._recursive_generate_tree(np.array(data_splits[i]),
                                                        new_feature_ids, new_level, new_label, verbose)
                if child != None:
                    if i == 0:
                        node.children['left'] = child
                    else:
                        node.children['right'] = child

        return node

    def _check_one_class_remains(self, data):
        uniques = np.unique(data[:,-1])
        if len(uniques) <= 1:
            return True

        return False

    def _get_major_class_index(self, data):
        #print('classes', self.classes_)
        freq = [0]*len(self.classes_)
        #print('classes', self.classes_)

        freq = np.array(freq)
        for row in data:
            #print('row-1, row:', row[-1], row)
            index = np.where(self.classes_ == row[-1])
            freq[index] += 1
        max_ind = np.where(freq == max(freq))

        #print('freqs', freq)
        return max_ind

    def _get_class_frequencies(self, data):
        #print('classes', self.classes_)
        freqs = [0]*len(self.classes_)
        #print('classes', self.classes_)

        freqs = np.array(freqs)
        for row in data:
            #print('row-1, row:', row[-1], row)
            index = np.where(self.classes_ == row[-1])
            freqs[index] += 1
        #print('freqs', freq)
        return freqs

    def printTree(self):
        print('--- Tree ---')
        self.printNode(self.tree)

    def printNode(self, node, indent=""):

        print(indent + 'Label:', node.label)
        print(indent + 'Description:', node.description)

        total_entries = 0
        classes_count_str = 'Classes count: ['
        for c, p in node.counts.items():
            classes_count_str += str(c) + ':' + str(p) + '; '
            total_entries += p
        classes_count_str += ']'

        classes_prob_str = 'Classes probs: ['
        for c, p in node.probabilities.items():
            classes_prob_str += str(c) + ':' + str(p) + '; '
#            classes_prob_str += str(self.feature_names_[c]) + ':' + str(p / total_entries) + ';'
        classes_prob_str += ']'

        print(indent + classes_count_str)
        print(indent + classes_prob_str)
        print(indent + 'Best feature to split:', self.feature_names_[node.best_feature_id])
        print(indent + 'Best feature threshold:', node.best_feature_threshold)

        for link, child in node.children.items():
            print(indent+'Link to child', link)
            self.printNode(child, indent + '\t')


import os
from setuptools import setup

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name = "hello_python",
    version = "0.0.1",
    author = "Asif Raza",
    author_email = "raoasifraza1@hotmail.com",
    description = ("An demonstration of how to create, document, and publish "
                                   "to the cheese shop a5 test.pypi.org."),
    license = "BSD",
    keywords = "example documentation tutorial",
    url = "https://github.com/raoasifraza11/hello_pyton.git",
    long_description=read('README.md'),
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Topic :: Utilities",
        "License :: OSI Approved :: BSD License",
    ],
)


#!/usr/bin/env python3
# MRMS grib2 data fetching script for python-based HDWX
# Created 29 April 2022 by Sam Gardner <stgardner4@tamu.edu>

from os import path, remove, listdir
from pathlib import Path
import pandas as pd
from datetime import datetime as dt, timedelta
import json
import requests
import gzip
import shutil
import urllib

basePath = path.realpath(path.dirname(__file__))

def downloadFile(fileName):
    output = path.join(basePath, "input", fileName)
    print("Downloading "+fileName)
    urlToFetch = "https://mrms.ncep.noaa.gov/data/2D/ReflectivityAtLowestAltitude/"+fileName
    try:
        mrmsData = requests.get(urlToFetch)
        if mrmsData.status_code == 200:
            mrmsData = mrmsData.content
    except requests.exceptions.SSLError:
        import subprocess
        mrmsDataProc = subprocess.run(["curl", urlToFetch], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        mrmsData = mrmsDataProc.stdout
    with open(output, "wb") as fileWrite:
        fileWrite.write(mrmsData)
    with gzip.open(output, "rb") as f_in:
        with open(output.replace(".gz", ""), "wb") as f_out:
            shutil.copyfileobj(f_in, f_out)
    remove(output)

if __name__ == "__main__":
    Path(path.join(basePath, "input")).mkdir(parents=True, exist_ok=True)
    try:
        gribList = pd.read_html("https://mrms.ncep.noaa.gov/data/2D/ReflectivityAtLowestAltitude/")[0].dropna(how="any")
    except urllib.error.URLError:
        import subprocess
        from io import BytesIO
        gribListCurlProc = subprocess.run(["curl", "https://mrms.ncep.noaa.gov/data/2D/ReflectivityAtLowestAltitude/"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        gribList = pd.read_html(BytesIO(gribListCurlProc.stdout))[0].dropna(how="any")
    gribList = gribList[~gribList.Name.str.contains("latest") == True].reset_index()
    gribList["pyDateTimes"] = [dt.strptime(filename, "MRMS_ReflectivityAtLowestAltitude_00.50_%Y%m%d-%H%M%S.grib2.gz") for filename in gribList["Name"]]
    gribList = gribList.set_index(["pyDateTimes"])
    alreadyDownloaded = listdir(path.join(basePath, "input"))
    for mosaicTime in reversed(gribList.index):
        if mosaicTime > dt.utcnow() - timedelta(hours=1):
            runFile = path.join(basePath, "output", "metadata", "products", "3", mosaicTime.strftime("%Y%m%d%H00.json"))
            if path.exists(runFile):
                with open(runFile) as jsonLoad:
                    runData = json.load(jsonLoad)
                alreadyHavePlotted = False
                for productFrame in runData["productFrames"]:
                    if mosaicTime.strftime("%Y%m%d%H%M") == productFrame["valid"]:
                        alreadyHavePlotted = True
                if not alreadyHavePlotted:
                    fileToDownload = gribList[gribList.index == mosaicTime]["Name"][0]
                    if fileToDownload.replace(".gz", "") not in alreadyDownloaded:
                        downloadFile(fileToDownload)
            else:
                fileToDownload = gribList[gribList.index == mosaicTime]["Name"][0]
                if fileToDownload.replace(".gz", "") not in alreadyDownloaded:
                    downloadFile(fileToDownload)


import asyncio
import bakkesmod
from bmutils import logger, decorators

logger.redirectStandardStreams()

@decorators.plugin("AsyncTestPlugin", decorators.PLUGINTYPE_ASYNC)
class AsyncTest(bakkesmod.BakkesModPlugin):
    def __init__(self):
        super().__init__()
        print("init class")

    def __del__(self):
        print("del class")

    @decorators.onLoad
    async def onLoad(self):
        print("onLoad class 1")
        await asyncio.sleep(3)
        print("onLoad class 2")

    @decorators.onUnload
    def onUnload(self):
        print("onUnload class 3")


"""
Fit Gaussian Mixture by YUAN Ruihong
"""

import argparse
import sys
from random import random

import numpy as np
import pandas as pd
from scipy.stats import norm

from yasim.helper.gmm import GaussianMixture1D
from yasim.helper.plot_utils import plot

def _main():
    parser = argparse.ArgumentParser()
    parser.add_argument('data')
    parser.add_argument('-c', '--numComponents', type=int, required=False, default=2)
    parser.add_argument('--columnIndex', type=int, required=False, default=-1)
    parser.add_argument('--header', type=int, required=False)
    parser.add_argument('--transform', nargs='+', required=False,
                        choices=('scale', 'log'), default=[])
    parser.add_argument('--numIters', type=int, required=False, default=100)

    args = parser.parse_args()
    if args.data == 'dummy':
        mu1, sigma1 = random() * 100, random() * 10
        mu2, sigma2 = random() * 100, random() * 10
        p = random()
        model1 = norm(mu1, sigma1)
        model2 = norm(mu2, sigma2)
        data = np.append(model1.rvs(size=int(p * 200)), model2.rvs(size=int((1 - p) * 200)))
        print(f"Ground truth is {p}*N({mu1}, {sigma1}) + {1 - p}*N({mu2}, {sigma2})")
    else:
        df = pd.read_table(args.data, header=args.header or 'infer')
        df = df[df.iloc[:, args.columnIndex] != 0]  # filter zeros
        data = np.asarray(df.iloc[:, args.columnIndex])

    for transform_method in args.transform:
        if transform_method == 'log':
            data = np.log(data + 1)
    #
    # scaler = StandardScaler().fit(data)
    # if 'scale' in args.transform:
    #     data = scaler.transform(data)
    #

    model = GaussianMixture1D(n_components=args.numComponents,
                              n_iter=args.numIters).fit(data)
    print(f"Fitted to Gaussian mixture model ({args.numComponents} components)")

    plot(data, model)

if __name__ == '__main__':
    sys.exit(_main())


import logging
import time
import json
from datetime import datetime
from threading import current_thread
from resilient_circuits import ResilientComponent, function, handler, StatusMessage, FunctionResult, FunctionError
from resilient_lib import RequestsCommon
from fn_mcafee_esm.util.helper import check_config, get_authenticated_headers, check_status_code

log = logging.getLogger(__name__)

def case_get_case_detail(rc, options, headers, id):
    url = options["esm_url"] + "/rs/esm/v2/caseGetCaseDetail"

    payload = {
        "id": id
    }

    r = rc.execute_call_v2('post', url, headers=headers, data=json.dumps(payload), verify=options["trust_cert"],
                           proxies=rc.get_proxies())
    log.debug(r.content)
    if r.status_code == 400:
        raise FunctionError(r.content)
    check_status_code(r.status_code)

    return r.json()

class FunctionComponent(ResilientComponent):
    """Component that implements Resilient function 'mcafee_esm_get_case_detail"""

    def __init__(self, opts):
        """constructor provides access to the configuration options"""
        super(FunctionComponent, self).__init__(opts)
        self.opts = opts
        self.options = opts.get("fn_mcafee_esm", {})

        # Check config file and change trust_cert to Boolean
        self.options = check_config(self.options)

    @handler("reload")
    def _reload(self, event, opts):
        """Configuration options have changed, save new values"""
        self.opts = opts
        self.options = opts.get("fn_mcafee_esm", {})

    @function("mcafee_esm_get_case_detail")
    def _mcafee_esm_get_case_detail_function(self, event, *args, **kwargs):
        """Function: Calls the caseGetCaseDetail endpoint and returns all the details of a case."""
        try:
            start_time = time.time()
            yield StatusMessage("starting...")

            options = self.options

            # Instantiate RequestsCommon object
            rc = RequestsCommon(opts=self.opts, function_opts=self.options)

            authenticated_headers = get_authenticated_headers(rc, options["esm_url"], options["esm_username"],
                                               options["esm_password"], options["trust_cert"])

            # Get the function parameters:
            mcafee_esm_case_id = kwargs.get("mcafee_esm_case_id")  # number

            log = logging.getLogger(__name__)
            if not mcafee_esm_case_id:
                raise ValueError("mcafee_esm_case_id is required")
            log.info("mcafee_esm_case_id: %s", mcafee_esm_case_id)

            # Get case details

            details = case_get_case_detail(rc, options, authenticated_headers, mcafee_esm_case_id)

            end_time = time.time()
            results = {
                "inputs": {
                    "mcafee_esm_case_id": mcafee_esm_case_id
                },
                "metrics": {
                    "execution_time": str(end_time - start_time),
                    "function": "mcafee_esm_get_case_detail",
                    "thread": current_thread().name,
                    "timestamp": datetime.fromtimestamp(end_time).strftime("%Y-%m-%d %H:%M:%S")
                },
                "details": details
            }

            yield StatusMessage("done...")
            # Produce a FunctionResult with the results
            yield FunctionResult(results)
        except Exception as e:
            yield FunctionError(e)


# -*- coding: utf-8 -*-
"""
Created on Thu Sep 27 14:08:12 2018

@author: hong
"""

import requests
from lxml import etree
from saver import get_all
import logging

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(thread)s - %(message)s"
logging.basicConfig(filename='my.log', level=logging.ERROR, format=LOG_FORMAT)
headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"}

a = requests.get("https://www.kujiale.com/huxing/china?kpm=9V8.ZQA.731697a.1535902181154", headers=headers).content.decode("utf")
html = etree.HTML(a)
#所有的城市名称, 注意这里有一个全部是不要的
li =  html.xpath("//li[@class='item ']/a/text()")

result = get_all()

for i in range(len(result)):
    try:
        imgName = './img/'+repr(result[i]['id'])+"_"+result[i]['location']+"_"+result[i]['hourse_name']+"_"+result[i]['house_type']+'_'+result[i]['construction_area']+'.jpg'
        img = requests.get(result[i]['src'], headers=headers)
        with open(imgName, 'ab') as f:
            f.write(img.content)
            f.close()
    except Exception as e:
        logging.error("id 为%s的图片获取失败")


import datetime, pytz
from datetime import datetime
from django.db.models import Sum, OuterRef, Subquery, Func, Count, F

from rest_framework import status, filters
from rest_framework.permissions import IsAuthenticated
from rest_framework.views import APIView
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView

from books.models import Book
from helpers.csv_reader import CSVReader
from utils.permissions import CustomBasePermission

from .paginations import BuybackPagination
from .models import Buyback, BuybackOrder
from .serializers import BuybackOrderSerializer

class ListCreateBuybackAPIView(ListCreateAPIView):
    permission_classes = [CustomBasePermission]
    serializer_class = BuybackOrderSerializer
    queryset = BuybackOrder.objects.all()
    pagination_class = BuybackPagination
    filter_backends = [filters.OrderingFilter, filters.SearchFilter]
    ordering_fields = '__all__'
    ordering = ['-date']

    def paginate_queryset(self, queryset):
        if 'no_pagination' in self.request.query_params:
            return None
        else:
            return super().paginate_queryset(queryset)

    def create(self, request, *args, **kwargs):
        # Add User to BuybackOrder
        request.data['user'] = request.user.id

        serializer = BuybackOrderSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        saved_buyback_order = serializer.save()

        response_data = serializer.data
        response_data['id'] = saved_buyback_order.id
        return Response(response_data, status=status.HTTP_201_CREATED)

    def get_queryset(self):
        default_query_set = BuybackOrder.objects.all()

        # Create a subquery to aggregate the 'revenue' value for each buyback in BuybackOrder
        revenue_subquery = Buyback.objects.filter(buyback_order=OuterRef('id')).values_list(Func(
            'revenue',
            function='SUM',
        ),)

        default_query_set = default_query_set.annotate(total_revenue=Subquery(revenue_subquery))

        # Filter by quantity of books in BuybackOrder
        num_books_subquery = Buyback.objects.filter(buyback_order=OuterRef('id')).values_list(Func('quantity', function='SUM'),)

        default_query_set = default_query_set.annotate(num_books=Subquery(num_books_subquery))

        default_query_set = default_query_set.annotate(num_unique_books=Count('buybacks__book', distinct=True))

        default_query_set = default_query_set.annotate(vendor_name=F('vendor__name'))

        default_query_set = default_query_set.annotate(username=F('user__username'))

        # Filter by vendor
        vendor = self.request.GET.get('vendor')
        if vendor is not None:
            default_query_set = default_query_set.filter(vendor=vendor)

        # Filter by date
        start_date = self.request.GET.get('start')
        end_date = self.request.GET.get('end')
        if start_date is not None and end_date is not None:
            default_query_set = default_query_set.filter(date__range=(start_date, end_date))
        elif start_date is not None:
            default_query_set = default_query_set.filter(date__range=(start_date, datetime.datetime.now(pytz.timezone('US/Eastern'))))

        # Filter by book
        book = self.request.GET.get('book')
        if book is not None:
            default_query_set = default_query_set.filter(buybacks__book=book).distinct()

        # Filter by >= revenue
        buyback_revenue__gte = self.request.GET.get('buyback_revenue__gte')
        if buyback_revenue__gte is not None:
            default_query_set = default_query_set.filter(buybacks__revenue__gte=buyback_revenue__gte).distinct()

        # Filter by <= revenue
        buyback_revenue__lte = self.request.GET.get('buyback_revenue__lte')
        if buyback_revenue__lte is not None:
            default_query_set = default_query_set.filter(buybacks__revenue__lte=buyback_revenue__lte).distinct()

        # Filter by <= revenue
        buyback_revenue = self.request.GET.get('buyback_revenue')
        if buyback_revenue is not None:
            default_query_set = default_query_set.filter(buybacks__revenue=buyback_revenue).distinct()

        # Filter by >= quantity
        buyback_quantity__gte = self.request.GET.get('buyback_quantity__gte')
        if buyback_quantity__gte is not None:
            default_query_set = default_query_set.filter(buybacks__quantity__gte=buyback_quantity__gte).distinct()

        # Filter by <= quantity
        buyback_quantity__lte = self.request.GET.get('buyback_quantity__lte')
        if buyback_quantity__lte is not None:
            default_query_set = default_query_set.filter(buybacks__quantity__lte=buyback_quantity__lte).distinct()

        # Filter by == quantity
        buyback_quantity = self.request.GET.get('buyback_quantity')
        if buyback_quantity is not None:
            default_query_set = default_query_set.filter(buybacks__quantity=buyback_quantity).distinct()

        # Filter by >= unit_buyback_price
        buyback_unit_buyback_price__gte = self.request.GET.get('buyback_unit_buyback_price__gte')
        if buyback_unit_buyback_price__gte is not None:
            default_query_set = default_query_set.filter(buybacks__unit_buyback_price__gte=buyback_unit_buyback_price__gte).distinct()

        # Filter by <= unit_buyback_price
        buyback_unit_buyback_price__lte = self.request.GET.get('buyback_unit_buyback_price__lte')
        if buyback_unit_buyback_price__lte is not None:
            default_query_set = default_query_set.filter(buybacks__unit_buyback_price__lte=buyback_unit_buyback_price__lte).distinct()

        # Filter by == unit_buyback_price
        buyback_unit_buyback_price = self.request.GET.get('buyback_unit_buyback_price')
        if buyback_unit_buyback_price is not None:
            default_query_set = default_query_set.filter(buybacks__unit_buyback_price=buyback_unit_buyback_price).distinct()

        return default_query_set

class RetrieveUpdateDestroyBuybackAPIView(RetrieveUpdateDestroyAPIView):
    permission_classes = [CustomBasePermission]
    serializer_class = BuybackOrderSerializer
    lookup_field = 'id'
    pagination_class = BuybackPagination

    def get_queryset(self):
        return BuybackOrder.objects.filter(id=self.kwargs['id'])

    def retrieve(self, request, *args, **kwargs):
        invalid_id_response = self.verify_existance()
        if invalid_id_response:
            return invalid_id_response
        (buyback_order,) = self.get_queryset()
        serializer = self.get_serializer(buyback_order)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def update(self, request, *args, **kwargs):
        invalid_id_response = self.verify_existance()
        if invalid_id_response:
            return invalid_id_response
        partial = kwargs.pop('partial', False)
        (buyback_order,) = self.get_queryset()
        serializer = self.get_serializer(buyback_order, data=request.data, partial=partial)
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return Response(serializer.data, status=status.HTTP_200_OK)

    def verify_existance(self):
        if (len(self.get_queryset()) == 0):
            return Response({"id": "No buyback order with queried id."}, status=status.HTTP_400_BAD_REQUEST)
        return None

    def destroy(self, request, *args, **kwargs):
        buyback_book_quantities = Buyback.objects.filter(buyback_order=self.get_object().id).values('book').annotate(num_books=Sum('quantity')).values('book', 'num_books')
        for buyback_book_quantity in buyback_book_quantities:
            book_to_remove_buyback = Book.objects.filter(id=buyback_book_quantity['book']).get()
            book_to_remove_buyback.stock += buyback_book_quantity['num_books']
            book_to_remove_buyback.save()
        return super().destroy(request, *args, **kwargs)

class CSVBuybackAPIView(APIView):
    permission_classes = [CustomBasePermission]

    def post(self, request: Request):
        csv_reader = CSVReader("buybacks")
        return csv_reader.read_csv(request)


import cv2

nfile=input("Number of files?")
nfile=int(nfile)
img_array=[]
for k in range(nfile):
    fname=str(k+1)+".png"
    img=cv2.imread(fname)
    height,width,layers=img.shape
    size=(width,height)

    img_array.append(img)

name="snaps.mp4"
out=cv2.VideoWriter(name,cv2.VideoWriter_fourcc(*'MP4V'),3.0,size)

for i in range(len(img_array)):
    out.write(img_array[i])
out.release()


#!/usr/bin/env/python3
#OCS Console Explorer
#Requires python3
#Requires the following pip3 packages: console-menu, requests
import json
import requests
import terminalplot
from consolemenu import *
from consolemenu.items import *

def get_token(base_url, creds):
    """Returns a bearer token for use with OCS."""
    client_id = creds[2].strip()
    client_secret = creds[3].strip()

    tok_post = {'client_id':client_id, 'client_secret': client_secret, 'grant_type':'client_credentials'}
    resp = requests.post(base_url + '/identity/connect/token', data=tok_post)
    return resp.json()['access_token']

def get_streams(namespace_url):
    """Returns a list of streams for a namespace."""
    streams = requests.get(namespace_url + '/Streams', headers=headers)
    return streams.json()

def get_count():
    possible_counts= [10, 100, 1000]
    return possible_counts

def get_data(stream_url, count, is_graph):
    """Returns a list of data for a stream."""
    data = requests.get(stream_url + '/Data' + '?count=' + str(count) + '&Startindex=1970-01-01T00:00:00Z', headers=headers)
    Values = []
    for event in data.json():
        Values.append(event['Value'])
    if is_graph:
        terminalplot.plot(range(len(Values)), Values)
    else:
        print(Values)
    input("Press any key to continue...")
    return

def namespace_selection(base_url, namespaces):
    """Presents a menu for selection of a namespace."""
    top_menu = ConsoleMenu("Select Namespace", "")
    for ns in namespaces:
        namespace_url = ns['Self']
        namespace_menu = FunctionItem(ns['Id'], stream_selection, [namespace_url, get_streams(namespace_url)])
        top_menu.append_item(namespace_menu)
    top_menu.show()

def stream_selection(namespace_url, streams):
    """Presents a menu for selection of a stream."""
    top_menu = ConsoleMenu("Select Stream", "")
    for stream in streams:
        stream_url = namespace_url + '/Streams/' + stream['Id']
        stream_menu = FunctionItem(stream['Id'], data_selection, [stream_url, get_count()])
        top_menu.append_item(stream_menu)
    top_menu.show()

def data_selection(stream_url, counts):
    """Presents a menu for selection of a data range (10, 100, or 1000 data points since 1970)."""
    top_menu = ConsoleMenu("How many Data point would you like?", "")
    for count in counts:
        count_option = FunctionItem(str(count), get_data, [stream_url, count, False])
        count_option2 = FunctionItem(str(count) + ' (Graph)', get_data, [stream_url, count, True])
        top_menu.append_item(count_option)
        top_menu.append_item(count_option2)
    top_menu.show()

if __name__ == "__main__":
    with open("credentials.txt") as f:
        creds = f.readlines()
    base_url = creds[0].strip()
    tenant_id = creds[1].strip()
    headers = {'Authorization': 'Bearer ' + get_token(base_url, creds)}

    menu = ConsoleMenu("OSIsoft Cloud Services Explorer", "Welcome to OCS Explorer!")

    namespaces = requests.get(base_url + '/api/v1/Tenants/' + tenant_id + '/Namespaces', headers=headers)
    token_menu = FunctionItem("Enter", namespace_selection, [base_url, namespaces.json()])
    menu.append_item(token_menu)

    menu.show()


from pymongo import MongoClient
import json

DB_IP = "35.194.174.101"  # 35.194.174.101
DB_PORT = 27017  # default MongoDB port
DB_NAME = "beautybot"  # use the collection

class PixnetDatabase(object):

    def __init__(self):
        pass

    def search_article(self, collection_pixnet, search_rule):
        article_list = []
        for article in collection_pixnet.find(search_rule):
            print(article['title'])
            article_list.append(article)
        return article_list

def main():
    client = MongoClient(DB_IP, DB_PORT)
    collection_pixnet = client[DB_NAME]["pixnet"]
    print("Total Pixnet articles: ", collection_pixnet.count())

    search_rule = {"category": "lips", "title": {"$regex": "霧"}}
    pixnet_db = PixnetDatabase()
    pixnet_db.search_article(collection_pixnet, search_rule)

if __name__ == '__main__':
    main()


##
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag
from nltk.corpus import wordnet as wn
from nltk.metrics import jaccard_distance
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn import model_selection

from scipy.stats import pearsonr
from nltk.tag.perceptron import PerceptronTagger
from nltk import download
from nltk.corpus import treebank
from xgboost import XGBRegressor

import re
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
download('treebank')

stop_words = stopwords.words('english')
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
wnl = WordNetLemmatizer()
filenames_train = ['MSRpar', 'MSRvid', 'SMTeuroparl']
filenames_test = ['MSRpar', 'MSRvid', 'SMTeuroparl',
                  'surprise.OnWN', 'surprise.SMTnews']
train_data = treebank.tagged_sents()

# Perceptron tagger
per = PerceptronTagger(load='false')
per.train(train_data)

file_prefix = lambda stage: stage + '/STS'

""" more orthodox and robust implementation """
def dice_coefficient(s1, s2, n=2):
    """dice coefficient 2nt/na + nb."""
    if isinstance((s1), list):
        a = ' '.join(s1)
    elif isinstance(s1, str):
        a = s1
    if isinstance((s2), list):
        b = ' '.join(s2)
    elif isinstance(s2, str):
        b = s2
    if not len(a) or not len(b): return 0.0
    if len(a) == 1:  a = a + u'.'
    if len(b) == 1:  b = b + u'.'

    a_bigram_list = []
    for i in range(len(a) - 1):
        a_bigram_list.append(a[i:i + n])
    b_bigram_list = []
    for i in range(len(b) - 1):
        b_bigram_list.append(b[i:i + n])

    a_bigrams = set(a_bigram_list)
    b_bigrams = set(b_bigram_list)
    overlap = len(a_bigrams & b_bigrams)
    dice_coeff = overlap * 2.0 / (len(a_bigrams) + len(b_bigrams))
    return dice_coeff

def getData(filenames, file_prefix, **removals):
    input_prefix = file_prefix + '.input.'
    gs_prefix = file_prefix + '.gs.'
    input = pd.DataFrame()
    for filename in filenames:
        sentences = pd.read_csv(input_prefix + filename + '.txt',
                                sep='\t', names=['sentence1', 'sentence2'],
                                quoting=3)
        golden_standards = pd.read_csv(gs_prefix + filename + '.txt',
                                       names=['golden_standard'])
        dfX = pd.concat([sentences, golden_standards], axis=1)
        input = pd.concat([input, dfX])

    input.reset_index(drop=True)
    # Remove punctuation (and numbers)
    input[['sentence1', 'sentence2']] = input[['sentence1', 'sentence2']].apply(lambda col: sentClean(col, **removals))
    # Tokenize (remove stop words)
    input[['sentence1', 'sentence2']] = input[['sentence1', 'sentence2']].apply(lambda col: sentTokenize(col, **removals))
    return input

def sentClean(sents, **removals):
    new_sent = []
    for sent in sents:
        if sent is np.NaN:
            pass#print(sent)
        if removals.get('numbers'):
            mod_sent = re.sub(r'[^A-Z a-z]', ' ', sent)
        else:
            mod_sent = re.sub(r'[^\w]', ' ', sent)  # |\b\w\b -> to take out single chars
        clean_sent = re.sub(r'[ ]+', ' ', mod_sent.strip())
        if len(clean_sent) == 0:
            clean_sent = sent
        new_sent.append(clean_sent)
    return new_sent

def sentTokenize(sents, **removals):
    new_sent = []
    for sent in sents:
        tokens = nltk.word_tokenize(sent)
        if removals.get('stop_words'):
            tokens = [token for token in tokens if token not in stop_words]
        new_sent.append(tokens)
    return new_sent

def lemmatize(p):
    if p[1][0] in {'N','V'}:
        return wnl.lemmatize(p[0].lower(), pos=p[1][0].lower())
    return p[0].lower()

def lemmas(sents):
    new_sent = []
    for sent in sents:
        #pairs = pos_tag(sent)
        pairs = per.tag(sent)
        new_sent.append([lemmatize(pair) for pair in pairs])
    return new_sent

def ngrams(sents, n, word=True):
    new_sent = []
    n_old = n
    for sent in sents:
        n = n_old
        if len(sent) < n:
            n_old = n
            n = len(sent)
        if word:
            grams_lst = [w for w in nltk.ngrams(sent, n)]
            new_sent.append(grams_lst)
        else:
            sent_joined = ' '.join(sent)
            grams_lst = [''.join(ch) for ch in nltk.ngrams(sent_joined, n)]
            new_sent.append(grams_lst)
    return new_sent

def mylesk(context_sentence, ambiguous_word, pos=None, synsets=None):
    context = set(context_sentence)
    # context = ' '.join(context_sentence)
    hyper = lambda s: s.hypernyms()
    if synsets is None:
        synsets = wn.synsets(ambiguous_word)

    if pos:
        synsets = [ss for ss in synsets if str(ss.pos()) == pos]

    if not synsets:
        return ambiguous_word

    max_sense = []
    for ss in synsets:
        ss_lst = list(ss.closure(hyper))
        ss_lst.append(ss)
        for ss_hyper in ss_lst:
            max_sense.append(
                (len(context.intersection(ss_hyper.definition().split())), ss_hyper, ss)
                # (self.dice_coefficient(context, ss_hyper.definition()), ss_hyper, ss)
            )
            for ex in ss_hyper.examples():
                if ex:
                    max_sense.append(
                        (len(context.intersection(ex.split())), ss_hyper, ss)
                        # (self.dice_coefficient(context, ex), ss_hyper, ss)
                    )
    _, hyper, sense = max(max_sense)
    return hyper.lemmas()[0].name()

def myWSD(pt_pair, context):
    '''
    Returns the synset if it's a noun, a verb, an adberv or an adjective or the lowered word
    '''
    wn_postag_map = {'N': 'n',
             'V': 'v',
             'J': 'a',
             'R': 'r'
            }

    pt = wn_postag_map.get(pt_pair[1][0])
    word = pt_pair[0]
    if pt:
        return mylesk(context, word, pt)
    else:
        return word

def lesk(sents):
    new_sent = []
    for sent in sents:
        pairs = pos_tag(sent)
        new_sent.append([myWSD(pair,sent) for pair in pairs])
    return new_sent

def modelFeatures(df_sents, files='', stage=''):
    Xlesk = df_sents.apply(lesk)

    Mdice_lem = df_sents.apply(lambda x: dice_coefficient(x['sentence1'], x['sentence2']), axis=1)

    Mjac_lem = df_sents.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)

    #Mlesk = Xlesk.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)
    Mlesk = Xlesk.apply(lambda x: dice_coefficient(x['sentence1'], x['sentence2']), axis=1)

    Xngram = df_sents.apply(lambda col: ngrams(col, 2))
    Mngram2 = Xngram.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)

    Xngram = df_sents.apply(lambda col: ngrams(col, 4))
    Mngram4 = Xngram.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)

    removals = {
        'stop_words': True,
        'numbers': True
    }
    df_XtrainSW = getData(files, file_prefix(stage), **removals)
    #labels_trn = df_Xtrain.iloc[:, -1]
    df_XtrainSW = df_XtrainSW.drop(df_XtrainSW.columns[len(df_XtrainSW.columns) - 1], axis=1)

    Xlem = df_XtrainSW.apply(lemmas)

    Xngram = Xlem.apply(lambda col: ngrams(col, 1))
    Mngram1_sw = Xngram.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)

    Xngram = Xlem.apply(lambda col: ngrams(col, 3))
    Mngram2_sw = Xngram.apply(lambda x: jaccard_distance(set(x['sentence1']), set(x['sentence2'])), axis=1)

    df = pd.concat([Mdice_lem, Mjac_lem, Mlesk, Mngram2, Mngram4, Mngram1_sw, Mngram2_sw], axis=1)
    #df = pd.concat([Mdice_lem, Mjac_lem, Mlesk, Mngram2, Mngram4], axis=1)
    return df

def fit_model2(X):
    bow = CountVectorizer(lowercase=False,
                          analyzer=lambda x: x)
    join_X = X['sentence1'] + X['sentence2']
    bow_Xtrn = bow.fit_transform(join_X)
    sents_tfidf = TfidfTransformer()
    tfidf_Xtrn = sents_tfidf.fit_transform(bow_Xtrn)
    return bow, sents_tfidf, tfidf_Xtrn
#input_prefix = lambda stage:  file_prefix(stage) + '.input.'
#gs_prefix = lambda stage: file_prefix(stage) + '.gs.'

removals = {
            'stop_words': False,
            'numbers': True
            }
# Tokenized raw train data
df_Xtrain = getData(filenames_train, file_prefix('train'), **removals)
#df_Xtrain.sample(frac=1)
labels_trn = df_Xtrain.iloc[:, -1]
df_Xtrain = df_Xtrain.drop(df_Xtrain.columns[len(df_Xtrain.columns) - 1], axis=1)

# Tokenized raw test data
df_Xtest = getData(filenames_test, file_prefix('test'), **removals)
labels_tst = df_Xtest.iloc[:, -1]
df_Xtest = df_Xtest.drop(df_Xtest.columns[len(df_Xtest.columns) - 1], axis=1)

'''
Xtrn, Xval, ytrn, yval = train_test_split(
                                        df_Xtrain[['sentence1', 'sentence2']],
                                        df_Xtrain['golden_standard'],
                                        test_size=0.4)
                                        #random_state=0)
'''
Xlem = df_Xtrain.apply(lemmas)
Xlem_tst = df_Xtest.apply(lemmas)
##
#print(Xlem.head())
#Xngram = Xlem.apply(lambda col: ngrams(col, n))

transformed_train = modelFeatures(Xlem, filenames_train, 'train')
#reg = LinearRegression().fit(transformed_train, labels_trn)
#reg_model = AdaBoostRegressor().fit(transformed_train, labels_trn)
reg_model = AdaBoostRegressor()
kfold = model_selection.KFold(n_splits=10)
results_model1 = model_selection.cross_val_predict(reg_model, transformed_train, labels_trn, cv=kfold)

transformed_test = modelFeatures(Xlem_tst, filenames_test, 'test')
pred_model = reg_model.fit(transformed_train, labels_trn).predict(transformed_test)
print(pearsonr(pred_model.T.tolist(), labels_tst.tolist())[0])

#bow = CountVectorizer(lowercase=False,
#                      analyzer=lambda x: x)
#join_Xlem = Xlem['sentence1'] + Xlem['sentence2']
#bow_Xtrn = bow.fit_transform(join_Xlem)
#sents_tfidf = TfidfTransformer()
#tfidf_Xtrn = sents_tfidf.fit_transform(bow_Xtrn)
#reg_bow = AdaBoostRegressor().fit(tfidf_Xtrn, labels_trn)

bow, sents_tfidf, tfidf_Xtrn = fit_model2(Xlem)
reg_bow = GradientBoostingRegressor()#.fit(tfidf_Xtrn, labels_trn)

results_model2 = model_selection.cross_val_predict(reg_bow, tfidf_Xtrn, labels_trn, cv=kfold)
join_Xlem_tst = Xlem_tst['sentence1'] + Xlem_tst['sentence2']
bow_Xtst = bow.transform(join_Xlem_tst)
tfidf_Xtst = sents_tfidf.transform(bow_Xtst)
pred_bow = reg_bow.fit(tfidf_Xtrn, labels_trn).predict(tfidf_Xtst)
print(pearsonr(pred_bow.T.tolist(), labels_tst.tolist())[0])
##
#pred_model_trn = reg_model.predict(transformed_train)

#pred_bow_trn = reg_bow.predict(tfidf_Xtrn)

# ensemble
reg_final = XGBRegressor().fit(np.concatenate([results_model1.reshape(-1,1), results_model2.reshape(-1,1)], axis=1), labels_trn)
pred_final = reg_final.predict(np.concatenate([pred_model.reshape(-1,1), pred_bow.reshape(-1,1)], axis=1))
print(pearsonr(pred_final.T.tolist(), labels_tst.tolist())[0])