from pyomo.environ import Block, Expression, units as pyunits
from watertap3.utils import financials
from watertap3.wt_units.wt_unit import WT3UnitProcess

## REFERENCE: ADD REFERENCE HERE

module_name = 'irwin_brine_management'
basis_year = 2019
tpec_or_tic = 'TPEC'

class UnitProcess(WT3UnitProcess):

    def fixed_cap(self):
        time = self.flowsheet().config.time.first()
        self.flow_in = pyunits.convert(self.flow_vol_in[time], to_units=pyunits.m ** 3 / pyunits.hr)
        self.conc_mass_tot = 0
        for constituent in self.config.property_package.component_list:
            self.conc_mass_tot += self.conc_mass_in[time, constituent]
        self.density = 0.6312 * self.conc_mass_tot + 997.86  # kg/m3 # assumption from Tim's reference (ask Ariel for Excel if needed)
        self.total_mass = self.density * self.flow_in  # kg/hr to tons for Mike's Excel needs
        self.chem_dict = {}
        self.capacity_basis = 9463.5
        self.base_fixed_cap_cost = 31
        self.cap_scaling_exp = 0.873
        irwin_bm_cap = self.base_fixed_cap_cost * (self.total_mass / self.capacity_basis) ** self.cap_scaling_exp
        return irwin_bm_cap  # M$

    def elect(self):
        self.lift_height = 100 * pyunits.ft
        self.pump_eff = 0.9 * pyunits.dimensionless
        self.motor_eff = 0.9 * pyunits.dimensionless
        flow_in_gpm = pyunits.convert(self.flow_in, to_units=pyunits.gallons / pyunits.minute)
        electricity = (0.746 * flow_in_gpm * self.lift_height / (3960 * self.pump_eff * self.motor_eff)) / self.flow_in  # kWh/m3
        return electricity

    def get_costing(self, unit_params=None, year=None):
        '''
        Initialize the unit in WaterTAP3.
        '''
        financials.create_costing_block(self, basis_year, tpec_or_tic)
        self.costing.fixed_cap_inv_unadjusted = Expression(expr=self.fixed_cap(),
                                                           doc='Unadjusted fixed capital investment')  # $M
        self.electricity = Expression(expr=self.elect(),
                                      doc='Electricity intensity [kwh/m3]')  # kwh/m3
        financials.get_complete_costing(self.costing)


from django.contrib import admin
from django.utils import timezone
from django.urls import path
from django.shortcuts import render
from django import forms
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from django.contrib import messages
from rest_framework import generics, permissions, serializers
from rest_framework.reverse import reverse

from collectionjson import services

from .models import PluginMeta, Plugin, ComputeResource
from .serializers import ComputeResourceSerializer, PluginSerializer
from .services.manager import PluginManager

plugin_readonly_fields = [fld.name for fld in Plugin._meta.fields if
                          fld.name != 'compute_resources']

class ComputeResourceAdmin(admin.ModelAdmin):
    readonly_fields = ['creation_date', 'modification_date']
    list_display = ('name', 'compute_url', 'description', 'id')
    list_filter = ['name', 'creation_date', 'modification_date']
    search_fields = ['name', 'description']

    def add_view(self, request, form_url='', extra_context=None):
        """
        Overriden to only show the required fields in the add compute resource page.
        """
        self.fields = ['name', 'compute_url', 'description']
        return admin.ModelAdmin.add_view(self, request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        """
        Overriden to show all compute resources's fields in the compute resource page.
        """
        self.fields = ['name', 'compute_url', 'description', 'creation_date',
                       'modification_date']
        return admin.ModelAdmin.change_view(self, request, object_id, form_url,
                                            extra_context)

    def save_model(self, request, obj, form, change):
        """
        Overriden to set the modification date.
        """
        if change:
            obj.modification_date = timezone.now()
        super().save_model(request, obj, form, change)

    def delete_model(self, request, obj):
        """
        Overriden to prevent deleting compute resources that would leave orphan
        plugins after the operation.
        """
        try:
            super().delete_model(request, obj)
        except ValidationError as e:
            messages.set_level(request, messages.ERROR)
            messages.error(request, e.message)

    def delete_queryset(self, request, queryset):
        """
        Overriden to prevent deleting compute resources that would leave orphan
        plugins after the operation. This customizes the deletion process for the
        “delete selected objects” action.
        """
        for compute_resource in queryset:
            plg_ids = compute_resource.get_plugins_with_self_as_single_compute_resource()
            if plg_ids:
                messages.set_level(request, messages.ERROR)
                msg = "Can not delete compute resource '%s'. Please do not check it " \
                      "for the delete or otherwise first register its associated " \
                      "plugins with IDs %s with another compute resource."
                messages.error(request, msg % (compute_resource, plg_ids))
                return None
        super().delete_queryset(request, queryset)

class PluginMetaAdmin(admin.ModelAdmin):
    readonly_fields = ['name', 'title', 'stars', 'public_repo', 'license', 'type', 'icon',
                       'category', 'authors', 'documentation', 'creation_date',
                       'modification_date']
    list_display = ('name', 'type', 'id')
    list_filter = ['type', 'creation_date', 'modification_date', 'category']
    search_fields = ['name']

    def has_change_permission(self, request, obj=None):
        """
        Overriden to disable the editing of fields in the view plugin meta page.
        """
        return False

    def has_add_permission(self, request):
        """
        Overriden to remove the add plugin meta button.
        """
        return False

class UploadFileForm(forms.Form):
    file = forms.FileField()

class PluginAdminForm(forms.ModelForm):
    name = forms.CharField(max_length=100, required=False)
    version = forms.CharField(max_length=10, required=False)
    url = forms.URLField(max_length=300, required=False)

    def clean(self):
        """
        Overriden to validate the full set of plugin descriptors and save the newly
        created plugin to the DB.
        """
        if self.instance.pk is None:  # create plugin operation
            pl_manager = PluginManager()
            compute_resources = self.cleaned_data.get('compute_resources')
            if compute_resources is None:
                raise forms.ValidationError('Please choose a compute resource')
            url = self.cleaned_data.pop('url', None)
            cr = compute_resources[0]
            if url:
                try:
                    self.instance = pl_manager.register_plugin_by_url(url, cr.name)
                except Exception as e:
                    raise forms.ValidationError(e)
            else:
                name = self.cleaned_data.get('name')
                if not name:
                    raise forms.ValidationError("A plugin's name or url is required")
                # get user-provided version (can be blank)
                version = self.cleaned_data.get('version')
                try:
                    self.instance = pl_manager.register_plugin(name, version, cr.name)
                except Exception as e:
                    raise forms.ValidationError(e)
            # reset form validated data
            self.cleaned_data['name'] = self.instance.meta.name
            self.cleaned_data['version'] = self.instance.version
            current_compute_resources = list(self.instance.compute_resources.all())
            compute_resources = list(compute_resources) + current_compute_resources
            self.instance.compute_resources.set(compute_resources)
            self.cleaned_data['compute_resources'] = self.instance.compute_resources.all()

class PluginAdmin(admin.ModelAdmin):
    form = PluginAdminForm
    list_display = ('meta', 'version', 'id')
    search_fields = ['meta', 'version']
    list_filter = ['meta', 'creation_date']
    change_form_template = 'admin/plugins/plugin/change_form.html'
    change_list_template = 'admin/plugins/plugin/change_list.html'

    def add_view(self, request, form_url='', extra_context=None):
        """
        Overriden to only show the required fields in the add plugin page.
        """
        self.readonly_fields = []
        self.fieldsets = [
            ('Compute resources', {'fields': ['compute_resources']}),
            ('Identify plugin by name and version', {'fields': [('name', 'version')]}),
            ('Or identify plugin by url', {'fields': ['url']}),
        ]
        return admin.ModelAdmin.add_view(self, request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        """
        Overriden to show all plugin's fields in the view plugin page.
        """
        self.readonly_fields = [fl for fl in plugin_readonly_fields]
        self.readonly_fields.append('get_registered_compute_resources')
        self.fieldsets = [
            ('Compute resources', {'fields': ['compute_resources',
                                              'get_registered_compute_resources']}),
            ('Plugin properties', {'fields': plugin_readonly_fields}),
        ]
        return admin.ModelAdmin.change_view(self, request, object_id, form_url,
                                            extra_context)

    # def save_model(self, request, obj, form, change):
    #     """
    #     Overriden to set the modification date..
    #     """
    #     if change:
    #         obj.modification_date = timezone.now()
    #     super().save_model(request, obj, form, change)

    def get_urls(self):
        urls = super(PluginAdmin, self).get_urls()
        custom_urls = [
            path('add_plugins/',
                 self.admin_site.admin_view(self.add_plugins_from_file_view),
                 name="add_plugins"),
        ]
        return custom_urls + urls

    def delete_model(self, request, obj):
        """
        Overriden to delete the associated meta if this is the last associated plugin.
        """
        meta = obj.meta
        super().delete_model(request, obj)
        if meta.plugins.count() == 0:
            meta.delete()  # delete the meta if this is the last associated plugin

    def delete_queryset(self, request, queryset):
        """
        Overriden to delete plugin metas if their last associated plugin is deleted.
        This customizes the deletion process for the “delete selected objects” action.
        """
        pl_metas = []
        for plugin in queryset:
            if plugin.meta.plugins.count() == 1:
                pl_metas.append(plugin.meta)
        super().delete_queryset(request, queryset)
        for meta in pl_metas:
            meta.delete()

    def add_plugins_from_file_view(self, request):
        """
        Custom view to handle the add plugins from file page.
        """
        # custom view should return an HttpResponse
        context = dict(
            # Include common variables for rendering the admin template.
            self.admin_site.each_context(request),
            # Anything else you want in the context...
            opts=self.opts,
        )
        if request.method == 'POST':
            file_form = UploadFileForm(request.POST, request.FILES)
            if file_form.is_valid():
                # handle uploaded file (request.FILES['file'])
                summary = self.register_plugins_from_file(request.FILES['file'])
                context['summary'] = summary
                return render(request,
                              'admin/plugins/plugin/add_plugins_from_file_result.html',
                              context)
        else:
            file_form = UploadFileForm()
        context['file_form'] = file_form
        return render(request,
                      'admin/plugins/plugin/add_plugins_from_file.html',
                      context)

    def register_plugins_from_file(self, f):
        """
        Custom method to register plugins from a text file f. The first string of each
        line of the file is the plugin's name or url, the second string is the version
        (can be empty) and the third string is the associated compute environment. Any
        plugin in the file must already be previously uploaded to the ChRIS store.
        """
        summary = {'success': [], 'error': []}
        val = URLValidator()
        pl_manager = PluginManager()
        for line in f:
            try:  # check file must be a text file
                strings = line.decode().strip().split()
            except UnicodeDecodeError:
                summary = {'success': [], 'error': []}
                break
            if len(strings) == 1:
                summary['error'].append({'plugin_name': strings[0],
                                         'code': 'Missing compute resource identifier.'})
            elif len(strings) > 1:
                try:
                    val(strings[0])  # check whether the first string is a url
                except ValidationError:
                    # register by name
                    plg_name = strings[0]
                    if len(strings) == 2:
                        plg_version = None
                        compute_resource = strings[1]
                    else:
                        plg_version = strings[1]
                        compute_resource = strings[2]
                    try:
                        pl_manager.register_plugin(plg_name, plg_version,
                                                   compute_resource)
                    except Exception as e:
                        summary['error'].append({'plugin_name': plg_name, 'code': str(e)})
                    else:
                        summary['success'].append({'plugin_name': plg_name})
                else:
                    # register by url
                    plg_url = strings[0]
                    compute_resource = strings[1]
                    try:
                        pl_manager.register_plugin_by_url(plg_url, compute_resource)
                    except Exception as e:
                        summary['error'].append({'plugin_name': plg_url, 'code': str(e)})
                    else:
                        summary['success'].append({'plugin_name': plg_url})
        return summary

class ComputeResourceAdminList(generics.ListCreateAPIView):
    """
    A JSON view for the collection of compute resources that can be used by ChRIS admins
    to add a new compute resource through a REST API (alternative to the HTML-based admin
    site).
    """
    http_method_names = ['get', 'post']
    serializer_class = ComputeResourceSerializer
    queryset = ComputeResource.objects.all()
    permission_classes = (permissions.IsAdminUser,)

    def list(self, request, *args, **kwargs):
        """
        Overriden to append a collection+json template to the response.
        """
        response = super(ComputeResourceAdminList, self).list(request, *args, **kwargs)
        # append write template
        template_data = {'name': '', 'compute_url': '', 'description': ''}
        return services.append_collection_template(response, template_data)

class PluginAdminSerializer(PluginSerializer):
    """
    A Plugin serializer for the PluginAdminList JSON view.
    """
    plugin_store_url = serializers.URLField(write_only=True)
    compute_name = serializers.CharField(max_length=100, write_only=True)
    version = serializers.CharField(required=False)
    dock_image = serializers.CharField(required=False)
    execshell = serializers.CharField(required=False)
    selfpath = serializers.CharField(required=False)
    selfexec = serializers.CharField(required=False)

    class Meta(PluginSerializer.Meta):
        fields = PluginSerializer.Meta.fields + ('plugin_store_url', 'compute_name')

    def validate(self, data):
        """
        Overriden to validate and register a plugin from a ChRIS store with a compute
        resource associated with this ChRIS instance.
        """
        # remove write_only fields that not part of the Plugin model
        cr_name = data.pop('compute_name')
        plugin_store_url = data.pop('plugin_store_url')
        pl_manager = PluginManager()
        try:
            self.instance = pl_manager.register_plugin_by_url(plugin_store_url, cr_name)
        except Exception as e:
            raise serializers.ValidationError({'non_field_errors': [str(e)]})
        return data

class PluginAdminList(generics.ListCreateAPIView):
    """
    A JSON view for the collection of plugins that can be used by ChRIS admins to
    register plugins through a REST API (alternative to the HTML-based admin site).
    """
    http_method_names = ['get', 'post']
    serializer_class = PluginAdminSerializer
    queryset = Plugin.objects.all()
    permission_classes = (permissions.IsAdminUser,)

    def list(self, request, *args, **kwargs):
        """
        Overriden to append document-level link relations and a collection+json template
        to the response.
        """
        response = super(PluginAdminList, self).list(request, *args, **kwargs)
        # append document-level link relations
        links = {'compute_resources': reverse('admin-computeresource-list',
                                              request=request)}
        response = services.append_collection_links(response, links)
        # append write template
        template_data = {'plugin_store_url': '', 'compute_name': ''}
        return services.append_collection_template(response, template_data)

admin.site.register(ComputeResource, ComputeResourceAdmin)
admin.site.register(PluginMeta, PluginMetaAdmin)
admin.site.register(Plugin, PluginAdmin)


import face_recognition
import cv2
import numpy as np
import config
import time, math
from datetime import datetime, timedelta

NEW_EVENT_MINS = 5

video_capture = cv2.VideoCapture(0)
video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, 400)
video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 300)

names, faces = config.loadJson()
print(str(len(names)) + ' items loaded')
print('names loaded')
print(names)
print('faces loaded')
print(faces)

# Create arrays of known face encodings and their names
known_face_encodings = []
known_face_names = []
known_face_metadata = []

def register_new_face(face_encoding, face_image):
    known_face_encodings.append(face_encoding)
    known_face_metadata.append({
        "first_seen": datetime.now(),
        "first_seen_this_interaction": datetime.now(),
        "last_seen": datetime.now(),
        "seen_count": 1,
        "seen_frames": 1,
        "face_image": face_image,
    })

def lookup_known_face(face_encoding):
    metadata = None

    if len(known_face_encodings) == 0:
        return metadata

    face_distances = face_recognition.face_distance(
        known_face_encodings,
        face_encoding
    )

    best_match_index = np.argmin(face_distances)

    if face_distances[best_match_index] < 0.65:
        metadata = known_face_metadata[best_match_index]
        metadata["last_seen"] = datetime.now()
        metadata["seen_frames"] += 1

        if datetime.now() - metadata["first_seen_this_interaction"] > timedelta(minutes=NEW_EVENT_MINS):
            metadata["first_seen_this_interaction"] = datetime.now()
            metadata["seen_count"] += 1

    return metadata

# Initialize some variables
face_locations = []
face_encodings = []
face_names = []
process_this_frame = True

fps = video_capture.get(cv2.CAP_PROP_FPS)
fps_count = 0

print(fps)
second = math.floor(time.time())
while True:
    fps_count += 1
    if second != math.floor(time.time()):
        fps = fps_count
        fps_count = 0
        second = math.floor(time.time())

    # Grab a single frame of video
    ret, frame = video_capture.read()

    # Resize frame of video to 1/4 size for faster face recognition processing
    small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)

    # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
    rgb_small_frame = small_frame[:, :, ::-1]

    # Only process every other frame of video to save time
    if process_this_frame:
        # Find all the faces and face encodings in the current frame of video
        face_locations = face_recognition.face_locations(rgb_small_frame)
        face_encodings = face_recognition.face_encodings(rgb_small_frame, face_locations)

        # face_names = []
        # for face_encoding in face_encodings:
        #     # See if the face is a match for the known face(s)
        #     matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
        #     name = "Unknown"
        #
        #     # # If a match was found in known_face_encodings, just use the first one.
        #     # if True in matches:
        #     #     first_match_index = matches.index(True)
        #     #     name = known_face_names[first_match_index]
        #
        #     # Or instead, use the known face with the smallest distance to the new face
        #     face_distances = face_recognition.face_distance(known_face_encodings, face_encoding)
        #     best_match_index = np.argmin(face_distances)
        #     if matches[best_match_index]:
        #         name = known_face_names[best_match_index]
        #
        #     face_names.append(name)
        #     print(name + ' detectd at ' + str(second))

        face_labels = []
        for face_location, face_encoding in zip(face_locations, face_encodings):
            # See if this face is in our list of known faces.
            metadata = lookup_known_face(face_encoding)

            # If we found the face, label the face with some useful information.
            if metadata is not None:
                time_at_location = datetime.now() - metadata['first_seen_this_interaction']
                face_label = f"At location {int(time_at_location.total_seconds())}s"

            # If this is a brand new face, add it to our list of known faces
            else:
                face_label = "New person!"

                # Grab the image of the the face from the current frame of video
                top, right, bottom, left = face_location
                face_image = small_frame[top:bottom, left:right]
                face_image = cv2.resize(face_image, (150, 150))

                # Add the new face to our known face data
                register_new_face(face_encoding, face_image)

            face_labels.append(face_label)
            #print(face_label)

    process_this_frame = not process_this_frame

    font = cv2.FONT_HERSHEY_DUPLEX

    # Display the results
    for (top, right, bottom, left), face_label in zip(face_locations, face_labels):
        # Scale back up face locations since the frame we detected in was scaled to 1/4 size
        top *= 4
        right *= 4
        bottom *= 4
        left *= 4

        # Draw a box around the face
        cv2.rectangle(frame, (left, top), (right, bottom), (255, 255, 255), 2)

        # Draw a label with a name below the face
        #cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (255, 255, 255), cv2.FILLED)

        #cv2.putText(frame, face_label, (left + 6, bottom - 6), font, 0.8, (255, 255, 255), 1)

    cv2.putText(frame, str(fps) + ' fps', (5, 22), font, 0.5, (0, 255, 0), 1)
    # Display the resulting image
    cv2.imshow('Video', frame)

    # Hit 'q' on the keyboard to quit!
    if cv2.waitKey(1) & 0xFF == ord('q'):
        #save_known_faces()
        break

    # if len(face_locations) > 0 and number_of_faces_since_save > 100:
    #     save_known_faces()
    #     number_of_faces_since_save = 0
    # else:
    #     number_of_faces_since_save += 1

# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()


from keras.models import Model
from keras.layers import Input, concatenate, SeparableConv2D, MaxPooling2D, Conv2DTranspose, SpatialDropout2D, BatchNormalization
from keras.optimizers import Adam
from keras import backend as K

smooth = 1

#Cálculo do Coeficiente de Dice para uma máscara prevista pelo modelo

def dice_coef(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)

    return (2. * intersection + smooth) / (K.sum(y_true_f) +
            K.sum(y_pred_f) + smooth)

#Valor do coeficiente de Dice a passar para a avaliação do modelo
#Como se pretende minimizar a função de custo (loss) deve tomar-se o simétrico
#do coeficiente de Dice

def dice_coef_loss(y_true, y_pred):
    return -dice_coef(y_true, y_pred)

#Rede convolucional unet, com:
#spatial_dropout na quarta e quinta camadas para reduzir o nível de overfit
#otimizador - Adam
#função de custo (loss_function) - Coeficiente de Dice
#Inicializador de pesos (weights ou kernel) - he_normal

def unet(vis):
    input_size = (256,256,1)
    inputs = Input(input_size)

    #Encoder

    conv1_1 = SeparableConv2D(64, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(inputs)
    conv1_1 = BatchNormalization()(conv1_1)
    conv1_2 = SeparableConv2D(64, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv1_1)
    conv1_2 = BatchNormalization()(conv1_2)
    pool1 = MaxPooling2D(pool_size=(2,2))(conv1_2)

    conv2_1 = SeparableConv2D(128, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(pool1)
    conv2_1 = BatchNormalization()(conv2_1)
    conv2_2 = SeparableConv2D(128, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv2_1)
    conv2_2 = BatchNormalization()(conv2_2)
    pool2 = MaxPooling2D(pool_size=(2,2))(conv2_2)

    conv3_1 = SeparableConv2D(256, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(pool2)
    conv3_1 = BatchNormalization()(conv3_1)
    conv3_2 = SeparableConv2D(256, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv3_1)
    conv3_2 = BatchNormalization()(conv3_2)
    pool3 = MaxPooling2D(pool_size=(2,2))(conv3_2)

    conv4_1 = SeparableConv2D(512, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(pool3)
    conv4_1 = BatchNormalization()(conv4_1)
    conv4_2 = SeparableConv2D(512, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv4_1)
    conv4_2 = BatchNormalization()(conv4_2)
    drop4_2 = SpatialDropout2D(0.5)(conv4_2)
    pool4 = MaxPooling2D(pool_size=(2,2))(drop4_2)

    conv5_1 = SeparableConv2D(1024, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(pool4)
    conv5_1 = BatchNormalization()(conv5_1)
    conv5_2 = SeparableConv2D(1024, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv5_1)
    conv5_2 = BatchNormalization()(conv5_2)
    drop5_2 = SpatialDropout2D(0.5)(conv5_2)

    #Decoder

    up6_1 = Conv2DTranspose(512, (2,2), strides = (2,2), padding = 'same',
                          activation = 'relu',
                          kernel_initializer='he_normal')(drop5_2)
    up6_1 = BatchNormalization()(up6_1)
    up6_2 = concatenate([drop4_2, up6_1], axis = 3)
    conv6_1 = SeparableConv2D(512, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(up6_2)
    conv6_1 = BatchNormalization()(conv6_1)
    conv6_2 = SeparableConv2D(512, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv6_1)
    conv6_2 = BatchNormalization()(conv6_2)

    up7_1 = Conv2DTranspose(256, (2,2), strides = (2,2), padding = 'same',
                          activation = 'relu',
                          kernel_initializer='he_normal')(conv6_2)
    up7_1 = BatchNormalization()(up7_1)
    up7_2 = concatenate([conv3_2, up7_1], axis = 3)
    conv7_1 = SeparableConv2D(256, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(up7_2)
    conv7_1 = BatchNormalization()(conv7_1)
    conv7_2 = SeparableConv2D(256, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv7_1)
    conv7_2 = BatchNormalization()(conv7_2)

    up8_1 = Conv2DTranspose(128, (2,2), strides = (2,2), padding = 'same',
                          activation = 'relu',
                          kernel_initializer='he_normal')(conv7_2)
    up8_1 = BatchNormalization()(up8_1)
    up8_2 = concatenate([conv2_2, up8_1], axis = 3)
    conv8_1 = SeparableConv2D(128, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(up8_2)
    conv8_1 = BatchNormalization()(conv8_1)
    conv8_2 = SeparableConv2D(128, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv8_1)
    conv8_2 = BatchNormalization()(conv8_2)

    up9_1 = Conv2DTranspose(64, (2,2), strides = (2,2), padding = 'same',
                          activation = 'relu',
                          kernel_initializer='he_normal')(conv8_2)
    up9_1 = BatchNormalization()(up9_1)
    up9_2 = concatenate([conv1_2, up9_1], axis = 3)
    conv9_1 = SeparableConv2D(64, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(up9_2)
    conv9_1 = BatchNormalization()(conv9_1)
    conv9_2 = SeparableConv2D(64, (3,3), padding = 'same', activation = 'relu',
                   kernel_initializer='he_normal')(conv9_1)
    conv9_2 = BatchNormalization()(conv9_2)
    conv9_3 = SeparableConv2D(2, 3, activation = 'relu', padding = 'same',
                   kernel_initializer = 'he_normal')(conv9_2)
    conv9_3 = BatchNormalization()(conv9_3)

    conv10 = SeparableConv2D(1, 1, activation = 'sigmoid')(conv9_3)

    if(vis):
        outputs = [conv1_1, conv1_2, pool1, conv2_1, conv2_2, pool2, conv3_1,
                   conv3_2, pool3, conv4_1, conv4_2, drop4_2, pool4, conv5_1,
                   conv5_2, drop5_2, up6_1, up6_2,conv6_1, conv6_2, up7_1,
                   up7_2, conv7_1, conv7_2, up8_1, up8_2, conv8_1, conv8_2,
                   up9_1, up9_2, conv9_1, conv9_2, conv9_3, conv10]

        model = Model(inputs = [inputs], outputs = outputs)
        model.summary()
    else:
        model = Model(inputs = [inputs], outputs = [conv10])
        model.summary()

    model.compile(optimizer=Adam(lr=1e-3), loss=dice_coef_loss,
                  metrics=[dice_coef, 'accuracy'])

    return model


# -*- coding: cp1251 -*-
import urllib

from django.shortcuts import render, HttpResponse, redirect
from main.models import Carpet, Size, Product, Shape, Country, Order, Color, Collection, Discount, News
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.mail import send_mail
from main.forms import OrderForm
from cart import Cart
from django.template.loader import render_to_string
from django.db.models import Q
import json

def pag(request, objects):
    paginator = Paginator(objects, 12)
    page = request.GET.get('page')
    try:
        objects = paginator.page(page)
    except PageNotAnInteger:
        objects = paginator.page(1)
    except EmptyPage:
        objects = paginator.page(paginator.num_pages)
    return objects

class MyPaginator:
    def __init__(self, request, objects):
        self.paginator = Paginator(objects, 12)
        self.page = request.GET.get('page')
        self.begin = None
        self.middle = None
        self.end = None
        try:
            self.objects = self.paginator.page(self.page)
            self.page = int(self.page)
        except PageNotAnInteger:
            self.objects = self.paginator.page(1)
            self.page = 1
        except EmptyPage:
            self.objects = self.paginator.page(self.paginator.num_pages)
            self.page = self.paginator.num_pages

    @staticmethod
    def collides(firstlist, secondlist):
        return any(item in secondlist for item in firstlist)

    def calculate(self):
        self.begin = self.paginator.page_range[:2]
        self.end = self.paginator.page_range[-2:]
        before = max(self.page - 4, 0)
        after = self.page + 3
        self.middle = self.paginator.page_range[before:after]
        last_page_number = self.end[-1]

        if self.collides(self.middle, self.end):
            self.end = range(max(self.page-3, 1), last_page_number+1)
            self.middle = []

        if self.collides(self.begin, self.middle):
            self.begin = range(1, min(self.page + 3, last_page_number)+1)
            self.middle = []

        if self.collides(self.begin, self.end):
            self.begin = range(1, last_page_number+1)
            self.end = []

def index(request):
    carpets = Carpet.objects.all()[:6]
    cart = Cart(request)
    discounts = Discount.objects.all()[:6]
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'index.html', {'carpets': carpets, 'cart': cart, 'discounts': discounts, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections})

def catalog(request):
    carpets = Carpet.objects.all()
    paginator = None

    widthmin = request.GET.get('widthmin')
    widthmax = request.GET.get('widthmax')
    lengthmin = request.GET.get('lengthmin')
    lengthmax = request.GET.get('lengthmax')

    color_id = request.GET.get('color')
    shape_id = request.GET.get('shape')
    country_id = request.GET.get('country')
    collection_id = request.GET.get('collection')

    if widthmin and widthmax and lengthmin and lengthmax:
        no_sizes = False
        try:
            getsizes = Size.objects.all()
            getsizes = getsizes.filter(width__gte=widthmin)
            getsizes = getsizes.filter(length__lte=lengthmax)
            getsizes = getsizes.filter(width__lte=widthmax)
            getsizes = getsizes.filter(length__gte=lengthmin)
        except:
            no_sizes = True
            getsizes = []
        if not no_sizes:
            x = Q()
            for size in getsizes:
                x = x | Q(sizes=size)
            carpets = carpets.filter(x).distinct()
        else:
            carpets = []
    else:
        getsizes = []
    if carpets.count() != 0:
        color_all = False
        shape_all = False
        country_all = False
        collection_all = False

        try:
            color = Color.objects.get(id=color_id)
        except:
            color_all = True
        try:
            shape = Shape.objects.get(id=shape_id)
        except:
            shape_all = True
        try:
            country = Country.objects.get(id=country_id)
        except:
            country_all = True
        try:
            collection = Collection.objects.get(id=collection_id)
        except:
            collection_all = True

        if not color_all:
            carpets = carpets.filter(color=color)
        if not shape_all:
            carpets = carpets.filter(shape=shape)
        if not country_all:
            carpets = carpets.filter(country=country)
        if not collection_all:
            carpets = carpets.filter(collection=collection)
        paginator = MyPaginator(request, carpets)
        paginator.calculate()
        carpets = paginator.objects

    cart = Cart(request)
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    newss = News.objects.all()
    copy_request = request.GET.copy()
    page = copy_request.get('page')
    if page:
        del copy_request['page']
    params = urllib.urlencode(copy_request)
    return render(request, 'catalog.html',
                  {'carpets': carpets, 'cart': cart, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'lengthmin': lengthmin, 'lengthmax': lengthmax, 'widthmin': widthmin, 'widthmax': widthmax,
                   'collections': collections, 'params': params, 'p': request.GET, 'newss': newss, 'paginator': paginator})

def order(request):
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'order.html', {'cart': cart, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections})

def children(request):
    carpets = Carpet.objects.filter(is_children=True)
    carpets = pag(request, carpets)
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'carpets.html', {'carpets': carpets, 'cart': cart, 'shape': shape, 'newss': newss, 'children': True, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections,})

def delivery(request):
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'delivery.html', {'cart': cart, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections,})

def contacts(request):
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'contacts.html', {'cart': cart, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections,})

def shape(request, shape_id):
    cart = Cart(request)
    shape = Shape.objects.get(id=shape_id)
    carpets = Carpet.objects.filter(shape=shape)
    paginator = MyPaginator(request, carpets)
    paginator.calculate()
    carpets = paginator.objects
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'carpets.html', {'carpets': carpets, 'cart': cart, 'shape': shape, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections, 'paginator': paginator})

def country(request, country_id):
    cart = Cart(request)
    country = Country.objects.get(id=country_id)
    carpets = Carpet.objects.filter(country=country)
    paginator = MyPaginator(request, carpets)
    paginator.calculate()
    carpets = paginator.objects
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'carpets.html', {'carpets': carpets, 'cart': cart, 'country': country, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections, 'paginator': paginator})

def carpet(request, carpet_id):
    carpet = Carpet.objects.get(id=carpet_id)
    if request.path.find(carpet.name.replace(' ', '_')) == -1:
        return redirect(u"/carpet/" + unicode(carpet.id) + u"/" + unicode(carpet.name).replace(' ', '_'),
                        permanent=True)
    sizes = []
    for s in carpet.sizes.all().order_by('width', 'length'):
        price = s.square() * carpet.collection.sq_meter_cost
        price = price / 100
        price = round(price) * 100
        price = int(price)
        try:
            discount = Discount.objects.get(size=s, carpet=carpet)
            q = {'name': s, 'price': discount.price, 'id': s.id, 'is_discount': True, 'old_price': price}
        except:
            q = {'name': s, 'price': price, 'id': s.id, 'is_discount': False}
        sizes.append(q)
    carpets = Carpet.objects.all().filter(color=carpet.color).exclude(id=carpet_id)[:3]
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'carpet.html', {'carpet': carpet, 'sizes': sizes, 'carpets': carpets, 'cart': cart, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections,})

def price(request):
    cid = request.POST.get('cid')
    sid = request.POST.get('sid')
    size = Size.objects.get(id=sid)
    carpet = Carpet.objects.get(id=cid)
    try:
        discount = Discount.objects.get(carpet=carpet, size=size)
    except:
        discount = False
    cost = size.square() * carpet.collection.sq_meter_cost
    cost = cost / 100
    cost = round(cost) * 100
    cost = int(cost)
    if discount != False:
        resp = json.dumps({'price': discount.price, 'old_price': cost, 'is_discount': True})
    else:
        resp = json.dumps({'price': cost, 'is_discount': False})
    return HttpResponse(resp, content_type='application/json')

def add_to_cart(request):
    cid = request.POST.get('cid')
    sid = request.POST.get('sid')
    try:
        size = Size.objects.get(id=sid)
        carpet = Carpet.objects.get(id=cid)
    except:
        resp = json.dumps({'resp': 'fail'})
        return HttpResponse(resp, content_type='application/json')
    try:
        p = Product.objects.get(carpet=carpet, size=size)
    except:
        p = Product.objects.create(carpet=carpet, size=size)
    cart = Cart(request)
    cart.add(p, p.price)
    count = cart.count()
    resp = json.dumps({'resp': count})
    return HttpResponse(resp, content_type='application/json')

def remove_from_cart(request):
    pid = request.POST.get('pid')
    product = Product.objects.get(id=pid)
    cart = Cart(request)
    cart.remove(product)
    return redirect('/cart/')

def cart(request):
    cart = Cart(request)
    form = OrderForm()
    m = ''
    if request.POST:
        name = request.POST.get('name')
        phone = request.POST.get('phone')
        email = request.POST.get('email')
        adress = request.POST.get('adress')
        order = Order.objects.create(name=name, phone=phone, email=email, adress=adress)
        for item in cart:
            for i in range(0, item.quantity):
                order.products.add(item.product)
        order.save()
        message_me = render_to_string('email.txt', {'order': order, 'cart': cart})
        message_to = render_to_string('emailtouser.txt', {'order': order, 'cart': cart})
        cart.clear()
        send_mail(u"Заказ:", message_me, 'woolkover@gmail.com', ['sermalenk@gmail.com', '19742205@bk.ru'])
        send_mail(u"Заказ:", message_to, 'woolkover@gmail.com', [email])
        m = u"Заказ совершен. На Вашу электронную почту было отправлено письмо с заказом. В ближайшее время с Вами свяжется менеджер для уточнения деталей."
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'cart.html', {'cart': cart, 'form': form, 'message': m, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections,})

def feedback(request):
    if request.POST:
        name = request.POST.get('name')
        phone = request.POST.get('phone')
        subject = request.POST.get('subject')
        message = render_to_string('feedback.txt', {'name': name, 'phone': phone, 'subject': subject})
        send_mail(u"Обратная связь", message, 'woolkover@gmail.com', ['sermalenk@gmail.com'])
    resp = json.dumps({'resp': 'ok'})
    return HttpResponse(resp, content_type='application/json')

def search(request):
    name = request.GET.get('name')
    if name:
        carpets = Carpet.objects.filter(name__icontains=name)
        if carpets.count() != 0:
            paginator = MyPaginator(request, carpets)
            paginator.calculate()
            carpets = paginator.objects
            newss = News.objects.all()
            colors = Color.objects.all()
            shapes = Shape.objects.all()
            countries = Country.objects.all()
            collections = Collection.objects.all()
            copy_request = request.GET.copy()
            page = copy_request.get('page')
            if page:
                del copy_request['page']
            params = urllib.urlencode(copy_request)
            return render(request, 'carpets.html', {'carpets': carpets, 'name': name, 'search': True, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections, 'paginator': paginator, 'params': params})
    message = u"По вашему запросу ничего не найдено"
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    copy_request = request.GET.copy()
    page = copy_request.get('page')
    if page:
        del copy_request['page']
    params = urllib.urlencode(copy_request)
    return render(request, 'carpets.html', {'message': message, 'name': name, 'search': True, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
           'collections': collections, 'params': params})

def discounts(request):
    discounts = Discount.objects.all()
    paginator = MyPaginator(request, discounts)
    paginator.calculate()
    discounts = paginator.objects
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'discounts.html', {'discounts': discounts, 'cart': cart, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections, 'paginator': paginator})

def sales(request):
    cart = Cart(request)
    newss = News.objects.all()
    colors = Color.objects.all()
    shapes = Shape.objects.all()
    countries = Country.objects.all()
    collections = Collection.objects.all()
    return render(request, 'sales.html', {'cart': cart, 'newss': newss, 'colors': colors, 'shapes': shapes, 'countries': countries,
                   'collections': collections})


from pathlib import Path
from torch.utils.data import Dataset, ConcatDataset, DataLoader
from torchvision import transforms as trans
from torchvision.datasets import ImageFolder
from PIL import Image, ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True
import numpy as np
import cv2
import bcolz
import pickle

from tqdm import tqdm
import random

def de_preprocess(tensor):
    return tensor * 0.5 + 0.5

def get_train_dataset(imgs_folder, train_transforms=None):
    if not train_transforms:
        train_transforms = trans.Compose([
            trans.RandomHorizontalFlip(),
            trans.ToTensor(),
            trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
        ])
    ds = ImageFolder(imgs_folder, train_transforms)
    class_num = ds[-1][1] + 1
    return ds, class_num

def get_custom_train_dataset(imgs_folder, train_transforms=None):
    class CustomDataset(ImageFolder):
        """__init__ and __len__ functions are the same as in TorchvisionDataset"""

        def __init__(self, root, transform=None):
            super(CustomDataset, self).__init__(root, transform=transform)

        def __getitem__(self, index):
            """
            Args:
                index (int): Index

            Returns:
                tuple: (sample, target) where target is class_index of the target class.
            """

            while True:
                try:
                    path, target = self.samples[index]
                    sample = np.array(self.loader(path))
                    if self.transform is not None:
                        sample = self.transform(image=sample)['image']
                    return sample, target
                except Exception as e:
                    import traceback
                    traceback.print_exc()
                    # traceback.print_exc()
                    print(str(e), path)
                    index = random.randint(0, len(self) - 1)

    ds = CustomDataset(imgs_folder, train_transforms)
    class_num = ds[-1][1] + 1
    return ds, class_num

def get_dacon_landmark_train_dataset(imgs_folder, train_transforms=None, label_file=None):
    import torch
    import json
    import os
    import glob

    class CustomDataset(torch.utils.data.Dataset):
        """__init__ and __len__ functions are the same as in TorchvisionDataset"""

        def __init__(self, root, transform=None, label_file=None):
            self.transform = transform
            labels = json.load(open(label_file))
            label_map = {}
            for label in labels['categories']:
                label_map[label['landmark_name']] = label['landmark_id']

            samples = []
            for dir in glob.glob(os.path.join(root, '*')):
                label_name = os.path.basename(dir)
                label_idx = label_map[label_name]
                for path in glob.glob(os.path.join(dir, "*")):
                    samples.append([path, label_idx])
            self.samples = samples

        def __getitem__(self, index):
            """
            Args:
                index (int): Index

            Returns:
                tuple: (sample, target) where target is class_index of the target class.
            """

            while True:
                try:
                    path, target = self.samples[index]
                    sample = np.array(Image.open(path).convert("RGB"))
                    if self.transform is not None:
                        sample = self.transform(image=sample)['image']
                    return sample, target
                except Exception as e:
                    # traceback.print_exc()
                    print(str(e), path)
                    index = random.randint(0, len(self) - 1)

        def __len__(self):
            return len(self.samples)

    ds = CustomDataset(imgs_folder, train_transforms, label_file)
    class_num = ds[-1][1] + 1
    return ds, class_num

def get_train_loader(conf, train_transforms=None):
    if conf.data_mode == 'common':
        ds, class_num = get_custom_train_dataset(conf.train_img_dir, train_transforms)
    elif conf.data_mode == 'dacon_landmark':
        ds, class_num = get_custom_train_dataset(conf.train_img_dir, train_transforms, conf.label_file)
    else:
        if conf.data_mode in ['ms1m', 'concat']:
            ms1m_ds, ms1m_class_num = get_train_dataset(conf.ms1m_folder / 'imgs')
            print('ms1m loader generated')
        if conf.data_mode in ['vgg', 'concat']:
            vgg_ds, vgg_class_num = get_train_dataset(conf.vgg_folder / 'imgs')
            print('vgg loader generated')
        if conf.data_mode == 'vgg':
            ds = vgg_ds
            class_num = vgg_class_num
        elif conf.data_mode == 'ms1m':
            ds = ms1m_ds
            class_num = ms1m_class_num
        elif conf.data_mode == 'concat':
            for i, (url, label) in enumerate(vgg_ds.imgs):
                vgg_ds.imgs[i] = (url, label + ms1m_class_num)
            ds = ConcatDataset([ms1m_ds, vgg_ds])
            class_num = vgg_class_num + ms1m_class_num
        elif conf.data_mode == 'emore':
            ds, class_num = get_train_dataset(conf.emore_folder / 'imgs')
    loader = DataLoader(ds, batch_size=conf.batch_size, shuffle=True, pin_memory=conf.pin_memory,
                        num_workers=conf.num_workers)
    return loader, class_num

def load_bin(path, rootdir, transform, image_size=[112, 112]):
    import mxnet as mx
    if not rootdir.exists():
        rootdir.mkdir()
    bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
    data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]], dtype=np.float32, rootdir=rootdir, mode='w')
    for i in range(len(bins)):
        _bin = bins[i]
        img = mx.image.imdecode(_bin).asnumpy()
        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
        img = Image.fromarray(img.astype(np.uint8))
        data[i, ...] = transform(img)
        i += 1
        if i % 1000 == 0:
            print('loading bin', i)
    print(data.shape)
    np.save(str(rootdir) + '_list', np.array(issame_list))
    return data, issame_list

def get_val_pair(path, name):
    carray = bcolz.carray(rootdir=path / name, mode='r')
    issame = np.load(path / '{}_list.npy'.format(name))
    return carray, issame

def get_common_val_data(data_path, max_positive_cnt, batch_size, pin_memory, num_workers, val_transforms=None,
                        use_pos=True, use_neg=True, val_smapling_type=None, use_keras_model=False):
    class ValDataset(Dataset):
        """__init__ and __len__ functions are the same as in TorchvisionDataset"""

        def __init__(self, files, transform=None):
            self.files = files
            self.transform = transform

        def __getitem__(self, index):
            """
            Args:
                index (int): Index

            Returns:
                tuple: (sample, target) where target is class_index of the target class.
            """
            file = self.files[index]
            im = Image.open(file).convert("RGB")
            sample = np.array(im)
            if self.transform is not None:
                sample = self.transform(image=sample)['image']

            if use_keras_model:
                from tensorflow.keras.preprocessing.image import img_to_array
                img = im
                w, h = img.size
                maxsz = max(w, h)
                scale = maxsz / 400

                img = img.resize((
                    int(w / scale), int(h / scale)
                ))

                return sample, img_to_array(img)
            else:
                return sample

        def __len__(self):
            return len(self.files)

    import glob
    import os

    if not use_pos and not use_neg:
        raise Exception("use_pos and use_neg all false")

    label_dirs = glob.glob(os.path.join(data_path, "*"))
    total_positive_cnt = 0
    label_files_list = []
    for label_dir in label_dirs:
        file_cnt = len(glob.glob(os.path.join(label_dir, "*")))
        total_positive_cnt += file_cnt * (file_cnt - 1) / 2
        if total_positive_cnt < 1:
            continue
        label_files_list.append(glob.glob(os.path.join(label_dir, "*")))
    print("total_positive_cnt", total_positive_cnt)
    if not max_positive_cnt or max_positive_cnt > total_positive_cnt:
        max_positive_cnt = total_positive_cnt

    if max_positive_cnt < 1:
        raise Exception("max_positive_cnt is 0")

    positive_files = []
    issame = []

    if val_smapling_type:
        unq_dirs = list({"_".join(d.split("_")[:-1]): True for d in label_dirs}.keys())
        if use_pos:
            each_cnt = max_positive_cnt / len(label_dirs)
            for label_idx, label_files in enumerate(label_files_list):
                cur_cnt = 0
                try:
                    for i in range(0, len(label_files) - 1):
                        for j in range(i + 1, len(label_files)):
                            positive_files.append(label_files[i])
                            positive_files.append(label_files[j])
                            cur_cnt += 1
                            if cur_cnt >= each_cnt:
                                raise
                except:
                    print("val positive label cnt", cur_cnt, os.path.basename(label_dirs[label_idx]))
                    pass
            max_positive_cnt = len(positive_files) // 2
            issame += [True] * int(len(positive_files) / 2)
            print("val positive cnt", len(positive_files))
        if val_smapling_type == "left_right_eye_half_same_dog":
            max_positive_cnt = max_positive_cnt // 2

        negative_files = []
        if use_neg:
            total_negative_cnt = 0
            idx_map = {}
            dir_map = {}
            while max_positive_cnt > total_negative_cnt:
                target_label_idx = random.randint(0, len(unq_dirs) - 1)
                left_dir = unq_dirs[target_label_idx] + "_left"
                right_dir = unq_dirs[target_label_idx] + "_right"
                if os.path.basename(left_dir) not in dir_map:
                    dir_map[os.path.basename(left_dir)] = glob.glob(os.path.join(left_dir, "*"))
                if os.path.basename(right_dir) not in dir_map:
                    dir_map[os.path.basename(right_dir)] = glob.glob(os.path.join(right_dir, "*"))
                if len(dir_map[os.path.basename(left_dir)]) < 1:
                    print("empty", left_dir)
                    continue
                if len(dir_map[os.path.basename(right_dir)]) < 1:
                    print("empty", right_dir)
                    continue
                left_item_idx = random.randint(0, len(dir_map[os.path.basename(left_dir)]) - 1)
                right_item_idx = random.randint(0, len(dir_map[os.path.basename(right_dir)]) - 1)
                if "{}_{}_{}".format(target_label_idx, left_item_idx, right_item_idx) in idx_map:
                    continue
                idx_map["{}_{}_{}".format(target_label_idx, left_item_idx, right_item_idx)] = True
                negative_files.append(dir_map[os.path.basename(left_dir)][left_item_idx])
                negative_files.append(dir_map[os.path.basename(right_dir)][right_item_idx])
                print("================")
                print(dir_map[os.path.basename(left_dir)][left_item_idx])
                print(dir_map[os.path.basename(right_dir)][right_item_idx])
                total_negative_cnt += 1

            if val_smapling_type == "left_right_eye":
                issame += [False] * int(len(negative_files) / 2)
                print("val negative cnt", len(negative_files))

        if val_smapling_type == "left_right_eye_half_same_dog":
            idx_map = {}
            total_negative_cnt = 0
            while max_positive_cnt > total_negative_cnt:
                target_label_idx = random.randint(0, len(label_files_list) - 1)
                if len(label_files_list[target_label_idx]) < 1:
                    continue
                target_item_idx = random.randint(0, len(label_files_list[target_label_idx]) - 1)
                neg_label_idx = random.randint(0, len(label_files_list) - 1)
                while target_label_idx == neg_label_idx:
                    neg_label_idx = random.randint(0, len(label_files_list) - 1)

                if len(label_files_list[neg_label_idx]) < 1:
                    continue
                neg_item_idx = random.randint(0, len(label_files_list[neg_label_idx]) - 1)

                if "{}_{}_{}_{}".format(target_label_idx, target_item_idx, neg_label_idx, neg_item_idx) in idx_map:
                    continue
                idx_map["{}_{}_{}_{}".format(target_label_idx, target_item_idx, neg_label_idx, neg_item_idx)] = True
                negative_files.append(label_files_list[target_label_idx][target_item_idx])
                negative_files.append(label_files_list[neg_label_idx][neg_item_idx])
                print("================")
                print(label_files_list[target_label_idx][target_item_idx])
                print(label_files_list[neg_label_idx][neg_item_idx])
                total_negative_cnt += 1
            issame += [False] * int(len(negative_files) / 2)
            print("val negative cnt", len(negative_files))
    else:
        if use_pos:
            each_cnt = max_positive_cnt / len(label_dirs)
            for label_idx, label_files in enumerate(label_files_list):
                cur_cnt = 0
                try:
                    for i in range(0, len(label_files) - 1):
                        for j in range(i + 1, len(label_files)):
                            positive_files.append(label_files[i])
                            positive_files.append(label_files[j])
                            cur_cnt += 1
                            if cur_cnt >= each_cnt:
                                raise
                except:
                    print("val positive label cnt", cur_cnt, os.path.basename(label_dirs[label_idx]))
                    pass
            max_positive_cnt = len(positive_files) // 2
            issame += [True] * int(len(positive_files) / 2)
            print("val positive cnt", len(positive_files))
        negative_files = []
        if use_neg:
            total_negative_cnt = 0
            idx_map = {}
            while max_positive_cnt > total_negative_cnt:
                target_label_idx = random.randint(0, len(label_files_list) - 1)
                if len(label_files_list[target_label_idx]) < 1:
                    continue
                target_item_idx = random.randint(0, len(label_files_list[target_label_idx]) - 1)
                neg_label_idx = random.randint(0, len(label_files_list) - 1)
                while target_label_idx == neg_label_idx:
                    neg_label_idx = random.randint(0, len(label_files_list) - 1)

                if len(label_files_list[neg_label_idx]) < 1:
                    continue
                neg_item_idx = random.randint(0, len(label_files_list[neg_label_idx]) - 1)

                if "{}_{}_{}_{}".format(target_label_idx, target_item_idx, neg_label_idx, neg_item_idx) in idx_map:
                    continue
                idx_map["{}_{}_{}_{}".format(target_label_idx, target_item_idx, neg_label_idx, neg_item_idx)] = True
                negative_files.append(label_files_list[target_label_idx][target_item_idx])
                negative_files.append(label_files_list[neg_label_idx][neg_item_idx])
                total_negative_cnt += 1
            issame += [False] * int(len(negative_files) / 2)
            print("val negative cnt", len(negative_files))

    loader = DataLoader(ValDataset(positive_files + negative_files, val_transforms), batch_size=batch_size,
                        shuffle=False, pin_memory=pin_memory,
                        num_workers=num_workers)
    return loader, issame

def get_val_data(data_path):
    agedb_30, agedb_30_issame = get_val_pair(data_path, 'agedb_30')
    cfp_fp, cfp_fp_issame = get_val_pair(data_path, 'cfp_fp')
    lfw, lfw_issame = get_val_pair(data_path, 'lfw')
    return agedb_30, cfp_fp, lfw, agedb_30_issame, cfp_fp_issame, lfw_issame

def load_mx_rec(rec_path):
    import mxnet as mx
    save_path = rec_path / 'imgs'
    if not save_path.exists():
        save_path.mkdir()
    imgrec = mx.recordio.MXIndexedRecordIO(str(rec_path / 'train.idx'), str(rec_path / 'train.rec'), 'r')
    img_info = imgrec.read_idx(0)
    header, _ = mx.recordio.unpack(img_info)
    max_idx = int(header.label[0])
    for idx in tqdm(range(1, max_idx)):
        img_info = imgrec.read_idx(idx)
        header, img = mx.recordio.unpack_img(img_info)
        label = int(header.label)
        img = Image.fromarray(img)
        label_path = save_path / str(label)
        if not label_path.exists():
            label_path.mkdir()
        img.save(label_path / '{}.jpg'.format(idx), quality=95)

# class train_dataset(Dataset):
#     def __init__(self, imgs_bcolz, label_bcolz, h_flip=True):
#         self.imgs = bcolz.carray(rootdir = imgs_bcolz)
#         self.labels = bcolz.carray(rootdir = label_bcolz)
#         self.h_flip = h_flip
#         self.length = len(self.imgs) - 1
#         if h_flip:
#             self.transform = trans.Compose([
#                 trans.ToPILImage(),
#                 trans.RandomHorizontalFlip(),
#                 trans.ToTensor(),
#                 trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
#             ])
#         self.class_num = self.labels[-1] + 1

#     def __len__(self):
#         return self.length

#     def __getitem__(self, index):
#         img = torch.tensor(self.imgs[index+1], dtype=torch.float)
#         label = torch.tensor(self.labels[index+1], dtype=torch.long)
#         if self.h_flip:
#             img = de_preprocess(img)
#             img = self.transform(img)
#         return img, label


# -*- coding: utf-8 -*-
import os
import sys

from setuptools import setup, find_packages

here = os.path.dirname(__file__)
version = __import__('faq').get_version()

def get_long_desc():
    return open(os.path.join(here, 'README.rst')).read()

if sys.argv[-1] == 'publish':
    os.system('python setup.py bdist_wheel upload -r natgeo')
    print("You probably want to also tag the version now:")
    print("  python setup.py tag")
    sys.exit()
elif sys.argv[-1] == 'tag':
    cmd = "git tag -a %s -m 'version %s';git push --tags" % (version, version)
    os.system(cmd)
    sys.exit()

setup(
    name='django-faq',
    version=version.replace(' ', '-'),
    description='Frequently Asked Question (FAQ) management for Django apps.',
    url='https://github.com/natgeosociety/django-faq/',
    author='Ben Spaulding',
    author_email='ben@benspaulding.us',
    license='BSD',
    long_description=get_long_desc(),
    packages=find_packages(exclude=['example*']),
    package_data={
        'faq': [
            'fixtures/*',
            'locale/*/LC_MESSAGES/*',
            'templates/faq/*',
            'templates/search/indexes/faq/*',
        ],
    },
    install_requires=['future'],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Internet :: WWW/HTTP :: Site Management',
    ],
)


#!/usr/bin/env python3
from arp.arp import ARProcess
from baselines.common.cmd_util import make_mujoco_env
from common.utils import mujoco_arg_parser
from baselines.common import tf_util as U
import time
import matplotlib.pyplot as plt
import numpy as np
from rl_experiments.normalized_env import NormalizedEnv

def train(env_id, num_timesteps, seed, p, alpha):
    from common import ar_mlp_policy
    from ar_ppo import ar_pposgd_simple
    U.make_session(num_cpu=1).__enter__()
    env = make_mujoco_env(env_id, seed)
    env = NormalizedEnv(env)
    ar = ARProcess(p, alpha, size=env.action_space.shape[-1])
    def policy_fn(name, ob_space, ac_space):
        return ar_mlp_policy.ARMlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space,
           hid_size=64, num_hid_layers=2, phi=ar.phi, sigma_z=ar.sigma_z)

    # Plot the learning curve
    plt.ion()
    fig = plt.figure(figsize=(10, 6))
    ax1 = fig.add_subplot(111)
    hl1, = ax1.plot([], [], markersize=10, color='r')
    ax1.set_xlabel("# time steps")
    ax1.set_ylabel("Average return")
    ax1.set_title("Learning progress")
    def mujoco_callback(locals, globals):
        if not 'seg' in locals:
            return
        episodic_returns = locals['episodic_returns']
        episodic_lengths = locals['episodic_lengths']
        ep_rets = locals['seg']['ep_rets']
        ep_lens = locals['seg']['ep_lens']
        if len(ep_rets):
            episodic_returns += ep_rets
            episodic_lengths += ep_lens
            window_size_steps = 5000
            x_tick = 1000
            if episodic_lengths:
                ep_lens = np.array(episodic_lengths)
                returns = np.array(episodic_returns)
            cum_episode_lengths = np.cumsum(ep_lens)
            if cum_episode_lengths[-1] >= x_tick:
                steps_show = np.arange(x_tick, cum_episode_lengths[-1] + 1, x_tick)
                rets = []

                for i in range(len(steps_show)):
                    rets_in_window = returns[(cum_episode_lengths > max(0, x_tick * (i + 1) - window_size_steps)) *
                                             (cum_episode_lengths < x_tick * (i + 1))]
                    if rets_in_window.any():
                        rets.append(np.mean(rets_in_window))
                if len(rets) > 0:
                    hl1.set_xdata(np.arange(1, len(rets) + 1) * x_tick)
                    ax1.set_xlim([x_tick, len(rets) * x_tick])
                    hl1.set_ydata(rets)
                    ax1.set_ylim([np.min(rets), np.max(rets) + 50])
            time.sleep(0.01)
            fig.canvas.draw()
            fig.canvas.flush_events()

    ar_pposgd_simple.learn(env, policy_fn,
                           callback=mujoco_callback,
                           max_timesteps=num_timesteps,
                           timesteps_per_actorbatch=2048,
                           clip_param=0.2, entcoeff=0.0,
                           optim_epochs=10, optim_stepsize=3e-4, optim_batchsize=64,
                           gamma=0.995, lam=0.995, schedule='linear', ar=ar
                           )
    env.close()

def main():
    args = mujoco_arg_parser().parse_args()
    train(args.env, num_timesteps=args.num_timesteps, seed=args.seed, p=args.p, alpha=args.alpha)

if __name__ == '__main__':
    main()


#########################################################################
################ IV - WORKING WITH A DIFFERENT DATASET ##################
#########################################################################

#first, we are going to need a new module so let's start by importing it
import matplotlib.pyplot as plt
from sklearn import datasets
import numpy as np

#let's load this differente dataset I was talking about
digits = datasets.load_digits()

#and let's get a good look at it to spot the differences
print(digits.DESCR)

#let's see how our dataset is organized so we can better explore it latter
print(digits.images.shape)
print(digits.data.shape)

#let's explore one of this matrixes so we can better understand the situation
plt.imshow(digits.images[23], cmap=plt.cm.gray_r, interpolation='nearest')
plt.show()

#########################################################################
################ V - UNDER AND OVER FITTING YOUR MODEL ##################
#########################################################################

#on my last chapter with you I want to talk about the risks of under e over fitting
#for this let's change our neighbors number between 1 and 9
neighbors =np.arange(1, 9)

#for this we'll plot a graph so let's store our data on
train_accuracy = np.empty(len(neighbors))
test_accuracy = np.empty(len(neighbors))

#as we did on Iris dataset let's create our X_train, X_test, y_train and y_test
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
X = digits.data
y = digits.target

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25,
                                                    random_state = 10,
                                                    stratify = y)

#to generate the values 9 times let's loop over the various values of k
for i, k in enumerate(neighbors):
    #let's create a classifier with k neighbors
    knn = KNeighborsClassifier(n_neighbors = k)

    #let's do the fitting
    knn.fit(X_train, y_train)

    #Compute the accuracy on the training set
    train_accuracy[i] = knn.score(X_train, y_train)

    #Compute the accuracy on the test set
    test_accuracy[i] = knn.score(X_test, y_test)

#to end your time with me, unfortunatelly, let's plot our final graph and take some conclusions
plt.title('k-NN: Varying Number of Neighbors')
plt.plot(neighbors, test_accuracy, label = 'Testing Accuracy')
plt.plot(neighbors, train_accuracy, label = 'Training Accuracy')
plt.legend()
plt.xlabel('Number of Neighbors')
plt.ylabel('Accuracy')
plt.show()


#!/usr/bin/python
import logging
from copy import deepcopy

from advent import Advent

"""
-- Day 15: Beverage Bandits ---
Having perfected their hot chocolate, the Elves have a new problem: the Goblins
that live in these caves will do anything to steal it. Looks like they're here
for a fight.

You scan the area, generating a map of the walls (#), open cavern (.), and
starting position of every Goblin (G) and Elf (E) (your puzzle input).

Combat proceeds in rounds; in each round, each unit that is still alive takes a
turn, resolving all of its actions before the next unit's turn begins. On each
unit's turn, it tries to move into range of an enemy (if it isn't already)
and then attack (if it is in range).

All units are very disciplined and always follow very strict combat rules. Units
never move or attack diagonally, as doing so would be dishonorable. When
multiple choices are equally valid, ties are broken in reading order:

top-to-bottom, then left-to-right. For instance, the order in which units take
their turns within a round is the reading order of their starting positions in
that round, regardless of the type of unit or whether other units have moved
after the round started. For example:

                   would take their
  These units:   turns in this order:
    #######           #######
    #.G.E.#           #.1.2.#
    #E.G.E#           #3.4.5#
    #.G.E.#           #.6.7.#
    #######           #######

Each unit begins its turn by identifying all possible targets (enemy units). If
no targets remain, combat ends.

Then, the unit identifies all of the open squares (.) that are in range of each
target; these are the squares which are adjacent (immediately up, down, left, or
right) to any target and which aren't already occupied by a wall or
another unit. Alternatively, the unit might already be in range of a target. If
the unit is not already in range of a target, and there are no open squares
which are in range of a target, the unit ends its turn.

If the unit is already in range of a target, it does not move, but continues its
turn with an attack. Otherwise, since it is not in range of a target, it moves.

To move, the unit first considers the squares that are in range and determines
which of those squares it could reach in the fewest steps. A step is a single
movement to any adjacent (immediately up, down, left, or right) open (.)
square. Units cannot move into walls or other units. The unit does this while
considering the current positions of units and does not do any prediction about
where units will be later. If the unit cannot reach (find an open path to)
any of the squares that are in range, it ends its turn. If multiple squares are
in range and tied for being reachable in the fewest steps, the square which is
first in reading order is chosen. For example:

  Targets:      In range:     Reachable:    Nearest:      Chosen:
  #######       #######       #######       #######       #######
  #E..G.#       #E.?G?#       #E.@G.#       #E.!G.#       #E.+G.#
  #...#.#  -->  #.?.#?#  -->  #.@.#.#  -->  #.!.#.#  -->  #...#.#
  #.G.#G#       #?G?#G#       #@G@#G#       #!G.#G#       #.G.#G#
  #######       #######       #######       #######       #######

In the above scenario, the Elf has three targets (the three Goblins):

  - Each of the Goblins has open, adjacent squares which are in range (marked
  with a ? on the map).
  - Of those squares, four are reachable (marked @); the other two (on the
  right) would require moving through a wall or unit to reach.
  - Three of these reachable squares are nearest, requiring the fewest steps
  (only 2) to reach (marked !).
  - Of those, the square which is first in reading order is chosen (+).

The unit then takes a single step toward the chosen square along the shortest
path to that square. If multiple steps would put the unit equally closer to its
destination, the unit chooses the step which is first in reading order. (This
requires knowing when there is more than one shortest path so that you can
consider the first step of each such path.) For example:

  In range:     Nearest:      Chosen:       Distance:     Step:
  #######       #######       #######       #######       #######
  #.E...#       #.E...#       #.E...#       #4E212#       #..E..#
  #...?.#  -->  #...!.#  -->  #...+.#  -->  #32101#  -->  #.....#
  #..?G?#       #..!G.#       #...G.#       #432G2#       #...G.#
  #######       #######       #######       #######       #######

The Elf sees three squares in range of a target (?), two of which are nearest
(!), and so the first in reading order is chosen (+). Under "Distance", each
open square is marked with its distance from the destination square; the two
squares to which the Elf could move on this turn (down and to the right) are
both equally good moves and would leave the Elf 2 steps from being in range of
the Goblin. Because the step which is first in reading order is chosen, the Elf
moves right one square.

Here's a larger example of movement:

  Initially:
  #########
  #G..G..G#
  #.......#
  #.......#
  #G..E..G#
  #.......#
  #.......#
  #G..G..G#
  #########

  After 1 round:
  #########
  #.G...G.#
  #...G...#
  #...E..G#
  #.G.....#
  #.......#
  #G..G..G#
  #.......#
  #########

  After 2 rounds:
  #########
  #..G.G..#
  #...G...#
  #.G.E.G.#
  #.......#
  #G..G..G#
  #.......#
  #.......#
  #########

  After 3 rounds:
  #########
  #.......#
  #..GGG..#
  #..GEG..#
  #G..G...#
  #......G#
  #.......#
  #.......#
  #########

Once the Goblins and Elf reach the positions above, they all are either in range
of a target or cannot find any square in range of a target, and so none of the
units can move until a unit dies.

After moving (or if the unit began its turn in range of a target), the unit
attacks.

To attack, the unit first determines all of the targets that are in range of it
by being immediately adjacent to it.  If there are no such targets, the unit
ends its turn. Otherwise, the adjacent target with the fewest hit points is
selected; in a tie, the adjacent target with the fewest hit points which is
first in reading order is selected.

The unit deals damage equal to its attack power to the selected target, reducing
its hit points by that amount. If this reduces its hit points to 0 or fewer, the
selected target dies: its square becomes . and it takes no further turns.

Each unit, either Goblin or Elf, has 3 attack power and starts with 200 hit
points.

For example, suppose the only Elf is about to attack:

         HP:            HP:
  G....  9       G....  9
  ..G..  4       ..G..  4
  ..EG.  2  -->  ..E..
  ..G..  2       ..G..  2
  ...G.  1       ...G.  1

The "HP" column shows the hit points of the Goblin to the left in the
corresponding row. The Elf is in range of three targets: the Goblin above it
(with 4 hit points), the Goblin to its right (with 2 hit points), and the Goblin
below it (also with 2 hit points). Because three targets are in range, the ones
with the lowest hit points are selected: the two Goblins with 2 hit points each
(one to the right of the Elf and one below the Elf). Of those, the Goblin first
in reading order (the one to the right of the Elf) is selected. The selected
Goblin's hit points (2) are reduced by the Elf's attack power (3), reducing its
hit points to -1, killing it.

After attacking, the unit's turn ends. Regardless of how the unit's turn ends,
the next unit in the round takes its turn. If all units have taken turns in this
round, the round ends, and a new round begins.

The Elves look quite outnumbered. You need to determine the outcome of the
battle: the number of full rounds that were completed (not counting the round in
which combat ends) multiplied by the sum of the hit points of all remaining
units at the moment combat ends. (Combat only ends when a unit finds no targets
during its turn.)

Below is an entire sample combat. Next to each map, each row's units' hit points
are listed from left to right.

  Initially:
  #######
  #.G...#   G(200)
  #...EG#   E(200), G(200)
  #.#.#G#   G(200)
  #..G#E#   G(200), E(200)
  #.....#
  #######

  After 1 round:
  #######
  #..G..#   G(200)
  #...EG#   E(197), G(197)
  #.#G#G#   G(200), G(197)
  #...#E#   E(197)
  #.....#
  #######

  After 2 rounds:
  #######
  #...G.#   G(200)
  #..GEG#   G(200), E(188), G(194)
  #.#.#G#   G(194)
  #...#E#   E(194)
  #.....#
  #######

  Combat ensues; eventually, the top Elf dies:

  After 23 rounds:
  #######
  #...G.#   G(200)
  #..G.G#   G(200), G(131)
  #.#.#G#   G(131)
  #...#E#   E(131)
  #.....#
  #######

  After 24 rounds:
  #######
  #..G..#   G(200)
  #...G.#   G(131)
  #.#G#G#   G(200), G(128)
  #...#E#   E(128)
  #.....#
  #######

  After 25 rounds:
  #######
  #.G...#   G(200)
  #..G..#   G(131)
  #.#.#G#   G(125)
  #..G#E#   G(200), E(125)
  #.....#
  #######

  After 26 rounds:
  #######
  #G....#   G(200)
  #.G...#   G(131)
  #.#.#G#   G(122)
  #...#E#   E(122)
  #..G..#   G(200)
  #######

  After 27 rounds:
  #######
  #G....#   G(200)
  #.G...#   G(131)
  #.#.#G#   G(119)
  #...#E#   E(119)
  #...G.#   G(200)
  #######

  After 28 rounds:
  #######
  #G....#   G(200)
  #.G...#   G(131)
  #.#.#G#   G(116)
  #...#E#   E(113)
  #....G#   G(200)
  #######

  More combat ensues; eventually, the bottom Elf dies:

  After 47 rounds:
  #######
  #G....#   G(200)
  #.G...#   G(131)
  #.#.#G#   G(59)
  #...#.#
  #....G#   G(200)
  #######

Before the 48th round can finish, the top-left Goblin finds that there are no
targets remaining, and so combat ends.  So, the number of full rounds that were
completed is 47, and the sum of the hit points of all remaining units is
200+131+59+200 = 590. From these, the outcome of the battle is 47 * 590 = 27730.

  Here are a few example summarized combats:

  #######       #######
  #G..#E#       #...#E#   E(200)
  #E#E.E#       #E#...#   E(197)
  #G.##.#  -->  #.E##.#   E(185)
  #...#E#       #E..#E#   E(200), E(200)
  #...E.#       #.....#
  #######       #######

  Combat ends after 37 full rounds
  Elves win with 982 total hit points left
  Outcome: 37 * 982 = 36334
  #######       #######
  #E..EG#       #.E.E.#   E(164), E(197)
  #.#G.E#       #.#E..#   E(200)
  #E.##E#  -->  #E.##.#   E(98)
  #G..#.#       #.E.#.#   E(200)
  #..E#.#       #...#.#
  #######       #######

  Combat ends after 46 full rounds
  Elves win with 859 total hit points left
  Outcome: 46 * 859 = 39514
  #######       #######
  #E.G#.#       #G.G#.#   G(200), G(98)
  #.#G..#       #.#G..#   G(200)
  #G.#.G#  -->  #..#..#
  #G..#.#       #...#G#   G(95)
  #...E.#       #...G.#   G(200)
  #######       #######

  Combat ends after 35 full rounds
  Goblins win with 793 total hit points left
  Outcome: 35 * 793 = 27755
  #######       #######
  #.E...#       #.....#
  #.#..G#       #.#G..#   G(200)
  #.###.#  -->  #.###.#
  #E#G#G#       #.#.#.#
  #...#G#       #G.G#G#   G(98), G(38), G(200)
  #######       #######

  Combat ends after 54 full rounds
  Goblins win with 536 total hit points left
  Outcome: 54 * 536 = 28944
  #########       #########
  #G......#       #.G.....#   G(137)
  #.E.#...#       #G.G#...#   G(200), G(200)
  #..##..G#       #.G##...#   G(200)
  #...##..#  -->  #...##..#
  #...#...#       #.G.#...#   G(200)
  #.G...G.#       #.......#
  #.....G.#       #.......#
  #########       #########

Combat ends after 20 full rounds
Goblins win with 937 total hit points left
Outcome: 20 * 937 = 18740
What is the outcome of the combat described in your puzzle input?

--- Part Two ---
According to your calculations, the Elves are going to lose badly. Surely, you
won't mess up the timeline too much if you give them just a little advanced
technology, right?

You need to make sure the Elves not only win, but also suffer no losses: even
the death of a single Elf is unacceptable.

However, you can't go too far: larger changes will be more likely to permanently alter spacetime.

So, you need to find the outcome of the battle in which the Elves have the
lowest integer attack power (at least 4) that allows them to win without a
single death. The Goblins always have an attack power of 3.

In the first summarized example above, the lowest attack power the Elves need to
win without losses is 15:

#######       #######
#.G...#       #..E..#   E(158)
#...EG#       #...E.#   E(14)
#.#.#G#  -->  #.#.#.#
#..G#E#       #...#.#
#.....#       #.....#
#######       #######

Combat ends after 29 full rounds
Elves win with 172 total hit points left
Outcome: 29 * 172 = 4988
In the second example above, the Elves need only 4 attack power:

#######       #######
#E..EG#       #.E.E.#   E(200), E(23)
#.#G.E#       #.#E..#   E(200)
#E.##E#  -->  #E.##E#   E(125), E(200)
#G..#.#       #.E.#.#   E(200)
#..E#.#       #...#.#
#######       #######

Combat ends after 33 full rounds
Elves win with 948 total hit points left
Outcome: 33 * 948 = 31284
In the third example above, the Elves need 15 attack power:

#######       #######
#E.G#.#       #.E.#.#   E(8)
#.#G..#       #.#E..#   E(86)
#G.#.G#  -->  #..#..#
#G..#.#       #...#.#
#...E.#       #.....#
#######       #######

Combat ends after 37 full rounds
Elves win with 94 total hit points left
Outcome: 37 * 94 = 3478
In the fourth example above, the Elves need 12 attack power:

#######       #######
#.E...#       #...E.#   E(14)
#.#..G#       #.#..E#   E(152)
#.###.#  -->  #.###.#
#E#G#G#       #.#.#.#
#...#G#       #...#.#
#######       #######

Combat ends after 39 full rounds
Elves win with 166 total hit points left
Outcome: 39 * 166 = 6474
In the last example above, the lone Elf needs 34 attack power:

#########       #########
#G......#       #.......#
#.E.#...#       #.E.#...#   E(38)
#..##..G#       #..##...#
#...##..#  -->  #...##..#
#...#...#       #...#...#
#.G...G.#       #.......#
#.....G.#       #.......#
#########       #########

Combat ends after 30 full rounds
Elves win with 38 total hit points left
Outcome: 30 * 38 = 1140
After increasing the Elves' attack power until it is just barely enough for them
to win without any Elves dying, what is the outcome of the combat described in
your puzzle input?

"""

logging.basicConfig(level=logging.INFO)
log = logging.getLogger()

class NoMoreEnemies(Exception):
    pass

class DeadElve(Exception):
    pass

class Position(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __str__(self):
        return '<Pos %s,%s>' % (self.x, self.y)

    def __hash__(self):
        return hash(self.pos)

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y

    @property
    def pos(self):
        return (self.x, self.y)

    def neighbours(self):
        yield Position(self.x, self.y - 1)
        yield Position(self.x - 1, self.y)
        yield Position(self.x + 1, self.y)
        yield Position(self.x, self.y + 1)

    def neighbours_on_grid(self, grid):
        """all neighbour positions that are on the grid and empty"""
        for n in self.neighbours():
            if n not in grid:
                continue  # not part of the grid (wall or outside)
            if grid[n] is None:
                yield n

    def distance(self, other):
        """manhatten distance"""
        return abs(self.x - other.x) + abs(self.y - other.y)

class Unit(Position):
    def __init__(self, x, y, race):
        super(Unit, self).__init__(x, y)
        self.hp = 200
        self.power = 3
        self.race = race

    def __str__(self):
        return '<Unit %s at %s,%s>' % (self.race, self.x, self.y)

    def free(self):
        if self.race == '.':
            return True
        if self.hp <= 0:
            return True
        return False

    def is_enemy(self, other):
        test = (self.race, other.race)
        return test in (('E', 'G'), ('G', 'E'))

    def attack(self, grid):
        enemy = None
        for n in self.neighbours():
            # we rely on neighbours to yield in preferred attack order
            # N, W, E, S
            if n not in grid:
                continue
            if grid[n] is None:
                continue
            if grid[n].race == self.race:
                continue
            if enemy is None:
                enemy = grid[n]
            else:
                if grid[n].hp < enemy.hp:
                    enemy = grid[n]
        if enemy is None:
            log.debug('%s has no target in range', self)
            return
        log.debug('%s attacks %s', self, enemy)
        enemy.hp -= self.power
        if enemy.hp <= 0:
            log.debug('%s has been killed', enemy)
            grid[enemy] = None

    def search_enemy(self, grid):
        """search the closest reachable enemy"""
        log.debug('%s is searching for enemies', self)
        targets = []
        units = filter(None, grid.values())
        units = filter(lambda u: u.race != self.race, units)
        units = list(units)  # otherwise we have filter expression
        if not units:
            raise NoMoreEnemies()
        for unit in units:
            for pos in unit.neighbours():
                try:
                    node = grid[pos]
                except KeyError:
                    # a wall or outside
                    continue
                if node is None:
                    targets.append(pos)
                    continue
                if node.pos == self.pos:
                    targets.append(pos)
                    continue
        # remove duplicates
        targets = set(targets)
        distance = 0
        seen = set()  # nodes visited
        options = [self]
        while options:
            # check if any option is good
            for p in sorted(options, key=lambda n: (n.y, n.x)):
                seen.add(p)
                if p in targets:
                    log.debug('found enemy %s with distance %s', p, distance)
                    return p, distance
            # increment distance, add all new options
            distance += 1
            work = options[:]
            options = []
            for p in work:
                for n in p.neighbours_on_grid(grid):
                    if n in seen:
                        continue
                    if n in options:
                        continue
                    options.append(n)
        return None, None  # no enemy found

    def step(self, enemy, grid):
        """walk back from enemy to one of our neighbour fields
        our neighbour fields will be sorted in preferred order
        """
        seen = set()
        targets = list(self.neighbours_on_grid(grid))
        options = [enemy]
        while options:
            for p in sorted(options, key=lambda n: (n.y, n.x)):
                seen.add(p)
                if p in targets:
                    log.debug('%s moving to %s', self, p)
                    grid[self] = None  # leave field
                    self.x = p.x
                    self.y = p.y
                    grid[self] = self
                    return
            work = options[:]
            for p in work:
                for n in p.neighbours_on_grid(grid):
                    if n in seen:
                        continue
                    if n in options:
                        continue
                    options.append(n)

def print_grid(grid):
    print()
    max_x = max([p.x for p in grid]) + 1
    max_y = max([p.y for p in grid]) + 1
    for y in range(max_y):
        for x in range(max_x):
            try:
                node = grid[Position(x, y)]
            except KeyError:
                print('#', end='')  # noqa
                continue
            if node is None:
                print('.', end='')  # noqa
            else:
                print(node.race, end='')  # noqa
        print()

class Day(Advent):
    lines = True

    def prepare(self):
        super(Day, self).prepare()
        grid = {}
        for y, line in enumerate(self.data):
            for x, value in enumerate(line):
                if value == '#':  # do not note walls
                    continue
                p = Position(x, y)
                if value == '.':
                    grid[p] = None
                else:
                    grid[p] = Unit(x, y, value)
        self.grid = grid

    def units(self, grid):
        units = filter(None, grid.values())
        units = sorted(units, key=lambda n: (n.y, n.x))
        return units

    def battle(self, grid, raise_elve=False):
        """execute one round of the battle"""
        units = self.units(grid)
        for unit in units:
            if unit.hp <= 0:
                continue  # dead, killed this round
            enemy, distance = unit.search_enemy(grid)
            if enemy is None:
                continue
            if distance > 0:
                unit.step(enemy, grid)
            unit.attack(grid)
        if raise_elve:
            for unit in units:
                if unit.hp <= 0 and unit.race == 'E':
                    raise DeadElve()

    def hastings(self, grid, raise_elve=False):
        """execute a complete battle
        (or wait for an elve to die)"""
        result = 0
        while True:
            try:
                self.battle(grid, raise_elve=raise_elve)
            except NoMoreEnemies:
                return result
            result += 1

    def solve1(self):
        grid = deepcopy(self.grid)
        battleround = self.hastings(grid)
        hp = sum([u.hp for u in self.units(grid)])
        return hp * battleround

    def solve2(self):
        power = 4
        while True:
            grid = deepcopy(self.grid)
            # upgrade elves
            for unit in self.units(grid):
                if unit.race == 'E':
                    unit.power = power
            # fight
            try:
                battleround = self.hastings(grid, raise_elve=True)
            except DeadElve:
                power += 1
                continue
            break
        hp = sum([u.hp for u in self.units(grid)])
        return hp * battleround

Day.main()