"""
Copyright (c) Facebook, Inc. and its affiliates.

This source code is licensed under the MIT license found in the
LICENSE file in the root directory of this source tree.
"""

import pytest
import torch
from torch import nn

import neuralcompression.functional as ncF
from neuralcompression.models import ScaleHyperprior

@pytest.mark.parametrize(
    "batch_size, dim_in, dim_out",
    [(1, 1, 1), (2, 1, 10), (3, 10, 10), (4, 10, 1), (5, 6, 7)],
)
def test_flop_count_ffn(batch_size, dim_in, dim_out):
    # Testing the flop counter counts simple modules as expected.

    ffn_model = nn.Sequential(
        nn.Linear(dim_in, dim_out, bias=False), nn.Linear(dim_out, dim_out, bias=False)
    )
    ffn_inputs = (torch.randn(batch_size, dim_in),)

    total_flops, _, unsupported_ops = ncF.count_flops(ffn_model, ffn_inputs)
    assert total_flops == batch_size * dim_in * dim_out + batch_size * dim_out * dim_out
    assert len(unsupported_ops) == 0

@pytest.mark.parametrize(
    "input_shape, layer_channels, layer_strides, layer_kernels, layer_padding",
    (
        ((1, 3, 64, 64), (10, 6, 11), (1, 2, 1), (1, 2, 3), (0, 1, 5)),
        ((3, 10, 11, 127), (2, 4, 8, 1), (2, 2, 2, 2), (4, 7, 1), (0, 2, 3, 0)),
    ),
)
def test_flop_count_conv(
    input_shape, layer_channels, layer_strides, layer_kernels, layer_padding
):
    # Testing that the flop counter counts a convolutional model as expected
    # (i.e. when compared to a reference value).

    class ConvModel(nn.Module):
        def __init__(self):
            super().__init__()
            self.layers = nn.ModuleList()
            cin = input_shape[1]

            for cout, stride, kernel, padding in zip(
                layer_channels, layer_strides, layer_kernels, layer_padding
            ):
                self.layers.append(
                    nn.Conv2d(
                        cin, cout, kernel_size=kernel, stride=stride, padding=padding
                    )
                )
                cin = cout

        def forward(self, inp):
            flops = torch.tensor(0)
            out = inp
            for layer, k_size in zip(self.layers, layer_kernels):
                cin = out.shape[1]
                out = layer(out)
                # +3 for counting the additions to the flops variable itself
                flops += out.numel() * k_size * k_size * cin + 3

            return out, flops

    model = ConvModel()
    inp = torch.randn(input_shape)
    _, correct_flops = model(inp)

    counted_flops, _, unsupported_ops = ncF.count_flops(model, (inp,))
    assert torch.allclose(torch.tensor(counted_flops).long(), correct_flops)
    assert len(unsupported_ops) == 0

@pytest.mark.parametrize("input_shape", ((10, 3), (1, 1), (5, 9), (10, 9, 8)))
def test_flop_count_elementwise(input_shape):
    # Tests that the flop counter counts flops correctly (i.e.
    # when compared to a reference value) for a model involving
    # lots of binary and elementwise operations.

    class ElementwiseModel(nn.Module):
        def forward(self, inp):
            flops = torch.tensor(0)
            out = inp

            out = out * 2
            # +1 here and below for counting the additions to the flop
            # variable itself.
            flops += out.numel() + 1

            out = out.pow(2).exp().log()
            flops += out.numel() * 3 + 1

            out = out.view(-1, 1) @ out.view(1, -1)
            flops += out.numel() + 1

            out = out + out
            flops += out.numel() + 1

            return out, flops

    model = ElementwiseModel()
    inp = torch.randn(input_shape)
    _, correct_flops = model(inp)

    counted_flops, _, unsupported_ops = ncF.count_flops(
        model, (inp,), use_single_flop_estimates=True
    )
    assert torch.allclose(torch.tensor(counted_flops).long(), correct_flops)
    assert len(unsupported_ops) == 0

@pytest.mark.parametrize(
    "batch_size, network_channels, compression_channels, img_size", [(1, 8, 16, 128)]
)
def test_flop_count_hyperprior(
    batch_size, network_channels, compression_channels, img_size
):
    # Testing that the flop counter doesn't crash and records all operators
    # on a more complicated model, i.e. the scale hyperprior model.
    model = ScaleHyperprior(
        network_channels=network_channels, compression_channels=compression_channels
    )
    inputs = (torch.randn(batch_size, 3, img_size, img_size),)
    _, _, unsupported_ops = ncF.count_flops(
        model, inputs, use_single_flop_estimates=True
    )
    assert len(unsupported_ops) == 0

def test_flop_counter_overrides():
    # Testing that one can override the counter's default implementation for
    # an operation with a custom counter function.

    class Addition(nn.Module):
        def forward(self, x, y):
            return x + y

    add_override_value = 1234567.0

    count, _, _ = ncF.count_flops(
        Addition(),
        (torch.tensor(5.0), torch.tensor(6.0)),
        counter_overrides={"aten::add": lambda inputs, outputs: add_override_value},
    )

    assert torch.allclose(
        torch.tensor(count).float(), torch.tensor(add_override_value).float()
    )


from typing import Callable, Tuple, Dict, Union

import ptan
import torch
from torch import Tensor
from torch.nn import Module, functional as F

from autograph.lib.util import one_hot_multi
from autograph.lib.util.trace_info_processor import TraceInfoPreprocessor

LossFunction = Callable[[TraceInfoPreprocessor], Tuple[Tensor, Dict[str, Union[float, Tensor]]]]

class PPOLossFunction:
    def __init__(self, net: Module, device, discount: float, epsilon: float = .2, sync_old_net_every: int = 10,
                 **kwargs):
        self.discount = discount
        self.device = device
        self.epsilon = epsilon
        self.net = net
        self.old_net = ptan.agent.TargetNet(net)
        self.sync_every = sync_old_net_every
        self.sync_counter = 0

    def __call__(self, trace_info: TraceInfoPreprocessor):
        self.sync_counter += 1
        if self.sync_counter % self.sync_every == 0:
            self.sync_counter = 0
            self.old_net.sync()

        states = trace_info("state")
        actions = trace_info("action_selected")
        rewards = trace_info("reward")
        values = trace_info("value")
        ns = trace_info("next_state")
        orig_action_probs = trace_info("action_probs")
        total_return = trace_info("discounted_return")
        # edit
        rewards = total_return

        net_policy, net_values = self.net(states)
        net_policy_probs = F.softmax(net_policy, dim=-1)

        _, next_net_values = self.net(ns)
        advantages = rewards + (self.discount * next_net_values) - net_values

        old_net_policy, _ = self.old_net.target_model(states)
        old_net_policy_probs = F.softmax(old_net_policy, dim=-1).detach()

        action_space = net_policy_probs.size(-1)
        actions_one_hot = one_hot_multi(actions, action_space, self.device)

        old_net_policy_probs_selected = torch.masked_select(old_net_policy_probs, actions_one_hot)
        net_policy_probs_selected = torch.masked_select(net_policy_probs, actions_one_hot)

        # Add the tiny amount in case probability dropped near zero
        ratio = (net_policy_probs_selected + 1e-6) / (old_net_policy_probs_selected + 1e-6)
        surr_obj = advantages * ratio
        clipped_surr_obj = advantages * torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon)
        policy_loss = -torch.min(surr_obj, clipped_surr_obj).mean()

        value_loss = F.mse_loss((values[0] + advantages[0]).reshape((100, 1)), net_values)

        agent_loss = value_loss + policy_loss

        return agent_loss, {
            "loss/value_loss": value_loss,
            "loss/policy_loss": policy_loss,
            "loss/agent_loss": agent_loss
        }

class AdvantageActorCriticLossFunction:
    def __init__(self, net: Module, discount: float, device, **kwargs):
        self.net = net
        self.discount = discount
        self.device = device

    def __call__(self, trace_info: TraceInfoPreprocessor):
        states = trace_info("state")
        actions = trace_info("action_selected")
        values = trace_info("value")
        ns = trace_info("next_state")
        rewards = trace_info("reward")
        orig_action_probs = trace_info("action_probs")
        total_return = trace_info("discounted_return")

        net_policy, net_values = self.net(states)
        # _, next_net_values = self.net(ns)

        advantages = total_return - net_values

        net_policy_logprobs = -F.log_softmax(net_policy, dim=-1)
        # softmax: 0 to 1
        # log: -inf to 0
        # net_policy_logprobs: 0 to inf (reversed on policy)

        value_loss = F.mse_loss(total_return, net_values)
        action_space = net_policy.size(-1)
        actions_one_hot = one_hot_multi(actions, action_space, self.device)

        net_policy_logprobs_selected = torch.masked_select(net_policy_logprobs, actions_one_hot)

        importance = torch.masked_select(F.softmax(net_policy, dim=-1) / F.softmax(orig_action_probs, dim=-1),
                                         actions_one_hot)

        policy_loss = (advantages.detach() * importance.detach() * net_policy_logprobs_selected).mean()
        # Positive advantage, likely action: low positive
        # Positive advantage, rare action: high positive
        # Negative advantage, likely action: low negative
        # Negative advantage, rare action: high negative
        # So we want to decrease this amount

        agent_loss = value_loss + policy_loss

        # TODO anchor value to 0 if done is true

        return agent_loss, {
            "loss/value_loss": value_loss,
            "loss/policy_loss": policy_loss,
            "loss/agent_loss": agent_loss
        }

class TakeSimilarActionsLossFunction:
    def __init__(self, net: Module, device, no_adv=False, **kwargs):
        self.device = device
        self.net = net
        self.no_adv = no_adv

    def __call__(self, trace_info):
        disc_return = trace_info("discounted_return")
        values = trace_info("value")

        states = trace_info("state")
        actions = trace_info("action_selected")

        net_policy, net_values = self.net(states)

        if self.no_adv:
            value_loss = F.mse_loss(values, net_values)
        else:
            value_loss = F.mse_loss(disc_return, net_values.squeeze(-1))

        policy_loss = F.cross_entropy(net_policy, actions.long())

        agent_loss = value_loss + policy_loss

        # -1 * probability * log(prob)

        entropy = (-F.log_softmax(net_policy, dim=-1) * F.softmax(net_policy, dim=-1)).sum(dim=-1).mean()

        # entropy_loss = (entropy - log(6))^2
        # loss + (const * entropy_loss)

        return agent_loss, {
            "loss/value_loss": value_loss,
            "loss/policy_loss": policy_loss,
            "loss/agent_loss": agent_loss,
            "loss/entropy": entropy
        }


# -*- coding: utf-8 -*- #
# Copyright 2017 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command to update labels for addresses."""

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from googlecloudsdk.api_lib.compute import base_classes
from googlecloudsdk.api_lib.compute.operations import poller
from googlecloudsdk.api_lib.util import waiter
from googlecloudsdk.calliope import base
from googlecloudsdk.calliope import exceptions as calliope_exceptions
from googlecloudsdk.command_lib.compute import flags as compute_flags
from googlecloudsdk.command_lib.compute.addresses import flags
from googlecloudsdk.command_lib.util.args import labels_util

@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
class Update(base.UpdateCommand):
  r"""Update a Google Compute Engine address.

  *{command}* updates labels for a Google Compute Engine
  address.

  ## EXAMPLES

  To add/update labels 'k0' and 'k1' and remove labels with key 'k3' for address
  'example-address', run:

    $ {command} example-address --region=us-central1 \
      --update-labels=k0=value1,k1=value2 --remove-labels=k3

  Labels can be used to identify the address and to filter them as in:

    $ {parent_command} list --filter='labels.k1:value2'

  To list existing labels for address 'example-address', run:

    $ {parent_command} describe example-address --format='default(labels)'

  """

  ADDRESS_ARG = None

  @classmethod
  def Args(cls, parser):
    cls.ADDRESS_ARG = flags.AddressArgument(plural=False)
    cls.ADDRESS_ARG.AddArgument(parser)
    labels_util.AddUpdateLabelsFlags(parser)

  def Run(self, args):
    holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
    client = holder.client.apitools_client
    messages = holder.client.messages

    address_ref = self.ADDRESS_ARG.ResolveAsResource(
        args,
        holder.resources,
        scope_lister=compute_flags.GetDefaultScopeLister(holder.client))

    labels_diff = labels_util.Diff.FromUpdateArgs(args)
    if not labels_diff.MayHaveUpdates():
      raise calliope_exceptions.RequiredArgumentException(
          'LABELS', 'At least one of --update-labels or '
          '--remove-labels must be specified.')

    if address_ref.Collection() == 'compute.globalAddresses':
      address = client.globalAddresses.Get(
          messages.ComputeGlobalAddressesGetRequest(**address_ref.AsDict()))
      labels_value = messages.GlobalSetLabelsRequest.LabelsValue
    else:
      address = client.addresses.Get(
          messages.ComputeAddressesGetRequest(**address_ref.AsDict()))
      labels_value = messages.RegionSetLabelsRequest.LabelsValue

    labels_update = labels_diff.Apply(labels_value, address.labels)

    if not labels_update.needs_update:
      return address

    if address_ref.Collection() == 'compute.globalAddresses':
      request = messages.ComputeGlobalAddressesSetLabelsRequest(
          project=address_ref.project,
          resource=address_ref.Name(),
          globalSetLabelsRequest=messages.GlobalSetLabelsRequest(
              labelFingerprint=address.labelFingerprint,
              labels=labels_update.labels))

      operation = client.globalAddresses.SetLabels(request)
      operation_ref = holder.resources.Parse(
          operation.selfLink, collection='compute.globalOperations')

      operation_poller = poller.Poller(client.globalAddresses)
    else:
      request = messages.ComputeAddressesSetLabelsRequest(
          project=address_ref.project,
          resource=address_ref.Name(),
          region=address_ref.region,
          regionSetLabelsRequest=messages.RegionSetLabelsRequest(
              labelFingerprint=address.labelFingerprint,
              labels=labels_update.labels))

      operation = client.addresses.SetLabels(request)
      operation_ref = holder.resources.Parse(
          operation.selfLink, collection='compute.regionOperations')

      operation_poller = poller.Poller(client.addresses)

    return waiter.WaitFor(
        operation_poller, operation_ref,
        'Updating labels of address [{0}]'.format(address_ref.Name()))


import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, roc_auc_score, matthews_corrcoef
import numpy as np

cvCount = 10
iterations = 4

clf = RandomForestClassifier(random_state=42, max_depth=31, max_features='sqrt', min_samples_leaf=1,
                             min_samples_split=2, n_estimators=166, bootstrap=False)

colsToBeDropped = ['pdbId']

def getPredictionsGivenThreshold(myMatrix, th):
    myList = []
    for i in range(myMatrix.shape[0]):
        p1 = myMatrix[i, 1]
        if p1>= th:
            myList.append(1)
        else:
            myList.append(0)
    return np.asarray(myList)

myDataframe = pd.read_csv('NetFV.csv')
myDataframe.drop(columns=colsToBeDropped, inplace=True)
mesoDf = myDataframe[myDataframe['label'] == 0]
thermoDf = myDataframe[myDataframe['label'] == 1]

m = 508
t = 575
kf = StratifiedKFold(n_splits=cvCount, random_state=42, shuffle = True)

listOfMesoDfs = []
for i in range(iterations):
    mesoSubDf = mesoDf.sample(n=m, replace=False, random_state=42)
    listOfMesoDfs.append(mesoSubDf)
    mesoDf = mesoDf.drop(index=mesoSubDf.index)

thresholdRange = np.linspace(start=0.49, stop=0.51, num=20)

thresholdList = []
precisionList = []
recallList = []
f1list = []
aucList = []
accuracyList = []
mcList = []
for threshold in thresholdRange:
    print(threshold)
    pit = 0
    rit = 0
    f1it = 0
    accit = 0
    aurocit = 0
    mccit = 0
    for mesoSubDf in listOfMesoDfs:
        tempDf = pd.concat([mesoSubDf, thermoDf], axis=0, ignore_index=True)
        labels = tempDf['label']
        tempDf.drop(columns=['label'], inplace=True)
        tempDf = tempDf.values
        labels = labels.values

        pcv = 0
        rcv = 0
        f1scv = 0
        acccv = 0
        auroccv = 0
        mcccv = 0
        for train_index, test_index in kf.split(tempDf, labels):
            X_train, X_test = tempDf[train_index], tempDf[test_index]
            Y_train, Y_test = labels[train_index], labels[test_index]
            clf.fit(X_train, Y_train)
            predictions_prob = clf.predict_proba(X_test)
            predictions = getPredictionsGivenThreshold(predictions_prob, threshold)
            pcv += precision_score(Y_test, predictions)
            rcv += recall_score(Y_test, predictions)
            f1scv += f1_score(Y_test, predictions)
            acccv += accuracy_score(Y_test, predictions)
            auroccv += roc_auc_score(Y_test, predictions_prob[:, 1])
            mcccv += matthews_corrcoef(Y_test, predictions)
        pit += (pcv/cvCount)
        rit += (rcv/cvCount)
        f1it += (f1scv/cvCount)
        accit += (acccv/cvCount)
        aurocit += (auroccv/cvCount)
        mccit += (mcccv/cvCount)
    thresholdList.append(threshold)
    precisionList.append(pit/iterations)
    recallList.append(rit/iterations)
    f1list.append(f1it/iterations)
    accuracyList.append(accit/iterations)
    aucList.append(aurocit/iterations)
    mcList.append(mccit/iterations)

df = pd.DataFrame()
df['Threshold'] = thresholdList
df['Precision'] = precisionList
df['F1'] = f1list
df['Recall'] = recallList
df['AUROC'] = aucList
df['Accuracy'] = accuracyList
df['MC'] = mcList
df.to_csv('Thresholding.csv', index=False)
print('Done')


from django.contrib.auth.models import User
from rest_framework.authtoken.models import Token
from rest_framework import permissions

class IsOwner(permissions.BasePermission):
    def has_permission(self, request, view):
        return super().has_permission(request, view)

    def has_object_permission(self, request, view, obj):
        print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", obj)
        if request.user == obj or request.user == "admin":
            return True
        else:
            return False
        # if :
        #     return True

        # w_u = request.user.websiteuser_set.filter(user_id=request.user.id, email_settings_id=obj.id).first()
        # if w_u:
        #     return True
        # else:
        #     if request.method.lower() == 'get':
        #         return True

        # return False


import re

def main():
    rules = {}
    input = open("input.txt", "r")
    (input_rules, input_messages) = input.read().split("\n\n")

    input_rules = input_rules.splitlines()
    while len(rules) < len(input_rules):
        for input_rule in input_rules:
            (rule, expr) = input_rule.split(": ")
            if expr == '"a"' or expr == '"b"':
                rules[rule] = expr[1]
            else:
                found = True
                built = []
                for block in expr.split(" | "):
                    pointers = block.split(" ")
                    if not all(elem in rules.keys() for elem in pointers):
                        found = False
                        break
                    else:
                        built.append(pointers)
                if found:
                    regex = "" if len(built) == 1 else "("
                    for group in built:
                        for pointer in group:
                            regex = regex + rules[pointer]
                        regex = regex + "|"
                    regex = regex[:-1]
                    if len(built) > 1:
                        regex = regex + ")"
                    rules[rule] = regex

    total = 0
    input_messages = input_messages.splitlines()
    for input_message in input_messages:
        if re.search("^" + rules["0"] + "$", input_message) != None:
            total = total + 1
    print(total)

if __name__ == "__main__":
    main()


import YouTubeMusicClient

'''
INFORMATION TO FILL OUT
'''

#enter spotify information here
user_id = ""
spotify_token = ""
#enter youtube information here
youtube_key = ''

#enter youtube playlist id and the title you want for the new playlist
youtube_playlistId = ""
spotify_playlistTitle = ""

client = YouTubeMusicClient.YouTubeMusic(user_id,spotify_token,youtube_key)
client.playlisttransfer(youtube_playlistId, spotify_playlistTitle)


'''
#include<stdio.h>
int palindrome(int n)
{
  int t,r = 0;
  t=n;
  while(t!=0)
  {
    r=r*10;
    r=r+(t%10);
    t=t/10;
  }
  if(r==n)
    return 1;
  else
    return 0;
}

def palindrome(n):
    t = n
    r = 0
    while t != 0:
        r *= 10
        r += (t % 10)
        t //= 10
        if r == n and len(str(r))%2== 0:
            return 1
    return 0

int palindrome(int n)
{
  int t,r = 0;
  t=n;
  while(t!=0)
  {
    r=r*10;
    r=r+(t%10);
    t=t/10;
  }
  if(r==n)
    return 1;
  else
    return 0;
}

int main()
{
  int k,d=0,i;
  scanf("%d",&k);
  i = k + 1;
  while(1)
  {
    d=palindrome(i);
    //printf("%d ", i);
    if(d==1)
      break;
    i++;
  }
  printf("%d",i);
  return 0;
}
'''
def palindrome(n):
    t = n
    r = 0
    while t != 0:
        r *= 10
        r += (t % 10)
        t //= 10
        if r == n :
            return 1
    return 0

for i in range(int(input())):
    k = int(input())
    k-= 1
    while k > 0:
        if len(str(k))%2:
            k -=1
            continue
        if len(set(str(k))) == len(str(k))//2:
            print(k)
            break
        k -= 1


import MLKit

class Column:

    """
    A DataTable Column.

    Attributes:
        name        The name of the column.
        values      The values of the column.
        attributes  The attributes of the column. Computed by calling the compute_attributes method.
    """

    def __init__(self, name, values):
        self.name = name
        self.values = values
        self.scaled_values = []
        self.attributes = None

    def compute_attributes(self, model):
        """Compute the column attributes"""
        self.attributes = MLKit.ColumnAttributes(self)
        if model and self.name in list(model["attributes"]["mean"].keys()):
            self.__scale_values(model)

    def __scale_values(self, model):
        for value in self.values:
            if value == None:
                self.scaled_values.append(None)
                continue
            numeric_value = self.attributes.numeric_value_for_value(value)
            scaled_value = (numeric_value - model["attributes"]["mean"][self.name]) / model["attributes"]["std"][self.name]
            self.scaled_values.append(scaled_value)


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import DataLoader,TensorDataset
import numpy as np
import scipy.io as sio
import math
import argparse
import random
import os
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import normalize
from sklearn.metrics import accuracy_score
from gzslcub2 import zsl_NShot
from sklearn import svm
torch.cuda.set_device(2)
from    torch import optim

# step 1: init dataset
print("init dataset")

dataroot = '/data'
dataset = 'CUB1_data'
image_embedding = 'res101'
class_embedding = 'att_splits'

matcontent = sio.loadmat(dataroot + "/" + dataset + "/" + image_embedding + ".mat")
feature = matcontent['features'].T
label = matcontent['labels'].astype(int).squeeze() - 1
matcontent = sio.loadmat(dataroot + "/" + dataset + "/" + class_embedding + ".mat")
# numpy array index starts from 0, matlab starts from 1
trainval_loc = matcontent['trainval_loc'].squeeze() - 1
test_seen_loc = matcontent['test_seen_loc'].squeeze() - 1
test_unseen_loc = matcontent['test_unseen_loc'].squeeze() - 1

attribute = matcontent['att'].T

x = feature[trainval_loc] # train_features
train_label = label[trainval_loc].astype(int)  # train_label
att = attribute[train_label] # train attributes

x_test = feature[test_unseen_loc]  # test_feature
test_label = label[test_unseen_loc].astype(int) # test_label
x_test_seen = feature[test_seen_loc]  #test_seen_feature
test_label_seen = label[test_seen_loc].astype(int) # test_seen_label
test_id = np.unique(test_label)   # test_id
att_pro = attribute[test_id]      # test_attribute

import numpy as np
path=dataroot
file='/cub_attributes_reed.npy'
attribute=15*np.load(file) # it scale the attribute in the some higher range since such
#that its attribute is in the AWA dataset attribute range since our model is optimized for the AWA and
# we are using the same parameter for all dataset. it scale the attribute in range of 100

# train set
#train_features=torch.from_numpy(x)
train_features=x
print('train_features.shape: ' + str(train_features.shape))

train_label=np.array(torch.from_numpy(train_label).unsqueeze(1))
#train_label=torch.from_numpy(train_label).unsqueeze(1)
print('train_label.shape:  '+str(train_label.shape))

# attributes
all_attributes=np.array(attribute)
print('all_attributes.shape:  '+str(all_attributes.shape))

attributes = torch.from_numpy(attribute)
# test set

# test_features=torch.from_numpy(x_test)
test_features=x_test
print('test_features.shape:  '+ str(test_features.shape))

test_label=np.array(torch.from_numpy(test_label).unsqueeze(1))
print('test_label.shape:  ' +str(test_label.shape))

testclasses_id = np.array(test_id)
print('testclasses_id.shape:  ' +str(testclasses_id.shape))

test_attributes = torch.from_numpy(att_pro).float()
print('test_attributes.shape:  ' +str(test_attributes.shape))

test_seen_features = torch.from_numpy(x_test_seen)
print('test_seen_features.shape:  ' +str(test_seen_features.shape))

test_seen_label = torch.from_numpy(test_label_seen)

train_data = [train_features,train_label]
test_data = [test_features,test_label]

unq_train_labels=np.unique(train_label)
unq_test_labels=np.unique(test_label)
#train_data = TensorDataset(train_features,train_label)

def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.normal_(m.weight, mean=0, std=0.002)
        m.bias.data.fill_(0.002)

class RelationNetwork(nn.Module):
    """Graph Construction Module"""

    def __init__(self):
        super(RelationNetwork, self).__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(64, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, padding=1))
        self.layer2 = nn.Sequential(
            nn.Conv2d(32, 1, kernel_size=3, padding=1),
            nn.BatchNorm2d(1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, padding=1))

        self.fc3 = nn.Linear(2 * 2, 8)
        self.fc4 = nn.Linear(8, 1)

        self.m0 = nn.MaxPool2d(2)  # max-pool without padding
        self.m1 = nn.MaxPool2d(2, padding=1)  # max-pool with padding

    def forward(self, x, rn):
        x = x.view(160, 64, 4, 4)

        out = self.layer1(x)
        out = self.layer2(out)
        # flatten
        out = out.view(out.size(0), -1)
        out = F.relu(self.fc3(out))
        out = self.fc4(out)  # no relu

        out = out.view(out.size(0), -1)  # bs*1

        return out

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()

        self.modelG = nn.Sequential(
            nn.Linear(args.attri_dim + args.noise_dim, 2048),
            nn.Dropout(0.5),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(2048, 2048),
            nn.BatchNorm1d(2048, 0.8),
            nn.Dropout(0.5),
            nn.Linear(2048, 2048),
            nn.BatchNorm1d(2048, 0.8),
            nn.Dropout(0.5),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(2048, args.input_shape)
        )

    def forward(self, noise, attri, weightsM=None):
        # Concatenate label embedding and image to produce input
        gen_input = torch.cat((attri, noise), -1)
        if weightsM is None:
            img = self.modelG(gen_input)
        else:
            i = 0
            weights = weightsM[0]
            for m in self.modelG.modules():
                if isinstance(m, nn.Linear):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
                if isinstance(m, nn.BatchNorm1d):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
            img = self.modelG(gen_input)

        return img

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        self.modelD = nn.Sequential(
            nn.Linear(args.input_shape + args.attri_dim, 2048),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(2048, 1024),
            nn.Dropout(0.4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(1024, 1024),
            nn.Dropout(0.4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(1024, 1)
        )

    #         self.modelD.apply(init_weights)

    def forward(self, img, attri, weightsM=None):
        d_in = torch.cat((img.view(img.size(0), -1), attri), -1)
        if weightsM is None:
            real_fake = self.modelD(d_in)
        else:
            i = 0
            weights = weightsM[1]
            for m in self.modelD.modules():
                if isinstance(m, nn.Linear):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
                if isinstance(m, nn.BatchNorm1d):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
            real_fake = self.modelD(d_in)

        return real_fake

class mapping(nn.Module):
    def __init__(self):
        super(mapping, self).__init__()

        self.modelC = nn.Sequential(
            nn.Linear(2048, 1024),
            nn.Dropout(0.4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(1024,1024),
            #
            # nn.Linear(512, 128)
        )

    #self.modelC.apply(init_weights)

    def forward(self, img_gen, weightsM=None):
        # Concatenate label embedding and image to produce input
        if weightsM is None:
            output = self.modelC(img_gen)
        else:
            i = 0
            weights = weightsM[2]
            for m in self.modelC.modules():
                if isinstance(m, nn.Linear):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
                if isinstance(m, nn.BatchNorm1d):
                    m.weight.data = weights[i]
                    m.bias.data = weights[i + 1]
                    i = i + 2
            output = self.modelC(img_gen)
        return output

class all_arguments():
    n_way = 10
    k_spt = 5
    k_qry = 3

    imgsz = 2048
    sigma_ts = 0.25
    sigma_tr = 0.5

    meta_lr = 1e-5
    meta_lrD = 1e-3
    update_lr = 1e-3
    update_step = 5

    input_shape = 2048
    num_class = 200
    attri_dim = 1024
    noise_dim = 512
    clssifier_weight = 0.05

    # Get options
    attributes = attributes
    cuda = True if torch.cuda.is_available() else False

args = all_arguments()
# Loss functions
adversarial_loss = torch.nn.MSELoss()
auxiliary_loss = torch.nn.CrossEntropyLoss()

# Initialize generator and discriminator
# generator = Generator()
# relationnetwork=RelationNetwork()
# discriminator = Discriminator()
# classifier = Classifier()
#
# if args.cuda:
#     generator.cuda()
#     discriminator.cuda()
#     classifier.cuda()
#     adversarial_loss.cuda()
#     auxiliary_loss.cuda()
#     relationnetwork.cuda()
#
# para = list(classifier.parameters())
# print(classifier)

#
# glen=len(list(generator.parameters()))
# gen_para=list(generator.parameters())+ list(classifier.parameters())
#
# disc_optim = optim.SGD(discriminator.parameters(), lr=args.meta_lrD,weight_decay=1e-6)
# gen_optim = optim.Adam(gen_para, lr=args.meta_lr,betas=(0.9, 0.99),weight_decay=1e-6)
#
# disc_schedular = StepLR(disc_optim,step_size=100,gamma=0.95)
# gen_schedular = StepLR(gen_optim,step_size=100,gamma=0.95)

class Meta(nn.Module):
    """
    Meta Learner
    """

    def __init__(self,args):
        """
        :param args:
        """
        super(Meta, self).__init__()

        self.update_lr = args.update_lr
        self.meta_lr = args.meta_lr
        self.n_way = args.n_way
        self.k_spt = args.k_spt
        self.k_qry = args.k_qry
        self.cls_weight = args.clssifier_weight
        self.update_step = args.update_step
        cuda = args.cuda
        self.noise_dim = args.noise_dim
        # self.all_loss=all_loss()

        self.FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
        self.LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor

        #self.glen = glen
        #self.gen_para = gen_para
        #self.disc_optim = disc_optim
        #self.gen_optim = gen_optim

        #self.disc_schedular = disc_schedular
        #self.gen_schedular = gen_schedular

        self.generator = Generator()
        self.generator.cuda()
        self.relationnetwork = RelationNetwork()
        self.relationnetwork.cuda()
        self.discriminator = Discriminator()
        self.discriminator.cuda()
        self.mapping = mapping()
        self.mapping.cuda()

    def clip_grad_by_norm_(self, grad, max_norm):
        """
        in-place gradient clipping.
        :param grad: list of gradients
        :param max_norm: maximum norm allowable
        :return:
        """
        total_norm = 0
        counter = 0
        for g in grad:
            param_norm = g.data.norm(2)
            total_norm += param_norm.item() ** 2
            counter += 1
        total_norm = total_norm ** (1. / 2)

        clip_coef = max_norm / (total_norm + 1e-6)
        if clip_coef < 1:
            for g in grad:
                g.data.mul_(clip_coef)

        return total_norm / counter

    def all_loss(self, img_feature, img_labels, qurey_feature, qurey_labels,fast_weight=None):
        batch_size = img_feature.shape[0]
        FloatTensor = self.FloatTensor
        LongTensor = self.LongTensor
        img_labels = img_labels.type(LongTensor)
        img_labels1 = qurey_labels.type(LongTensor)

        valid = Variable(FloatTensor(batch_size+30, 1).fill_(1.0), requires_grad=False)
        fake = Variable(FloatTensor(batch_size+30, 1).fill_(0.0), requires_grad=False)

        labels = torch.cat((img_labels, img_labels1))

        # Configure input
        feature = torch.cat((img_feature, qurey_feature))
        real_imgs = Variable(feature.type(FloatTensor))
        gen_attri = Variable(attributes[labels].type(FloatTensor))

        z = Variable(FloatTensor(np.random.normal(0, args.sigma_tr, (batch_size+30, self.noise_dim))))

        # Generate a batch of images
        gen_imgs = self.generator(z, gen_attri, fast_weight)
        validity = self.discriminator(gen_imgs, gen_attri, fast_weight)

        g_loss = adversarial_loss(validity, valid)
        # print('g_loss: '+str(g_loss))

        validity_real = self.discriminator(real_imgs, gen_attri, fast_weight)
        d_real_loss = adversarial_loss(validity_real, valid)
        # print('d_real_loss: '+str(d_real_loss))

        # Loss for fake images
        validity_fake = self.discriminator(gen_imgs.detach(), gen_attri, fast_weight)
        d_fake_loss = adversarial_loss(validity_fake, fake)
        # print('d_fake_loss: '+str(d_fake_loss))

        reconstruction_criterion = nn.L1Loss(size_average=False)
        lossrecon = reconstruction_criterion(gen_imgs, real_imgs)
        # Total discriminator loss
        d_loss = (d_real_loss + d_fake_loss) / 2

        inp1 = torch.cat((feature,gen_imgs), 0)
        inp=self.mapping(inp1)

        emb_all = inp.view(-1,1024)
        N, d    = emb_all.shape[0], emb_all.shape[1]
        sigma = self.relationnetwork(inp, 300)

        eps = np.finfo(float).eps
        emb_all = emb_all / (sigma + eps)  # N*d
        emb1 = torch.unsqueeze(emb_all, 1)  # N*1*d
        emb2 = torch.unsqueeze(emb_all, 0)  # 1*N*d
        W = ((emb1 - emb2) ** 2).mean(2)  # N*N*d -> N*N
        W = torch.exp(-W / 2)

        ## keep top-k values
        k=20

        if k > 0:
            topk, indices = torch.topk(W, k)
            mask = torch.zeros_like(W)
            mask = mask.scatter(1, indices, 1)
            mask = ((mask + torch.t(mask)) > 0).type(torch.float32)  # union, kNN graph
            # mask = ((mask>0)&(torch.t(mask)>0)).type(torch.float32)  # intersection, kNN graph
            W = W * mask

            ## normalize
        D = W.sum(0)
        D_sqrt_inv = torch.sqrt(1.0 / (D + eps))
        D1 = torch.unsqueeze(D_sqrt_inv, 1).repeat(1, N)
        D2 = torch.unsqueeze(D_sqrt_inv, 0).repeat(N, 1)
        S = D1 * W * D2

        # Step3: Label Propagation, F = (I-\alpha S)^{-1}Y
        ys = img_labels
        ys = torch.eye(200)[ys, :].cuda()
        labels = torch.eye(200)[labels, :].cuda()

        yu = (torch.ones(30, 200)/200).cuda()
        y1 = torch.cat((ys, yu), 0)
        y = torch.cat((y1, labels), 0)
        F = torch.matmul(torch.inverse(torch.eye(N).cuda() - 0.99 * S + eps), y)
        Fq = F[50:80, :]  # query predictions

        # Step4: Cross-Entropy Loss
        ce = nn.CrossEntropyLoss().cuda()
        ## both support and query loss
        gt = torch.argmax(torch.cat((labels,labels), 0), 1)
        loss = ce(F, gt)

        ## acc
        predq = torch.argmax(Fq, 1)

        correct = (predq == qurey_labels).sum()
        total = 30
        acc = 1.0 * correct.float() / float(total)

        return g_loss, d_loss, loss,acc

    def one_hot(self,x):
        return torch.eye(200)[x, :]

    def forward(self, x_spt, y_spt, x_qry, y_qry):
        """
        :param x_spt:   [b, setsz, d]
        :param y_spt:   [b, setsz]
        :param x_qry:   [b, querysz, d]
        :param y_qry:   [b, querysz]
        :return:
        """

        g_loss, d_loss, loss,acc = self.all_loss(x_spt, y_spt, x_qry, y_qry)

        return  g_loss, d_loss, loss,acc

def test_zsl(genunseen_input, test_labels_repeat, test_features, testLabels):
    """
    """
    with torch.no_grad():
        gen_imgs = maml.generator(genunseen_input[:, args.attri_dim:], genunseen_input[:, :args.attri_dim])
        gen_imgs  = maml.mapping(gen_imgs)
        pseudoTrainData = normalize(gen_imgs.cpu(), axis=1)
        #testData = normalize(test_features, axis=1)
        pseudoTrainLabels = test_labels_repeat
        testLabels = testLabels

        clf5 = svm.SVC(C=1, kernel='linear', class_weight='balanced')

        clf5.fit(pseudoTrainData, pseudoTrainLabels)
        # print 'Predicting...'
        testf=torch.from_numpy(test_features).to(torch.float32).cuda()
        testData = maml.mapping(testf)
        testData = testData.cpu().numpy()
        pred = clf5.predict(testData)
        unseen_accuracy = accuracy_score(testLabels, pred)

    return unseen_accuracy

def gradient_penalty(D, xr, xf):
    """
    :param D:
    :param xr:[b,2]
    :param xf:[b,2]
    :return:
    """

    # only constrait for Discriminator
    #    xf = xf.detach()
    #    xr = xr.detach()

    # [b, 1] => [b, 2]
    t = torch.rand(batchsz, 1).cuda()
    t = t.expand_as(xr)

    mid = t * xr + ((1 - t) * xf)
    mid.requires_grad_()
    pred = D(mid)
    grads = autograd.grad(outputs=pred, inputs=mid,
                          grad_outputs=torch.ones_like(pred),
                          create_graph=True, retain_graph=True, only_inputs=True)[0]
    gp = torch.pow((grads.norm(2, dim=1) - 1), 2).mean()

    return gp

def test():
    testunq_labels = np.unique(test_label)
    lab_attribute = np.reshape(attribute[testunq_labels[0]], [1, args.attri_dim])
    test_attri_repeat = []
    test_labels_repeat = []
    for i in testunq_labels:
        lab_attribute = np.reshape(attribute[i], [1, args.attri_dim])
        test_attri_repeat.append(np.repeat(lab_attribute, 50, axis=0))
        test_labels_repeat.append(np.repeat(i, 50, axis=0))
    test_attri_repeat = np.concatenate(test_attri_repeat, 0)
    test_labels_repeat = np.concatenate(test_labels_repeat, 0)
    z = np.random.normal(0, args.sigma_ts, (2500, args.noise_dim))
    genunseen_input = np.concatenate((test_attri_repeat, z), 1)

    FloatTensor = torch.cuda.FloatTensor
    genunseen_input = (torch.from_numpy(genunseen_input)).type(FloatTensor)

    unseen_accuracy = test_zsl(genunseen_input, test_labels_repeat, test_features, test_label)
    return unseen_accuracy

db = zsl_NShot(train_data, test_data, batchsz=10, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry)
x_spt, y_spt, x_qry, y_qry = db.next('train')
x_spt = torch.from_numpy(x_spt).cuda()
x_qry = torch.from_numpy(x_qry).cuda()
y_spt = torch.from_numpy(y_spt).cuda()
y_qry = torch.from_numpy(y_qry).cuda()
print('train :  ' + str(x_spt.shape))
print('train_labels :  ' + str(y_spt.shape))
print('test :  ' + str(x_qry.shape))
print('test_labels :  ' + str(y_qry.shape))

best = 0
maml = Meta(args).cuda()
# optimizer
# model_optim = torch.optim.Adam(maml.parameters(), lr=0.001)

gen_para=list(maml.generator.parameters())+ list(maml.relationnetwork.parameters())+ list(maml.mapping.parameters())
maml.discriminator.parameters =maml.discriminator.parameters()

disc_optim = optim.Adam(maml.discriminator.parameters, lr=1e-3,betas=(0.9, 0.99),weight_decay=1e-6)
gen_optim = optim.Adam(gen_para, lr=1e-5,betas=(0.9, 0.99),weight_decay=1e-6)

for itr in range(12000):

    x_spt, y_spt, x_qry, y_qry = db.next('train')
    x_spt = torch.from_numpy(x_spt).cuda()
    x_qry = torch.from_numpy(x_qry).cuda()
    y_spt = torch.from_numpy(y_spt).cuda()
    y_qry = torch.from_numpy(y_qry).cuda()

    maml.train()

    for i in range(10):

        maml.train()

        for _ in range(5):

            for _ in range(5):

                g_loss, d_loss, loss, accs = maml(x_spt[i], y_spt[i], x_qry[i], y_qry[i])

                disc_optim.zero_grad()

                d_loss.backward()

                disc_optim.step()

            #gp = gradient_penalty(maml.discriminator, xr, xf.detach())

            for p in maml.discriminator.parameters:
                p.requires_grad = False

            g_loss, d_loss, loss, accs = maml(x_spt[i], y_spt[i], x_qry[i], y_qry[i])

            loss1 = 0.1*g_loss+loss
            #print(loss)
            #print(accs)
            gen_optim.zero_grad()
            loss1.backward()
            # torch.nn.utils.clip_grad_norm(model.parameters(), 4.0)
            gen_optim.step()

    if itr % 100 == 0:
        maml.eval()
        unseen_accuracy = test()
        if unseen_accuracy >= best:
            best = unseen_accuracy
        print(best)


class Solution:
    def rotate(self, matrix):
        """
        :type matrix: List[List[int]]
        :rtype: void Do not return anything, modify matrix in-place instead.
        """
        def rotateLayer(matrix, l, r):
            if l >= r:
                return
            for i in range(r - l):
                tmp = matrix[l][l + i]
                matrix[l][l + i] = matrix[r - i][l]
                matrix[r - i][l] = matrix[r][r - i]
                matrix[r][r - i] = matrix[l + i][r]
                matrix[l + i][r] = tmp
            rotateLayer(matrix, l + 1, r - 1)
        rotateLayer(matrix, 0, len(matrix) - 1)


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# __author__ = "Q1mi"
# Date: 2018/8/22
from django import forms
from app01 import models

class BookForm(forms.Form):
    title = forms.CharField(max_length=32)
    price = forms.DecimalField(max_digits=5, decimal_places=2)
    publish_date = forms.DateField(
        widget=forms.widgets.DateInput(attrs={"type": "date"})
    )
    publisher = forms.ModelChoiceField(
        queryset=models.Publisher.objects.all()
    )
    authors = forms.ModelMultipleChoiceField(
        queryset=models.Author.objects.all()
    )

class BookModelForm(forms.ModelForm):
    class Meta:
        model = models.Book
        fields = "__all__"


from jaeschke import t_2
from utils import readfile

bases = [2, 3, 5, 7, 11, 13, 17, 19,
         23, 29, 31, 37, 41, 43, 47, 53]

primes = readfile("primes/primes_1m.txt")

if __name__ == "__main__":
    print(3, 12)
    t_2(bases[:3], 10 ** 12, primes)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.profile, name='profile'),
    path('order_history/<order_number>', views.order_history,
         name='order_history'),
    path('favorites/', views.view_favorites, name='view_favorites'),
    path('add/<product_id>/', views.add_to_favorites, name='add_to_favorites'),
]


import sys
import requests
import json
data="";
# GET
search=sys.argv[1]
res = requests.get('https://api.dictionaryapi.dev/api/v2/entries/en/'+str(search))

if str(res.status_code)=="404":
    data="404"
else:
    val=json.loads(res.text)
    data=val[0]["meanings"][0]["definitions"][0]["definition"]
f = open("./test.txt", 'w')
f.write(data)
f.close()


from PIL import Image
from tqdm import tqdm
import pandas as pd
import config as C
import numpy as np
import os
import copy
import random
from skimage.util import random_noise
import cv2
random.seed = C.SEED

np.random.seed(C.SEED)

def resize_square_img(img):
    """
    改成正方形
    :param img:
    :return:
    """
    h = img.shape[0]
    w = img.shape[1]
    value = (0, 0)
    if h > w:
        le = int((h - w) / 2)
        r = h - w - le
        border = [0, 0, le, r]
    else:
        t = int((w - h) / 2)
        b = w - h - t
        border = [t, b, 0, 0]
    img = cv2.copyMakeBorder(img, border[0], border[1], border[2], border[3], cv2.BORDER_CONSTANT, value=value)
    # print(img.shape)
    return img

def threshold(img):
    """
    二值化
    :param img:
    :return:
    """
    img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    ret, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
    img = cv2.medianBlur(img, C.BLUR_SIZE)
    # print("阈值 ", ret)
    return img

df_char = pd.read_csv(C.CHARS_FREQ_FILE)

argument_num = 0
def crop_all_char(image_name, labels, is_train):
    """
    将图片中的全部文字剪裁出来保存到csv中, label 0 是少见类, 所以label从1开始
    :param image_name:
    :param labels:
    :return:
    """
    global argument_num
    img_path = os.path.join(C.TRAIN_IMAGES, f'{image_name}.jpg')
    labels = np.array(labels.split(' ')).reshape(-1, 5)
    img = cv2.imread(img_path)
    n = 0
    for idx, (codepoint, x_min, y_min, w, h) in enumerate(labels):

        crop_img_name = f'{image_name}_{idx}'
        crop_img_path = os.path.join(C.SAVE_ALL_CHARS, f'{crop_img_name}.jpg')
        if os.path.exists(crop_img_path):
            continue
        n += 1
        x_min, y_min, w, h = int(x_min), int(y_min), int(w), int(h)
        # 稍微扩展一下裁剪范围
        # x_min, y_min, w, h = int(x_min-C.CROP_PAD_SIZE), int(y_min-C.CROP_PAD_SIZE), int(w+C.CROP_PAD_SIZE*2), int(h+C.CROP_PAD_SIZE*2)
        label_index = df_char[df_char['char'] == codepoint]['idx'].values[0]
        jp_char = df_char[df_char['char'] == codepoint]['jp_char'].values[0]
        freq = df_char[df_char['char'] == codepoint]['count'].values[0]
        crop_img = img[y_min:y_min + h, x_min:x_min + w]
        crop_img = threshold(crop_img)
        crop_img = resize_square_img(crop_img)
        cv2.imwrite(crop_img_path, crop_img)
        save_content = {
            'img_name': crop_img_name,
            'label': label_index + 1,
            'jp_char': jp_char,
            'is_train': 1 if is_train else 0
        }
        # save_content = [crop_img_name, label_index+1, jp_char, is_train]
        char_dict = pd.DataFrame(save_content, index=[0])  # , columns=['img_name', 'label', 'jp_char', 'is_train'])
        char_dict.to_csv(C.CHARS_DICT_FILE, mode='a', header=None, index=False)
        if is_train:
            if freq < C.NUM_PER_CLASS:
                need_argument = C.NUM_PER_CLASS // freq + 1
                for i in range(need_argument):
                    crop_img_path = os.path.join(C.SAVE_ALL_CHARS, f'{crop_img_name}_{argument_num}.jpg')
                    _crop_img = data_argument(crop_img)
                    cv2.imwrite(crop_img_path, _crop_img)
                    save_content = {
                        'img_name': f'{crop_img_name}_{argument_num}',
                        'label': label_index + 1,
                        'jp_char': jp_char,
                        'is_train': 1 if is_train else 0
                    }
                    # save_content = [crop_img_name, label_index+1, jp_char, is_train]
                    char_dict = pd.DataFrame(save_content,
                                             index=[0])  # , columns=['img_name', 'label', 'jp_char', 'is_train'])
                    char_dict.to_csv(C.CHARS_DICT_FILE, mode='a', header=None, index=False)
                    argument_num += 1
                    n += 1

    return n

def data_argument(image):
    h, w = image.shape
    img = copy.deepcopy(image)
    # 二值化，无需噪声
    # if random.random() < 0.5:
    #     img = random_noise(img, mode='gaussian', clip=True) * 255
    if random.random() < 0.7:
        # 随机增加外边距
        top_size = np.random.randint(0, int(0.2 * h))
        bottom_size = np.random.randint(0, int(0.2 * h))
        right_size = np.random.randint(0, int(0.2 * w))
        left_size = np.random.randint(0, int(0.2 * w))
        img = cv2.copyMakeBorder(img, top_size, bottom_size, left_size, right_size,
                                 cv2.BORDER_CONSTANT, value=(0, 0))

    return img

def main():
    with open(C.TRAIN_FILE, 'r') as train:
        train_ids = list(map(lambda x: x.strip(), train.readlines()))
    df_train = pd.read_csv(C.TRAIN_CSV).dropna()
    sum = len(os.listdir(C.SAVE_ALL_CHARS))
    for idx in tqdm(range(len(df_train))):  #
        img_name, labels = df_train.values[idx]
        is_train = img_name in train_ids
        sum += crop_all_char(img_name, labels, is_train)
        print(sum)

if __name__ == '__main__':
    main()


# coding=utf-8

import tensorflow as tf
import os
import cifar10

image_size = 224
channel_num = 3
class_num = 10
dropout_keep_prob = 0.7
batch_size = 32

num_examples_per_epoch_for_train = cifar10.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN

dataset_dir = '../../dataset/cifar10'
model_save_path = './model/'
model_name = 'model.ckpt'

def get_train_data(data_dir, batch_size):
    cifar10.maybe_download_and_extract(data_dir)
    images, labels = cifar10.get_distorted_train_batch(dataset_dir, batch_size)
    images = tf.image.resize_images(images=images, size=[image_size, image_size])
    return images, labels

def get_eval_data(data_dir, batch_size):
    cifar10.maybe_download_and_extract(data_dir)
    images, labels = cifar10.get_undistorted_eval_batch(dataset_dir, batch_size)
    images = tf.image.resize_images(images=images, size=[image_size, image_size])
    return images, labels

def variable_summary(value):
    with tf.name_scope(name='summaries'):
        mean=tf.reduce_mean(value)
        stddev=tf.sqrt(tf.reduce_mean(tf.square(value-mean)))
        tf.summary.scalar(name='mean', tensor=mean)
        tf.summary.scalar(name='stddev', tensor=stddev)
        tf.summary.scalar(name='max', tensor=tf.reduce_max(value))
        tf.summary.scalar(name='min', tensor=tf.reduce_min(value))
        tf.summary.histogram(name='histogram', values=value)

def weights_variable(shape, stddev=0.1, name='weights'):
    initial = tf.truncated_normal(shape=shape, stddev=stddev, dtype=tf.float32)
    return tf.Variable(initial_value=initial, name=name)

def biases_variable(shape, name='biases'):
    initial = tf.constant(value=0.1, dtype=tf.float32, shape=shape)
    return tf.Variable(initial_value=initial, name=name)

def conv2d(input, filter_height, filter_width, num_channels, num_filters, name):
    with tf.name_scope(name):
        weigths = weights_variable(shape=[filter_height, filter_width, num_channels, num_filters],
                                   stddev=0.1)
        biases = biases_variable(shape=[num_filters])
        conv = tf.nn.conv2d(input=input,
                            filter=weigths,
                            strides=[1, 1, 1, 1],
                            padding='SAME',
                            name=name)
        conv_out = tf.nn.relu(features=tf.nn.bias_add(value=conv, bias=biases), name='relu')
        return conv_out

def pool_max(input, name):
    with tf.name_scope(name):
        return tf.nn.max_pool(value=input,
                              ksize=[1, 2, 2, 1],
                              strides=[1, 2, 2, 1],
                              padding='SAME',
                              name='pool')

def fc(input, num_in, num_out, name, active=True):
    with tf.name_scope(name):
        weights = weights_variable(shape=[num_in, num_out])
        biases = biases_variable(shape=[num_out])
        fc = tf.matmul(input, weights) + biases
        if active:
            fc_out = tf.nn.relu(features=fc, name='relu')
            return fc_out
        else:
            return fc

def dropout(input, keep_prob, name):
    with tf.name_scope(name):
        return tf.nn.dropout(x=input, keep_prob=keep_prob, name='dropout')

def inference(inputs, train=True):
    with tf.name_scope('conv1'):
        conv1_1 = conv2d(input=inputs, filter_height=3, filter_width=3, num_channels=3, num_filters=64,
                         name='conv1_1')
        conv1_2 = conv2d(input=conv1_1, filter_height=3, filter_width=3, num_channels=64, num_filters=64,
                         name='conv1_2')
        pool1 = pool_max(input=conv1_2, name='pool1')

    with tf.name_scope('conv2'):
        conv2_1 = conv2d(input=pool1, filter_height=3, filter_width=3, num_channels=64, num_filters=128,
                         name='conv2_1')
        conv2_2 = conv2d(input=conv2_1, filter_height=3, filter_width=3, num_channels=128, num_filters=128,
                         name='conv2_2')
        pool2 = pool_max(input=conv2_2, name='pool2')

    with tf.name_scope('conv3'):
        conv3_1 = conv2d(input=pool2, filter_height=3, filter_width=3, num_channels=128, num_filters=256,
                         name='conv3_1')
        conv3_2 = conv2d(input=conv3_1, filter_height=3, filter_width=3, num_channels=256, num_filters=256,
                         name='conv3_2')
        conv3_3 = conv2d(input=conv3_2, filter_height=3, filter_width=3, num_channels=256, num_filters=256,
                         name='conv3_3')
        conv3_4 = conv2d(input=conv3_3, filter_height=3, filter_width=3, num_channels=256, num_filters=256,
                         name='conv3_4')
        pool3 = pool_max(conv3_4, name='pool3')

    with tf.name_scope('conv4'):
        conv4_1 = conv2d(input=pool3, filter_height=3, filter_width=3, num_channels=256, num_filters=512,
                         name='conv4_1')
        conv4_2 = conv2d(input=conv4_1, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv4_2')
        conv4_3 = conv2d(input=conv4_2, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv4_3')
        conv4_4 = conv2d(input=conv4_3, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv4_4')
        pool4 = pool_max(conv4_4, name='pool4')

    with tf.name_scope('conv5'):
        conv5_1 = conv2d(input=pool4, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv5_1')
        conv5_2 = conv2d(input=conv5_1, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv5_2')
        conv5_3 = conv2d(input=conv5_2, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv5_3')
        conv5_4 = conv2d(input=conv5_3, filter_height=3, filter_width=3, num_channels=512, num_filters=512,
                         name='conv5_4')
        pool5 = pool_max(conv5_4, name='pool5')

    with tf.name_scope('fc6') as scope:
        pool5_shape = pool5.get_shape().as_list()
        nodes = pool5_shape[1] * pool5_shape[2] * pool5_shape[3]
        pool5_flat = tf.reshape(tensor=pool5, shape=[-1, nodes], name='flat')
        fc6 = fc(input=pool5_flat, num_in=nodes, num_out=4096, name=scope)
        if train:
            fc6_dropout = dropout(input=fc6, keep_prob=dropout_keep_prob, name=scope)
        else:
            fc6_dropout = dropout(input=fc6, keep_prob=1.0, name=scope)

    with tf.name_scope('fc7') as scope:
        fc7 = fc(input=fc6_dropout, num_in=4096, num_out=4096, name=scope)
        if train:
            fc7_dropout = dropout(input=fc7, keep_prob=dropout_keep_prob, name=scope)
        else:
            fc7_dropout = dropout(input=fc7, keep_prob=1.0, name=scope)

    with tf.name_scope('fc8') as scope:
        fc8 = fc(input=fc7_dropout, num_in=4096, num_out=class_num, name=scope)

    return fc8

if __name__ == '__main__':
    with tf.name_scope('Train_Batch'):
        train_x, train_y = get_train_data(data_dir=dataset_dir, batch_size=batch_size)

    with tf.name_scope('Valid_Batch'):
        valid_x, valid_y = get_eval_data(data_dir=dataset_dir, batch_size=batch_size)

    with tf.name_scope('Inputs'):
        x = tf.placeholder(
            dtype=tf.float32,
            shape=[batch_size, image_size, image_size, channel_num],  # 第四维表示图片的深度，对于RBG格式的图片，深度为5
            name='X')
        y = tf.placeholder(
            dtype=tf.int32,  # 不是one-hot时要为整型
            shape=[batch_size],
            name='Y')

    with tf.name_scope('Inference'):
        y_hat = inference(inputs=x)

    with tf.name_scope('Loss'):
        # 在没有one-hot时使用此种损失计算方法
        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=y_hat)
        loss = tf.reduce_mean(cross_entropy)

    with tf.name_scope('Train'):
        global_step = tf.Variable(initial_value=0, trainable=False)
        optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
        train = optimizer.minimize(loss=loss, global_step=global_step)

    with tf.name_scope('Evaluate'):
        top_k = tf.nn.in_top_k(predictions=y_hat, targets=y, k=1, name='top_k')
        # correct_pred = tf.equal(tf.argmax(y_hat, 1), tf.argmax(y, 1))
        accuracy = tf.reduce_mean(tf.cast(top_k, tf.float32))

    init = tf.global_variables_initializer()
    # 初始化Tensorflow持久化类
    saver = tf.train.Saver()

    with tf.Session() as sess:
        sess.run(init)
        print('===>>>>>>>==开始训练集上训练模型==<<<<<<<=====')
        total_batches = int(num_examples_per_epoch_for_train / batch_size)
        print('Per batch Size:,', batch_size)
        print('Train sample Count Per Epoch:', num_examples_per_epoch_for_train)
        print('Total batch Count Per Epoch:', total_batches)

        # 启动数据读取队列
        tf.train.start_queue_runners()
        for epoch in range(10):
            for i in range(total_batches):
                batch_x, batch_y = sess.run(fetches=[train_x, train_y])
                # print(batch_x.shape())
                _, step = sess.run(fetches=[train, global_step],
                                   feed_dict={x: batch_x, y: batch_y})
                if step % 10 == 0:
                    train_loss, train_eval = sess.run(fetches=[loss, accuracy],
                                                      feed_dict={x: batch_x, y: batch_y})
                    print(str(step) + '训练集损失：' + str(train_loss))
                    print(str(step) + '训练集评估：' + str(train_eval))

                    batch_x, batch_y = sess.run(fetches=[train_x, train_y])
                    validation_loss, validation_eval = sess.run(fetches=[loss, accuracy],
                                                                feed_dict={x: batch_x, y: batch_y})
                    print(str(step) + '验证集损失：' + str(validation_loss))
                    print(str(step) + '验证集评估：' + str(validation_eval))

                    # 输出当前的训练情况。这里只输出了模型在当前训练batch上的损失函数大小。
                    # 在验证数据集上的正确率信息会有一个单独的程序来生成。
                    # print("After %d training step(s), loss on training batch is %f." % (step, loss_value))
                    # 保存当前的模型。注意这里隔出了global_step参数，这样可以让每个被保存模型的文件名末尾加上训练的轮数，比如“model.ckpt-1000”表示训练1000轮后得到的模型
                    saver.save(sess, os.path.join(model_save_path, model_name), global_step=global_step)


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk

from queue import Queue
from square import Square

class Board(Gtk.Box):

  def __init__(self, maze, player):
    Gtk.Box.__init__(self, orientation = Gtk.Orientation.VERTICAL, spacing = 0)

    self.height = maze.height * 2
    self.width = maze.width * 2
    self.maze = maze
    self.player = player

    self.square = []
    for i in range(self.height):
      self.square.append([])
      for j in range(self.width):
        if maze[i//2][j//2] == 1:
          sq = Square(Type = "player")
        else:
          sq = Square(Type = "enemy")
        self.square[i].append(sq)
        sq.update(1, 1)

    for i in range(self.height):
      box = Gtk.Box(orientation = Gtk.Orientation.HORIZONTAL, spacing = 0)
      for j in range(self.width):
        box.pack_start(self.square[i][j], True, True, 0)
      self.pack_start(box, True, True, 0)

    self.dist = [[-1 for i in range(self.width)] for j in range(self.height)]

  def inside(self, x, y):
    return (0 <= x and x < self.height) and (0 <= y and y < self.width)

  def __getitem__(self, i):
    return self.square[i]

  def bfs(self, s_x, s_y):
    self.dist = [[-1 for i in range(self.width)] for j in range(self.height)]

    if s_x >= 0:
      self.dist[s_x][s_y] = 0
    else:
      self.dist[-s_x][-s_y] = 0
      for i in range(self.height):
        for j in range(self.width):
          if self[i][j].Type == "enemy":
            if self[i][j].Occupant != -1:
              self.dist[i][j] = 0

    Q = Queue()
    for i in range(self.height):
      for j in range(self.width):
        if self.dist[i][j] == 0:
          Q.put([i, j])

    while not Q.empty():
      u = Q.get()
      for d in [[0, 1], [1, 0], [0, -1], [-1, 0]]:
        if self.inside(u[0] + d[0], u[1] + d[1]):
          if self.dist[u[0] + d[0]][u[1] + d[1]] == -1:
            if self[u[0] + d[0]][u[1] + d[1]].Type == "enemy":
              self.dist[u[0] + d[0]][u[1] + d[1]] = self.dist[u[0]][u[1]] + 1
              Q.put([u[0] + d[0], u[1] + d[1]])

    return True


import json
import sys

arg=sys.argv

with open('package.json', "r") as f:
	data=json.load(f)

# 1st argument is the version of the current package
data['version']=arg[1]

# Setting dependencies versions
if len(arg)>=4:
	for i in range (2,len(arg),2):
		data['dependencies'][arg[i]]=arg[i+1]

with open('package.json', "w") as f:
	json.dump(data,f)


# Acoustic wave equations with Devito
# Forward/adjoint nonlinear and Born modeling
# Authors: Mathias Louboutin, Philipp Witte
# Date: November 2017
#

# Import modules
from __future__ import print_function
import numpy as np
import gc, os, psutil
from numpy.random import randint
from sympy import solve, cos, sin
from sympy import Function as fint
from devito.logger import set_log_level
from devito import Eq, Function, TimeFunction, Dimension, Operator, clear_cache, first_derivative
from PySource import PointSource, Receiver
from PyModel import Model
from checkpoint import DevitoCheckpoint, CheckpointOperator
from pyrevolve import Revolver

def forward_modeling(model, src_coords, wavelet, rec_coords, save=False, space_order=8, nb=40, op_return=False, dt=None):
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create the forward wavefield
    if save is False:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    else:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order, save=nt)

    # Set up PDE and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil = solve(eqn, u.forward)[0]
    expression = [Eq(u.forward, stencil)]

    # Source symbol with input wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Data is sampled at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=u, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += src_term + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Forward%s" % randint(1e5))
    if op_return is False:
        op(dt=dt)
        return rec.data, u
    else:
        return op

def adjoint_modeling(model, src_coords, rec_coords, rec_data, space_order=8, nb=40, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create the adjoint wavefield
    v = TimeFunction(name="v", grid=model.grid, time_order=2, space_order=space_order)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Adjoint source is injected at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec.data[:] = rec_data[:]
    adj_src = rec.inject(field=v.backward, offset=model.nbpml, expr=rec * dt**2 / m)

    # Data is sampled at source locations
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    adj_rec = src.interpolate(expr=v, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + adj_rec
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Backward%s" % randint(1e5))
    op(dt=dt)

    return src.data

def forward_born(model, src_coords, wavelet, rec_coords, space_order=8, nb=40, isic=False, dt=None):
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, dm, damp = model.m, model.dm, model.damp

    # Create the forward and linearized wavefield
    u = TimeFunction(name="u", grid=model.grid, time_order=2, space_order=space_order)
    du = TimeFunction(name="du", grid=model.grid, time_order=2, space_order=space_order)
    if len(model.shape) == 2:
        x,y = u.space_dimensions
    else:
        x,y,z = u.space_dimensions

    # Set up PDEs and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil1 = solve(eqn, u.forward)[0]
    if isic is not True:
        eqn_lin = m * du.dt2 - du.laplace + damp * du.dt + dm * u.dt2   # born modeling
    else:
        du_aux = sum([first_derivative(first_derivative(u, dim=d, order=space_order//2) * dm, order=space_order//2, dim=d) for d in u.space_dimensions])
        eqn_lin = m * du.dt2 - du.laplace + damp * du.dt + (dm * u.dt2 * m - du_aux)

    if isic is not True:
        stencil2 = solve(eqn_lin, du.forward)[0]
    else:
        stencil2 = solve(eqn_lin, du.forward, simplify=False, rational=False)[0]

    expression_u = [Eq(u.forward, stencil1)]
    expression_du = [Eq(du.forward, stencil2)]

    # Define source symbol with wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Define receiver symbol
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=du, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression = expression_u + src_term + expression_du + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Born%s" % randint(1e5))
    op(dt=dt)

    return rec.data

def adjoint_born(model, rec_coords, rec_data, u=None, op_forward=None, is_residual=False, space_order=8, nb=40, isic=False, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create adjoint wavefield and gradient
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    gradient = Function(name='gradient', grid=model.grid)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Data at receiver locations as adjoint source
    rec_g = Receiver(name='rec_g', grid=model.grid, ntime=nt, coordinates=rec_coords)
    if op_forward is None:
        rec_g.data[:] = rec_data[:]
    adj_src = rec_g.inject(field=v.backward, offset=model.nbpml, expr=rec_g * dt**2 / m)

    # Gradient update
    if u is None:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)

    if isic is not True:
        gradient_update = [Eq(gradient, gradient - u * v.dt2)]  # zero-lag cross-correlation imaging condition
    else:
        # linearized inverse scattering imaging condition (Op't Root et al. 2010; Whitmore and Crawley 2012)
        if len(model.shape) == 2:
            gradient_update = [Eq(gradient, gradient - (u * v.dt2 * m + u.dx * v.dx + u.dy * v.dy))]
        else:
            gradient_update = [Eq(gradient, gradient - (u * v.dt2 * m + u.dx * v.dx + u.dy * v.dy + u.dz * v.dz))]

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + gradient_update
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Gradient%s" % randint(1e5))

    # Optimal checkpointing
    if op_forward is not None:
        rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
        cp = DevitoCheckpoint([u])
        n_checkpoints = None
        wrap_fw = CheckpointOperator(op_forward, u=u, m=model.m.data, rec=rec, dt=dt)
        wrap_rev = CheckpointOperator(op, u=u, v=v, m=model.m.data, rec_g=rec_g, dt=dt)

        # Run forward
        wrp = Revolver(cp, wrap_fw, wrap_rev, n_checkpoints, nt-2)
        wrp.apply_forward()

        # Residual and gradient
        if is_residual is True:  # input data is already the residual
            rec_g.data[:] = rec_data[:]
        else:
            rec_g.data[:] = rec.data[:] - rec_data[:]   # input is observed data
            fval = .5*np.linalg.norm(rec_g.data[:])**2
        wrp.apply_reverse()
    else:
        op(dt=dt)
    clear_cache()

    if op_forward is not None and is_residual is not True:
        return fval, gradient.data
    else:
        return gradient.data

########################################################################################################################

def forward_freq_modeling(model, src_coords, wavelet, rec_coords, freq, space_order=8, nb=40, dt=None):
    # Forward modeling with on-the-fly DFT of forward wavefields
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp
    freq_dim = Dimension(name='freq_dim')
    time = model.grid.time_dim

    # Create wavefields
    nfreq = freq.shape[0]
    u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(freq_dim,), shape=(nfreq,))
    f.data[:] = freq[:]
    ufr = Function(name='ufr', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)
    ufi = Function(name='ufi', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)

    # Set up PDE and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil = solve(eqn, u.forward)[0]
    expression = [Eq(u.forward, stencil)]
    expression += [Eq(ufr, ufr + u*cos(2*np.pi*f*time*dt))]
    expression += [Eq(ufi, ufi - u*sin(2*np.pi*f*time*dt))]

    # Source symbol with input wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Data is sampled at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=u, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += src_term + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Forward%s" % randint(1e5))
    op(dt=dt)

    return rec.data, ufr, ufi

def adjoint_freq_born(model, rec_coords, rec_data, freq, ufr, ufi, space_order=8, nb=40, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp
    nfreq = ufr.shape[0]
    time = model.grid.time_dim

    # Create the forward and adjoint wavefield
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(ufr.indices[0],), shape=(nfreq,))
    f.data[:] = freq[:]
    gradient = Function(name="gradient", grid=model.grid)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Data at receiver locations as adjoint source
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec.data[:] = rec_data[:]
    adj_src = rec.inject(field=v.backward, offset=model.nbpml, expr=rec * dt**2 / m)

    # Gradient update
    gradient_update = [Eq(gradient, gradient + (2*np.pi*f)**2/nt*(ufr*cos(2*np.pi*f*time*dt) - ufi*sin(2*np.pi*f*time*dt))*v)]

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + gradient_update
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Gradient%s" % randint(1e5))
    op(dt=dt)
    clear_cache()

    return gradient.data


import datetime

year, month, day = map(int, input().split(" "))

date1 = datetime.date(year, month, day)

days = int(input())

date2 = date1 + datetime.timedelta(days)

print("{} {} {}".format(date2.year, date2.month, date2.day))


#!/usr/bin/env python
# coding: utf-8

###############################################################################
# Link of our dataset:
# http://www2.isprs.org/commissions/comm3/wg4/semantic-labeling.html
##############################################################################

from PIL import Image
import numpy as np
import os.path
import argparse

parser = argparse.ArgumentParser('image processing')
parser.add_argument('--fold_A', dest='fold_A', help='input directory for images', type=str, default='./datasets/images')
parser.add_argument('--fold_B', dest='fold_B', help='input directory for labels', type=str, default='./datasets/labels')
parser.add_argument('--fold_AB', dest='fold_AB', help='output directory', type=str, default='./datasets/satellite/')
parser.add_argument('--num_imgs', dest='num_imgs', help='number of images', type=int, default=1000000)
args = parser.parse_args()

# define the funtion of combining image
def image_compose():
    to_image = Image.new('RGB', (IMAGE_COLUMN * IMAGE_SIZE, IMAGE_ROW * IMAGE_SIZE)) #create a new image

    IMAGE_SAVE_PATH_TRAIN = os.path.join(IMAGE_SAVE_PATH, 'train')
    if not os.path.exists(IMAGE_SAVE_PATH_TRAIN):
        os.makedirs(IMAGE_SAVE_PATH_TRAIN)

    IMAGE_SAVE_PATH_TEST = os.path.join(IMAGE_SAVE_PATH, 'test')
    if not os.path.exists(IMAGE_SAVE_PATH_TEST):
        os.makedirs(IMAGE_SAVE_PATH_TEST)

    num_train = 0
    num_test = 0
    for name in image_names:
            from_image_a_row = Image.open(IMAGES_PATH_1 + name)
            from_image_b_row = Image.open(IMAGES_PATH_2 + name)

            from_image_width,from_image_height = from_image_a_row.size

            for i in range(int(from_image_width/IMAGE_SIZE)):
                for j in range(int(from_image_height / IMAGE_SIZE)):

                    from_image_a = from_image_a_row.crop((i*IMAGE_SIZE,j*IMAGE_SIZE,(i+1)*IMAGE_SIZE,(j+1)*IMAGE_SIZE))
                    from_image_b = from_image_b_row.crop((i*IMAGE_SIZE,j*IMAGE_SIZE,(i+1)*IMAGE_SIZE,(j+1)*IMAGE_SIZE))

                    to_image.paste(from_image_a, (0 * IMAGE_SIZE, 0 * IMAGE_SIZE))
                    to_image.paste(from_image_b, (1 * IMAGE_SIZE, 0 * IMAGE_SIZE))

                    if np.random.random() < 0.8:
                        to_image.save(os.path.join(IMAGE_SAVE_PATH_TRAIN, str(num_train+1)+'.jpg'),"JPEG") # save the new image as train sample
                        num_train = num_train + 1

                    else:

                        to_image.save(os.path.join(IMAGE_SAVE_PATH_TEST, str(num_test + 1) + '.jpg'), "JPEG")  # save the new image as test sample
                        num_test = num_test + 1

# convert the semantic image and ground_truth image to one paired image
# attention: the name of the paired semantic image and ground_truth image should be the same

# dataset of Vaihingen
IMAGES_PATH_1 = args.fold_A  # ground truth immage, path should be changed
IMAGES_PATH_2 = args.fold_B  # semantic image , path should be changed
IMAGES_FORMAT = ['.tif']
IMAGE_SIZE = 800
IMAGE_ROW = 1
IMAGE_COLUMN = 2
IMAGE_SAVE_PATH = args.fold_AB

# get all the name in the imageset
image_names = [name for name in os.listdir(IMAGES_PATH_2) for item in IMAGES_FORMAT if
               os.path.splitext(name)[1] == item]

image_compose() # call the function to get the aligned image


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.shortcuts import render, HttpResponse, redirect
from .models import User
from django.contrib import messages
# Create your views here.
def index(request):
    return render(request, 'login/index.html')

def register(request):
    if request.method == 'POST':
        newUser = User.objects.add_user(request.POST)
        if newUser['status']:
            request.session["current_user"] = {
                'id': newUser['user'].id,
            }
            return redirect('wishlist:index')

        else:
            for error in newUser['errors']:
                messages.error(request, error)
            return redirect('user:index')

    else:
        return redirect('user:index')

def login(request):
    if request.method == 'POST':
        user = User.objects.check_user(request.POST)
        if user['status']:
            request.session["current_user"] = {
                'id': user['user'].id,
            }
            return redirect('wishlist:index')
        else:
            messages.error(request, user['errors'])
            return redirect('user:index')

def logout(request):
    request.session.clear()
    return redirect('user:index')


# Copyright (c) 2019 Vlad Balmos <vladbalmos@yahoo.com>
# Author: Vlad Balmos <vladbalmos@yahoo.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import urwid

class BaseWidgetsFactory:
    def __init__(self, state_machine):
        self._state_machine = state_machine

        # Contains signal name, handler and parameters for each created widget
        # Used by the destructor to disconnect any event handlers
        self._signal_handlers = []

        # Widgets cache
        self._widgets = {}
        self._factory_methods = self._get_factory_methods()

    def _get_factory_methods(self):
        raise NotImplementedError()

    def __del__(self):
        while True:
            try:
                obj, name, callback, kwargs = self._signal_handlers.pop()
                urwid.disconnect_signal(obj, name, callback, **kwargs)
            except IndexError:
                break

    def change_fsm_state(self, input, *args, **kwargs):
        self._state_machine.change_state(input, *args, **kwargs)

    def create(self, name, *args, **kwargs):
        cache_widget = True
        if 'cache' in kwargs:
            cache_widget = kwargs['cache']
            del kwargs['cache']

        if name not in self._widgets or cache_widget is False:
            factory_method = self._factory_methods[name]
            widget = factory_method(*args, **kwargs)

            if cache_widget is False:
                return widget
            self._widgets[name] = widget
        else:
            # Refresh the widget if it was cached
            widget = self._widgets[name]
            refresh_method = getattr(widget, 'refresh', None)
            if callable(refresh_method):
                refresh_method(*args, **kwargs)
        return widget

    def _connect_signal(self, obj, name, callback, **kwargs):
        key = urwid.connect_signal(obj, name, callback, **kwargs)
        self._signal_handlers.append((obj, name, callback, kwargs))


import time
import numpy as np
from module_velocity_plotter import ModuleVelocityPlotter
from ros2_state_loader import get_states

# data = get_states("data/diffyjr_2022-06-08-23-52-28_0.json")
# data = get_states("data/diffyjr_2022-06-09-22-42-13_0.json")  # perimeter
# data = get_states("data/diffyjr_2022-06-09-23-07-18_0.json")  # reference recorded (wrong units)
# data = get_states("data/diffyjr_2022-06-09-23-41-58_0.json")  # figure 8's

# no angle controller
# data = get_states("data/diffyjr_2022-06-09-23-55-06_0.json")  # 10.5 V
# data = get_states("data/diffyjr_2022-06-11-00-38-10_0.json")  # 10.5 V
# data = get_states("data/diffyjr_2022-06-11-00-44-19_0.json")  # 11.0 V
# data = get_states("data/diffyjr_2022-06-11-00-46-46_0.json")  # 12.0 V

# no voltage recorded
# data = get_states("data/diffyjr_2022-06-11-12-54-00_0.json")
# data = get_states("data/diffyjr_2022-06-11-14-31-53_0.json")
# data = get_states("data/diffyjr_2022-06-11-14-34-45_0.json")
data = get_states("data/diffyjr_2022-06-11-14-39-19_0.json")

playback_real_time = False
# playback_real_time = True
# plot_rate = 1.0 / 10.0
plot_rate = 0.0

plotter = ModuleVelocityPlotter(4.76, 47.37, 2 * np.pi, history=None, playback_real_time=playback_real_time)
plotter.draw_perimeter_from_file("data/perimeter.csv")
plotter.init()

prev_draw_time = time.time()

sim_start_t = data[0][0]["time"]
real_start_t = time.time()

length = min([len(data[module_index]) for module_index in range(len(data))])

for index in range(length):
    current_time = time.time()
    pause_start_t = current_time
    while plotter.is_paused:
        plotter.pause()
        current_time = time.time()
    real_start_t += current_time - pause_start_t
    sim_time = data[0][index]["time"]
    real_time = current_time
    sim_duration = sim_time - sim_start_t
    real_duration = real_time - real_start_t
    for module_index in range(len(data)):
        module_data = data[module_index][index]
        plotter.draw(sim_duration, module_index, module_data)

    if playback_real_time:
        if sim_duration >= real_duration:  # if simulation time has caught up to real time, spend some time drawing
            plotter.pause()
    else:
        if plot_rate > 0.0 and time.time() - prev_draw_time > plot_rate:
            plotter.pause()
            prev_draw_time = time.time()

print("Bag finished")
plotter.stop()


#!/usr/bin/python3

''' from tutorial:
https://www.tensorflow.org/tutorials/mnist/pros/
'''

import tensorflow as tf

tf.set_random_seed(0)

BATCH_SIZE = 100
NUM_TRAINING_ITERATIONS = 10000

def get_data():
  from tensorflow.examples.tutorials.mnist import input_data
  mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
  return mnist

def main():
  mnist = get_data()
  sess = tf.InteractiveSession()

  # placeholders for x and y
  x = tf.placeholder(tf.float32, shape=[None, 784])
  y_ = tf.placeholder(tf.float32, shape=[None, 10])

  # Variables for weights and bias
  W1 = tf.Variable(tf.truncated_normal([784,200], stddev=0.1))
  W2 = tf.Variable(tf.truncated_normal([200,100], stddev=0.1))
  W3 = tf.Variable(tf.truncated_normal([100,60], stddev=0.1))
  W4 = tf.Variable(tf.truncated_normal([60,30], stddev=0.1))
  W5 = tf.Variable(tf.truncated_normal([30,10], stddev=0.1))

  B1 = tf.Variable(tf.zeros([200]))
  B2 = tf.Variable(tf.zeros([100]))
  B3 = tf.Variable(tf.zeros([60]))
  B4 = tf.Variable(tf.zeros([30]))
  B5 = tf.Variable(tf.zeros([10]))

  # regression model
  y1 = tf.nn.sigmoid(tf.matmul(x,W1) + B1)
  y2 = tf.nn.sigmoid(tf.matmul(y1,W2) + B2)
  y3 = tf.nn.sigmoid(tf.matmul(y2,W3) + B3)
  y4 = tf.nn.sigmoid(tf.matmul(y3,W4) + B4)
  y_logits = tf.matmul(y4,W5) + B5
  y = tf.nn.softmax(y_logits)

  # cost function -- cross-entropy between target
  # and softmax activation function
  cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_logits, labels=y_))*100.0

  # training via gradient descent
  train_step = tf.train.GradientDescentOptimizer(0.003).minimize(cross_entropy)

  # initialize variables for use within session
  sess.run(tf.global_variables_initializer())

  # run training iterations
  for i in range(NUM_TRAINING_ITERATIONS):
    batch = mnist.train.next_batch(BATCH_SIZE)
    # print('batch[0] shape: %s, batch[1] shape: %s' % (batch[0].shape, batch[1].shape))
    train_step.run(feed_dict={ x: batch[0], y_: batch[1] })

  # get boolean array of true/false for equality of predicted (y) equals actual (y_)
  correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))

  # cast to float and get mean of correct predictions
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  # accuracy should be about 92 %
  print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))

if __name__ == '__main__':
  main()


"""ROSARIO VALERO MIRANDA - 1º DAW - PRACTICA6 - EJERCICIO 2
Escribe un programa que permita crear una lista de palabras y que, a continuación,
pida una palabra y diga cuántas veces aparece esa palabra en la lista
"""

print("Dime cuántas palabras tiene la lista: ")
num=int(input())

if num <1:
    print("No posible!")
else:
    lista=[]
    for i in range(num):
        print("Dígame la palabra", str(i + 1) + ": ")
        palabra = input()
        lista += [palabra]
    print("La lista creada es:", lista)

    print("Dime la palabra: ")
    buscar=input()
    cont = 0
    for i in lista:
        if i == buscar:
            cont+= 1;
    if cont == 0:
        print("La palabra '" + buscar + "' no aparece en la lista.")
    elif cont == 1:
        print("La palabra '" + buscar + "' aparece una vez en la lista.")
    else:
        print("La palabra '" + buscar + "' aparece", cont, "veces en la lista.")


import argparse
import os
import pickle
import random
from collections import defaultdict

import networkx as nx

def keep_distinct_node(g):
    '''
    Keep nodes with distinct neighborhood in the graph
    https://github.com/ylaboratory/ETNA/blob/master/src/load_data.py
    '''
    A = nx.adjacency_matrix(g).toarray()
    a_dict = defaultdict(set)
    node_list = list(g.nodes())
    for i in range(A.shape[0]):
        a_dict[tuple(A[i])].add(node_list[i])
    distinct_nodes = set()
    for s in a_dict.values():
        if len(s) < 2:
            distinct_nodes.add(list(s)[0])
    removed_nodes = set(g.nodes()).difference(distinct_nodes)
    g.remove_nodes_from(removed_nodes)

def map_nodes(g):
    '''
    map from node name to node index & node index to node name
    '''
    map_n2i, map_i2n = dict({}), dict({})
    for i,j in enumerate(g.nodes()):
        map_n2i[j] = i
        map_i2n[i] = j
    return map_n2i, map_i2n

def load_ppi_graph(file):
    '''
    load and preprocess ppi graph
    modified from https://github.com/ylaboratory/ETNA/blob/master/src/load_data.py
    '''

    nx_g = nx.read_edgelist(file, create_using=nx.DiGraph())
    for edge in nx_g.edges():
        nx_g[edge[0]][edge[1]]['weight'] = 1 # weight for node2vec
    print(f'Read as directed: {len(nx_g)} nodes, {nx_g.size()} edges')

    nx_g.remove_edges_from(nx.selfloop_edges(nx_g))
    nx_g.remove_nodes_from(list(nx.isolates(nx_g)))
    print(f'Self loop and isolates removed: {len(nx_g)} nodes, {nx_g.size()} edges')

    nx_g = nx_g.to_undirected()
    print(f'Convert to undirected: {len(nx_g)} nodes, {nx_g.size()} edges')

    node_len = 0
    while node_len != len(nx_g.nodes()):
        node_len = len(nx_g.nodes())
        keep_distinct_node(nx_g)
        nx_g.remove_nodes_from(list(nx.isolates(nx_g)))
    print(f'Keep distinct nodes: {len(nx_g)} nodes, {nx_g.size()} edges')

    return nx_g

def load_both_ppi(species1, species2, parent_dir='data'):
    '''
    load data for GraNA
    '''

    f_ppi1 = os.path.join(parent_dir, 'physical_interaction', species1+'_physical_pairs.txt')
    f_ppi2 = os.path.join(parent_dir, 'physical_interaction', species2+'_physical_pairs.txt')

    nx_g1 = load_ppi_graph(f_ppi1)
    nx_g2 = load_ppi_graph(f_ppi2)

    nx_g12 = nx.union(nx_g1, nx_g2, rename=(species1, species2))
    map_n2i, map_i2n = map_nodes(nx_g12)

    nx_g12_relabeled = nx.relabel_nodes(nx_g12, map_n2i)

    print(f'Integrated graph: {len(nx_g12_relabeled.nodes())} nodes, {nx_g12_relabeled.size()} edges')

    # save edgelist of the integrated graph
    nx.write_edgelist(nx_g12_relabeled, os.path.join(parent_dir, 'physical_interaction/'+species1+'_'+species2+'.edgelist'))

    # nx.write_adjlist(nx_g12_relabeled, os.path.join(parent_dir, 'physical_interaction/'+species1+'_'+species2+'.adjlist'))

    pickle.dump(map_n2i, open(os.path.join(parent_dir, 'emb/'+species1+'_'+species2+'map_n2i.pkl'), 'wb'))
    pickle.dump(map_i2n, open(os.path.join(parent_dir, 'emb/'+species1+'_'+species2+'map_i2n.pkl'), 'wb'))

    return nx_g1, nx_g2, nx_g12_relabeled

def load_orthologs(species1='sce', species2='spo', parent_dir='data'):
    '''
    load ortholog files
    modified from https://github.com/ylaboratory/ETNA/blob/master/src/load_data.py
    '''

    orth_file = os.path.join(parent_dir, 'ortholog/'+species1+'_'+species2+'_orthomcl.txt')
    with open(orth_file, 'r') as f:
        data = f.readlines()[1:]

    new_file = os.path.join(parent_dir,'ortholog/'+species1+'_'+species2+'.edgelist')
    with open(new_file, 'w') as f:
        for d in data:
            p1, p2, weight = d.strip().split()
            f.write(' '.join([species1+p1, species2+p2, '{\'weight\': '+weight+'}'])+'\n')

    nx_g_orth = nx.read_edgelist(os.path.join(parent_dir, 'ortholog/'+species1+'_'+species2+'.edgelist'))

    map_n2i = pickle.load(open(os.path.join(parent_dir, 'emb/'+species1+'_'+species2+'map_n2i.pkl'), 'rb'))

    removed_nodes = set(nx_g_orth.nodes()).difference(map_n2i.keys())
    nx_g_orth.remove_nodes_from(removed_nodes)

    nx_g_orth = nx.relabel_nodes(nx_g_orth, map_n2i)

    nx.write_edgelist(nx_g_orth, os.path.join(parent_dir, 'ortholog/'+species1+'_'+species2+'_relabeled.edgelist'))

    return nx_g_orth

def load_ontology(species1='sce', species2='spo', parent_dir='data'):
    '''
    load ontology file
    modified from https://github.com/ylaboratory/ETNA/blob/master/src/load_data.py
    '''
    onto_file = os.path.join(parent_dir, 'sce_spo/'+species1+'_'+species2+'_ontology_pairs_expert.txt')
    with open(onto_file, 'r') as f:
        data = f.readlines()

    new_file = os.path.join(parent_dir, 'sce_spo/'+species1+'_'+species2+'_ontology.edgelist')
    with open(new_file, 'w') as f:
        for d in data:
            p1, p2 = d.strip().split()
            f.write(' '.join([species1+p1, species2+p2])+'\n')

    nx_g_onto = nx.read_edgelist(os.path.join(parent_dir, 'sce_spo/'+species1+'_'+species2+'_ontology.edgelist'))

    map_n2i = pickle.load(open(os.path.join(parent_dir, 'emb/'+species1+'_'+species2+'map_n2i.pkl'), 'rb'))

    removed_nodes = set(nx_g_onto.nodes()).difference(map_n2i.keys())
    nx_g_onto.remove_nodes_from(removed_nodes)

    nx_g_onto = nx.relabel_nodes(nx_g_onto, map_n2i)

    nx.write_edgelist(nx_g_onto, os.path.join(parent_dir, 'sce_spo/'+species1+'_'+species2+'_relabeled.edgelist'))

def split_data_final(species1, species2, nx_g1, nx_g2, g12, nx_g_orth, seed, ratio=0.7, parent_dir='data'):
    '''
    split data for training and testing
    '''

    total_number = len(g12.nodes)
    split_number = len(nx_g1.nodes)

    map_n2i = pickle.load(open(os.path.join(parent_dir, 'emb/'+species1+'_'+species2+'map_n2i.pkl'), 'rb'))

    num_training_1 = int(ratio * len(nx_g1.nodes))
    num_training_2 = int(ratio * len(nx_g2.nodes))

    valid_ratio = 0.8 - ratio
    num_valid_1 = int(valid_ratio * len(nx_g1.nodes))
    num_valid_2 = int(valid_ratio * len(nx_g2.nodes))

    # sample training samples
    random.seed(seed)
    train_nodes_g1 = random.sample(range(split_number), num_training_1)
    random.seed(seed)
    train_nodes_g2 = random.sample(range(split_number, total_number), num_training_2)

    # sample valid samples
    random.seed(seed)
    valid_nodes_g1 = random.sample(set(range(split_number)).difference(set(train_nodes_g1)), num_valid_1)
    random.seed(seed)
    valid_nodes_g2 = random.sample(set(range(split_number, total_number)).difference(set(train_nodes_g2)), num_valid_2)

    # acquire test samples
    test_nodes_g1 = list(set(range(split_number)).difference(train_nodes_g1).difference(valid_nodes_g1))
    test_nodes_g2 = list(set(range(split_number, total_number)).difference(train_nodes_g2).difference(valid_nodes_g2))

    train_nodes = list(sorted([n for n in train_nodes_g1 + train_nodes_g2]))
    valid_nodes = list(sorted([n for n in valid_nodes_g1 + valid_nodes_g2]))
    test_nodes = list(sorted([n for n in test_nodes_g1 + test_nodes_g2]))

    assert len(train_nodes) + len(valid_nodes) + len(test_nodes) == total_number
    assert len(train_nodes_g1) + len(valid_nodes_g1) + len(test_nodes_g1) == split_number

    train_mask, valid_mask, test_mask = train_nodes, valid_nodes, test_nodes

    print(len(train_mask), len(valid_mask), len(test_mask))

    pickle.dump([split_number, total_number, train_mask, valid_mask, test_mask], open(os.path.join(parent_dir, 'split/'+species1+'_'+species2+'_seed'+str(seed)+'.pkl'), 'wb'))

def preprocess(species1='sce', species2='spo'):

    print('Loading both PPI...')
    g1, g2, g12 = load_both_ppi(species1, species2)

    print('Loading orthologs...')
    g_orth = load_orthologs(species1, species2)

    print('Loading ontology...')
    load_ontology(species1, species2)

    print('Splitting data...')
    # split_data(species1, species2, total_number)
    for seed in range(5):
        split_data_final(species1, species2, g1, g2, g12, g_orth, seed)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--first', '-p1', default='sce', type=str, help='first ppi network')
    parser.add_argument('--second', '-p2', default='spo', type=str, help='second ppi network')
    args = parser.parse_args()
    preprocess(args.first, args.second)