"""
Multilayer perceptron for cycloid and Lorenz attractor.

"Eugene Morozov"<Eugene ~at~ HiEugene.com>
"""

import matplotlib.pyplot as plt
import torch.nn
import torch.nn as nn
import torch.optim as optim
import scipy.linalg
from scipy.integrate import odeint
from util import *

v = 1
epoch_sec = 1 # int(time.time())
start_time, device = preamble(epoch_sec)

rnd = 0.4
do_dropout = False
dropout_prob_zero = 0.5
do_batch_normalization = False
do_regularization = False
L2_lambda = 0.01
do_early_stopping = False
early_stop_patience = 2
info_step = 50
max_epoch = 75
batch_size = 20
train_data_batches = 80

if v:
  plt.ion()
  fig_grad = plt.figure(); ax_grad = fig_grad.gca()
  plt.show(); plt.tight_layout()
  move_plot(fig_grad, 0, 0, 1000, 500); fig_grad.canvas.flush_events()
  i_grad = 0
  def fig_grad_init():
    global i_grad
    ax_grad.cla()
    ax_grad.grid()
    ax_grad.set_title(f"grad norm-2")
    i_grad = 0

  fig_w, ax_w = plt.subplots(3, 4)
  fig_w.show(); plt.tight_layout()
  move_plot(fig_w, 1000, 0, 1600, 1000); fig_w.canvas.flush_events()
  def fig_w_init():
    pass

  fig_loss = plt.figure(); ax_loss = fig_loss.gca()
  plt.show(); plt.tight_layout()
  move_plot(fig_loss, 0, 500, 1000, 800); fig_loss.canvas.flush_events()
  i_loss = 0
  def fig_loss_init(lr):
    global i_loss
    ax_loss.cla()
    ax_loss.grid()
    ax_loss.set_title(f"loss; lr={lr}")
    i_loss = 0

  fig_ver = plt.figure()
  ax_ver = fig_ver.gca()
  # ax_ver = fig_ver.add_subplot(projection="3d")
  fig_ver.show(); plt.tight_layout()
  move_plot(fig_ver, 600, 200, 1000, 800); fig_ver.canvas.flush_events()
  def fig_ver_init():
    ax_ver.cla()
    ax_ver.grid()
    ax_ver.set_title(f"cycloid; rnd={rnd}")
    # ax_ver.set_title(f"Lorenz attractor; rnd={rnd}")

  fig_lr = plt.figure(); ax_lr = fig_lr.gca()
  fig_lr.show() #; plt.tight_layout()
  move_plot(fig_lr, 1200, 200, 1000, 800); fig_lr.canvas.flush_events()
  def fig_lr_init():
    ax_lr.cla()
    ax_lr.grid()
    ax_lr.set_xlabel("learning rate")
    ax_lr.set_ylabel("loss")
  fig_lr_init()

def cycloid(t, rnd, r=0.5):
  x = r*(t-np.sin(t))
  y = r*(1-np.cos(t))
  if rnd > 0:
    noise = np.random.uniform(low=-rnd, high=rnd+1e-8, size=len(x))
    y += noise
  return x, y

def cycloid_normalize_x(x):
  # already in [0, 1]
  return x

### Lorenz attractor ###
Lorenz_rho   = 28.0
Lorenz_sigma = 10.0
Lorenz_beta  = 8.0 / 3.0

def Lorenz_derivatives(state, t):
  x, y, z = state # unpack the state vector
  return Lorenz_sigma*(y-x), x*(Lorenz_rho-z)-y, x*y-Lorenz_beta*z

def Lorenz(t, rnd, x0=np.random.uniform(low=-10, high=10+1e-8, size=3)):
  xx = odeint(Lorenz_derivatives, x0, t)
  xx /= 20
  if rnd > 0:
    noise = np.random.uniform(low=-rnd, high=rnd+1e-8, size=xx.shape[0])
    xx[:,2] += noise
  return xx[:,[0,1]], xx[:,2]

def Lorenz_normalize_x(x):
  return x

if v and False:
  t_num = 600
  delta_t = 0.05
  t_start = 100*np.random.rand()
  t_end = t_start + t_num*delta_t
  t = np.linspace(t_start, t_end, t_num)
  x, y = Lorenz(t, rnd=0)
  x = Lorenz_normalize_x(x)
  fig = plt.figure()
  ax = fig.add_subplot(projection="3d")
  ax.plot(x[:,0], x[:,1], y, "-o")
  ax.set_xlabel("x")
  ax.set_ylabel("y")
  ax.set_zlabel("z")
  plt.show()

class MLP_NN(nn.Module):
  def __init__(self):
    super().__init__()
    self.l1 = nn.Linear(in_features=1, out_features=64, bias=True) # 2 for Lorenz
    self.l2 = nn.Linear(64, 32)
    if do_dropout:
      self.l2a = nn.Dropout(p=dropout_prob_zero)
    if do_batch_normalization:
      self.l2a = nn.BatchNorm1d(32) # batch normalization (especially for convolutional networks and networks with sigmoidal nonlinearities) (allows dropout to be omitted)
    self.l3 = nn.Linear(32, 1)
    # self.leakyReLU = nn.LeakyReLU(0.01)
    # self.softmax = nn.Softmax(dim=1)

  def forward(self, x):
    x = torch.flatten(x, 1) # flatten all dimensions except batch (which is 1st dimension of the tensor)
    x = torch.relu(self.l1(x))
    # x = self.leakyReLU(self.l1(x))
    # x = self.softmax(self.l1(x))
    x = self.l2(x)
    if do_dropout:
      x = self.l2a(x)
    if do_batch_normalization:
      x = self.l2a(x) # we add the BN transform immediately before the nonlinearity
    x = torch.relu(x)
    # x = self.leakyReLU(x)
    # x = self.softmax(x)
    x = self.l3(x)
    return x

# verify gradient by using complex numbers trick
def verify_grad():
  # note: not all methods in torch 1.10 are supported for complex numbers
  try:
    m = 1
    epsilon = 1e-20
    # true mse = mean(abs(h_calc-h_true).^2); % where h is m x 1 matrix of complex numbers
    def loss_function(a, b):
      # if using only torch functions then fine (otherwise need a custom class)
      loss = torch.mean((a-b)**2)
      return loss
    # loss_function = nn.L1Loss()

    t = np.random.uniform(low=0, high=2*np.pi+1e-8, size=m*1)
    x, y = cycloid(t, rnd=rnd)
    x = cycloid_normalize_x(x)
    xc = np.zeros(m, dtype=np.csingle)
    xc.real = x
    yc = np.zeros(m, dtype=np.csingle)
    yc.real = y
    inputs, targets = torch.from_numpy(xc), torch.from_numpy(yc)
    inputs = torch.reshape(inputs, (m,1))
    targets = torch.reshape(targets, (m,1))

    l1 = nn.Linear(1, 64, dtype=torch.complex64)
    l1.weight.data.fill_(0.01)
    l1.bias.data.fill_(0.01)
    l2 = nn.Linear(64, 32, dtype=torch.complex64)
    l2.weight.data.fill_(0.01)
    l2.bias.data.fill_(0.01)
    l3 = nn.Linear(32, 1, dtype=torch.complex64)
    l3.weight.data.fill_(0.01)
    l3.bias.data.fill_(0.01)

    l1.weight.data[0] += 0 + 1j*epsilon

    out = torch.tanh(l1(inputs))
    out = torch.tanh(l2(out))
    out = l3(out)
    loss = loss_function(out, targets)

    deriv = loss.data.numpy().imag / epsilon

    loss.backward()
    auto_deriv = l1.weight.grad.data.numpy()[0].real[0]
    if np.abs(deriv - auto_deriv) > epsilon:
      warn(f"derivatives differ: delta = {np.abs(deriv - auto_deriv)}")
  except Exception as e:
    warn(f"exception: {e}")

def print_weight(model, x):
  if v:
    for i in range(3):
      for j in range(4):
        ax_w[i][j].cla()
        ax_w[i][j].grid()
    ax_w[0][0].hist(model.l1.weight.data.cpu().numpy().flatten()); ax_w[0][1].hist(model.l1.bias.data.cpu().numpy().flatten())
    ax_w[1][0].hist(model.l2.weight.data.cpu().numpy().flatten()); ax_w[1][1].hist(model.l2.bias.data.cpu().numpy().flatten())
    ax_w[2][0].hist(model.l3.weight.data.cpu().numpy().flatten()); ax_w[2][1].hist(model.l3.bias.data.cpu().numpy().flatten())
    x = model.l1(x)
    ax_w[0][2].hist(x.data.cpu().numpy().flatten())
    x = torch.relu(x)
    ax_w[0][3].hist(x.data.cpu().numpy().flatten())
    x = model.l2(x)
    ax_w[1][2].hist(x.data.cpu().numpy().flatten())
    x = torch.relu(x)
    ax_w[1][3].hist(x.data.cpu().numpy().flatten())
    x = model.l3(x)
    ax_w[2][2].hist(x.data.cpu().numpy().flatten())
    for i in range(3):
      if i == 0:   suffix = "st"
      elif i == 1: suffix = "nd"
      elif i == 2: suffix = "rd"
      else:        suffix = "th"
      ax_w[i][0].set_title(f"{i+1}{suffix} layer weights")
      ax_w[i][1].set_title(f"{i+1}{suffix} layer biases")
      ax_w[i][2].set_title(f"{i+1}{suffix} layer preactivation")
      ax_w[i][3].set_title(f"{i+1}{suffix} layer activation")

def print_grad(model):
  l1_weight_norm = torch.sqrt(torch.sum(model.l1.weight.grad.mul(model.l1.weight.grad))).data.cpu().numpy()
  l1_bias_norm   = torch.sqrt(torch.sum(model.l1.bias.grad.mul(model.l1.bias.grad))).data.cpu().numpy()
  l2_weight_norm = torch.sqrt(torch.sum(model.l2.weight.grad.mul(model.l2.weight.grad))).data.cpu().numpy()
  l2_bias_norm   = torch.sqrt(torch.sum(model.l2.bias.grad.mul(model.l2.bias.grad))).data.cpu().numpy()
  l3_weight_norm = torch.sqrt(torch.sum(model.l3.weight.grad.mul(model.l3.weight.grad))).data.cpu().numpy()
  l3_bias_norm   = torch.sqrt(torch.sum(model.l3.bias.grad.mul(model.l3.bias.grad))).data.cpu().numpy()
  print(f"l1 grad weight abs max = {model.l1.weight.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l1.weight.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l1.weight.grad.abs(), 1e-10).sum().int() / (model.l1.weight.grad.shape[0]*model.l1.weight.grad.shape[1])):2d}%), l1.weight 2-norm = {l1_weight_norm:0.3f}")
  print(f"l1 grad bias   abs max = {model.l1.bias.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l1.bias.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l1.bias.grad.abs(), 1e-10).sum().int() / model.l1.bias.grad.shape[0]):2d}%), l1.bias   2-norm = {l1_bias_norm:0.3f}")
  print(f"l2 grad weight abs max = {model.l2.weight.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l2.weight.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l2.weight.grad.abs(), 1e-10).sum().int() / (model.l2.weight.grad.shape[0]*model.l2.weight.grad.shape[1])):2d}%), l2.weight 2-norm = {l2_weight_norm:0.3f}")
  print(f"l2 grad bias   abs max = {model.l2.bias.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l2.bias.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l2.bias.grad.abs(), 1e-10).sum().int() / model.l2.bias.grad.shape[0]):2d}%), l2.bias   2-norm = {l2_bias_norm:0.3f}")
  print(f"l3 grad weight abs max = {model.l3.weight.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l3.weight.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l3.weight.grad.abs(), 1e-10).sum().int() / (model.l3.weight.grad.shape[0]*model.l3.weight.grad.shape[1])):2d}%), l3.weight 2-norm = {l3_weight_norm:0.3f}")
  print(f"l3 grad bias   abs max = {model.l3.bias.grad.abs().max().float():0.3f}, 0 # = {torch.le(model.l3.bias.grad.abs(), 1e-10).sum().int():4d} ({int(100*torch.le(model.l3.bias.grad.abs(), 1e-10).sum().int() / model.l3.bias.grad.shape[0]):2d}%), l3.bias   2-norm = {l3_bias_norm:0.3f}")
  # gradient/parameter_value should ~= 1% over a minibatch
  a = torch.abs(model.l1.weight.grad / model.l1.weight).data.cpu().numpy().flatten()
  a1 = (a > 0.01).sum()
  a10 = (a > 0.1).sum()
  if a10 > 0:
    warn(f"L1 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  else:
    print(f"L1 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  a = torch.abs(model.l2.weight.grad / model.l2.weight).data.cpu().numpy().flatten()
  a1 = (a > 0.01).sum()
  a10 = (a > 0.1).sum()
  if a10 > 0:
    warn(f"L2 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  else:
    print(f"L2 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  a = torch.abs(model.l3.weight.grad / model.l3.weight).data.cpu().numpy().flatten()
  a1 = (a > 0.01).sum()
  a10 = (a > 0.1).sum()
  if a10 > 0:
    warn(f"L3 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  else:
    print(f"L3 grad > 1%: {a1}, > 10%: {a10} out of {len(a)}")
  if v:
    # A test that can rule out local minima as the problem is plotting the norm of the gradient over time.
    global i_grad, l1_weight_norm_prev, l1_bias_norm_prev, l2_weight_norm_prev, l2_bias_norm_prev, l3_weight_norm_prev, l3_bias_norm_prev
    if i_grad == 0:
      ax_grad.plot(i_grad, l1_weight_norm, color="blue",  linestyle='-', label="l1 weight norm") # marker='*'
      ax_grad.plot(i_grad, l1_bias_norm,   color=lblue,   linestyle='-', label="l1 bias norm")
      ax_grad.plot(i_grad, l2_weight_norm, color="green", linestyle='-', label="l2 weight norm")
      ax_grad.plot(i_grad, l2_bias_norm,   color=lgreen,  linestyle='-', label="l2 bias norm")
      ax_grad.plot(i_grad, l3_weight_norm, color="red",   linestyle='-', label="l3 weight norm")
      ax_grad.plot(i_grad, l3_bias_norm,   color=lred,    linestyle='-', label="l3 bias norm")
      ax_grad.legend(loc="best", ncol=1, scatterpoints=1, numpoints=1)
    else:
      ax_grad.plot([i_grad-1, i_grad], [l1_weight_norm_prev, l1_weight_norm], color="blue",  linestyle='-', label="l1 weight norm") # marker='*'
      ax_grad.plot([i_grad-1, i_grad], [l1_bias_norm_prev,   l1_bias_norm],   color=lblue,   linestyle='-', label="l1 bias norm")
      ax_grad.plot([i_grad-1, i_grad], [l2_weight_norm_prev, l2_weight_norm], color="green", linestyle='-', label="l2 weight norm")
      ax_grad.plot([i_grad-1, i_grad], [l2_bias_norm_prev,   l2_bias_norm],   color=lgreen,  linestyle='-', label="l2 bias norm")
      ax_grad.plot([i_grad-1, i_grad], [l3_weight_norm_prev, l3_weight_norm], color="red",   linestyle='-', label="l3 weight norm")
      ax_grad.plot([i_grad-1, i_grad], [l3_bias_norm_prev,   l3_bias_norm],   color=lred,    linestyle='-', label="l3 bias norm")
    l1_weight_norm_prev = l1_weight_norm
    l1_bias_norm_prev   = l1_bias_norm
    l2_weight_norm_prev = l2_weight_norm
    l2_bias_norm_prev   = l2_bias_norm
    l3_weight_norm_prev = l3_weight_norm
    l3_bias_norm_prev   = l3_bias_norm
    i_grad += 1
    fig_grad.canvas.flush_events()

def check_hessian(model, loss_function):
  if do_batch_normalization:
    m = 20
  else:
    m = 1
  t = np.random.uniform(low=0, high=2*np.pi+1e-8, size=m)
  t = t.astype(np.float32)
  x, y = cycloid(t, rnd=rnd)
  x = cycloid_normalize_x(x)

  # t_num = m
  # delta_t = 0.05
  # t_start = 100*np.random.rand()
  # t_end = t_start + t_num*delta_t
  # t = np.linspace(t_start, t_end, t_num)
  # t = t.astype(np.float32)
  # x, y = Lorenz(t, rnd=rnd)
  # x = Lorenz_normalize_x(x)
  # x = x.astype(np.float32)
  # y = y.astype(np.float32)

  inputs, targets = torch.from_numpy(x), torch.from_numpy(y)
  inputs = torch.reshape(inputs, (m,1)) # (m,2) for Lorenz
  targets = torch.reshape(targets, (m,1))
  inputs, targets = inputs.to(device), targets.to(device)
  outputs = model(inputs)
  h = torch.autograd.functional.hessian(loss_function, (outputs, targets))
  print(f"Hessian = {h}")
  hm = np.array([[h[0][0].data.cpu().numpy()[0][0][0][0], h[0][1].data.cpu().numpy()[0][0][0][0]], # [0][0][0][0] is for 1 sample
                 [h[1][0].data.cpu().numpy()[0][0][0][0], h[1][1].data.cpu().numpy()[0][0][0][0]]])
  w, _ = scipy.linalg.eig(hm)
  if (w[0].real < -1e-8 and w[1].real > 1e-8) or (w[0].real > 1e-8 and w[1].real < -1e-8):
    warn(f"At a saddle point, the Hessian matrix has both positive and negative eigenvalues: {w}")

def MLP_train(model, loss_function, optimizer):
  start = time.time()
  running_loss_prev = 0.0
  i_loss_valid_prev = 0
  if do_early_stopping:
    validation_loss_prev = 1e8
    early_stop_trigger_times = 0

  def calc():
    nonlocal loss

    outputs = model(inputs)
    loss = loss_function(outputs, targets)

    if do_regularization:
      L2_reg = torch.tensor(0.0).to(device)
      for param in model.parameters():
        L2_reg += torch.pow(param,2).sum()/2
      loss += L2_lambda * L2_reg

    optimizer.zero_grad()
    loss.backward()
    return loss

  for epoch in range(1,max_epoch+1):
    hours, minutes, seconds = sec2hms((max_epoch - epoch) * (time.time() - start) / epoch)
    print(f"starting epoch {epoch}; ETA = {hours:02d}:{minutes:02d}:{seconds:02d}")
    running_loss = 0.0
    train_data_size = train_data_batches*batch_size
    print(f"train_data_size = {train_data_size:,d}")
    for i in range(train_data_size//batch_size):
      t = np.random.uniform(low=0, high=2*np.pi+1e-8, size=batch_size*1)
      t = t.astype(np.float32)
      x, y = cycloid(t, rnd=rnd)
      x = cycloid_normalize_x(x)

      # t_num = batch_size
      # delta_t = 0.05
      # t_start = 100*np.random.rand()
      # t_end = t_start + t_num*delta_t
      # t = np.linspace(t_start, t_end, t_num)
      # t = t.astype(np.float32)
      # x, y = Lorenz(t, rnd=rnd)
      # x = Lorenz_normalize_x(x)
      # x = x.astype(np.float32)
      # y = y.astype(np.float32)

      inputs, targets = torch.from_numpy(x), torch.from_numpy(y)
      inputs = torch.reshape(inputs, (batch_size,1)) # 2 for Lorenz
      targets = torch.reshape(targets, (batch_size,1))
      inputs, targets = inputs.to(device), targets.to(device)

      if optimizer.__repr__().startswith("LBFGS"):
        optimizer.step(calc)
      else:
        loss = calc()
        optimizer.step()

      running_loss += loss.item()
      if i % info_step == info_step-1:
        running_loss = running_loss / info_step / batch_size
        print(f"loss after mini-batch {i+1:5d}: {running_loss:.05f}")
        if v:
          global i_loss
          if i_loss == 0:
            ax_loss.plot(i_loss, running_loss, color="blue", linestyle='-', label="training loss")
          else:
            ax_loss.plot([i_loss-1, i_loss], [running_loss_prev, running_loss], color="blue", linestyle='-') # , label="training loss")
          running_loss_prev = running_loss
          i_loss += 1
          fig_loss.canvas.flush_events()
        running_loss = 0.0
        print_grad(model)
        print_weight(model, inputs)
    if v:
      ax_loss.annotate(f"epoch={epoch}", xy=(i_loss, running_loss_prev), rotation=60)

    if do_early_stopping or v:
      validation_loss, _, _, _, _, _ = MLP_validate(model, loss_function)
      if v:
        if i_loss_valid_prev == 0:
          ax_loss.plot(i_loss, validation_loss, color="red", linestyle='-', label="validation loss")
          ax_loss.legend(loc="best", ncol=1, scatterpoints=1, numpoints=1)
        else:
          ax_loss.plot([i_loss_valid_prev, i_loss], [validation_loss_prev, validation_loss], color="red", linestyle='-', label="validation loss")
        i_loss_valid_prev = i_loss
      if do_early_stopping:
        if validation_loss > validation_loss_prev:
          # or save model to file when < validation_loss_best
          early_stop_trigger_times += 1
          if early_stop_trigger_times >= early_stop_patience:
            print(f"early stopping triggered at epoch={epoch}")
            break
        else:
          early_stop_trigger_times = 0
      validation_loss_prev = validation_loss

  print("finished training")
  check_hessian(model, loss_function)

def report():
  pass

def MLP_validate(model, loss_function):
  t = np.arange(0, 2*np.pi+0.1, 0.1)
  t = t.astype(np.float32)
  xs, ys = cycloid(t, rnd=rnd)
  xs = cycloid_normalize_x(xs)

  # t_num = 300
  # delta_t = 0.05
  # t_start = 100*np.random.rand()
  # t_end = t_start + t_num*delta_t
  # t = np.linspace(t_start, t_end, t_num)
  # t = t.astype(np.float32)
  # x0=np.random.uniform(low=-10, high=10+1e-8, size=3)
  # x, y = Lorenz(t, rnd=rnd, x0=x0)
  # x = Lorenz_normalize_x(x)
  # xs = x.astype(np.float32)
  # ys = y.astype(np.float32)

  length = t.shape[0]
  print(f"t_validate length = {length}")
  with torch.no_grad():
    inputs, targets = torch.from_numpy(xs), torch.from_numpy(ys)
    inputs = torch.reshape(inputs, (length,1)) # 2 for Lorenz
    targets = torch.reshape(targets, (length,1))
    inputs, targets = inputs.to(device), targets.to(device)
    outputs = model(inputs)
    sample_loss = loss_function(outputs, targets)
    sample_loss /= length
    print(f"validation samples loss = {sample_loss:.05f}")
    zs = outputs.data.cpu().numpy()
    # zs = zs.flatten() # for Lorenz, no need for cycloid
    x0 = None
  return sample_loss.data.cpu().numpy().flatten()[0], t, xs, ys, zs, x0

def MLP_verify(model, loss_function):
  sample_loss, t, xs, ys, zs, x0 = MLP_validate(model, loss_function)
  _, ss = cycloid(t, rnd=0)
  # _, ss = Lorenz(t, rnd=0, x0=x0)
  with torch.no_grad():
    zst, sst = torch.from_numpy(zs), torch.from_numpy(ss)
    zst = torch.reshape(zst, (zs.shape[0],1))
    sst = torch.reshape(sst, (ss.shape[0],1))
    true_loss = loss_function(zst, sst)
    # true_loss = true_loss.item() / ss.shape[0]
  print(f"verification true  loss = {true_loss:.05f}")
  if v:
    ax_ver.plot(xs, ys, "b-*", label="samples")
    ax_ver.plot(xs, zs, "r-o", label="predicted")
    ax_ver.plot(xs, ss, "g-x", label="truth")
    # ax_ver.plot(xs[:,0], xs[:,1], ys, "b-*", label="samples")
    # ax_ver.plot(xs[:,0], xs[:,1], zs, "r-o", label="predicted")
    # ax_ver.plot(xs[:,0], xs[:,1], ss, "g-x", label="truth")
    ax_ver.legend()
  report()
  return sample_loss, true_loss

def MLP():
  # tune the learning rate
  # lrs = [0.0001, 0.0005, 0.0010, 0.0025, 0.0050, 0.0075, 0.0100, 0.0500] # SGD; lr=0.001
  # lrs = [0.0001, 0.0002, 0.0006, 0.0006, 0.0008, 0.0010, 0.0015, 0.0020, 0.0025, 0.0050, 0.0075, 0.0100, 0.0500] # Adam; lr=0.001
  lrs = [0.001]
  lr_prev = lrs[0]
  sample_loss_prev, true_loss_prev = None, None

  model = MLP_NN().to(device)
  if v: print(model); # assert(all(p.is_cuda for p in model.parameters()))
  loss_function = nn.MSELoss()
  # loss_function = nn.L1Loss()
  verify_grad()

  for lr in lrs:
    print(f"learning rate = {lr}")
    if v:
      fig_grad_init()
      fig_w_init()
      fig_loss_init(lr)

    # optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) # lr=0.001, momentum=0.9 # weight_decay is L2 regularization
    optimizer = optim.Adam(params=model.parameters(), lr=lr)
    # optimizer = optim.LBFGS(params=model.parameters(), lr=lr, line_search_fn="strong_wolfe")
    MLP_train(model, loss_function, optimizer)
    if v: fig_ver_init()
    sample_loss, true_loss = MLP_verify(model, loss_function)

    if v:
      if not sample_loss_prev: sample_loss_prev = sample_loss
      if not true_loss_prev: true_loss_prev = true_loss
      ax_lr.plot([lr_prev, lr], [sample_loss_prev, sample_loss], color="red",  linestyle='-', label="validation sample loss")
      ax_lr.plot([lr_prev, lr], [true_loss_prev, true_loss], color="green",  linestyle='-', label="validation truth loss")
      if lr == lrs[0]: ax_lr.legend(loc="best", ncol=1, scatterpoints=1, numpoints=1)
      lr_prev = lr
      sample_loss_prev = sample_loss
      true_loss_prev = true_loss
      fig_lr.canvas.flush_events()
    print()

MLP()

postscript(start_time)


import json  # JSON modülünü içe aktar

from kafka import KafkaConsumer  # kafka kütüphanesinden KafkaConsumer sınıfını içe aktar

kafka_bootstrap_servers = 'localhost:9092'  # Kafka başlangıç sunucularını belirle
kafka_topic = 'your_kafka_topic'  # Kafka konusunu belirle

consumer = KafkaConsumer(  # KafkaConsumer nesnesi oluştur
    kafka_topic,  # Abone olunacak konuyu belirt
    bootstrap_servers=kafka_bootstrap_servers,  # Kafka başlangıç sunucularını ayarla
    value_deserializer=lambda v: json.loads(v.decode('utf-8'))  # Değer çözümleyiciyi ayarla, JSON verilerini çözümlemek için
)

def consume_messages():
    for message in consumer:  # Her bir mesaj için döngüye gir
        print(message.value)  # Mesajın değerini ekrana yazdır

if __name__ == '__main__':
    consume_messages()  # Mesajları tüketmek için consume_messages fonksiyonunu çağır


import base64
import datetime
import glob
import logging
import os
import re
import time
import types
from importlib import util
from random import Random, shuffle

import toml
import yaml
from kapitan import cached, defaults, utils
from kapitan.errors import CompileError
from six import string_types

logger = logging.getLogger(__name__)

def load_jinja2_filters(env):
    """Load Jinja2 custom filters into env"""
    env.filters["sha256"] = utils.sha256_string
    env.filters["b64encode"] = base64_encode
    env.filters["b64decode"] = base64_decode
    env.filters["yaml"] = to_yaml
    env.filters["toml"] = to_toml
    env.filters["fileglob"] = fileglob
    env.filters["bool"] = to_bool
    env.filters["to_datetime"] = to_datetime
    env.filters["strftime"] = strftime
    env.filters["regex_replace"] = regex_replace
    env.filters["regex_escape"] = regex_escape
    env.filters["regex_search"] = regex_search
    env.filters["regex_findall"] = regex_findall
    env.filters["reveal_maybe"] = reveal_maybe
    env.filters["ternary"] = ternary
    env.filters["shuffle"] = randomize_list

def load_module_from_path(env, path):
    """
    Loads a python module from provided path and adds it to jinja2 environment
    filter name is same as that of function
    """
    try:
        module_name = os.path.basename(path).split(".")[0]
        custom_filter_spec = util.spec_from_file_location(module_name, path)
        custom_filter_module = util.module_from_spec(custom_filter_spec)
        custom_filter_spec.loader.exec_module(custom_filter_module)
        for function in dir(custom_filter_module):
            if isinstance(getattr(custom_filter_module, function), types.FunctionType):
                logger.debug("custom filter loaded from %s", path)
                env.filters[function] = getattr(custom_filter_module, function)
    except Exception as e:
        raise IOError("jinja2 failed to render, could not load filter at {}: {}".format(path, e))
        logger.debug("failed to find custom filter from path %s", path)

def load_jinja2_filters_from_file(env, jinja2_filters):
    """
    if filter points to default file and in case it doesn't exist then proceed silently, no error
    else try to load module (which will throw error in case of non existence of file)
    """
    jinja2_filters = os.path.normpath(jinja2_filters)
    if jinja2_filters == defaults.DEFAULT_JINJA2_FILTERS_PATH:
        if not os.path.isfile(jinja2_filters):
            return
    load_module_from_path(env, jinja2_filters)

# Custom filters
def reveal_maybe(ref_tag):
    "Will reveal ref_tag if valid and --reveal flag is used"
    if cached.args["compile"].reveal:
        return cached.revealer_obj.reveal_raw(ref_tag)
    else:
        return ref_tag

def base64_encode(string):
    return base64.b64encode(string.encode("UTF-8")).decode("UTF-8")

def base64_decode(string):
    return base64.b64decode(string).decode("UTF-8")

def to_yaml(obj):
    return yaml.safe_dump(obj, default_flow_style=False)

def to_toml(obj):
    return toml.dumps(obj)

# Following filters are from https://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/filter/core.py
def fileglob(pathname):
    """return list of matched regular files for glob"""
    return [g for g in glob.glob(pathname) if os.path.isfile(g)]

def to_bool(a):
    """return a bool for the arg"""
    if a is None or isinstance(a, bool):
        return a
    if isinstance(a, string_types):
        a = a.lower()
    if a in ("yes", "on", "1", "true", 1):
        return True
    return False

def to_datetime(string, format="%Y-%m-%d %H:%M:%S"):
    return datetime.datetime.strptime(string, format)

def strftime(string_format, second=None):
    """return current date string for format. See https://docs.python.org/3/library/time.html#time.strftime for format"""
    if second is not None:
        try:
            second = int(second)
        except Exception:
            raise CompileError("Invalid value for epoch value ({})".format(second))
    return time.strftime(string_format, time.localtime(second))

def regex_replace(value="", pattern="", replacement="", ignorecase=False):
    """Perform a `re.sub` returning a string"""
    if ignorecase:
        flags = re.I
    else:
        flags = 0
    _re = re.compile(pattern, flags=flags)
    return _re.sub(replacement, value)

def regex_escape(string):
    """Escape all regular expressions special characters from STRING."""
    return re.escape(string)

def regex_search(value, regex, *args, **kwargs):
    """Perform re.search and return the list of matches or a backref"""
    groups = list()
    for arg in args:
        if arg.startswith("\\g"):
            match = re.match(r"\\g<(\S+)>", arg).group(1)
            groups.append(match)
        elif arg.startswith("\\"):
            match = int(re.match(r"\\(\d+)", arg).group(1))
            groups.append(match)
        else:
            raise CompileError("Unknown argument")

    flags = 0
    if kwargs.get("ignorecase"):
        flags |= re.I
    if kwargs.get("multiline"):
        flags |= re.M

    match = re.search(regex, value, flags)
    if match:
        if not groups:
            return match.group()
        else:
            items = list()
            for item in groups:
                items.append(match.group(item))
            return items

def regex_findall(value, regex, multiline=False, ignorecase=False):
    """Perform re.findall and return the list of matches"""
    flags = 0
    if ignorecase:
        flags |= re.I
    if multiline:
        flags |= re.M
    return re.findall(regex, value, flags)

def ternary(value, true_val, false_val, none_val=None):
    """value ? true_val : false_val"""
    if value is None and none_val is not None:
        return none_val
    elif bool(value):
        return true_val
    else:
        return false_val

def randomize_list(mylist, seed=None):
    try:
        mylist = list(mylist)
        if seed:
            r = Random(seed)
            r.shuffle(mylist)
        else:
            shuffle(mylist)
    except Exception:
        pass
    return mylist


from django.shortcuts import render
from django.http import HttpResponse
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
from surprise import Reader, Dataset, SVD
from surprise.model_selection import KFold
import pickle
from surprise.model_selection.validation import cross_validate
import copy
from datetime import datetime

# In[3]:

meta = pd.read_csv('Dataset/movies_metadata.csv')

# In[4]:

# Rating
ratings = pd.read_csv('Dataset/ratings_small.csv')

# In[5]:

links = pd.read_csv('Dataset/links_small.csv')

# In[6]:

keywords = pd.read_csv('Dataset/keywords.csv')

# In[7]:

# -- Content filtering based Recommender

meta['overview'] = meta['overview'].fillna('')

# In[8]:

pd.DataFrame({'feature': meta.dtypes.index, 'dtype': meta.dtypes.values})

# In[9]:

meta = meta.drop([19730, 29503, 35587])

meta['id'] = pd.to_numeric(meta['id'])

# In[10]:

pd.DataFrame({'feature': links.dtypes.index, 'dtype': links.dtypes.values})

# In[11]:

col = np.array(links['tmdbId'], np.int64)
links['tmdbId'] = col

# In[12]:

meta.rename(columns={'id': 'tmdbId'}, inplace=True)
meta = pd.merge(meta, links, on='tmdbId')
meta.drop(['imdb_id'], axis=1, inplace=True)
meta.head()

# In[13]:

tfidf = TfidfVectorizer(stop_words='english')
tfidf_matrix = tfidf.fit_transform(meta['overview'])

# In[14]:

# Compute cosine similarity
cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
indices = pd.Series(meta.index, index=meta['original_title']).drop_duplicates()

# In[15]:

def recommend(title, cosine_sim=cosine_sim):
    idx = indices[title]
    # pairwise similarity scores of movies with given movie
    sim_scores = list(enumerate(cosine_sim[idx]))
    # Sort the movies on similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    # 15 most similar movies
    sim_scores = sim_scores[1:16]
    movie_indices = [i[0] for i in sim_scores]
    # Remove low-rated
    for i in movie_indices:
        pop = meta.at[i, 'vote_average']
        if pop < 5 or pop > 10:
            movie_indices.remove(i)

    return meta[['original_title', 'vote_average']].iloc[movie_indices]

# In[16]:

# In[17]:

# Collaborative Filtering based Recommender

reader = Reader()
df = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)
kf = KFold(n_splits=5)
kf.split(df)  # Split the data into folds

# In[18]:

# Use Single Value Decomposition (SVD) for cross-validation and fitting
svd = pickle.load(open("movie_ml_model.sav", "rb"))
# In[19]:

# In[20]:

# reload files
links_df = pd.read_csv('Dataset/links_small.csv')
col = np.array(links_df['tmdbId'], np.int64)
links_df['tmdbId'] = col

links_df = links_df.merge(meta[['title', 'tmdbId']], on='tmdbId').set_index('title')
links_index = links_df.set_index('tmdbId')  # For label indexing

# In[21]:

def hybrid(userId, title):
    idx = indices[title]
    tmdbId = links_df.loc[title]['tmdbId']

    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:31]  # calculating score of 30 similar movies
    movie_indices = [i[0] for i in sim_scores]

    movies = meta.iloc[movie_indices][['title', 'vote_average', 'tmdbId']]
    movies['est'] = movies['tmdbId'].apply(
        lambda x: svd.predict(userId, links_index.loc[x]['movieId']).est)  # Estimated prediction using svd
    movies = movies.sort_values('est', ascending=False)  # Ranking movies according to predicted values
    movies.columns = ['Title', 'Vote Average', 'TMDb Id', 'Estimated Prediction']
    return movies.head(30)  # 30 similar movies

# necessary functions for contextual_update function

def day_time():
    now = datetime.now().time()
    morning = now.replace(hour=12, minute=0, second=0, microsecond=0)
    afternoon = now.replace(hour=16, minute=0, second=0, microsecond=0)
    evening = now.replace(hour=19, minute=0, second=0, microsecond=0)

    if now < morning:
        return "morning"
    elif now < afternoon:
        return "afternoon"
    elif now < evening:
        return "evening"
    else:
        return "night"

def season():
    month = datetime.now().month

    if month < 4:
        return "winter"
    elif month < 6:
        return "summer"
    elif month < 9:
        return "rainy"
    elif month < 11:
        return "autumn"
    else:
        return "winter"

def is_weekend():
    day = datetime.now().isoweekday()
    if day < 6:
        return False
    return True

# testing function
# day_time()
season()

# In[24]:

# Function to include movies on specific dates -
def special_date(recommended_list, date_passed):
    print("special date function reached")
    date_event = datetime.now().date()

    # Independence Day
    date_event = date_event.replace(month=8, day=15)
    new_list = recommended_list.copy()
    if date_event == date_passed:
        # Vote Average  TMDb Id  Estimated Prediction
        new_movie = pd.DataFrame({"Title": ["Border", "Uri:The Surgical Strike"],
                                  "Vote Average": [6.8, 7.1],
                                  "TMDb Id": [33125, 554600],
                                  "Estimated Prediction": [5.0, 5.0],
                                  "tmdbId": [33125, 554600],
                                  "genres": ["[{'name':'Action'},{'name':'History'},{'name':'War'}]",
                                             "[{'name':'Action'},{'name':'Drama'},{'name':'War'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # Repubic Day
    date_event = date_event.replace(month=1, day=26)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["Shaheed", "Border", "Uri:The Surgical Strike"],
                                  "Vote Average": [5.0, 6.8, 7.1],
                                  "TMDb Id": [498713, 33125, 554600],
                                  "Estimated Prediction": [5.0, 5.0, 5.0],
                                  "tmdbId": [498713, 33125, 554600],
                                  "genres": ["[{'name':'War'},{'name':'History'}]",
                                             "[{'name':'Action'},{'name':'History'},{'name:'War'}]",
                                             "[{'name':'Action'},{'name':'Drama'},{'name':'War'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # Teachers Day
    date_event = date_event.replace(month=9, day=5)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["Super 30", "Taare Zameen Par"],
                                  "Vote Average": [7.6, 8.0],
                                  "TMDb Id": [534075, 7508],
                                  "Estimated Prediction": [5.0, 5.0],
                                  "tmdbId": [534075, 7508],
                                  "genres": ["[{'name':'Drama'}]", "[{'name':'Drama'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # Children day
    date_event = date_event.replace(month=11, day=14)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["Taare Zameen Par", "Chillar Party"],
                                  "Vote Average": [8.0, 6.9],
                                  "TMDb Id": [7508, 69891],
                                  "Estimated Prediction": [5.0, 5.0],
                                  "tmdbId": [7508, 69891],
                                  "genres": ["[{'name':'Drama'}]",
                                             "[{'name':'Drama'},{'name':'Comedy'},{'name':'Family'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # Christmas
    date_event = date_event.replace(month=12, day=25)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["Let It Snow", "Home Alone"],
                                  "Vote Average": [6.1, 7.3],
                                  "TMDb Id": [295151, 771],
                                  "Estimated Prediction": [5.0, 5.0],
                                  "tmdbId": [295151, 771],
                                  "genres": ["[{'name':'Romance'},{'name':'Comedy'}]",
                                             "[{'name':'Comedy'},{'name':'Family'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # New Year
    date_event = date_event.replace(month=12, day=31)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["New Years Eve"],
                                  "Vote Average": [5.9],
                                  "TMDb Id": [62838],
                                  "Estimated Prediction": [5.0],
                                  "tmdbId": [62838],
                                  "genres": ["[{'name':'Comedy'},{'name':'Romance'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    date_event = date_event.replace(month=1, day=1)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["New Years Eve"],
                                  "Vote Average": [5.9],
                                  "TMDb Id": [62838],
                                  "Estimated Prediction": [5.0],
                                  "tmdbId": [62838],
                                  "genres": ["[{'name':'Comedy'},{'name':'Romance'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])
    # Valentine
    date_event = date_event.replace(month=2, day=14)
    if date_event == date_passed:
        new_movie = pd.DataFrame({"Title": ["The Notebook", "Titanic"],
                                  "Vote Average": [7.9, 7.9],
                                  "TMDb Id": [11036, 597],
                                  "Estimated Prediction": [5.0, 5.0],
                                  "tmdbId": [11036, 597],
                                  "genres": ["[{'name':'Romance'},{'name':'Drama'}]",
                                             "[{'name':'Drama'},{'name':'Romance'}]"]
                                  })
        new_list = pd.concat([new_movie, recommended_list])

    return new_list

# In[25]:

def recommendation_updater(recommended_list, genre_score):
    # print("reached recommendation updater - ")
    new_list = recommended_list.copy()
    for ind in recommended_list.index:
        new_score = 0
        movie_genre = list(eval(recommended_list['genres'][ind]))
        # print(recommended_list['genres'][ind])
        # print(type(recommended_list['genres'][ind]))
        # print(movie_genre)
        curr_genre_list = [li['name'] for li in movie_genre]
        # print(curr_genre_list)
        for genre in curr_genre_list:
            if genre in genre_score:
                new_score += genre_score[genre]
        # print(new_score)
        new_list['Estimated Prediction'][ind] = new_list['Estimated Prediction'][ind] + new_score
    return new_list

# In[26]:

def contextual_update(list_passed, family=False, device="Mobile", no_of_people=1, date_passed=15,
                      month_passed=8):
    # categories we have romance,action,comedy,drama ,crime and thriller ,documentary,sci-fi
    recommended_list = list_passed.copy()
    print("Before Context-Awareness based changes - ")
    print(list_passed)

    # Adding Genres for update
    recommended_list = pd.merge(recommended_list, meta[['tmdbId', 'genres']], left_on=['TMDb Id'],
                                right_on=['tmdbId']).dropna()

    # Special Days
    date_used = datetime.now().date()
    date_used = date_used.replace(month=int(month_passed), day=int(date_passed))
    recommended_list = special_date(recommended_list, date_used)
    recommended_list.reset_index(drop=True, inplace=True)

    # Reducing score to take account for contextual_update

    effect_rate = 0.75
    category = 4
    recommended_list['Estimated Prediction'] = recommended_list['Estimated Prediction'] - effect_rate

    # Timing based

    day_part = day_time()
    if day_part == "morning":
        scores = {
            'Romance': 0.24 * (effect_rate / category), 'Action': 0.18 * (effect_rate / category),
            'Comedy': 0.64 * (effect_rate / category), 'Drama': 0.24 * (effect_rate / category),
            'Crime': 0.17 * (effect_rate / category)
            , 'Thriller': 0.17 * (effect_rate / category), 'Documentary': 0.25 * (effect_rate / category),
            'Science Fiction': 0.28 * (effect_rate / category)
        }
    elif day_part == "afternoon":
        scores = {
            'Romance': 0.18 * (effect_rate / category), 'Action': 0.44 * (effect_rate / category),
            'Comedy': 0.48 * (effect_rate / category), 'Drama': 0.35 * (effect_rate / category),
            'Crime': 0.5 * (effect_rate / category)
            , 'Thriller': 0.5 * (effect_rate / category), 'Documentary': 0.24 * (effect_rate / category),
            'Science Fiction': 0.35 * (effect_rate / category)
        }
    elif day_part == "evening":
        scores = {
            'Romance': 0.4 * (effect_rate / category), 'Action': 0.34 * (effect_rate / category),
            'Comedy': 0.48 * (effect_rate / category), 'Drama': 0.3 * (effect_rate / category),
            'Crime': 0.4 * (effect_rate / category)
            , 'Thriller': 0.4 * (effect_rate / category), 'Documentary': 0.24 * (effect_rate / category),
            'Science Fiction': 0.32 * (effect_rate / category)
        }
    else:
        scores = {
            'Romance': 0.57 * (effect_rate / category), 'Action': 0.37 * (effect_rate / category),
            'Comedy': 0.42 * (effect_rate / category), 'Drama': 0.37 * (effect_rate / category),
            'Crime': 0.54 * (effect_rate / category)
            , 'Thriller': 0.54 * (effect_rate / category), 'Documentary': 0.31 * (effect_rate / category),
            'Science Fiction': 0.41 * (effect_rate / category)
        }
    recommended_list = recommendation_updater(recommended_list, scores)

    # Season based
    curr_season = season()
    if curr_season == "summer":
        scores = {
            'Romance': 0.32 * (effect_rate / category), 'Action': 0.48 * (effect_rate / category),
            'Comedy': 0.57 * (effect_rate / category), 'Drama': 0.5 * (effect_rate / category),
            'Crime': 0.6 * (effect_rate / category)
            , 'Thriller': 0.6 * (effect_rate / category), 'Documentary': 0.27 * (effect_rate / category),
            'Science Fiction': 0.47 * (effect_rate / category)
        }
    elif curr_season == "rainy":
        scores = {
            'Romance': 0.57 * (effect_rate / category), 'Action': 0.3 * (effect_rate / category),
            'Comedy': 0.52 * (effect_rate / category), 'Drama': 0.5 * (effect_rate / category),
            'Crime': 0.41 * (effect_rate / category)
            , 'Thriller': 0.41 * (effect_rate / category), 'Documentary': 0.14 * (effect_rate / category),
            'Science Fiction': 0.32 * (effect_rate / category)
        }
    elif curr_season == "autumn":
        scores = {
            'Romance': 0.41 * (effect_rate / category), 'Action': 0.37 * (effect_rate / category),
            'Comedy': 0.5 * (effect_rate / category), 'Drama': 0.48 * (effect_rate / category),
            'Crime': 0.52 * (effect_rate / category)
            , 'Thriller': 0.52 * (effect_rate / category), 'Documentary': 0.31 * (effect_rate / category),
            'Science Fiction': 0.44 * (effect_rate / category)
        }
    else:
        scores = {
            'Romance': 0.54 * (effect_rate / category), 'Action': 0.45 * (effect_rate / category),
            'Comedy': 0.51 * (effect_rate / category), 'Drama': 0.42 * (effect_rate / category),
            'Crime': 0.5 * (effect_rate / category)
            , 'Thriller': 0.5 * (effect_rate / category), 'Documentary': 0.21 * (effect_rate / category),
            'Science Fiction': 0.32 * (effect_rate / category)
        }
    recommended_list = recommendation_updater(recommended_list, scores)

    # Weekday based -

    if is_weekend():
        scores = {
            'Romance': 0.41 * (effect_rate / category), 'Action': 0.48 * (effect_rate / category),
            'Comedy': 0.54 * (effect_rate / category), 'Drama': 0.38 * (effect_rate / category),
            'Crime': 0.7 * (effect_rate / category)
            , 'Thriller': 0.7 * (effect_rate / category), 'Documentary': 0.28 * (effect_rate / category),
            'Science Fiction': 0.41 * (effect_rate / category)
        }
    else:
        scores = {
            'Romance': 0.37 * (effect_rate / category), 'Action': 0.32 * (effect_rate / category),
            'Comedy': 0.51 * (effect_rate / category), 'Drama': 0.32 * (effect_rate / category),
            'Crime': 0.48 * (effect_rate / category)
            , 'Thriller': 0.48 * (effect_rate / category), 'Documentary': 0.21 * (effect_rate / category),
            'Science Fiction': 0.38 * (effect_rate / category)
        }

    recommended_list = recommendation_updater(recommended_list, scores)

    # Device Based

    if device == "phone":
        scores = {
            'Romance': 0.36 * (effect_rate / category), 'Action': 0.24 * (effect_rate / category),
            'Comedy': 0.66 * (effect_rate / category), 'Drama': 0.44 * (effect_rate / category),
            'Crime': 0.38 * (effect_rate / category)
            , 'Thriller': 0.38 * (effect_rate / category), 'Documentary': 0.2 * (effect_rate / category),
            'Science Fiction': 0.21 * (effect_rate / category)
        }
    elif device == "tablet":
        scores = {
            'Romance': 0.34 * (effect_rate / category), 'Action': 0.37 * (effect_rate / category),
            'Comedy': 0.43 * (effect_rate / category), 'Drama': 0.43 * (effect_rate / category),
            'Crime': 0.42 * (effect_rate / category)
            , 'Thriller': 0.42 * (effect_rate / category), 'Documentary': 0.22 * (effect_rate / category),
            'Science Fiction': 0.36 * (effect_rate / category)
        }
    else:
        scores = {
            'Romance': 0.33 * (effect_rate / category), 'Action': 0.6 * (effect_rate / category),
            'Comedy': 0.24 * (effect_rate / category), 'Drama': 0.3 * (effect_rate / category),
            'Crime': 0.66 * (effect_rate / category)
            , 'Thriller': 0.66 * (effect_rate / category), 'Documentary': 0.21 * (effect_rate / category),
            'Science Fiction': 0.58 * (effect_rate / category)
        }
    recommended_list = recommendation_updater(recommended_list, scores)

    # Based on Number of people and Family -

    if no_of_people > 1:
        if family:
            scores = {
                'Romance': 0.1 * (effect_rate / category), 'Action': 0.43 * (effect_rate / category),
                'Comedy': 0.66 * (effect_rate / category), 'Drama': 0.49 * (effect_rate / category),
                'Crime': 0.26 * (effect_rate / category)
                , 'Thriller': 0.26 * (effect_rate / category), 'Documentary': 0.36 * (effect_rate / category),
                'Science Fiction': 0.29 * (effect_rate / category)
            }
        else:
            scores = {
                'Romance': 0.33 * (effect_rate / category), 'Action': 0.63 * (effect_rate / category),
                'Comedy': 0.54 * (effect_rate / category), 'Drama': 0.33 * (effect_rate / category),
                'Crime': 0.61 * (effect_rate / category)
                , 'Thriller': 0.61 * (effect_rate / category), 'Documentary': 0.17 * (effect_rate / category),
                'Science Fiction': 0.54 * (effect_rate / category)
            }

        recommended_list = recommendation_updater(recommended_list, scores)

    # removing genre from table

    recommended_list.drop(['tmdbId', 'genres'], axis=1, inplace=True)

    # Sorting the list for final result and comparing
    # print(list_passed)
    recommended_list['Estimated Prediction'].clip(lower=0,upper =5,inplace=True)
    recommended_list.sort_values(by='Estimated Prediction', ascending=False, inplace=True)
    print(recommended_list)
    return recommended_list

def index(request):
    if request.method == 'POST':

        ## Values that can used to test server
        ##movies_id = [[862, 96], [8884, 95], [284, 93], [1907, 98], [1285, 95], [867, 87], [337, 95], [10527, 67],
        ##             [1998, 78], [580, 95], [10527, 95], [874, 95]]

        choice = request.POST['choice']
        movie = request.POST['movie']
        userId = request.POST['user_Id']
        numberOfPeople = request.POST['number_of_people']
        device = request.POST['device-type']
        mood = request.POST['mood']
        family = request.POST.getlist('family[]')
        day = request.POST['day']
        month = request.POST['month']
        year = request.POST['year']

        movie_list = hybrid(userId, movie)
        print(type(family))
        print(family)
        print(device)
        print(type(numberOfPeople))
        print(numberOfPeople)
        ## Apply contextual recommendation if true
        if choice == "Yes":
            if family == ['Yes']:
                family = True
            else:
                family = False
            numberOfPeople = int(numberOfPeople)
            movie_list = contextual_update(movie_list, family, device, numberOfPeople,day,month)

        ## FIXING FORMAT FOR DISPLAY
        print(movie_list)
        movie_list.drop(['Title', 'Vote Average'], axis=1, inplace=True)
        movie_list['Estimated Prediction'] = movie_list['Estimated Prediction'] * 20
        movie_list['Estimated Prediction'] = round(movie_list['Estimated Prediction'], 2)
        movie_list = movie_list.values.tolist()
        print(movie_list)
        movies_id = movie_list
        return render(request, 'rec/index.html', {'display_rec': 'inline-block', 'movies_id': movies_id})

    return render(request, 'rec/index.html',
                  {'yes': False, 'no': False, 'movie': "Hello", 'number': 0, 'display_rec': 'none'})

def movie(request, id):
    return render(request, 'rec/movie.html', {'id': id})


# coding=utf-8
from __future__ import absolute_import

__author__ = "Gina Häußge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import logging
from flask import jsonify, make_response

import octoprint.plugin

from octoprint.server import admin_permission

class NetconnectdSettingsPlugin(octoprint.plugin.SettingsPlugin,
                                octoprint.plugin.TemplatePlugin,
                                octoprint.plugin.SimpleApiPlugin,
                                octoprint.plugin.AssetPlugin):

	def __init__(self):
		self.address = None

	def initialize(self):
		self.address = self._settings.get(["socket"])

	@property
	def hostname(self):
		hostname = self._settings.get(["hostname"])
		if hostname:
			return hostname
		else:
			import socket
			return socket.gethostname() + ".local"

	##~~ SettingsPlugin

	def on_settings_save(self, data):
		octoprint.plugin.SettingsPlugin.on_settings_save(self, data)
		self.address = self._settings.get(["socket"])

	def get_settings_defaults(self):
		return dict(
			socket="/var/run/netconnectd.sock",
			hostname=None,
			timeout=10
		)

	##~~ TemplatePlugin API

	def get_template_configs(self):
		return [
			dict(type="settings", name="Network connection")
		]

	##~~ SimpleApiPlugin API

	def get_api_commands(self):
		return dict(
			start_ap=[],
			stop_ap=[],
			refresh_wifi=[],
			configure_wifi=[],
			forget_wifi=[],
			reset=[]
		)

	def is_api_adminonly(self):
		return True

	def on_api_get(self, request):
		try:
			status = self._get_status()
			if status["wifi"]["present"]:
				wifis = self._get_wifi_list()
			else:
				wifis = []
		except Exception as e:
			return jsonify(dict(error=str(e)))

		return jsonify(dict(
			wifis=wifis,
			status=status,
			hostname=self.hostname
		))

	def on_api_command(self, command, data):
		if command == "refresh_wifi":
			return jsonify(self._get_wifi_list(force=True))

		# any commands processed after this check require admin permissions
		if not admin_permission.can():
			return make_response("Insufficient rights", 403)

		if command == "configure_wifi":
			if data["psk"]:
				self._logger.info("Configuring wifi {ssid} and psk...".format(**data))
			else:
				self._logger.info("Configuring wifi {ssid}...".format(**data))

			self._configure_and_select_wifi(data["ssid"], data["psk"], force=data["force"] if "force" in data else False)

		elif command == "forget_wifi":
			self._forget_wifi()

		elif command == "reset":
			self._reset()

		elif command == "start_ap":
			self._start_ap()

		elif command == "stop_ap":
			self._stop_ap()

	##~~ AssetPlugin API

	def get_assets(self):
		return dict(
			js=["js/netconnectd.js"],
			css=["css/netconnectd.css"],
			less=["less/netconnectd.less"]
		)

	##~~ Private helpers

	def _get_wifi_list(self, force=False):
		payload = dict()
		if force:
			self._logger.info("Forcing wifi refresh...")
			payload["force"] = True

		flag, content = self._send_message("list_wifi", payload)
		if not flag:
			raise RuntimeError("Error while listing wifi: " + content)

		result = []
		for wifi in content:
			result.append(dict(ssid=wifi["ssid"], address=wifi["address"], quality=wifi["signal"], encrypted=wifi["encrypted"]))
		return result

	def _get_status(self):
		payload = dict()

		flag, content = self._send_message("status", payload)
		if not flag:
			raise RuntimeError("Error while querying status: " + content)

		return content

	def _configure_and_select_wifi(self, ssid, psk, force=False):
		payload = dict(
			ssid=ssid,
			psk=psk,
			force=force
		)

		flag, content = self._send_message("config_wifi", payload)
		if not flag:
			raise RuntimeError("Error while configuring wifi: " + content)

		flag, content = self._send_message("start_wifi", dict())
		if not flag:
			raise RuntimeError("Error while selecting wifi: " + content)

	def _forget_wifi(self):
		payload = dict()
		flag, content = self._send_message("forget_wifi", payload)
		if not flag:
			raise RuntimeError("Error while forgetting wifi: " + content)

	def _reset(self):
		payload = dict()
		flag, content = self._send_message("reset", payload)
		if not flag:
			raise RuntimeError("Error while factory resetting netconnectd: " + content)

	def _start_ap(self):
		payload = dict()
		flag, content = self._send_message("start_ap", payload)
		if not flag:
			raise RuntimeError("Error while starting ap: " + content)

	def _stop_ap(self):
		payload = dict()
		flag, content = self._send_message("stop_ap", payload)
		if not flag:
			raise RuntimeError("Error while stopping ap: " + content)

	def _send_message(self, message, data):
		obj = dict()
		obj[message] = data

		import json
		js = json.dumps(obj, encoding="utf8", separators=(",", ":"))

		import socket
		sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
		sock.settimeout(self._settings.get_int(["timeout"]))
		try:
			sock.connect(self.address)
			sock.sendall(js + '\x00')

			buffer = []
			while True:
				chunk = sock.recv(16)
				if chunk:
					buffer.append(chunk)
					if chunk.endswith('\x00'):
						break

			data = ''.join(buffer).strip()[:-1]

			response = json.loads(data.strip())
			if "result" in response:
				return True, response["result"]

			elif "error" in response:
				# something went wrong
				self._logger.warn("Request to netconnectd went wrong: " + response["error"])
				return False, response["error"]

			else:
				output = "Unknown response from netconnectd: {response!r}".format(response=response)
				self._logger.warn(output)
				return False, output

		except Exception as e:
			output = "Error while talking to netconnectd: {}".format(e)
			self._logger.warn(output)
			return False, output

		finally:
			sock.close()

__plugin_name__ = "Netconnectd Client"

def __plugin_check__():
	import sys
	if sys.platform == 'linux2':
		return True

	logging.getLogger("octoprint.plugins." + __name__).warn("The netconnectd plugin only supports Linux")
	return False

def __plugin_load__():
	# since we depend on a Linux environment, we instantiate the plugin implementation here since this will only be
	# called if the OS check above was successful
	global __plugin_implementation__
	__plugin_implementation__ = NetconnectdSettingsPlugin()
	return True


import logging
import time

from airhockey.utils import Timeout

class AwaitVideoHandler(object):
    SUCCESS = "SUCCESS"
    TIMEOUT = "TIMEOUT"

    def __init__(self, video_stream, timeout):
        self.video_stream = video_stream
        self.timeout = timeout
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)

    def __call__(self, *args, **kwargs):
        self.logger.info("Waiting for video stream...")
        t = Timeout(self.timeout)
        self.video_stream.start()
        while not self.video_stream.has_frame():
            if t.timeout():
                self.logger.info("Video stream timeout.")
                return self.TIMEOUT
            time.sleep(0.1)

        self.logger.info("Video stream OK.")
        return self.SUCCESS