'''Reinforcement learning (RL) agent and related utility functions.'''

# python
import time
from copy import copy
# scipy
from scipy.io import savemat
from numpy.linalg import inv, norm
from numpy.random import choice, permutation, randint, rand
from numpy import arccos, array, cos, cross, dot, eye, linspace, ones, pi, sin, vstack, zeros
# openrave
import openravepy
# caffe
import h5py
import caffe
# self
import point_cloud
from grasp import Grasp
from grasp_proxy_matlab import GraspProxyMatlab

# AGENT ============================================================================================

class RlAgent:

  def __init__(self, rlEnvironment):
    '''Initializes agent in the given environment.'''

    # parameters

    self.caffeDir = "/home/mgualti/mgualti/PickAndPlace/simulation/caffe/"
    self.caffeWeightsFilePrefix = self.caffeDir + "image_iter_"
    self.caffeModelFileName = self.caffeDir + "networkDeploy-image.prototxt"
    self.caffeSolverFileName = self.caffeDir + "solver-image.prototxt"
    self.caffeTrainFileName = self.caffeDir + "train.h5"
    self.caffeTestFileName = self.caffeDir + "test.h5"

    self.emptyGraspImage = zeros((12,60,60))
    self.emptyGrasp = Grasp(array([0,0,0]), array([0,0,0]), array([1,0,0]), array([0,1,0]), 0, [],
      array([0,0,1]), 0, self.emptyGraspImage)

    # simple assignments
    self.rlEnv = rlEnvironment
    self.env = self.rlEnv.env
    self.robot = self.rlEnv.robot
    self.plotCloudHandle = None
    self.plotGraspsHandle = None
    self.nViewSamples = 1000

    # get sensor from openrave
    self.sensor = self.env.GetSensors()[0]
    self.sTh = dot(inv(self.sensor.GetTransform()), self.robot.GetTransform())

    # possible sensor "up" choices
    theta = linspace(0, 2*pi, 20)
    x = cos(theta); y = sin(theta)

    self.sensorUpChoices = []
    for i in xrange(len(theta)):
      self.sensorUpChoices.append(array([x[i], y[i], 0]))

    # specify possible place configurations

    # In this case the configurations are relative to the sensor, which happens to be near the
    # center of the closing region of the hand. It's easier to set these positions relative to the
    # center of the closing region of the hand.

    placeOrientations = [ \
      array([[-1,0,0], [0,1, 0], [0,0,-1]]).T, \
      array([[ 0,0,1], [1,0, 0], [0,1, 0]]).T]
    sideOrientations = [0, 1]

    placePositions = zeros((21, 3))
    placePositions[:, 0] = -1.0
    placePositions[:, 2] = linspace(0.05, 0.25, 21)

    self.placePoses = []; self.isSidePlace = []
    for i in xrange(len(placeOrientations)):
      for j in xrange(len(placePositions)):
        P = eye(4)
        P[0:3,0:3] = placeOrientations[i]
        P[0:3,3] = placePositions[j]
        P = dot(P, self.sTh)
        self.MoveHandToPose(P)
        if not self.env.CheckCollision(self.robot):
          self.placePoses.append(P)
          self.isSidePlace.append(sideOrientations[i])

    print("There are {} place poses.".format(len(self.placePoses)))

    # initialize grasp proxy
    self.graspProxy = GraspProxyMatlab()

    # initialize caffe
    caffe.set_device(0)
    caffe.set_mode_gpu()
    self.caffeNet = caffe.Net(self.caffeModelFileName, caffe.TEST)
    self.caffeFirstTrain = True

  def DetectGrasps(self, cloud, viewPoints, viewPointIndices, nSamples, scoreThresh, detectMode):
    '''Detects grasps in the point cloud.
      - Input cloud: nx3 cloud of points in 3d.
      - Input viewPoints: nx3 list of view points from which cloud was taken.
      - Input viewPointIndices: 0-based index of *start* of range from corresponding viewpoint.
      - Input nSamples: number of points in cloud to sample for grasps.
      - Input scoreThresh: Grasp detector score threshold (only used for detectMode=1).
      - Input detectMode: 0 for sampling candidates only, 1 for using trained CNN to score grasps.
      - Returns grasps: List of grasps found by the detector.
    '''

    # setup
    offsets = [0.04, 0.0]

    viewPointIndices = viewPointIndices + 1
    tableUpAxis = 3
    tablePosition = copy(self.rlEnv.tablePosition)
    tablePosition[tableUpAxis-1] += 0.002
    tableFingerLength = 0.05
    minWidth = 0.002; maxWidth = 0.100

    # detect
    if detectMode == 0:
      grasps = self.graspProxy.SampleGrasps(cloud, viewPoints, viewPointIndices, nSamples, minWidth,
        maxWidth, tablePosition, tableUpAxis, tableFingerLength, offsets)
    elif detectMode == 1:
      grasps = self.graspProxy.DetectGrasps(cloud, viewPoints, viewPointIndices, nSamples,
        scoreThresh, minWidth, maxWidth, tablePosition, tableUpAxis, tableFingerLength, offsets)
    else:
      raise Exception("Unrecognized grasp detection mode {}.".format(detectMode))

    return grasps

  def DownsampleAndLabelData(self, D, batchSize):
    '''Samples data to a batch size, uniformly at random, and labels the data with the current value function approximation.'''

    if len(D) <= batchSize:
      return self.LabelData(D)

    idxs = choice(len(D), batchSize, replace=False)
    batchD = [D[i] for i in idxs]
    return self.LabelData(batchD)

  def FilterGraspsWithNoPoints(self, grasps, cloud):
    '''Gets rid of any grasps that do not have points between the fingers.'''

    keepGrasps = []
    for grasp in grasps:

      # parameters
      depth = norm(grasp.top-grasp.bottom)
      width = grasp.width / 2.0
      height = grasp.height

      # transform points into grasp frame
      C = vstack((cloud.T, ones(cloud.shape[0])))
      C = dot(inv(grasp.poses[1]), C)

      # determine which points are in the grasp
      mask = ((C[0,:] >= -height) & (C[0,:] <= height) & \
              (C[1,:] >= -width)  & (C[1,:] <= width)  & \
              (C[2,:] >= -depth)  & (C[2,:] <= 0)      )

      if sum(mask) > 0:
        keepGrasps.append(grasp)

    return keepGrasps

  def GetAction(self, state, grasps, epsilon):
    '''Chooses the next action from (grasp, place, end) with an epsilon-greedy policy.
      - Warning: Cannot place until grasped.
    '''

    # 1. Initialization
    grasp = None; place = None
    isGrasped = state[1][0]

    # 2. Epsilon
    if rand() <= epsilon:
      # first, choose an action type
      nChoices = 2 if isGrasped else 1
      choice = randint(nChoices)
      # now choose details of action
      if choice == 0:
        grasp = self.GetRandomGrasp(grasps)
      else:
        place = self.GetRandomPlacePose()
      # finished
      return grasp, place

    # 3. Greedy

    # evaluate grasps
    bestValue = float("-Inf")
    for g in grasps:
      s = self.rlEnv.GetState(self, g, None)
      self.caffeNet.blobs["imagestate"].data[0] = s[0].image
      self.caffeNet.blobs["state"].data[0] = s[1]
      self.caffeNet.forward()
      value = self.caffeNet.blobs["ip3"].data[0,0]
      if value > bestValue:
        grasp = g
        bestValue = value
        place = None

    # evaluate places
    if isGrasped:
      for p in self.placePoses:
        s = self.rlEnv.GetState(self, state[0], p)
        self.caffeNet.blobs["imagestate"].data[0] = s[0].image
        self.caffeNet.blobs["state"].data[0] = s[1]
        self.caffeNet.forward()
        value = self.caffeNet.blobs["ip3"].data[0,0]
        if value > bestValue:
          place = p
          bestValue = value
          grasp = None

    return grasp, place

  def GetCloud(self, workspace=None):
    '''Agent gets point cloud from its sensor from the current position.'''

    self.StartSensor()
    self.env.StepSimulation(0.001)

    data = self.sensor.GetSensorData(openravepy.Sensor.Type.Laser)
    cloud = data.ranges + data.positions[0]

    self.StopSensor()

    if workspace is not None:
      cloud = point_cloud.FilterWorkspace(cloud, workspace)

    return cloud

  def GetDualCloud(self, viewCenter, viewKeepout, workspace):
    '''Gets a point cloud combined form two views, 45 degrees above table and 90 degrees apart.
      - Returns cloud: nx3 combined point cloud.
      - Returns viewPoints:  transforms of sensor pose from where cloud was taken.
      - Returns viewPointIndices: Starting index into the points indicating which pose the points belong.
    '''

    poses = self.GetDualViewPoses(viewCenter, viewKeepout)

    C = []
    cloud = []
    viewPoints = []
    viewPointIndices = []

    for pose in poses:
      viewPointIndices.append(len(C))
      self.MoveSensorToPose(pose)
      C = self.GetCloud(workspace)
      cloud.append(C)
      viewPoints.append(pose[0:3,3].T)

    cloud = vstack(cloud)
    viewPoints = vstack(viewPoints)
    viewPointIndices = array(viewPointIndices)

    return cloud, viewPoints, viewPointIndices

  def GetDualViewPoses(self, viewCenter, viewKeepout):
    '''Gets standard dual poses, 45 degress from table and 90 degrees apart.'''

    p1 = viewCenter + viewKeepout*array([0, -cos(45*(pi/180)), sin(45*(pi/180))])
    p2 = viewCenter + viewKeepout*array([0,  cos(45*(pi/180)), sin(45*(pi/180))])

    upChoice = array([1,0,0])
    viewPose1 = GeneratePoseGivenUp(p1, viewCenter, upChoice)
    viewPose2 = GeneratePoseGivenUp(p2, viewCenter, upChoice)

    return viewPose1, viewPose2

  def GetGrasp(self, grasps, epsilon=0):
    '''Returns the best grasp according to V(s) or a random grasp with probability epsilon.'''

    # act randomly
    doRandomAction = rand() <= epsilon
    if doRandomAction:
      return self.GetRandomGrasp(grasps)

    # act according to value function
    bestGrasps = None; bestValue = float('-Inf')
    for grasp in grasps:
      s = self.rlEnv.GetState(self, grasp, None)
      self.caffeNet.blobs["imagestate"].data[0] = s[0].image
      self.caffeNet.blobs["state"].data[0] = s[1]
      self.caffeNet.forward()
      value = self.caffeNet.blobs["ip3"].data[0,0]
      if value > bestValue:
        bestGrasps = [grasp]
        bestValue = value
      elif value == bestValue:
        bestGrasps.append(grasp)

    # break ties randomly
    print("Best grasp value: {}.".format(bestValue))
    return bestGrasps[randint(len(bestGrasps))]

  def GetGraspCylinder(self, grasps, cloud, viewPoints, viewPointIndices, kClusters, maxGraspToCylinderAngle):
    '''Uses cylinder fitting and heuristics to select the grasp.'''

    # Fit cylinder.

    viewPointIndices = viewPointIndices + 1 # Matlab 1-indexed
    cylinder = self.graspProxy.FitCylinder(cloud, viewPoints, viewPointIndices, kClusters)

    # Find grasps aligned to cylinder.

    alignedGrasps = []
    for grasp in grasps:
      if arccos(min(1.0, abs(dot(grasp.axis, cylinder.axis)))) <= maxGraspToCylinderAngle:
        alignedGrasps.append(grasp)
    if len(alignedGrasps) == 0:
      alignedGrasps = grasps

    # Of the aligned grasps, choose the closest one.

    minDistance = float('inf'); minDistanceGrasp = None
    for grasp in grasps:
      d = norm(cylinder.center-grasp.center)
      if d < minDistance:
        minDistance = d
        minDistanceGrasp = grasp
    grasp = minDistanceGrasp

    if cloud.size == 0: return grasp, cylinder

    # Decide if the grasp should be flipped.

    # transform cloud into cylinder frame
    bTc = eye(4)
    cylinderApproach = array([cylinder.axis[1], -cylinder.axis[0], 0])
    cylinderApproach = cylinderApproach / norm(cylinderApproach) \
      if norm(cylinderApproach) > 0.001 else array([1,0,0])
    bTc[0:3, 0] = cylinderApproach
    bTc[0:3, 1] = cross(cylinder.axis, cylinderApproach)
    bTc[0:3, 2] = cylinder.axis
    bTc[0:3, 3] = cylinder.center
    C = vstack((cloud.T, ones(cloud.shape[0])))
    C = dot(inv(bTc), C)

    # keep points within cylinder caps
    C = C[:, (C[2,:] >= -cylinder.height) & (C[2,:] <= cylinder.height)]
    if C.size == 0: return grasp, cylinder

    # keep points close to cylinder axis
    distSquared = C[0, :]**2 + C[1, :]**2
    C = C[:, distSquared <= (cylinder.radius+0.01)**2]
    if C.size == 0: return grasp, cylinder

    # count the number of points in upper and lower halves of cylinder
    upperCount = sum(C[2,:] >= 0)
    lowerCount = sum(C[2,:] <= 0)

    # flip if the object is top-heavy
    cylinderAxis = cylinder.axis if lowerCount > upperCount else -cylinder.axis
    if dot(grasp.axis, cylinderAxis) < 0:
      grasp = grasp.Flip()

    return grasp, cylinder

  def GetPlacePose(self, grasp, epsilon=0):
    '''Returns the best place pose according to V(s) or a random pose with probability epsilon.'''

    # act randomly
    doRandomAction = rand() <= epsilon
    if doRandomAction:
      return self.GetRandomPlacePose()

    # act according to value function
    bestPoses = None; bestValue = float('-Inf')
    for i, pose in enumerate(self.placePoses):
      s = self.rlEnv.GetState(self, grasp, pose)
      self.caffeNet.blobs["imagestate"].data[0] = s[0].image
      self.caffeNet.blobs["state"].data[0] = s[1]
      self.caffeNet.forward()
      value = self.caffeNet.blobs["ip3"].data[0,0]
      #print("a={}, v={}".format(i+1, value))
      if value > bestValue:
        bestPoses = [pose]
        bestValue = value
      elif value == bestValue:
        bestPoses.append(pose)

    # break ties randomly
    print("Best place value: {}".format(bestValue))
    return bestPoses[randint(len(bestPoses))]

  def GetPlacePoseCylinder(self, grasp, cylinder, maxTableGap):
    '''Gets a place pose that should clear the cylinder from the table.'''

    targetHeight = (cylinder.height / 2.0) + (maxTableGap / 2.0)

    minDist = float('inf'); minDistPose = None
    for i, pose in enumerate(self.placePoses):
      if self.isSidePlace[i]:
        d = abs(targetHeight-pose[2,3])
        if d < minDist:
          minDist = d
          minDistPose = pose

    return minDistPose

  def GetRandomGrasp(self, grasps):
    '''Selects a random grasp from the provided list.'''

    if len(grasps) == 0:
      return None

    return grasps[randint(len(grasps))]

  def GetRandomPlacePose(self):
    '''Selects a random pose for the hand from the discrete list of allowed place poses.'''

    return self.placePoses[randint(len(self.placePoses))]

  def GetStandardViewPose(self, viewCenter, viewKeepout):
    '''Gets a standard pose for the viewer directly above viewCenter in the z direction.'''

    viewPose = eye(4)
    viewPose[0,0] = -1
    viewPose[1,1] = 1
    viewPose[2,2] = -1
    viewPose[0:3,3] = viewCenter
    viewPose[2,3] += viewKeepout
    return viewPose

  def LabelData(self, D, gamma=1.0):
    '''Given a database of (state, nextState, reward), use the network to compute one-step lookahead values.'''

    Dl = []
    for d in D:

      s = d[0]; ss = d[1]; r = d[2]

      if ss is None:
        vss = 0 # terminal state -- known to have 0 value
      else:
        self.caffeNet.blobs["imagestate"].data[0] = ss[0].image
        self.caffeNet.blobs["state"].data[0] = ss[1]
        self.caffeNet.forward()
        vss = self.caffeNet.blobs["ip3"].data[0,0]

      vs = r + gamma*vss # Bellman update
      Dl.append((s, vs))

    return Dl

  def LoadNetworkWeights(self, weightsFileName):
    '''Loads the network weights from the specified file name.'''

    self.caffeNet = caffe.Net(self.caffeModelFileName, caffe.TEST, weights=weightsFileName)
    print("Loaded file " + weightsFileName + " successfully.")

  def MoveHandToPose(self, T):
    '''Moves the hand of the robot to the specified pose.'''

    self.robot.SetTransform(T)
    self.env.UpdatePublishedBodies()

    return True # didMove

  def MoveObjectToHandAtGrasp(self, grasp, objectHandle):
    '''Aligns the grasp on the object to the current hand position and moves the object there.'''

    bTg = grasp.poses[0]
    bTo = objectHandle.GetTransform()
    bTs = self.sensor.GetTransform()

    gTo = dot(inv(bTg), bTo)
    bTo_new = dot(bTs, gTo)

    objectHandle.SetTransform(bTo_new)

  def MoveSensorToPose(self, T):
    '''Moves the hand of the robot to the specified pose.'''

    self.robot.SetTransform(dot(T, self.sTh))
    self.env.UpdatePublishedBodies()

    return True # didMove

  def PerformAction(self, state, grasp, place, objHandle):
    '''Performs the action given by self.GetAction and returns the next state and reward.'''

    isPlaced = state[1][1]

    if isPlaced:
      raise Exception("The object has already been placed when an action is being performed.")

    if grasp is not None and place is None:
      nextState = self.rlEnv.GetState(self, grasp, None)
    elif place is not None and grasp is None:
      self.MoveHandToPose(place)
      self.MoveObjectToHandAtGrasp(state[0], objHandle)
      nextState = self.rlEnv.GetState(self, state[0], place)
    else:
      raise Exception("Only one of grasp, place, or endEpisode actions must be selected.")

    return nextState, self.rlEnv.RewardMultiDetect(self, state, nextState, objHandle)

  def PlotCloud(self, cloud):
    '''Plots a cloud in the environment.'''

    if not self.rlEnv.showViewer:
      return

    if self.plotCloudHandle is not None:
      self.UnplotCloud()

    self.plotCloudHandle = self.env.plot3(\
      points=cloud, pointsize=0.001, colors=zeros(cloud.shape), drawstyle=1)

  def PlotGrasps(self, grasps):
    '''Visualizes grasps in openrave viewer.'''

    if not self.rlEnv.showViewer:
      return

    if self.plotGraspsHandle is not None:
      self.UnplotGrasps()

    if len(grasps) == 0:
      return

    graspLength = 0.06
    graspColor = [0,0,1]

    lineList = []; colorList = []
    for grasp in grasps:

      c = grasp.bottom
      a = c - graspLength*grasp.approach
      l = c - 0.5*grasp.width*grasp.binormal
      r = c + 0.5*grasp.width*grasp.binormal
      lEnd = l + graspLength*grasp.approach
      rEnd = r + graspLength*grasp.approach

      lineList.append(c); lineList.append(a)
      lineList.append(l); lineList.append(r)
      lineList.append(l); lineList.append(lEnd)
      lineList.append(r); lineList.append(rEnd)

    for i in xrange(len(lineList)):
      colorList.append(graspColor)

    self.plotGraspsHandle = self.env.drawlinelist(\
      points=array(lineList), linewidth=3.0, colors=array(colorList))

  def PruneDatabase(self, replayDatabase, maxEntries):
    '''Removes oldest items in the database until the size is no more than maxEntries.'''

    if len(replayDatabase) <= maxEntries:
      return replayDatabase

    return replayDatabase[len(replayDatabase)-maxEntries:]

  def SaveCloud(self, cloud, viewPoints, viewPointIndices, fileName):
    '''Saves point cloud information for testing in Matlab.'''

    viewPointIndices = viewPointIndices + 1 # matlab is 1-indexed
    viewPoints = viewPoints.T
    cloud = cloud.T
    data = {"cloud":cloud, "viewPoints":viewPoints, "viewPointIndices":viewPointIndices}
    savemat(fileName, data)

  def StartSensor(self):
    '''Starts the sensor in openrave, displaying yellow haze.'''

    self.sensor.Configure(openravepy.Sensor.ConfigureCommand.PowerOn)
    self.sensor.Configure(openravepy.Sensor.ConfigureCommand.RenderDataOn)

  def StopSensor(self):
    '''Disables the sensor in openrave, removing the yellow haze.'''

    self.sensor.Configure(openravepy.Sensor.ConfigureCommand.PowerOff)
    self.sensor.Configure(openravepy.Sensor.ConfigureCommand.RenderDataOff)

  def Train(self, Dl, recordLoss=True, stepSize=100, nIterations=5000):
    '''Trains the network on the provided data labels.
      - Input Dl: List of tuples with (state,value).
      - Input recordLoss: If true, saves train and test return values (takes longer).
      - Input stepSize: Records loss values this often.
      - Input nIterations: Number of training iterations to run.
      - Returns: train loss, test loss.
    '''

    if len(Dl) == 0:
      return

    # 1. Load data

    # split data into train/test
    nTest = int(len(Dl)/4.0)
    nTrain = len(Dl) - nTest

    # shuffle data
    pdxs = permutation(len(Dl))
    idxs = pdxs[0:nTrain]
    jdxs = pdxs[nTrain:]

    sampleImage = Dl[0][0][0].image
    nx = sampleImage.shape[0]
    ny = sampleImage.shape[1]
    nz = sampleImage.shape[2]

    I = zeros((nTrain, nx, ny, nz))
    S = zeros((nTrain, len(Dl[0][0][1])))
    L = zeros(nTrain)
    for i, idx in enumerate(idxs):
      I[i, :, :, :] = Dl[idx][0][0].image
      S[i, :] = Dl[idx][0][1]
      L[i] = Dl[idx][1]

    with h5py.File(self.caffeTrainFileName, 'w') as fileHandle:
      fileHandle.create_dataset("imagestate", data=I)
      fileHandle.create_dataset("state", data=S)
      fileHandle.create_dataset("label", data=L)

    I = zeros((nTest, nx, ny, nz))
    S = zeros((nTest, len(Dl[0][0][1])))
    L = zeros(nTest)
    for j, jdx in enumerate(jdxs):
      I[j, :, :, :] = Dl[jdx][0][0].image
      S[j, :] = Dl[jdx][0][1]
      L[j] = Dl[jdx][1]

    with h5py.File(self.caffeTestFileName, 'w') as fileHandle:
      fileHandle.create_dataset("imagestate", data=I)
      fileHandle.create_dataset("state", data=S)
      fileHandle.create_dataset("label", data=L)

    # 2. Optimize

    weightsFileName = self.caffeWeightsFilePrefix + str(nIterations) + ".caffemodel"
    solver = caffe.SGDSolver(self.caffeSolverFileName)

    if self.caffeFirstTrain:
      self.caffeFirstTrain = False
    else:
      solver.net.copy_from(weightsFileName)

    trainLoss = []; testLoss = []

    if recordLoss:

      for iteration in xrange(int(nIterations/stepSize)):
        solver.step(stepSize)
        loss = float(solver.net.blobs["loss"].data)
        trainLoss.append(loss)

        loss = 0
        for testIteration in xrange(stepSize):
          solver.test_nets[0].forward()
          loss += float(solver.test_nets[0].blobs['loss'].data)
        loss /= stepSize
        testLoss.append(loss)

    else:

      solver.step(nIterations)

    self.caffeNet = caffe.Net(self.caffeModelFileName, caffe.TEST, weights=weightsFileName)

    return trainLoss, testLoss

  def UnplotCloud(self):
    '''Removes a cloud from the environment.'''

    if not self.rlEnv.showViewer:
      return

    if self.plotCloudHandle is not None:
      self.plotCloudHandle.Close()
      self.plotCloudHandle = None

  def UnplotGrasps(self):
    '''Removes any grasps drawn in the environment.'''

    if not self.rlEnv.showViewer:
      return

    if self.plotGraspsHandle is not None:
      self.plotGraspsHandle.Close()
      self.plotGraspsHandle = None

# UTILITIES ========================================================================================

def GeneratePoseGivenUp(sensorPosition, targetPosition, upAxis):
  '''Generates the sensor pose with the LOS pointing to a target position and the "up" close to a specified up.

  - Input sensorPosition: 3-element desired position of sensor placement.
  - Input targetPosition: 3-element position of object required to view.
  - Input upAxis: The direction the sensor up should be close to.
  - Returns T: 4x4 numpy array (transformation matrix) representing desired pose of end effector in the base frame.
  '''

  v = targetPosition - sensorPosition
  v = v / norm(v)

  u = upAxis - dot(upAxis, v) * v
  u = u / norm(u)

  t = cross(u, v)

  T = eye(4)
  T[0:3,0] = t
  T[0:3,1] = u
  T[0:3,2] = v
  T[0:3,3] = sensorPosition

  return T


import tushare as ts
import pandas as pd

def Multiple_stocks(tickers):
    def data(ticker):
        stocks = ts.get_k_data(ticker,start='2019-07-01',end='2019-08-18')
        stocks.set_index('date', inplace=True)
        stocks.index = pd.to_datetime(stocks.index)
        return stocks

    datas = map(data, tickers)

    return pd.concat(datas, keys=tickers, names=['Ticker', 'trade_date'])

tickers = ['600030','000679']
all_stocks = Multiple_stocks(tickers)
print(all_stocks.head(50))


from pyspark.sql import SparkSession
from pyspark.sql.functions import explode
from pprint import pprint

if __name__ == "__main__":

    session = SparkSession.builder.appName("Payload").getOrCreate()

    dataFrameReader = session.read

    responses = dataFrameReader \
        .option("header", "true") \
        .option("inferSchema", value = True) \
        .json("payload/payload500.json")

    print("=== Print out schema ===")
    responses.printSchema()
    pprint(responses.columns)
    df = responses.select(explode('events'))
    df.show()
    #pprint(df.collect())
    #responses.show()

    session.stop()


from lixian_plugins.api import command

from lixian_config import *
from lixian_encoding import default_encoding
from lixian_cli_parser import command_line_parser
from lixian_cli_parser import with_parser
from lixian_cli_parser import command_line_option, command_line_value
from lixian_commands.util import parse_login, create_client

import urllib2
import json

def get_download_task_info(args, client):

    import lixian_query
    tasks = lixian_query.search_tasks(client, args)
    files = []
    for task in tasks:
        if task['type'] == 'bt':
            subs, skipped, single_file = lixian_query.expand_bt_sub_tasks(task)
            if not subs:
                continue
            if single_file:
                files.append((subs[0]['xunlei_url'], subs[0]['name'], None))
            else:
                for f in subs:
                    files.append((f['xunlei_url'], f['name'], task['name']))
        else:
            files.append((task['xunlei_url'], task['name'], None))
    return files

def export_download_task_info(files, client):
    output = ''
    for url, name, dir in files:
        if type(url) == unicode:
            url = url.encode(default_encoding)
        output += url + '\n'
        output += '  out=' + name.encode(default_encoding) + '\n'
        if dir:
            output += '  dir=' + dir.encode(default_encoding) + '\n'
        output += '  header=Cookie: gdriveid=' + client.get_gdriveid() + '\n'
    return output

def execute_download_aria2rpc(args):
    client = create_client(args)
    task_list = get_download_task_info(args, client)
    # print(export_download_task_info(task_list, client))

    for url, name, dir in task_list:
        if type(url) == unicode:
            url = url.encode(default_encoding)
        if dir:
            dir = dir.encode(default_encoding)

        jsonreq = json.dumps({"jsonrpc": "2.0", "id": "qwer",
                              "method": "aria2.addUri",
                              "params": [
                                         [url],
                                         {
                                             "out": name.encode(default_encoding),
                                             "continue": "true",
                                             "header": ['Cookie: gdriveid=%s' % client.get_gdriveid()]
                                         }
                              ]
                              })
        # print(jsonreq)
        host = '127.0.0.1'
        if args.dev is True:
            host = '192.168.0.2'

        print("Add to aria2c server:%s" % host)
        c = urllib2.urlopen("http://%s:6800/jsonrpc" % host, jsonreq)
        # {u'jsonrpc': u'2.0', u'id': u'qwer', u'result': u'f1257fa333f235e6'}
        result = c.read()
        if result is None or result == "":
            print("\033[31mCann't add aria2 task %s\033[0m" % name)
        else:
            result = json.loads(result.decode(default_encoding))
            print("\033[32mAdd aria2 task[id= %s] %s\033[0m" % (result[u"result"], name))

@command(usage='concurrently download tasks in aria2 with rpc')
@command_line_parser()
@with_parser(parse_login)
@command_line_option('all')
@command_line_option('dev', default=False)
def download_aria2rpc(args):
    '''
    usage: lx download-aria2rpc -j 5 -h dev [id|name]...
    '''
    #print(args)
    execute_download_aria2rpc(args)


"""Tests for the fold_an_array kata"""

import unittest

from solutions.fold_an_array import fold_array, fold_array_once

class FoldAnArrayTest(unittest.TestCase):
    """Fold-an-array test suite"""

    def test_fold_array_once(self):
        """Test the single fold fuunction"""
        # cases = input, expected
        cases = [
            ([1, 2, 3, 4, 5], [6, 6, 3]),
            ([6, 6, 3], [9, 6])
        ]
        for inp, exp in cases:
            with self.subTest(inp=inp):
                self.assertEqual(fold_array_once(inp), exp)

    def test_fold_array_single_run(self):
        """Test fold function for a single fold"""
        self.assertEqual(fold_array([1, 2, 3, 4, 5], 1), [6, 6, 3])

    def test_fold_array_multi_run(self):
        """Test fold function for multiple folds"""
        self.assertEqual(fold_array([1, 2, 3, 4, 5], 2), [9, 6])


# -*- coding: utf-8 -*-
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os

from oslo_config import cfg
from oslo_log import log as logging

LOG = logging.getLogger(__name__)

def find_config(config_path):

    """Find a configuration file using the given hint.

    :param config_path: Full or relative path to the config.
    :returns: List of config paths
    """
    possible_locations = [
        config_path,
        os.path.join(cfg.CONF.pybasedir, "etc", "cue", config_path),
        os.path.join(cfg.CONF.pybasedir, "etc", config_path),
        os.path.join(cfg.CONF.pybasedir, config_path),
        "/etc/cue/%s" % config_path,
    ]

    found_locations = []

    for path in possible_locations:
        LOG.debug('Searching for configuration at path: %s' % path)
        if os.path.exists(path):
            LOG.debug('Found configuration at path: %s' % path)
            found_locations.append(os.path.abspath(path))

    return found_locations


import numpy as np
import config
import random
import json
import datetime
import matplotlib.pyplot as plt

def read_price_data(data_file, forex_config, min_len):
    # file_name = "data_price.json"
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    price_data = []
    for pair in forex_config.country_pair:
        price_data.append(data[pair][:min_len])
    price_data = np.transpose(np.asarray(price_data), [1, 0, 2])

    price_data = price_data.tolist()
    return price_data

def read_target_data(data_file):
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    return data["Target"], len(data["Target"])

def read_date_data(date_file):
    print('File ' + date_file + ' Loading.....')
    with open(date_file) as f:
        date = json.load(f)
    return date['Date']

def read_return_data(data_file):
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    return data['Return']

def year_sort(date_file, data_file, target_file, return_file):
    year_list = {}
    year_target_list = {}
    year_return_list = {}
    for i in range(len(data_file)):
        # print (date_file[str(i)])
        date = datetime.datetime.fromtimestamp(date_file[i] / 1000000000)
        if date.year not in year_list.keys():
            year_list[date.year] = []
            year_target_list[date.year] = []
            year_return_list[date.year] = []
        year_list[date.year].append(data_file[i])
        year_target_list[date.year].append(target_file[i])
        year_return_list[date.year].append(return_file[i])

    for key in year_list.keys():
        year_list[key] = zip(year_list[key], year_target_list[key], year_return_list[key])
    return year_list

def year_sort_rnn(date_file, data_file, target_file, return_file, forex_config):
    year_list = {}
    year_target_list = {}
    year_return_list = {}
    continuous_data_num = forex_config.continuous_period_num
    for i in range(len(data_file - continuous_data_num)):
        # print (date_file[str(i)])
        date = datetime.datetime.fromtimestamp(date_file[str(i)] / 1000)
        if date.year not in year_list.keys():
            year_list[date.year] = []
            year_target_list[date.year] = []
            year_return_list[date.year] = []
        year_list[date.year].append([])
        year_target_list[date.year].append([])
        year_return_list[date.year].append([])
        for j in range(continuous_data_num):
            year_list[date.year][-1].append(data_file[i + j])
            year_target_list[date.year][-1].append(target_file[i + j])
            year_return_list[date.year][-1].append(return_file[i + j])
    for key in year_list.keys():
        year_list[key] = zip(year_list[key], year_target_list[key], year_return_list[key])
    return year_list

#####  random pick train years, others will be viewed as testing set #####
def pick_train_test_pair(year_list,train_year_num ,forex_config):
    test_year_num = len(forex_config.years) - train_year_num
    test_years = list(np.random.choice(forex_config.years, test_year_num, replace=False))
    test_data = []
    for test_year in test_years:
        test_data.extend(year_list[test_year])
    key_list = []
    for key in year_list.keys():
        if key in test_years:
            continue
        key_list.append(key)
    print(key_list)
    random_select_num = np.random.choice(len(key_list), train_year_num, replace=False)
    train_data = []
    for i in random_select_num:
        train_data.extend(year_list[key_list[i]])
    return train_data, test_data, test_years

#####  pick train years in order, the year next to last train year will be viewed as testing set #####
def pick_train_test_pair_inorder(year_list,train_year_num ,forex_config):
    train_years = forex_config.years[:train_year_num]
    train_data = []
    for train_year in train_years:
        train_data.extend(year_list[train_year])
    test_data = []
    test_year = forex_config.years[train_year_num]
    test_data.extend(forex_config.year[test_year])
    return train_data,test_data,[test_year]

def get_batch(data, index, forex_config, is_back_test):
    end_index = index + forex_config.batch_size
    if end_index >= len(data[0]):
        end_index = len(data[0]) - 1
    batch_price = data[0][index:end_index]
    batch_price = np.transpose(np.asarray(batch_price), [1, 0, 2])
    if is_back_test:
        batch_target = data[2][index:end_index]
    else:
        batch_target = data[1][index:end_index]
    return batch_price, batch_target

def split_shuffle(train_data, test_data):
    train_data = list(train_data)
    random.shuffle(train_data)
    train_data = list(zip(*train_data))
    test_data = list(test_data)
    test_data = list(zip(*test_data))

    return train_data, test_data

def test(model, data, forex_config):
    total = 0
    correct_num = 0

    for i in range(int(len(data[0]) / forex_config.batch_size)):
        index = i * forex_config.batch_size
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=False)
        outputs = model.predict(batch_price)
        total += len(outputs)
        # total += np.sum(max_logit)
        # test_preference = np.zeros(42)
        for j in range(len(outputs)):
            #	test_preference[outputs[j]]+=1
            if batch_target[j][outputs[j]] == 1:  # and max_logit[j] == 1:
                correct_num += 1
                # print ("preference_test {}".format(test_preference))
    rate = float(correct_num) / float(total)
    print("total {}, correct {}, rate {}".format(total, correct_num, rate))

def back_test(model, test_years, data, forex_config, num_step):
    target_period = 48
    start_fund = 100
    base_fund_per_period = start_fund
    total_reward_rate = 0
    total_reward_rate_list = []
    single_return = []
    total_return = []
    temp_total_return = []
    for i in range(int(len(data[0]) / forex_config.batch_size)):
        index = i * forex_config.batch_size
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=True)
        outputs = model.predict(batch_price)
        for j in range(len(outputs)):
            total_reward_rate += batch_target[j][outputs[j]]  # reward
            # ---------- profit computation -----------
            single_return.append(batch_target[j][outputs[j]])
            tmp_total_reward_rate = total_reward_rate
            total_reward_rate_list.append(tmp_total_reward_rate)

    for i in range(len(single_return)):
        temp_total_return.append((base_fund_per_period / target_period) * (1 + single_return[i]))

        if i+1 % 48 == 0:
            print(i)
            temp = sum(temp_total_return)
            temp_return = (temp - start_fund)/ start_fund
            total_return.append(temp_return)
            base_fund_per_period = temp
            temp_total_return = []

            if start_fund < 0:
                print('strategy shut down')
                break

    print("total_reward_rate: {}".format(total_return[-1]))
    years_info = ""
    for year in test_years:
        years_info += str(year)[2:]
    fig = plt.figure()

    plt.plot(total_return)
    plt.xlabel('time tick')
    plt.ylabel('return rate')
    fig.savefig('graph/' + years_info + '_' + str(num_step+1) + '.png')

# plt.show()

def train(model, data, test_data, test_years, forex_config):
    limit_step = 5000
    for i in range(limit_step):
        index = (i * forex_config.batch_size) % len(data[0])
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=False)
        loss = model.step(batch_price, batch_target)
        print("step " + str(i) + ",loss " + str(loss))
        if (i + 1) % forex_config.steps_per_checkpoint == 0:
            print("test val set...")
            test(model, test_data, forex_config)
            back_test(model, test_years, test_data, forex_config, i)
            print ("test train set...")
            test(model,data,forex_config)
            #back_test(model,data,forex_config)
            model.save(test_years, forex_config)


# Functions for running an encryption or decryption.

# The values of the two jokers.
JOKER1 = 27
JOKER2 = 28

# A list containing all the alphabets in uppercase
char_list = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
             'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
             'Y', 'Z']

# A list containing all the corresponding numbers start from 0
num_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
            18, 19, 20, 21, 22, 23, 24, 25]

# Write your functions here:
def clean_message(msg):
    '''(str) -> str
    Given one line of string (msg), take out any nonalphabetical characters and
    convert the remaining characters to uppercase, then return the resulted
    message.
    >>> clean_message('I12 3love4 @56you 789')
    'ILOVEYOU'
    >>> clean_message('234567!53**97543%98777035')
    ''
    '''
    # create an ampty string (to store the new message)
    new_msg = ''

    # use elemental for loop to process each character in the string
    for char in msg:
        # use if statement to check whether the character is alphabetic
        if char.isalpha():
            # convert alphabetical character to uppercase and add to the new
            # string (message)
            new_msg += char.upper()

    return new_msg

def encrypt_letter(char, ks_val):
    ''' (str, int) -> str
    Given a single character (char) and a keystream value (ks_val), return the
    encrypted character by applying the keystream value.
    >>> encrypt_letter('X', 13)
    'K'
    >>> encrypt_letter('D', 5)
    'I'
    '''
    # find the index of the given character (from the alphabet list)
    char_index = char_list.index(char.upper())

    # get the corresponding number of that character (from the number list)
    char_num = num_list[char_index]

    # apply the keystream value (modulo 26 as there are 26 letters)
    encrypt_num = (char_num + ks_val) % 26

    # get the encrypted (corresponding) character
    encrypt_char = char_list[encrypt_num]

    return encrypt_char

def decrypt_letter(char, ks_val):
    ''' (str, int) -> str
    Given an uppercase character (char) and a keystream value (ks_val), return
    the decrypted character by applying the keystream value.
    >>> decrypt_letter('O', 7)
    'H'
    >>> decrypt_letter('W', 22)
    'A'
    '''
    # find the index of the given character (from the alphabet list)
    char_index = char_list.index(char.upper())

    # get the corresponding number of that character (from the number list)
    char_num = num_list[char_index]

    # apply the keystream value (modulo 26 as there are 26 letters)
    decrypt_num = (char_num - ks_val) % 26

    # get the decrypted (corresponding) character
    decrypt_char = char_list[decrypt_num]

    return decrypt_char

def swap_cards(deck_cards, index):
    '''(list of int, int) -> NoneType
    Given a deck of cards and an index, swap the card at that index with the
    card after it. If the chosen card is at the bottom, swaps it with the top
    card (the first card).
    REQ: index < len(deck_cards)
    >>> my_deck = [1, 2, 3, 4, 5]
    >>> swap_cards(my_deck, 2)
    >>> my_deck == [1, 2, 4, 3, 5]
    True
    >>> your_deck = [5, 7, 8, 2, 1, 9, 4, 3, 6]
    >>> swap_cards(your_deck, 8)
    >>> your_deck == [6, 7, 8, 2, 1, 9, 4, 3, 5]
    True
    '''
    # create a temporary variable to store the card (number) at the index
    temp = deck_cards[index]
    # use if statement to check whether the card (number) is at the bottom
    if index == len(deck_cards) - 1:
        # swap with the first card (number)
        deck_cards[index] = deck_cards[0]
        deck_cards[0] = temp
    else:
        # swap with the following card (number)
        deck_cards[index] = deck_cards[index + 1]
        deck_cards[index + 1] = temp

def move_joker_1(deck_cards):
    ''' (list of int) -> NoneType
    Given a deck of cards, swap JOKER1 with the card after it. If JOKER1 is
    at the bottom, swap it with the top card (the first card). This function
    is defined as step 1.
    >>> his_deck = [0, 3, 6, 9, 12, 15, 18, 21, 24, 27]
    >>> move_joker_1(his_deck)
    >>> his_deck == [27, 3, 6, 9, 12, 15, 18, 21, 24, 0]
    True
    >>> her_deck = [1, 30, 5, 27, 9, 24, 13, 20, 17]
    >>> move_joker_1(her_deck)
    >>> her_deck == [1, 30, 5, 9, 27, 24, 13, 20, 17]
    True
    '''
    # find the index of JOKER1
    index = deck_cards.index(JOKER1)
    # apply the swap_cards function
    swap_cards(deck_cards, index)

def move_joker_2(deck_cards):
    ''' (list of int) -> NoneType
    Given a deck of cards, move JOKER2 two cards down. Treat the deck as
    circular when JOKER2 is the last or the second-last card. This function
    is defined as step 2.
    >>> our_deck = [0, 4, 8, 12, 16, 20, 24, 28]
    >>> move_joker_2(our_deck)
    >>> our_deck == [4, 28, 8, 12, 16, 20, 24, 0]
    True
    >>> their_deck = [1, 32, 9, 28, 17, 24, 25, 20]
    >>> move_joker_2(their_deck)
    >>> their_deck == [1, 32, 9, 17, 24, 28, 25, 20]
    True
    '''
    # use indexed for loop to call the swap_cards function twice
    for i in range(2):
        # find the index of JOKER2
        index = deck_cards.index(JOKER2)
        swap_cards(deck_cards, index)

def triple_cut(deck_cards):
    ''' (list of int) -> NoneType
    Given a deck of cards, perform a triple cut after finding the two jokers.
    More specifically, move every card that is above the first joker to the
    bottom and at the same time move every card below the second joker to the
    top. This function is defined as step 3.
    >>> first_deck = [1, 2, 3, 28, 4, 5, 6, 27, 7, 8, 9]
    >>> triple_cut(first_deck)
    >>> first_deck == [7, 8, 9, 28, 4, 5, 6, 27, 1, 2, 3]
    True
    >>> second_deck = [5, 10, 15, 27, 28, 20, 25]
    >>> triple_cut(second_deck)
    >>> second_deck == [20, 25, 27, 28, 5, 10, 15]
    True
    '''
    # find the indices of JOKER1 and JOKER2
    index1 = deck_cards.index(JOKER1)
    index2 = deck_cards.index(JOKER2)
    # reassign the indices as first and second joker
    first_joker = min(index1, index2)
    second_joker = max(index1, index2)

    # store the list of cards before the first joker
    front_list = deck_cards[:first_joker]
    # store the list of cards after the second joker
    back_list = deck_cards[second_joker+1:]

    # use elemental for loop to remove numbers that were at the front and add
    # them to the back
    for number in front_list:
        deck_cards.remove(number)
        deck_cards.append(number)

    # initiate an index counter in order to prepend numbers in the right order
    index = 0
    # use elemental for loop to remove numbers that were at the back and add
    # them to the front
    for number in back_list:
        deck_cards.remove(number)
        deck_cards.insert(index, number)
        index += 1

def insert_top_to_bottom(deck_cards):
    '''(list of int) -> NoneType
    Given a deck of cards, take a number of cards from the top and place them
    right before the bottom card (the last card), where the quantity is
    indicated by the last card. If the last card is JOKER2, treat it as if it
    is JOKER1. This function is defined as step 4.
    >>> this_deck = [1, 2, 3, 5, 6, 7, 4]
    >>> insert_top_to_bottom(this_deck)
    >>> this_deck == [6, 7, 1, 2, 3, 5, 4]
    True
    >>> that_deck = [7, 24, 27, 5, 10, 2, 6, 4, 9, 21, 22, 15, 8, 11, 26, 14, \
    17, 12, 16, 19, 18, 20, 28, 3, 13, 25, 1, 23]
    >>> insert_top_to_bottom(that_deck)
    >>> that_deck == [3, 13, 25, 1, 7, 24, 27, 5, 10, 2, 6, 4, 9, 21, 22, 15, \
    8, 11, 26, 14, 17, 12, 16, 19, 18, 20, 28, 23]
    True
    '''
    # find the value of the bottom card
    last_num = deck_cards[-1]

    # use if statement to adjust the value of the last card when it is JOKER2
    if last_num == JOKER2:
        last_num = JOKER1

    # take the right number of cards as indicated by the last card and
    # rearrange them accordingly

    # store the list of numbers waiting to be arrange
    take_list = deck_cards[:last_num]

    # use elemental for loop to remove numbers that were taken and append them
    # right before the last card (one by one)
    for number in take_list:
        deck_cards.remove(number)
        deck_cards.insert(-1, number)

def get_card_at_top_index(deck_cards):
    ''' (list of int) -> int
    Given a deck of cards, return the card at the index which specified by the
    top card. If the top card is JOKER2, identify it as JOKER1. This function
    is defined as step 5.
    >>> these_deck = [5, 19, 22, 3, 2, 16, 1, 10]
    >>> get_card_at_top_index(these_deck)
    16
    >>> those_deck = [28, 5, 23, 3, 13, 25, 1, 7, 27, 24, 10, 2, 6, 4, 9, 21, \
    22, 15, 8, 11, 26, 14, 17, 12, 16, 19, 18, 20]
    >>> get_card_at_top_index(those_deck)
    20
    '''
    # find the value of the first card
    first_num = deck_cards[0]

    # use if statement to make adjustment to the value of the first card when
    # it is JOKER2
    if first_num == JOKER2:
        first_num = JOKER1

    # find the value at the specific index
    card_val = deck_cards[first_num]

    return card_val

def get_next_value(deck_cards):
    ''' (list of int) -> int
    Given a deck of cards, perform step 1 to 5 as defined and return the next
    potential keystream value.
    >>> family_deck = [9, 4, 7, 10, 13, 16, 19, 14, 23, 25, 3, 5, 8, 11, 12, \
    21, 24, 2, 20, 1, 28, 17, 22, 27, 6, 15, 26, 18]
    >>> get_next_value(family_deck)
    11
    >>> friend_deck = [10, 2, 6, 4, 9, 21, 22, 15, 8, 11, 26, 14, 17, 12, 16, \
    19, 18, 24, 5, 23, 3, 13, 25, 1, 28, 27, 20, 7]
    >>> get_next_value(friend_deck)
    16
    '''
    # calling functions to perform step 1 to 4
    move_joker_1(deck_cards)
    move_joker_2(deck_cards)
    triple_cut(deck_cards)
    insert_top_to_bottom(deck_cards)

    # return the value resulted from step 5
    return get_card_at_top_index(deck_cards)

def get_next_keystream_value(deck_cards):
    ''' (list of int) -> int
    Given a deck of cards, loop through step 1 to 5 by calling get_next_value
    until a valid keystream value is found (in the range of 1 to 26). Return
    that value.
    >>> good_deck = [11, 15, 8, 17, 3, 13, 25, 26, 12, 16, 2, 10, 18, 24, 1, \
    9, 6, 21, 20, 28, 14, 19, 7, 23, 5, 4, 22, 27]
    >>> get_next_keystream_value(good_deck)
    15
    >>> bad_deck = [26, 27, 28, 16, 12, 8, 15, 17, 2, 10, 18, 24, 1, 9, 6, \
    21, 20, 14, 19, 11, 5, 4, 22, 3, 13, 25, 23, 7]
    >>> get_next_keystream_value(bad_deck)
    26
    '''
    # get keystream value from the function get_next_value
    ks_val = get_next_value(deck_cards)

    # use while loop to repeat the process when keystream value is not valid
    while ks_val >= JOKER1:
        ks_val = get_next_value(deck_cards)

    return ks_val

def process_message(deck_cards, msg, mode):
    ''' (list of int, list of str, str) -> str
    Given a deck of cards, a message, and an indication to encrypt or decrypt
    that message ('e' for encrypt and 'd' for decrypt). Return the resulted
    message (string).
    REQ: mode in ['e', 'd']
    '''
    # clean the message to the desired form (i.e. alphabetic)
    msg = clean_message(msg)
    # get a valid keystream value
    ks_val = get_next_keystream_value(deck_cards)

    # create an empty string to store the resultant message
    res_msg = ''

    # use if statement to comfront cases for encryption and decryption
    if mode == 'e':
        # use elemental for loop to encrypt the message letter by letter
        for char in msg:
            res_msg += encrypt_letter(char, ks_val)
            # generate a new keystream value for the next letter
            ks_val = get_next_keystream_value(deck_cards)

    elif mode == 'd':
        # use elemental for loop to decrypt the message letter by letter
        for char in msg:
            res_msg += decrypt_letter(char, ks_val)
            # generate a new keystream value for the next letter
            ks_val = get_next_keystream_value(deck_cards)

    return res_msg

def process_messages(deck_cards, msgs, mode):
    ''' (list of int, list of str, str) -> list of str
    Given a deck of cards, a list of messages, and an indication to encrypt or
    decrypt those messages. Return the resulted list of messages.
    REQ: mode in ['e', 'd']
    '''
    # create an empty string for combining purpose
    msg_str = ''

    # create an almost empty list to store the length of each line (message).
    # Containing '0' for slicing purpose.
    len_msgs = ['0']

    # initiate a variable to store the current length (for accumulation)
    current_len = 0

    # create an empty list (to store the resultant message)
    res_msgs = []

    # use elemental for loop to process each line (message)
    for msg in msgs:
        # get the value of the accumulated length and store separately in list
        len_msgs.append(str(len(msg)+current_len))

        # update the current length
        current_len = len(msg) + current_len

        # concatenate all the messages into a single string
        msg_str += msg

    # process the single string by calling process_message function
    res_msg_str = process_message(deck_cards, msg_str, mode)

    # initiate a variable for index
    index = 0

    # create an empty list for conversion purpose
    len_msgs_int = []

    # use elemental for loop to convert list of string to list of integer
    for length in len_msgs:
        len_msgs_int += [int(length)]

    # use elemental for loop to slice the concatenated string back to the
    # original ones and add to the list (list for storing resultant messages)
    for length in len_msgs:
        # use if statement to check whether the process reaches the end of the
        # list
        if (index + 1) < len(len_msgs_int):
            # slice the string according to the original lengths
            res_msgs.append(res_msg_str[len_msgs_int[index]:
                                        len_msgs_int[index+1]])

        else:
            # last string is the string from the last length (index) to the end
            res_msgs.append(res_msg_str[len_msgs_int[index]:])

        # move on to the next length in the list
        index += 1

    return res_msgs

def read_messages(msg_file):
    ''' (file open for reading) -> list of str
    Given an open message file which contains messages (one per line). Convert
    lines of messages into a list while strip the newline from each line.
    '''
    # create an empty list
    msg_list = []

    # use elemental for loop to process line by line
    for line in msg_file:
        # strip newline and add to the list
        msg_list.append(line.strip('\n'))

    return msg_list

def read_deck(deck_file):
    ''' (file open for reading) -> list of int
    Given an open deck file contains numbers 1 to 28 in unknown order. Return
    these numbers as a list.
    '''
    # create two empty lists for conversion purpose
    deck_list_str = []
    deck_list_int = []

    # use elemental for loop to process line by line (into a list of string)
    for line in deck_file:
        deck_list_str = line.split()
        # use elemental for loop to convert the list of string into a list of
        # integer
        for number in deck_list_str:
            deck_list_int += [int(number)]

    return deck_list_int


# *******************************************************************************
# * Copyright (c) 2020, Lawrence Livermore National Security, LLC.
# * Produced at the Lawrence Livermore National Laboratory.
# *
# * Written by Suraj Kesavan <htpnguyen@ucdavis.edu>.
# *
# * LLNL-CODE-740862. All rights reserved.
# *
# * This file is part of CallFlow. For details, see:
# * https://github.com/LLNL/CallFlow
# * Please also read the LICENSE file for the MIT License notice.
# ******************************************************************************
# Library imports
import pandas as pd

class RuntimeScatterplot:
    def __init__(self, states, module):
        self.graph = state.new_gf.graph
        self.df = state.new_gf.df
        self.module = module
        self.entry_funcs = {}
        self.run(state)

    def run(self):
        ret = []

        # this should not work because there is no self.state
        entire_df = self.state.entire_df
        func_in_module = (
            self.df[self.df.module == self.module]["name"].unique().tolist()
        )

        for idx, func in enumerate(func_in_module):
            ret.append(
                {
                    "name": func,
                    "time (inc)": entire_df.loc[entire_df["name"] == func][
                        "time (inc)"
                    ].tolist(),
                    "time": entire_df.loc[entire_df["name"] == func]["time"].tolist(),
                    "rank": entire_df.loc[entire_df["name"] == func]["rank"].tolist(),
                }
            )
        ret_df = pd.DataFrame(ret)
        return ret_df.to_json(orient="columns")


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 19 21:17:34 2019

@author: Soriba
"""
import os
from PIL import Image
import numpy as np
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical
#data preprocessing
def process_data(data_path):
    classes=os.listdir(data_path)
    imgs=[]
    target=[]
    for classe in classes:
        #path to the class folder
        classe_path=os.path.join(data_path,classe)
        images=os.listdir(classe_path)
        for k in range(len(images)):
            img = Image.open(os.path.join(classe_path,images[k]))
            #normalize the pixels
            imgs.append(np.array(img)/255)
            #for each image, we store it's class into the target vector
            target.append(classe)

    nb_imgs=len(imgs)
    s=imgs[0].shape
    #tensor that will be feed to the network
    X= np.ndarray((nb_imgs,*s))
    for i in range(nb_imgs):
        X[i]=imgs[i]

    #target variable
    le=LabelEncoder()
    le.fit(target)
    #labelencoding
    encoded= le.transform(target)
    #OneHotEncoding
    y=to_categorical(encoded)

    return X, y


import PIL
import tensorflow as tf
import numpy as np
import os
import pickle

from tensorflow.python.keras.models import Model, Sequential
from tensorflow.python.keras.applications import VGG16
from sklearn.preprocessing import StandardScaler

VERBOSE = True

ad_images_path = "../../ad_images/"

def getFileNames(path):
    if VERBOSE:
        print("[MaherBot] Getting filenames from path: " + path)
    return sorted(os.listdir(path))

def loadImage(path):
    temp = PIL.Image.open(path)
    img = temp.copy()
    temp.close()
    return img

filenames = getFileNames(ad_images_path)

ids = list(map(lambda x: x.split(".")[0], filenames))

model = VGG16(include_top=True, weights='imagenet')

input_shape = model.layers[0].output_shape[1:3]

print(model.layers[0].output_shape)
print(input_shape)

imgs = list(map(lambda x: loadImage(ad_images_path + x), filenames))

r_imgs = list(map(lambda x: x.resize(input_shape,PIL.Image.LANCZOS), imgs))

np_imgs = list(map(lambda x: np.array(x), r_imgs))

np_imgs = list(map(lambda x: x.astype('float'), np_imgs))

# for n in np_imgs: print(n.shape)

ids_imgs = list(filter(lambda x: x[1].shape == (224, 224, 3), zip(ids, np_imgs)))

ids = list(map(lambda x: x[0], ids_imgs))

np_imgs = list(map(lambda x: x[1], ids_imgs))

np_imgs = np.array(np_imgs)
print("blabla")
print(np_imgs.shape)

# i.resize(input_shape,PIL.Image.LANCZOS)

# img = np.array(img)
# img = img.astype('float')

layer_name = 'fc2'
fc2_layer_model = Model(inputs=model.input,
                        outputs=model.get_layer(layer_name).output)

features = fc2_layer_model.predict(np_imgs)

scaler = StandardScaler().fit(features)
sfeatures = scaler.transform(features)

results = {}

for i in range(len(ids)):
	results[ids[i]] = sfeatures[i]

with open("data_img_features.pkl", 'wb') as f:
    pickle.dump(results, f, pickle.HIGHEST_PROTOCOL)


# baca citra konversi bit
import cv2
import mylib as my

img = cv2.imread('/root/PycharmProject/PemrogamanTingkatLanjut/lenacitra/lenalr.jpeg')
# baris, kolom, layer = img.shape
# pixel berapa komponen berapa
baris, kolom = img.shape[:2]  # slicing a value

stego = img.copy()

# pesan = "Samarinda kota tepian sudah punya bandara sejak tahun 2018"
fileteks = "/root/PycharmProject/PemrogamanTingkatLanjut/lenacitra/teksuji.txt"
pesan = my.readTextFile(fileteks)
print(pesan)
br = kl = 0

for ch in pesan:
    bits = format(ord(ch), '08b')  # biner dari huruf pada pesan

    for bit in bits:
        stgdat = format(stego[br, kl, 0], '08b')[:-1]  #ambil 7 bit dari citra
        stgdat = stgdat + bit  # tabahkan dengan 1 bit pesan
        stego[br, kl, 0] = int(stgdat,2)# simpan data

        kl += 1
        if kl == kolom:
            br += 1
            kl = 0

# save stegoimage
cv2.imwrite("lenastego.png", img)
# cv2.imshow('lenastego', img)
# cv2.imshow('lena', img)
# cv2.waitKey(0)
# cv2.destroyWindow()
# berapa lebar dan tingginya gunakan sh
# shape ( baris, kolom , layer ) | Shape( 255,255,3)
# 0011.0101
# msg.lsb
stegox = stego
# stegox = cv2.imread("lenastego.png")
# baris, kolom = stegox.shape[:2]

jmlhuruf = 647
pesan = ""

br = kl = 0
bits = ""

while len(pesan) < jmlhuruf:
    # ambil satu bit teakhir
    # print(br, kl, end="\t")
    bit = format(stegox[br, kl, 0], '08b')[-1]
    bits = bits + bit

    # konversi menjadi huruf, dan gabungkan
    if (len(bits)==8):
        huruf = chr(int(bits, 2))
        pesan = pesan + huruf
        bits = ""

    # ubah posisi kolom dan baris
    kl += 1
    if kl == kolom:
        br += 1
        kl = 0

print("\n\n")
print(pesan)


# -*- coding: utf-8 -*-
"""
Created on Fri Feb  2 10:35:42 2018

@author: hsf
"""

#from matplotlib import pyplot
#matplotlib.use('qt4agg')
import matplotlib
#matplotlib.use('qt4agg')
import matplotlib.pyplot as ply

#指定默认字体
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family']='sans-serif'
#解决负号'-'显示为方块的问题
matplotlib.rcParams['axes.unicode_minus'] = False
matplotlib.rcParams["figure.dpi"] = 100

#绘制柱状图
def bar_card_type(card_type_list, card_types):
    xticks = card_types
    cardGroup = {}
    #对每一类成绩进行频数统计
    for i in card_type_list:
        cardGroup[i] = cardGroup.get(i, 0) + 1
    #创建柱状图
    #第一个参数为柱的横坐标
    #第二个参数为柱的高度
    #参数align为柱的对齐方式，以第一个参数为参考标准
    rects = ply.bar(range(len(card_types)), [cardGroup.get(xtick, 0) for xtick in xticks], align='center')

    #设置柱的文字说明
    #第一个参数为文字说明的横坐标
    #第二个参数为文字说明的内容
    ply.xticks(range(len(card_types)), xticks)
    #设置横坐标的文字说明
    ply.xlabel('Card Type')
    #设置纵坐标的文字说明
    ply.ylabel('Frequency')
    #设置标题
    ply.title(u'Card Type Statistic')
    #绘图
    total = len(card_type_list)
    print(total)
    for rect in rects:
        height = rect.get_height()
        ply.text(rect.get_x() + rect.get_width() / 2, height, \
                 round(height/total*100,3), ha='center', va='bottom')
    ply.show()
#    print(cardGroup)

def bar_player(player_list, players):
    xticks = players
    playerGroup = {}
    #对每一类成绩进行频数统计
    for i in player_list:
        playerGroup[i] = playerGroup.get(i, 0) + 1
    #创建柱状图
    #第一个参数为柱的横坐标
    #第二个参数为柱的高度
    #参数align为柱的对齐方式，以第一个参数为参考标准
    rects = ply.bar(range(len(players)), [playerGroup.get(xtick, 0) for xtick in xticks], align='center')

    #设置柱的文字说明
    #第一个参数为文字说明的横坐标
    #第二个参数为文字说明的内容
    ply.xticks(range(len(playerGroup)), xticks)
    #设置横坐标的文字说明
    ply.xlabel('Card Type')
    #设置纵坐标的文字说明
    ply.ylabel('Frequency')
    #设置标题
    ply.title(u'Card Type Statistic')
    #绘图
    total = len(player_list)
    print(total)
    for rect in rects:
        height = rect.get_height()
        ply.text(rect.get_x() + rect.get_width() / 2, height, \
                 round(height/total*100,3), ha='center', va='bottom')
    ply.show()
#    print(playerGroup)


import random

def play_game():
    word_list = ['python', 'java', 'kotlin', 'javascript']
    word = list(random.choice(word_list))
    covered_word = list((len(word) * '-'))
    num_tries = 0
    previous_attempts = []

    def validate_input(user_input):
        if len(user_input) != 1:
            print('You should input a single letter')
            return False
        if ord(user_input) not in range(97, 123):
            print('It is not an ASCII lowercase letter')
            return False
        if user_input in previous_attempts:
            print('You already typed this letter')
            return False
        else:
            return True

    print('H A N G M A N')
    while num_tries < 8:
        if '-' not in covered_word:
            print('You guessed the word!\nYou survived!')
            exit()
        print('\n')
        print("".join(covered_word))
        guess = input('Input a letter:')
        if not validate_input(guess):
            continue
        previous_attempts.append(guess)

        if guess not in word:
            num_tries += 1
            print('No such letter in the word')
        elif guess in covered_word:
            num_tries += 1
            print('No improvements')

        for i in range(0, len(word)):
            if word[i] == guess:
                covered_word[i] = guess

    print("You are hanged!\n")

while True:
    game_selection = input('Type "play" to play the game, "exit" to quit:')
    if game_selection == "exit":
        exit()
    elif game_selection == 'play':
        play_game()
    else:
        continue


import string
from collections import Counter

from util.cipher_base import CipherBase

FREQUENCIES = {
	'E': 12.7, 'T': 9.06, 'A': 8.17, 'O': 7.51, 'I': 6.97, 'N': 6.75, 'S': 6.33, 'H': 6.09, 'R': 5.99,
	'D': 5.25, 'L': 4.03, 'C': 2.78, 'U': 2.76, 'M': 2.41, 'W': 2.36, 'F': 2.23, 'G': 2.02, 'Y': 1.97,
	'P': 1.93, 'B': 1.49, 'V': 0.98, 'K': 0.77, 'X': 0.15, 'J': 0.15, 'Q': 0.1, 'Z': 0.05,
}

class AffineCaesarCipher(CipherBase):

	def __init__(self):
		super().__init__()
		self.key_1 = None
		self.key_2 = None

	def encrypt(self, text):
		if not self.key_1 or not self.key_2:
			raise Exception('cipher: keys not set')
		cipher_text = ''
		for symbol in text.lower():
			if symbol in self.ALPHABET_EN:
				index = self.ALPHABET_EN.find(symbol)
				cipher_text += self.ALPHABET_EN[(index * self.key_1 + self.key_2) % self.en_alphabet_len]
			else:
				cipher_text += symbol
		return cipher_text

	def decrypt(self, text):
		if self.key_1 is None or self.key_2 is None:
			raise Exception('decipher: keys not set')
		plaintext = ''
		mod_inverse_of_key_1 = self.mod_inv(self.key_1, self.en_alphabet_len)
		for symbol in text.lower():
			if symbol in self.ALPHABET_EN:
				index = self.ALPHABET_EN.find(symbol)
				plaintext += self.ALPHABET_EN[(index - self.key_2) * mod_inverse_of_key_1 % self.en_alphabet_len]
			else:
				plaintext += symbol
		return plaintext

	def set_keys(self, key_1, key_2):
		self.check_keys(key_1, key_2)
		key_1 = int(key_1)
		key_2 = int(key_2)
		if key_1 < 0 or key_2 < 0 or key_2 > self.en_alphabet_len - 1:
			raise Exception('key_1 must be greater than 0 and key_2 must be between 0 and %s.' % (self.en_alphabet_len - 1))
		if self.e_gcd(key_1, self.en_alphabet_len)[0] != 1:
			raise Exception(
				'key_1 (%s) and the symbol set size (%s) are not relatively prime, choose a different key.'
				% (key_1, self.en_alphabet_len)
			)
		self.key_1 = key_1
		self.key_2 = key_2

	@staticmethod
	def check_keys(key_1, key_2):
		if isinstance(key_1, str) and isinstance(key_2, str):
			digits = string.digits
			for c in key_1:
				if c not in digits:
					raise Exception('first key contains non-numeric characters')

			for c in key_2:
				if c not in digits:
					raise Exception('second key contains non-numeric characters')
		elif isinstance(key_1, int) and isinstance(key_2, int):
			return
		else:
			raise TypeError('invalid key type')

	@staticmethod
	def mod_inv(num, mod):
		for x in range(0, mod + 1):
			if (num * x) % mod == 1:
				return x
		raise Exception('ERROR: modulo {} inverse of {} does not exists!'.format(mod, num))

	def e_gcd(self, a, b):
		if a == 0:
			return b, 0, 1
		else:
			g, y, x = self.e_gcd(b % a, a)
			return g, x - (b // a) * y, y

	@staticmethod
	def freq_attack(text, log):
		alphabet_len = len(AffineCaesarCipher.ALPHABET_EN)
		deciphered = ''
		key_1, key_2 = None, None
		max_sum = 0
		for i in range(alphabet_len):
			if (i % 2 != 0) and (i != 13):
				for j in range(0, alphabet_len):
					curr_sum = 0
					cipher_alg = AffineCaesarCipher()
					cipher_alg.set_keys(i, j)
					current_text = cipher_alg.decrypt(text)
					letters_freq = Counter(current_text.upper())
					for char in current_text.upper():
						if char in AffineCaesarCipher.ALPHABET_EN.upper():
							curr_sum += letters_freq[char] * FREQUENCIES[char]
					if curr_sum > max_sum:
						deciphered = current_text
						max_sum = curr_sum
						key_1 = i
						key_2 = j
		log('Keys <{}, {}>:\n\n{}'.format(key_1, key_2, deciphered))


import re

from PyQt5.QtCore import Qt

from backend import matting
from gui.general_window import *
from gui.general import *

import numpy as np

class matting_window(general_window):
    """
    This class describes a matting window.
    """

    def __init__(self, parent=None):
        general_window.__init__(self, lambda x: Image.fromarray((x * 255).astype(np.uint8)),
                                load_before=lambda: self.load_extra_now())
        self.method = matting.Matting()

    def get_int(self, field) -> int:
        """
        Makes sure the pixel format is correct

        Parameters
        ----------
        field : str
            Text to extract the number from

        Returns
        -------
        int
           int from the text
        """
        response = re.findall(r'\d+', field)
        if (len(response) == 0):
            return -1
        return int(response[0])

    def load_extra_now(self):
        """
        Showing the image
        """
        self.setAcceptDrops(True)
        self.label_, self.pixmap_ = self.add_image_draggable(self.method.source.data, (
            lambda x: self.pixmap_converter(x)) if not self.pixmap_converter is None else (
            lambda x: Image.fromarray(255 * x)), self)

        self.boundary_box, self.boundary_group_iamge_end = self.add_input_group("Start coordinates", [
            "x", "y"
        ], [
                lambda x: QTimer.singleShot(100,
                    lambda: self.update_image_size()),
                lambda x: QTimer.singleShot(100,
                    lambda: self.update_image_size())
        ])

        self.update_geometry(self.boundary_box.width(), 90, x=10)

        self.boundary_box_max, self.boundary_group_iamge = self.add_input_group("End coordinates", [
            "x1", "y1"
        ], [
            lambda x: QTimer.singleShot(100,
                lambda: self.update_image_size()),
            lambda x: QTimer.singleShot(100,
                lambda: self.update_image_size())
        ])
        self.update_geometry(self.boundary_box_max.width(), 90, x=10 + self.boundary_box.width(), y=-42)
        self.update_image_size()

    def update_image_size(self):
        """
        Update the image size based on the input size
        """
        x0, y0, x1, y1 = list(
            map(lambda x: self.get_int(x.text()), self.boundary_group_iamge_end + self.boundary_group_iamge))
        if x0 == -1:
            x0 = 0
        if y0 == -1:
            y0 = 0
        if x1 == -1:
            x1 = self.method.source.data.shape[1]
        if y1 == -1:
            y1 = self.method.source.data.shape[0]
        if x0 < x1 and y0 < y1:
            self.area = [[x0, x1], [y0, y1]]
            self.label_.setPixmap(
                pil2pixmap(Image.fromarray((255 * self.method.source.data[y0:y1, x0:x1]).astype(np.uint8))))

    def prepare(self):
        """
        Prepare the method before run
        """
        self.method.working_area = self.area
        self.method.padding = [self.label_.pos().x(), self.label_.pos().y()]
        self.label_.setVisible(False)

    def undo(self):
        """
        When the image is reset we need to show the source image again
        """
        self.label_.setVisible(True)

    def dragEnterEvent(self, event):
        """
        Drag event handler for the source image

        Parameters
        ----------
        event : QDragEnterEvent
            The drag event
        """
        event.accept()

    def dropEvent(self, event):
        """
        Drop event handler for the source image

        Parameters
        ----------
        event : QDropEvent
            The drop event
        """
        position = event.pos()
        self.label_.move(position)

        event.setDropAction(Qt.MoveAction)
        event.accept()


#!/usr/bin/env python3

# ------------
# Fibonacci.py
# ------------

from sys    import version
from timeit import timeit

# recursive procedure
# non-linear recursive process
def fibonacci_recursion (n) :
    assert n >= 0
    if n < 2 :
        return n
    return fibonacci_recursion(n - 1) + fibonacci_recursion(n - 2)

# recursive procedure
# linear iterative process
def fibonacci_tail_recursion (n) :
    assert n >= 0
    def f (n, x, y) :
        if n == 0 :
            return x
        return f(n - 1, y, x + y)
    return f(n, 0, 1)

# iterative procedure
# linear iterative process
def fibonacci_while (n) :
    assert n >= 0
    if n < 2 :
        return n
    x = 0
    y = 1
    i = 1
    while i != n :
        x, y = y, x + y
        i += 1
    return y

def fibonacci_range_for (n) :
    assert n >= 0
    if n < 2 :
        return n
    x = 0
    y = 1
    for i in range(1, n) :
        x, y = y, x + y
    return y

def testit (f) :
    f(15)

def test (f) :
    print(f.__name__)
    assert f(0) == 0
    assert f(1) == 1
    assert f(2) == 1
    assert f(3) == 2
    assert f(4) == 3
    assert f(5) == 5
    print("{:.2f} milliseconds".format(timeit("testit(" + f.__name__ + ")", "from __main__ import " + f.__name__ + ", testit", number = 1000) * 1000))
    print()

print("Factorial.py")
print()

print(version)
print()

test(fibonacci_recursion)
test(fibonacci_tail_recursion)
test(fibonacci_while)
test(fibonacci_range_for)

print("Done.")

"""
Fibonacci.py

3.4.2 (v3.4.2:ab2c023a9432, Oct  5 2014, 20:42:22)
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]

fibonacci_recursion
282.06 milliseconds

fibonacci_tail_recursion
2.76 milliseconds

fibonacci_while
1.59 milliseconds

fibonacci_range_for
1.21 milliseconds

Done.
"""


#!/usr/bin/python3
# longest_isoform_in_peptides.py by josh

import sys
import glob
import re
import subprocess
import os

IDPattern = re.compile("^>((TRINITY_DN\d+_c\d+_g\d+)_i\d+)(\.(p\d+)\s+.+){0,1}$")
seqPattern = re.compile("^[^>][A-Z*-]+$")

peptideFastas = [os.path.abspath(x) for x in glob.glob("./*_pep.fasta")]
peptideFastas.sort()

print(peptideFastas)
print(len(peptideFastas))

outDir = "longest_isoform_peptide_fastas"
if not os.path.isdir(outDir):
    os.makedirs(outDir)

for idx, peptideFasta in enumerate(peptideFastas):
    print(idx)
    species = "_".join(os.path.basename(peptideFasta).split("_")[0:2])
    print(species)
    transcriptome = "/data/autoScratch/monthly/btw915/chimaera_filtered_transcriptomes/sequential_fastas/{}_Peru_mega_transcriptome_Josh.fasta".format(species)
    if not os.path.isfile(transcriptome):
        sys.exit("Error, transcriptome file not found")
    print(transcriptome)
    lengthDict = {}
    inFile = open(transcriptome, "r")
    for line in inFile:
        line = line.strip()
        if IDPattern.match(line):
            trinityGene = IDPattern.match(line).group(2)
            isoform = IDPattern.match(line).group(1)
            sequence = inFile.readline().strip()
            if not seqPattern.match(sequence):
                print(sequence)
                sys.exit("Error, sequence not in expected pattern")
            seqLen = len(sequence)
            if trinityGene not in lengthDict:
                lengthDict[trinityGene] = [seqLen, isoform]
            elif seqLen > lengthDict[trinityGene][0]:
                # print("Longer isoform found; dictionary updated")
                lengthDict[trinityGene] = [seqLen, isoform]
            elif seqLen == lengthDict[trinityGene][0]:
                # print("Isoform length exactly the same!")
                pass
        else:
            print(line)
            sys.exit("Error, line not in expected format")
    inFile.close()
    print("Trinity genes processed:", len(lengthDict))
    outDict = {}
    print(peptideFasta)
    masterGeneList = []
    inFile = open(peptideFasta, "r")
    for line in inFile:
        line = line.strip()
        if IDPattern.match(line):
            trinityGene = IDPattern.match(line).group(2)
            isoform = IDPattern.match(line).group(1)
            peptide = IDPattern.match(line).group(4)
            outHeader = "_".join([isoform, peptide])
            sequence = inFile.readline().strip()
            if not seqPattern.match(sequence):
                print(sequence)
                sys.exit("Error, sequence not in expected pattern")
            seqLen = len(sequence)
            if trinityGene not in lengthDict:
                sys.exit("Error, gene not found in dictionary")
            if lengthDict[trinityGene][1] == isoform:
                # print("Longest isoform found")
                # print(outHeader)
                if trinityGene not in outDict:
                    masterGeneList.append(trinityGene)
                    outDict[trinityGene] = [seqLen, outHeader, sequence]
                elif seqLen > outDict[trinityGene][0]:
                    # print("Longer peptide found; dictionary updated")
                    outDict[trinityGene] = [seqLen, outHeader, sequence]
                elif seqLen == outDict[trinityGene][0]:
                    # print("Peptide length exactly the same!")
                    pass
    inFile.close()
    print("Trinity genes in peptide fasta:", len(masterGeneList))
    outFile = open("{}/{}".format(outDir, os.path.basename(peptideFasta)), "w")
    for trinityGene in masterGeneList:
        outLine = ">" + outDict[trinityGene][1]
        outFile.write(outLine + "\n")
        outFile.write(outDict[trinityGene][2] + "\n")
    outFile.close()

quit()


#!/usr/bin/python3

from sys import exit, platform

if platform == 'cygwin':
   from ctypes import CFUNCTYPE as WINFUNCTYPE, cdll as windll
else:
   from ctypes import WINFUNCTYPE, windll

from ctypes import (
   POINTER, Structure, addressof, byref, cast, create_string_buffer,
   c_bool, c_byte, c_char_p, c_long, c_ulong, c_ushort, c_void_p, c_wchar_p
)

class UNICODE_STRING(Structure):
   _fields_ = [
      ('Length',        c_ushort),
      ('MaximumLength', c_ushort),
      ('Buffer',        c_wchar_p),
   ]

class SYSTEM_PROCESS_INFORMATION(Structure):
   _fields_ = [
      ('NextEntryOffset', c_ulong),
      ('Skipped1',        c_byte * 52),
      ('ImageName',       UNICODE_STRING),
      ('BasePriority',    c_long),
      ('UniqueProcessId', c_ulong),
      ('Skippe2',         c_byte * 112),
   ]

EnumLoadedModulesCallback = WINFUNCTYPE(
   c_bool, c_char_p, c_ulong, c_ulong, c_void_p
)

def getmodules(m_name, m_base, m_size, usr_context):
   print('0x%-10.8x0x%-8x%s' % (m_base, m_size, m_name.decode('utf-8')))
   return True

SystemProcessInformation    = 5
STATUS_INFO_LENGTH_MISMATCH = c_long(0xC0000004).value
PROCESS_QUERY_INFORMATION   = 0x400

if __name__ == '__main__':
   sz = c_ulong()
   ptr = create_string_buffer(1024)

   nts = windll.ntdll.NtQuerySystemInformation(
      SystemProcessInformation, ptr, 1024, byref(sz)
   )
   if nts == STATUS_INFO_LENGTH_MISMATCH:
      ptr = create_string_buffer(sz.value)
      nts = windll.ntdll.NtQuerySystemInformation(
         SystemProcessInformation, ptr, sz.value, None
      )
      if nts != 0:
         msg = c_void_p()
         windll.kernel32.FormatMessageW(
            0x00001100, None, windll.ntdll.RtlNtStatusToDosError(nts),
            1024, byref(msg), 0, None
         )
         err = c_wchar_p(msg.value).value
         windll.kernel32.LocalFree(msg)
         print(err.strip() if err else 'Unknown error has been occured.')
         exit(1)
   tmp = cast(ptr, POINTER(SYSTEM_PROCESS_INFORMATION))
   while (1):
      spi = tmp.contents
      proc = windll.kernel32.OpenProcess(
         PROCESS_QUERY_INFORMATION, False, spi.UniqueProcessId
      )
      if not proc:
         pass
      else:
         print('-' * 78)
         print('%s pid: %lu\n' % (spi.ImageName.Buffer, spi.UniqueProcessId))
         print('Base        Size      Path')
         windll.dbghelp.EnumerateLoadedModules(
            proc, EnumLoadedModulesCallback(getmodules), None
         )
         windll.kernel32.CloseHandle(proc)
      if not spi.NextEntryOffset: break
      tmp = cast(addressof(spi) + spi.NextEntryOffset,
         POINTER(SYSTEM_PROCESS_INFORMATION)
      )


import os
import pandas as pd
import numpy as np
from tqdm import tqdm
import subprocess
import zipfile

PATH_TO_BUILDED_PROJECTS = os.path.abspath('./../../data/builded_projects/unpacked_zips')
OUTPUT_UNZIP = os.path.abspath('./../../data/builded_projects/unpacked_zips')
OUTPUT = os.path.abspath('./../../data/processed/new_class_graphs_10')
CSV_FILES = os.path.abspath('./../../data/processed/csv_files_traces')
PATH_TO_JAVA_PARSER = os.path.abspath('./../../../Java2Graph/javaparser-dloc/target/javaparser-dloc-3.5.14-SNAPSHOT-jar-with-dependencies.jar')
projects = list(filter(lambda x: x!= '.ipynb_checkpoints' and os.path.isdir(os.path.join(PATH_TO_BUILDED_PROJECTS, x)), os.listdir(PATH_TO_BUILDED_PROJECTS)))

for project_name in tqdm(projects):
    if project_name not in ['apache-ignite-2d2044a', 'flowable-flowable-engine-f9f4304']:
        continue

    directory_to_extract_to = os.path.join(OUTPUT_UNZIP, project_name)

    df = pd.read_csv('./../../data/builded_projects/good_methods_names_df.csv')
    df = df[df['repo_name'] == project_name]
    print(project_name)
    for g, df_ind in tqdm(df.groupby(np.arange(len(df)) // 100)):
        df_ind.to_csv(os.path.join(CSV_FILES, project_name + f'{g}.csv'), index=False)

        os.makedirs(os.path.join(OUTPUT, project_name), exist_ok=True)
#         os.makedirs(os.path.join(OUTPUT, project_name, 'call_graph'), exist_ok=True)
#         os.makedirs(os.path.join(OUTPUT, project_name, 'gml'), exist_ok=True)

        build_place = directory_to_extract_to
        output_folder = os.path.join(OUTPUT, project_name)
        csv = os.path.join(CSV_FILES, project_name + f'{g}.csv')
        run_arguments=[ 'java', '-Xmx12g', '-XX:-UseGCOverheadLimit', '-jar', PATH_TO_JAVA_PARSER ]
        run_arguments.append(build_place)
        run_arguments.append(output_folder)
        run_arguments.append(csv)
        run_arguments.append(build_place)
        run_arguments.append(project_name)

        subprocess.call(run_arguments)


class Solution:
    def reverse(self, x: int) -> int:
        reverse = 0
        returnNegative = False

        if x < 0:
            x *= -1
            returnNegative = True

        while (x > 0):
            remainder = x % 10

            if reverse > 214748364:
                reverse = 0
            else:
                reverse = (reverse * 10) + remainder

            x = x // 10

            #print(f"Remainder: {remainder}, Reverse: {reverse}, X: {x}")

        if returnNegative is True:
            return reverse*-1
        else:
            return reverse


#!/usr/bin/python3
import os
import re
import sys
import argparse
import configparser

def _argparse():
    parser = argparse.ArgumentParser(description="This is description")
    parser.add_argument('--input', action='store', dest='input_path', type=str, help = "the input folder with kraken2 log files.")
    parser.add_argument('--output', action='store', dest='output_path',type=str,  help = "the output file to write.")
    return parser.parse_args()

def main():
    parser = _argparse()
    abs_path = os.path.abspath(parser.input_path)
    output = open(parser.output_path, "w")
    output.write("sample\ttotal_reads\tconsumed_time\tclassified_reads\tpct\tunclassified_reads\tpct\n")
    pattern1 = re.compile('^(\d+).*in (\d+\.\d+\w).*')
    pattern2 = re.compile('\s+?(\d+)\ssequences\sclassified\s[(](\d+\.\d+%)[)]')
    pattern3 = re.compile('\s+?(\d+)\ssequences\sunclassified\s[(](\d+\.\d+%)[)]')
    files = [abs_path + '/' + each for each in os.listdir(abs_path) if each.endswith("log")]
    for each_file in files:
        print("Reading log file: {}".format(each_file))
        sample_name = os.path.basename(each_file).split(".")[0]
        output_content = []
        with open(each_file, "r") as f:
            for line in f:
                line = line.rstrip("\n")
                match1 = pattern1.match(line)
                if match1:
                    output_content.append(match1.group(1))
                    output_content.append(match1.group(2))
                    continue
                match2 = pattern2.match(line)
                if match2:
                    output_content.append(match2.group(1))
                    output_content.append(match2.group(2))
                    continue
                match3 = pattern3.match(line)
                if match3:
                    output_content.append(match3.group(1))
                    output_content.append(match3.group(2))
                    continue
        output.write("{}\t{}\n".format(sample_name, "\t".join(output_content)))
    print("Summary table written: {}".format(parser.output_path))
    output.close()

if __name__ == '__main__':
    main()


from django.shortcuts import render
from rest_framework import generics, status
from .models import Marafon, Category
from .serializers import MarafonSerializer, CreateMarafonSerializer, CategorySerializer, CreateCategorySerializer, UserSerializer
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from django.contrib.auth.models import User

# Create your views here.
class MarafonView(generics.ListAPIView):
    queryset = Marafon.objects.all()
    serializer_class = MarafonSerializer

class CreateMarafonView(APIView):
    serializer_class = CreateMarafonSerializer
    def get(self,request, *args, **kwargs):
        id = request.query_params["id"]
        pr
    def post(self, request, format=None):
        if request.user.is_authenticated:
            if not self.request.session.exists(self.request.session.session_key):
                self.request.session.create()
            serializer = self.serializer_class(data=request.data)
            if serializer.is_valid():
                title = serializer.data.get('title')
                description = serializer.data.get('description')
                category = serializer.data.get('category')
                owner = self.request.user
                marafon = Marafon(owner=owner, title=title,description=description,category=category)
                CreateCategory(category)
                marafon.save()
            return Response(MarafonSerializer(marafon).data, status=status.HTTP_200_OK)
        else:
            return Response({"Unregistered: cannot access anonymous"})

class CategoryView(generics.ListAPIView):
    queryset = Category.objects.all()
    serializer_class = CategorySerializer

def CreateCategory(label_create):
    if Category.objects.filter(label = label_create.strip().title()):
        return
    else:
        category = Category(label = label_create)
        category.save()

class UserCreate(APIView):
    """
    Creates the user.
    """
    def post(self, request, format='json'):
        serializer = UserSerializer(data=request.data)
        if serializer.is_valid():
            user = serializer.save()
            if user:
                token = Token.objects.create(user=user)
                json = serializer.data
                json['token'] = token.key
                return Response(json, status=status.HTTP_201_CREATED)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


import datetime
import math
from itertools import groupby
from operator import itemgetter
from datetime import timedelta
from openerp import models, fields, api, exceptions, _

def merge_lists(list1, list2):
    # pomozna funkcija, ki zdruzi attendances in leaves
    # list 1 in 2 sta seznama seznamov ali pa sta prazna
    list1_cols = len(list1[0][1:]) if list1 else 0
    list2_cols = len(list2[0][1:]) if list2 else 0
    dic1 = {col[0]: col[1:] for col in list1}
    dic2 = {col[0]: col[1:] for col in list2}
    for k, value in dic2.items():
        if dic1.has_key(k):
            dic1[k] += value
        else:
            dic1[k] = [0] * list1_cols + value
    list_result = []
    for key, value in dic1.items():
        if len(value) == list1_cols:
            list_result.append([key] + value + [0] * list2_cols)
        else:
            list_result.append([key] + value)
    list_result.sort()
    return list_result

class AttendanceReport(models.TransientModel):
    _inherit = 'attendance_report'

    @api.model
    def attendance_employees(self):
        employee_list = super(AttendanceReport, self).attendance_employees()
        if not self.employee_id:
            # todo: filtriraj se po state je validate
            HrHolidays = self.env['hr.holidays']
            holidays_obj = HrHolidays.search([
                '&',
                '|',
                '&',
                ('date_from', '>=', self.date_from),
                ('date_from', '<=', self.date_to),
                '&',
                ('date_to', '>=', self.date_from),
                ('date_to', '<=', self.date_to),
                ('state', '=', 'validate')
            ])
            holiday_employees = holidays_obj.mapped('employee_id')
            employee_list = list(set(employee_list + holiday_employees)) # odstranim duplikate
        return employee_list

    @api.model
    def was_on_leave(self, employee_id, datetime_day):
        '''
        To sem spremenil,ker moram upostevati, da je na en dan lahko vec leavov
        funkcija zdaj vrne vse leave na dolocen dan in njihovo trajanje
        '''

        day = datetime_day.strftime("%Y-%m-%d")
        holiday_ids = self.env['hr.holidays'].search([('state','=','validate'),
                                                      ('employee_id','=',employee_id),
                                                      ('type','=','remove'),
                                                      ('date_from','<=',day),('date_to','>=',day)])
        return  holiday_ids

    @api.model
    def attendance_employee_pivot_lines(self, employee_id):
        # linijam iz attendance, dodam tiste iz leavov
        self.ensure_one()
        start_hour, start_min, end_hour, end_min, working_hours_on_day = employee_id.get_working_hours()
        working_hours_on_day = working_hours_on_day * 60

        # preberem vrstice, ki jim bom dodal stolpce za leave
        header_line, attendance_lines, summary_line, recapitulation_lines = \
            super(AttendanceReport,self).attendance_employee_pivot_lines(employee_id)

        leave_temp_lines = []
        add_leave_lines = [] # vrstice, ki jih dodam attendances
        leave_types = set() # vsi razlicni leavi
        sum_validated_leaves = 0 # sestevek vseh leavov

        day_from = datetime.datetime.strptime(self.date_from,"%Y-%m-%d")
        day_to = datetime.datetime.strptime(self.date_to,"%Y-%m-%d")
        nb_of_days = (day_to - day_from).days + 1
        for day in range(0, nb_of_days):
            d = day_from + timedelta(days=day)
            leave_ids = self.was_on_leave(employee_id.id, d)
            for leave in leave_ids:
                leave_types.add(leave.holiday_status_id.name)
                leave_day_from = datetime.datetime.strptime(leave.date_from,'%Y-%m-%d %H:%M:%S')
                leave_day_to = datetime.datetime.strptime(leave.date_to,'%Y-%m-%d %H:%M:%S')
                hours = int(math.ceil( (leave_day_to - leave_day_from).seconds / 3600.00 ))

                if hours > working_hours_on_day: # leave je bil daljsi kot stevilo delovnih ur, lahko je trajal vec dni
                    hours = working_hours_on_day
                leave_temp_lines.append({
                    'name': datetime.datetime.strftime(d, '%Y-%m-%d'),
                    'action_desc' : leave.holiday_status_id.name,
                    'time_in_action': hours * 60,
                })
                sum_validated_leaves += hours * 60
        # zacasen seznam je napolnjen, sedaj, ga je potrebno pivotirati
        if not leave_types: # ce ni bilo nobene odsotnosti preprosto vrnem
            return (header_line, attendance_lines, summary_line, recapitulation_lines)

        leave_types = list(leave_types) # rabim list, ker set ne gre sortirat
        leave_types.sort()
        leave_temp_lines.sort(key=itemgetter('name'))
        add_header_line = leave_types                   # dodam obstojeci glavi
        add_summary_line = [0] * len(leave_types)       # dodam obstojecemu povzetku
        for leave_date, leave_group in groupby(leave_temp_lines, itemgetter('name')):
            result_line_head = [leave_date]
            result_line_tail = [0] * len(leave_types)
            for holiday in leave_group:
                position = leave_types.index(holiday['action_desc'])
                result_line_tail[position] += holiday['time_in_action']
                add_summary_line[position] += holiday['time_in_action']
            add_leave_lines.append(result_line_head + result_line_tail)

        header_line += add_header_line
        summary_line += add_summary_line
        attendance_lines = merge_lists(attendance_lines, add_leave_lines) # iz dveh seznamov naredim enega, mergam po datumu
        # na rekapitulacijo dodam sestevek vseh leavov za zaposlenenga
        recapitulation_lines.insert(len(recapitulation_lines) - 1, (_('Validated leaves'), sum_validated_leaves))
        # povecam razliko do polnega delovnega casa, ker grejo validated leavei vsi v delovni cas
        # ker ne morem spreminjati tupla, ga zamenjam z novim
        diff = recapitulation_lines[len(recapitulation_lines) -1][1] +  sum_validated_leaves
        recapitulation_lines[len(recapitulation_lines) - 1] =(_('Difference'), diff)
        return (header_line, attendance_lines, summary_line, recapitulation_lines)


# Copyright 2016 Joseph Wright <rjosephwright@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import contextlib
import json
import os
import sys

import click
import voluptuous as v
import yaml

import bossimage as b
import bossimage.core as bc

@click.group()
def main(): pass

@main.command()
@click.argument('instance')
@click.option('-v', '--verbosity', count=True,
              help='Verbosity, may be repeated up to 4 times')
def run(instance, verbosity):
    with load_config() as c:
        validate_instance(instance, c)
        sys.exit(bc.run(instance, c[instance], verbosity))

@main.command()
@click.argument('instance')
def image(instance):
    with load_config() as c:
        validate_instance(instance, c)
        bc.image(instance, c[instance])

@main.command()
@click.argument('instance')
def delete(instance):
    bc.delete(instance)

@main.command('list')
def lst():
    with load_config() as c:
        statuses = bc.statuses(c)
    longest = sorted(len(status[0]) for status in statuses)[-1]
    for instance, created in statuses:
        status = 'Created' if created else 'Not created'
        click.echo('{:{width}}{}'.format(instance, status, width=longest+4))

@main.command()
@click.argument('instance')
def login(instance):
    with load_config() as c:
        validate_instance(instance, c)
        if c[instance]['connection'] == 'winrm':
            click.echo('Login unsupported for winrm connections')
            raise click.Abort()
        bc.login(instance, c[instance])

@main.command()
@click.option('-a', '--attribute')
@click.argument('instance')
def info(attribute, instance):
    with load_config() as c:
        validate_instance(instance, c)
        if not attribute:
            click.echo(json.dumps(c[instance], indent=2, separators=(',', ': ')))
        else:
            if attribute not in c[instance]:
                click.echo('No such attribute {}'.format(attribute))
                raise click.Abort()
            else:
                click.echo(c[instance][attribute])

@main.command()
def version():
    click.echo(b.__version__)

def validate_instance(instance, config):
    if instance not in config:
        click.echo('No such instance {} configured'.format(instance))
        raise click.Abort()

@contextlib.contextmanager
def load_config():
    try:
        c = bc.load_config()
        yield c
    except bc.ConfigurationError as e:
        click.echo(e)
        raise click.Abort()


#coding=utf-8

from MyMatrix import Matrix
from copy import deepcopy
from MatrixAlghoritms import toTriangleShape, addColumn, getCol, printMatr, count
from collections import deque

class Gauss:
    def __init__(self, a, b):
        assert isinstance(a, Matrix) and len(a) == len(b)
        self.A = deepcopy(a)
        self.B = deepcopy(b)

    def getValues(self, matr):
        res = Matrix([])
        for i in range(len(matr))[::-1]:
            zero_zount = count(matr[i], 0.0)
            temp = []
            for j in range(zero_zount + 1, len(matr[i])):
                temp.append(matr[i][j])
            res.append(temp)
        return res

    def getResult(self):
        matr = toTriangleShape(addColumn(self.A, self.B))
        values = self.getValues(matr)
        res = deque(values[0])
        for i in range(1, len(values)):
            temp = 0.
            for j in range(1, len(values[i])):
                temp += res[j - 1] * values[i][j - 1]
            res.appendleft( values[i][-1] - temp )
        return list(res)

    @classmethod
    def getResultStatic(self, A, B):
        res = Gauss(A, B)
        return res.getResult()

if __name__ == '__main__':
    A = Matrix([ [1, 2, 3], [3, 2, 4], [2, -1, 0] ])
    B = Matrix([1, 2, -1])
    print(Gauss.getResultStatic(A, B))


__author__ = 'Bohdan Mushkevych'

import psutil
from subprocess import PIPE

from settings import settings
from system.process_context import ProcessContext
from workers.abstract_cli_worker import AbstractCliWorker

class PigDriver(AbstractCliWorker):
    """Python process that starts Pig processing job, supervises its execution and updates unit_of_work"""

    def __init__(self, process_name):
        super(PigDriver, self).__init__(process_name)

    def _start_process(self, start_timeperiod, end_timeperiod, arguments):
        try:
            input_file = ProcessContext.get_source(self.process_name)

            self.logger.info('start: %s {' % self.process_name)
            p = psutil.Popen([settings['bash_shell'],
                              settings['pig_command'],
                              '-f', '/home/bmushkevych/git/synergy-pig/script.pig',
                              '-p', 'input_file=' + input_file + '/' + start_timeperiod,
                              '-p', 'timeperiod=' + start_timeperiod],
                             close_fds=True,
                             cwd=settings['process_cwd'],
                             stdin=PIPE,
                             stdout=PIPE,
                             stderr=PIPE)
            self.cli_process = p
            self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True)
        finally:
            self.logger.info('}')