from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from django.shortcuts import render, get_list_or_404, get_object_or_404
from .models import Movie, Review, Genre
from django.contrib.auth import get_user_model
import jwt
from django.conf import settings
from .serializers import MovieListSerializer, MovieSerializer, ReviewListSerializer, ReviewSerializer
from rest_framework.decorators import authentication_classes, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework_jwt.authentication import JSONWebTokenAuthentication

# Create your views here.
@api_view(['GET'])
def movies(request):
    movie_list = get_list_or_404(Movie)
    serializer = MovieSerializer(movie_list, many=True)
    return Response(serializer.data)

@api_view(['GET'])
def movie_detail(request, movie_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    serializer = MovieSerializer(movie)
    return Response(serializer.data)

@api_view(['GET', 'POST'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def reviews(request, movie_pk):
    if request.method == 'GET':
        review_list = Review.objects.all().filter(movie_id=movie_pk)
        serializer = ReviewListSerializer(review_list, many=True)
        return Response(serializer.data)

    else:
        serializer = ReviewListSerializer(data=request.data)
        if serializer.is_valid(raise_exception=True):
            movie = get_object_or_404(Movie, pk=request.data.get('movie'))

            pre_point = movie.vote_average * movie.vote_count

            point = pre_point+int(request.data.get('rank'))
            count = movie.vote_count + 1
            new_vote_average = round(point/count, 2)

            movie.vote_average = new_vote_average
            movie.vote_count = count
            movie.save()
            # print(request.user)
            serializer.save(user=request.user)
            return Response(serializer.data, status=status.HTTP_201_CREATED)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def my_review(request, my_pk):
    if request.method == 'GET':
        review_list = Review.objects.all().filter(user_id=my_pk)
        serializer = ReviewListSerializer(review_list, many=True)
        return Response(serializer.data)

# authentication_classes 붙여줘야함!
@api_view(['PUT', 'DELETE'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def review_update_delete(request, review_pk):
  review = get_object_or_404(Review, pk=review_pk)
  if not request.user.reviews.filter(pk=review_pk).exists():
    return Response({'message': '권한이 없습니다.'})

  if request.method == 'PUT':
    serializer = ReviewListSerializer(review, data=request.data)

    # if serializer.is_valid(raise_exception=True):
    if serializer.is_valid(raise_exception=True):
      movie = get_object_or_404(Movie, pk=request.data.get('movie'))
      pre_point = movie.vote_average * (movie.vote_count - 1)
      point = pre_point+int(request.data.get('rank'))
      count = movie.vote_count
      new_vote_average = round(point/count, 2)
      movie.vote_average = new_vote_average
      movie.vote_count = count
      movie.save()
      serializer.save(user=request.user)
      return Response(serializer.data)

  else:
    review = get_object_or_404(Review, pk=review_pk)
    # token = request.headers['Authorization'].split()[1]
    # SECRET_KEY = settings.SECRET_KEY
    # print(token)
    # 디코드하려면 재료 3개
    # payload = jwt.decode(token,SECRET_KEY,algorithms=['HS256'])
    # print(payload['user_id'])
    movie = get_object_or_404(Movie, pk=review.movie_id)
    pre_point = movie.vote_average * (movie.vote_count)
    # pre_count = movie.vote_count
    point = pre_point - review.rank
    count = movie.vote_count-1
    new_vote_average = round(point/count, 2)
    movie.vote_average = new_vote_average
    movie.vote_count = count
    movie.save()
    # print(request.user) 익명으로 안받아와짐 그래서 토큰으로 받아줌
    review.delete()
    return Response({ 'id': review_pk })

@api_view(['POST'])
def recommend(request):
    me_like = request.data.get('me_like')

    # 인기순
    favorite_movies = Movie.objects.all().order_by('-vote_average')[:10]
    favorite_serialize = MovieSerializer(favorite_movies, many=True)
    # 리뷰 기반 장르기반 추천
    user_movies_review = []
    # 배우기반 추천
    user_movies_actor = []
    # 감독기반 추천
    user_movies_director = []
    # 개봉년도
    # 제작 국가
    # 리뷰 기반 장름별
    reviews = Review.objects.all()
    for review in reviews:
        movie = Movie.objects.get(pk=review.movie_id)
        if not movie in user_movies_review:
            user_movies_review.append(movie)

    # 좋아요 기반추천
    my_user_like_movies = []
    user_like_movies = []
    # 좋아요 기반 추천
    like_movies = request.data.get('like_movies')
    for like_movie in like_movies:
        movie = get_object_or_404(Movie, pk=like_movie)
        if not movie in my_user_like_movies:
            my_user_like_movies.append(movie)
    # 내가 좋아요 한 것 제거
    for like_movie in my_user_like_movies:
        # print(me_like)
        if like_movie.id not in me_like:
            user_like_movies.append(like_movie)
    # print(user_like_movies)

    # user_genre_serialize = MovieSerializer(user_movies_review, many=True)
    user_like_serialize = MovieSerializer(user_like_movies, many=True)

    # 연령대
    user_movies_age = []
    # 연령별 기반 추천
    me_age = request.data.get('me_age')
    for age in me_age:
        movie = get_object_or_404(Movie, pk=age)
        if not movie in user_movies_age:
            user_movies_age.append(movie)

    user_movies_age_serializer = MovieSerializer(user_movies_age, many=True)
    return Response([favorite_serialize.data, user_like_serialize.data, user_movies_age_serializer.data])

@api_view(['POST'])
# @authentication_classes([JSONWebTokenAuthentication])
# @permission_classes([IsAuthenticated])
def my_movie_like(request, my_pk):
    me = get_object_or_404(get_user_model(), pk=my_pk)
    # print(me)
    data = []
    movies = request.data
    for movie_pk in movies:
        movie = get_object_or_404(Movie, pk=movie_pk)
        serializer = MovieSerializer(movie)
        data.append(serializer.data)

    return Response(data)

@api_view(['POST'])
def my_movie_dislike(request, my_pk):
    me = get_object_or_404(get_user_model(), pk=my_pk)
    # print(me)
    data = []
    movies = request.data
    for movie_pk in movies:
        movie = get_object_or_404(Movie, pk=movie_pk)
        serializer = MovieSerializer(movie)
        data.append(serializer.data)

    return Response(data)

@api_view(['POST'])
def my_movie_wish(request, my_pk):
    me = get_object_or_404(get_user_model(), pk=my_pk)
    # print(me)
    data = []
    movies = request.data
    for movie_pk in movies:
        movie = get_object_or_404(Movie, pk=movie_pk)
        serializer = MovieSerializer(movie)
        data.append(serializer.data)

    return Response(data)

@api_view(['POST'])
def movie_like(request, my_pk, movie_title):
  movie = get_object_or_404(Movie, title=movie_title)
  me = get_object_or_404(get_user_model(), pk=my_pk)
  if me.like_movies.filter(pk=movie.pk).exists():
      me.like_movies.remove(movie.pk)
      liking = False

  else:
      me.like_movies.add(movie.pk)
      liking = True

  return Response(liking)

@api_view(['POST'])
def movie_dislike(request, my_pk, movie_title):
  movie = get_object_or_404(Movie, title=movie_title)
  me = get_object_or_404(get_user_model(), pk=my_pk)
  if me.dislike_movies.filter(pk=movie.pk).exists():
      me.dislike_movies.remove(movie.pk)
      disliking = False

  else:
      me.dislike_movies.add(movie.pk)
      disliking = True

  return Response(disliking)

@api_view(['POST'])
def movie_wish(request, my_pk, movie_title):
  movie = get_object_or_404(Movie, title=movie_title)
  me = get_object_or_404(get_user_model(), pk=my_pk)
  if me.wish_movies.filter(pk=movie.pk).exists():
      me.wish_movies.remove(movie.pk)
      wishing = False

  else:
      me.wish_movies.add(movie.pk)
      wishing = True

  return Response(wishing)

@api_view(['POST'])
def like_movie_users(request, my_pk):
  # print(request.data)
  users = []
  movies = request.data.get('movies')

  for movie in movies:
    movie = get_object_or_404(Movie, pk=movie)
    serializer = MovieSerializer(movie)

    for user in serializer.data.get('like_users'):
      if user not in users:
        users.append(user)

  return Response(users)

@api_view(['POST'])
def dislike_movie_users(request, my_pk):
  # print(request.data)
  users = []
  movies = request.data.get('movies')

  for movie in movies:
    movie = get_object_or_404(Movie, pk=movie)
    serializer = MovieSerializer(movie)

    for user in serializer.data.get('dislike_users'):
      if user not in users:
        users.append(user)

  return Response(users)

@api_view(['POST'])
def wish_movie_users(request, my_pk):
  # print(request.data)
  users = []
  movies = request.data.get('movies')

  for movie in movies:
    movie = get_object_or_404(Movie, pk=movie)
    serializer = MovieSerializer(movie)

    for user in serializer.data.get('wish_users'):
      if user not in users:
        users.append(user)

  return Response(users)


# https://docs.python.org/3/library/collections.html
from collections import defaultdict
import math

class NGram(object):

    def __init__(self, n, sents):
        """
        n -- order of the model.
        sents -- list of sentences, each one being a list of tokens.
        """
        assert n > 0
        self.n = n
        self.counts = counts = defaultdict(int)

        for sent in sents:
            for i in range(len(sent) - n + 1):
                ngram = tuple(sent[i: i + n])
                counts[ngram] += 1
                counts[ngram[:-1]] += 1

    def prob(self, token, prev_tokens=None):
        n = self.n
        if not prev_tokens:
            prev_tokens = []
        assert len(prev_tokens) == n - 1

        tokens = prev_tokens + [token]
        return float(self.counts[tuple(tokens)]) / self.counts[tuple(prev_tokens)]

    def count(self, tokens):
        """Count for an n-gram or (n-1)-gram.

        tokens -- the n-gram or (n-1)-gram tuple.
        """

    def cond_prob(self, token, prev_tokens=None):
        """Conditional probability of a token.

        token -- the token.
        prev_tokens -- the previous n-1 tokens (optional only if n = 1).
        """
        n = self.n
        if not prev_tokens:
            prev_tokens = []
        assert len(prev_tokens) == n - 1

        tokens = prev_tokens + [token]
        return float(self.counts[tuple(tokens)]) / self.counts[tuple(prev_tokens)]

    def sent_prob(self, sent):
        """Probability of a sentence. Warning: subject to underflow problems.

        sent -- the sentence as a list of tokens.
        """
        sent_prob = 1.0
        for i in range(len(sent)):
            sent_prob *= self.cond_prob(sent[i], sent[i-self.n:i])
        # VER CASOS i<n
        return sent_prob

    def sent_log_prob(self, sent):
        """Log-probability of a sentence.

        sent -- the sentence as a list of tokens.
        """
        sent_log_prob = 1.0
        for i in range(len(sent)):
            sent_log_prob *= math.log(self.cond_prob(sent[i], sent[i-self.n:i]))
        # VER CASOS i<n
        return sent_log_prob

class NGramGenerator:

    def __init__(self, model):
        """
        model -- n-gram model.
        """
        self.model = model

    def generate_sent(self):
        """Randomly generate a sentence."""
        sent = []
        prev_tokens = []
        i = 0
        while token != "STOP":
            prev_tokens = sent[i - self.model.n : i]
            token = self.generate_token(prev_tokens)
            sent.append(token)
            i += 1

        return sent

    def generate_token(self, prev_tokens=None):
        """
        Randomly generate a token, given prev_tokens.

        prev_tokens -- the previous n-1 tokens (optional only if n = 1).
        """

        # calcular cual de todas las palabras del vocabulario es la
        # mas probable que ocurra dado prev_tokens?


__author__ = '未昔'
__date__ = '2018/8/20 13:02'

from django.conf.urls import url,include
from news.views import NewView,NewContent
# from organization.views import OrgView,AddUserAskView,OrgHomeView,OrgCourseView

urlpatterns = [
    #课程机构首页
    url(r'^list/$', NewView.as_view(), name="new_list"),  # 新闻页面
    url(r'^new_content/(\d+)$', NewContent.as_view(), name="new_content"),  # (?P<org_id>\d+) 那个页面：取出数字
    # url(r'^home/(?P<org_id>\d+)/$', OrgHomeView.as_view(), name="org_home"),  # (?P<org_id>\d+) 那个页面：取出数字
    # url(r'^course/(?P<org_id>\d+)/$', OrgCourseView.as_view(), name="org_course"),  # 机构课程列表页url
]


def loadBeamModel(beamModelFileName, displayInfo=False):
    import pandas as pd
    if displayInfo:
        print('# Loading beam model from {:s}'.format(beamModelFileName), end=' ')
    try:
        beamModel=pd.read_csv(beamModelFileName, delim_whitespace=True, index_col='Energy')
        if {'Energy_MeV','dEnergy_pr','scalingFactor','FWHMX_cm','alphaX','betaX_cm','epsilonX_cm','FWHMY_cm','alphaY','betaY_cm','epsilonY_cm'}.issubset(beamModel.columns):
            if displayInfo:
                print('...OK')
            return beamModel
        else:
            print('\nError while loading beam model from \'{:s}\'. Missing columns or wrong column names.'.format(beamModelFileName))
            exit(1)
    except:
        print('\nError while loading beam model from \'{:s}\'. No such file.'.format(beamModelFileName))
        exit(1)

def loadDicomRT(dicomRTFileName, displayInfo=False):
    import pydicom as dicom
    if displayInfo:
        print('# Loading RT plan from {:s}'.format(dicomRTFileName), end=' ')
    try:
        dicomRT=dicom.read_file(dicomRTFileName)
        if dicomRT.SOPClassUID=='1.2.840.10008.5.1.4.1.1.481.8':
            if displayInfo:
                print('...OK')
            return dicomRT
        else:
            print('\nError while loading RT plan from {:s}. The file does not contain RT plan.'.format(dicomRTFileName))
            exit(1)
    except:
        print('\nError while loading RT plan from {:s}. No such file.'.format(dicomRTFileName))
        exit(1)

def interpolateBeamModel(beamModel, Energy, method='slinear'): # do not use methods 'linear'
    import pandas as pd
    beamModelEnergy=beamModel.reindex(beamModel.index.values.tolist()+[Energy]).sort_index().interpolate(method=method).loc[Energy]
    if isinstance(beamModelEnergy,pd.DataFrame):
        beamModelEnergy=beamModelEnergy.iloc[0]
    return beamModelEnergy

def buildPLAN(RTplanFileName, beamModelFileName, RSModelFileName, materialDefinitionFileName, beamModelInterpolationMethod='slinear', displayInfo=False):
    import sys, os
    import pandas as pd
    import pydicom as dicom
    import numpy as np
    from shutil import copyfile

    dicomRT=loadDicomRT(RTplanFileName, displayInfo=displayInfo)
    planInfo={'beamModelFileName': beamModelFileName,
              'RSModelFileName': RSModelFileName,
              'materialDefinitionFileName': materialDefinitionFileName,
              'RTplanFileName': RTplanFileName}

    # get field number and types
    NumberOfFields = dicomRT.FractionGroupSequence[0].NumberOfBeams
    NumberOfFieldsTreatment=0
    NumberOfFieldsSetup=0
    NumberOfFieldsOther=0
    for ifield in range(NumberOfFields):
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='TREATMENT':
            NumberOfFieldsTreatment+=1
            planInfo['TreatmentMachineName']=dicomRT.IonBeamSequence[ifield].TreatmentMachineName
        elif dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='SETUP':
            NumberOfFieldsSetup+=1
        else:
            NumberOfFieldsOther+=1
    # get number of fractions
    NumberOfFractionsPlanned = dicomRT.FractionGroupSequence[0].NumberOfFractionsPlanned

    # load beam model
    beamModel=loadBeamModel(beamModelFileName, displayInfo=displayInfo)

    if displayInfo:
        print('# Building plan:')
        print('# Nominal gantry: {:s}'.format(planInfo['TreatmentMachineName']))
        print('# Number of fields: {:d} treatment, {:d} setup,  {:d} other'.format(NumberOfFieldsTreatment, NumberOfFieldsSetup, NumberOfFieldsOther))
        print('# Number of fraction planned: {:d}'.format(NumberOfFractionsPlanned))

    planInfo['pencilBeamNumberAll']=0
    planInfo['protonNumberAll']=0
    planInfo['planTotalEnergy']=0
    planInfo['fieldIDList']=[]
    planInfo['fieldNameList']=[]

    fieldsInfo=[]
    for ifield in range(NumberOfFields):
        # convert if is a TREATMENT field (not SETUP or OTHER)
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType!='TREATMENT':
            continue;

        if displayInfo:
            print('Field {:d} / {:d}:'.format(ifield+1,NumberOfFieldsTreatment))

        IonBeamSequence = dicomRT.IonBeamSequence[ifield]

        # check if RadiationType is 'PROTON'
        if IonBeamSequence.RadiationType != 'PROTON':
            print('\t\tWarning: No PROTON plan. Field skipped.')
            npbVec.append(0)
            continue

        # check if the field is MODULATED which means that some properties are the same for all pb
        if IonBeamSequence.ScanMode != 'MODULATED' or IonBeamSequence.BeamType != 'STATIC':
            print('Error this RTPLAN is not for Raster Scan')
            continue;

        # check cumulative MU
        for ReferencedBeamSequence in dicomRT.FractionGroupSequence[0].ReferencedBeamSequence:
            if ReferencedBeamSequence.ReferencedBeamNumber==IonBeamSequence.BeamNumber:
                BeamMeterset=ReferencedBeamSequence.BeamMeterset
                break;
            else:
                BeamMeterset=np.nan
        if np.isnan(BeamMeterset):
            print('Error: cannot find BeamMeterset for beam number ID {:d}'.format(IonBeamSequence.BeamNumber))
            exit(1)

        # check presence of Range Shifter and Range Shifter ID
        if IonBeamSequence.NumberOfRangeShifters==1:
            if 'RangeShifterSequence' in IonBeamSequence:
                RangeShifterID=IonBeamSequence.RangeShifterSequence[0].RangeShifterID
        else:
            RangeShifterID=''

        # check distance of scanning magnets to isocentre
        for LateralSpreadingDeviceSettingsSequence in IonBeamSequence.IonControlPointSequence[0].LateralSpreadingDeviceSettingsSequence:
            for LateralSpreadingDeviceSequence in IonBeamSequence.LateralSpreadingDeviceSequence:
                if LateralSpreadingDeviceSettingsSequence.ReferencedLateralSpreadingDeviceNumber==LateralSpreadingDeviceSequence.LateralSpreadingDeviceNumber:
                    if LateralSpreadingDeviceSequence.LateralSpreadingDeviceID=='MagnetX':
                        MagnetX_mm=LateralSpreadingDeviceSettingsSequence.IsocenterToLateralSpreadingDeviceDistance
                    elif LateralSpreadingDeviceSequence.LateralSpreadingDeviceID=='MagnetY':
                        MagnetY_mm=LateralSpreadingDeviceSettingsSequence.IsocenterToLateralSpreadingDeviceDistance
        if MagnetX_mm<MagnetY_mm:
            print('Error: script assumes that the first scanning magnet is deflecting along X (MagnetX_mm={:.3f}, MagnetY_mm={:.3f},)'.format(MagnetX_mm,MagnetY_mm))
            exit(1)
        # define distance from field origin to isocentre
        fieldOriginToIsoDist_mm=MagnetY_mm

        # collect field information
        fieldInfo={'BeamNumber': int(IonBeamSequence.BeamNumber),  # check beam number ID (field ID)
                   'BeamName': IonBeamSequence.BeamName,
                   'RadiationType': IonBeamSequence.RadiationType, # type of radiation (PROTON)
                   'BeamType': IonBeamSequence.BeamType,
                   'ScanMode': IonBeamSequence.ScanMode,
                   'RangeShifterID': RangeShifterID,
                   'NumberOfControlPoints': int(IonBeamSequence.NumberOfControlPoints), # number of control points (2 x energy slice number) in field
                   'energySliceNumber': int(IonBeamSequence.NumberOfControlPoints/2), # number of energy slices in field
                   'MUsum': float(BeamMeterset), # cumulative MU for field
                   'WeightSum': float(IonBeamSequence.FinalCumulativeMetersetWeight), # cumulative weights for field (the sum of spot weights is used instead)
                   'GantryAngle_deg': float(IonBeamSequence.IonControlPointSequence[0].GantryAngle),
                   'CouchAngle_deg': float(IonBeamSequence.IonControlPointSequence[0].PatientSupportAngle),
                   'fieldOriginToIsoDist_mm': fieldOriginToIsoDist_mm, # distance of field origin point from Iso
                   'SnoutPosition': 368.5, # value taken from TPS eclipse (SnoutPosition 430 means 368.5 mm distance from Iso to RS surface) #IonBeamSequence.IonControlPointSequence[0].SnoutPosition, # snaut position
                   'IsocenterPosition_mm': np.array(IonBeamSequence.IonControlPointSequence[0].IsocenterPosition), # isocentre position
                   'VaccumExitToISODist_mm': 1680
                  }

        if displayInfo:
            print('\tBeam Type: {:s}'.format(fieldInfo['RadiationType']))
            print('\tBeam number ID = {:d}'.format(fieldInfo['BeamNumber']))
            print('\tBeam name = \'{:s}\''.format(fieldInfo['BeamName']))
            print('\tNumber of energy  = {:d}'.format(fieldInfo['energySliceNumber']))
            print('\tCumulative MU = {:.4f}'.format(fieldInfo['MUsum']))
            print('\tRange Shifter = {:s}'.format('None' if not fieldInfo['RangeShifterID'] else fieldInfo['RangeShifterID']))
            print('\tGantry Angle = {:.2f}°'.format(fieldInfo['GantryAngle_deg']))
            print('\tCouch Angle = {:.2f}°'.format(fieldInfo['CouchAngle_deg']))
            print('\tField Origin To Iso Dist = {:.2f} mm'.format(fieldInfo['fieldOriginToIsoDist_mm']))
            print('\tSnout position = {:.2f} mm'.format(fieldInfo['SnoutPosition']))
            print('\tIso position Vec. =              [ {:.2f} {:.2f} {:.2f}] mm'.format(fieldInfo['IsocenterPosition_mm'][0],fieldInfo['IsocenterPosition_mm'][1],fieldInfo['IsocenterPosition_mm'][2]))

        fieldInfo['slicesInfo']=[]
        for IonControlPointSequence in IonBeamSequence.IonControlPointSequence:
            if np.array(IonControlPointSequence.ScanSpotMetersetWeights).sum()==0:
                continue
            NumberOfScanSpotPositions=int(IonControlPointSequence.NumberOfScanSpotPositions)
            beamModelEnergy=interpolateBeamModel(beamModel, float(IonControlPointSequence.NominalBeamEnergy), method=beamModelInterpolationMethod)

            sliceInfo={ 'NominalBeamEnergy': float(IonControlPointSequence.NominalBeamEnergy),
                        'NumberOfPaintings': int(IonControlPointSequence.NumberOfPaintings),
                        'FRED_beamEnergy': beamModelEnergy.Energy_MeV,
                        'FRED_pSpread': beamModelEnergy.dEnergy_pr,
                        'FRED_epsilonX_cm': beamModelEnergy.epsilonX_cm,
                        'FRED_epsilonY_cm': beamModelEnergy.epsilonY_cm,
                        'FRED_alphaX': beamModelEnergy.alphaX,
                        'FRED_alphaY': beamModelEnergy.alphaY,
                        'FRED_betaX_cm': beamModelEnergy.betaX_cm,
                        'FRED_betaY_cm': beamModelEnergy.betaY_cm,
                        'FRED_scalingFactor': beamModelEnergy.scalingFactor,
                        'ScanSpotTuneID': float(IonControlPointSequence.ScanSpotTuneID)}

            # calculate spots parameters
            sliceSpots=pd.DataFrame({'ScanSpotWeight': IonControlPointSequence.ScanSpotMetersetWeights,
                                     'ScanSpotPositionX_mm': IonControlPointSequence.ScanSpotPositionMap[0::2],
                                     'ScanSpotPositionY_mm': IonControlPointSequence.ScanSpotPositionMap[1::2]})
            sliceSpots['ScanSpotMU'] = sliceSpots.ScanSpotWeight/fieldInfo['WeightSum']*fieldInfo['MUsum']
            sliceSpots['FRED_protonNumber'] = sliceSpots.ScanSpotMU*sliceInfo['FRED_scalingFactor']*sliceInfo['NumberOfPaintings']
            sliceSpots['FRED_energyDelivered'] = sliceSpots['FRED_protonNumber']*sliceInfo['FRED_beamEnergy']
            sliceInfo['PencilBeams']=len(sliceSpots.index)

            P_mm=np.zeros((3,sliceInfo['PencilBeams']))
            V_mm=np.array([-sliceSpots.ScanSpotPositionX_mm, sliceSpots.ScanSpotPositionY_mm, np.repeat(fieldInfo['fieldOriginToIsoDist_mm'],sliceInfo['PencilBeams'])])-P_mm
            V=V_mm/np.linalg.norm(V_mm, axis=0)
            sliceSpots['FRED_Px']=P_mm[0]/10
            sliceSpots['FRED_Py']=P_mm[1]/10
            sliceSpots['FRED_Pz']=P_mm[2]/10
            sliceSpots['FRED_Vx']=V[0]
            sliceSpots['FRED_Vy']=V[1]
            sliceSpots['FRED_Vz']=V[2]

            sliceInfo['sliceSpots']=sliceSpots

            fieldInfo['slicesInfo'].append(sliceInfo)

        fieldsInfo.append(fieldInfo)

        fieldInfo['PencilBeams']=0
        fieldInfo['PencilBeamsProtonNumber']=0
        fieldInfo['energyDelivered']=0
        for sliceInfo in fieldInfo['slicesInfo']:
            fieldInfo['PencilBeams']+=sliceInfo['PencilBeams']
            fieldInfo['PencilBeamsProtonNumber']+=sliceInfo['sliceSpots'].FRED_protonNumber.sum()
            fieldInfo['energyDelivered']=sliceInfo['sliceSpots'].FRED_energyDelivered.sum()

        planInfo['pencilBeamNumberAll']+=fieldInfo['PencilBeams']
        planInfo['protonNumberAll']+=fieldInfo['PencilBeamsProtonNumber']
        planInfo['planTotalEnergy']+=fieldInfo['energyDelivered']
        planInfo['fieldIDList'].append(fieldInfo['BeamNumber'])
        planInfo['fieldNameList'].append(fieldInfo['BeamName'])

        if displayInfo:
            print('\tPencil Beams loaded = {:d}'.format(fieldInfo['PencilBeams']))
    if displayInfo:
        print('# Total pencil beams loaded in the plan = {:d}'.format(planInfo['pencilBeamNumberAll']))
        print('# Total energy delivered by the plan = {:.20E}'.format(planInfo['planTotalEnergy']))

    return planInfo, fieldsInfo

def writeRtplan(rtplanfileName, planInfo, fieldsInfo, getPlanVersion='', fieldToSave=None, displayInfo=True):
    import sys, os, glob
    import pandas as pd
    import pydicom as dicom
    import numpy as np
    from shutil import copyfile
    Xsec='emittance'

    rtplanfileName=os.path.abspath(rtplanfileName)

    # remove old beam model file, RS model file and material definition file from simulation folder
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*beamModel.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*beamModel.txt'))[0])
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*RSModel.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*RSModel.txt'))[0])
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*materialDefinition.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*materialDefinition.txt'))[0])

    # copy current beam model file, RS model file and material definition file to simulation folder
    copyfile(planInfo['beamModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['beamModelFileName'])))
    copyfile(planInfo['RSModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['RSModelFileName'])))
    copyfile(planInfo['RSModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'regions.inp'))
    copyfile(planInfo['materialDefinitionFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['materialDefinitionFileName'])))
    copyfile(planInfo['materialDefinitionFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'materials.inp'))
    rtplan_H=open(rtplanfileName,'w')

    # choose field to save or save all fields
    if not fieldToSave==None:
        if   isinstance(fieldToSave, int):
            if fieldToSave in planInfo['fieldIDList']:
                fieldToSaveIdx=planInfo['fieldIDList'].index(fieldToSave)
                print('# Requested to save the rtplan only for the field ID {:d}'.format(fieldToSave))
            else:
                print('# Error: cannot find field ID {:d} in the treatment plan. Available field IDs are '.format(fieldToSave)+str(planInfo['fieldIDList']))
                exit(-1)
        elif isinstance(fieldToSave, str):
            if fieldToSave in planInfo['fieldNameList']:
                fieldToSaveIdx=planInfo['fieldNameList'].index(fieldToSave)

                print('# Requested to save the rtplan only for the field name \'{:s}\''.format(fieldToSave))
            else:
                print('# Error: cannot find field name \'{:s}\' in the treatment plan. Available field names are '.format(fieldToSave)+str(planInfo['fieldNameList']))
                exit(-1)
        fieldsInfo=[fieldsInfo[fieldToSaveIdx]]

        # write information header
    print('###############################################################################################################################################', file=rtplan_H)
    print('###### Beam plan for {:s} machine prepared in getPlan v.{:s}'.format(planInfo['TreatmentMachineName'], getPlanVersion), file=rtplan_H)
    print('###### DICOM plan {:s}'.format(planInfo['RTplanFileName']), file=rtplan_H)
    print('###### numPB {:d}, totNumParticles {:.20E}'.format(planInfo['pencilBeamNumberAll'],planInfo['protonNumberAll']), file=rtplan_H)
    print('###############################################################################################################################################', file=rtplan_H)

    # write field for G0
    for fieldInfo in fieldsInfo:
        print('field: {:d} ; '.format(fieldInfo['BeamNumber']), file=rtplan_H, end='')
        print('O=[ {:+.4f}, {:+.4f}, {:+.4f} ];\t'.format(0,-fieldInfo['fieldOriginToIsoDist_mm']/10,0), file=rtplan_H, end='')
        print('f=[ {:+.3f}, {:+.3f}, {:+.3f} ];\t'.format(0, +1, 0), file=rtplan_H, end='')
        print('u=[ {:+.3f}, {:+.3f}, {:+.3f} ];\t'.format(0, 0, +1), file=rtplan_H, end='')
        print('exitWindowPlane={:.4f};'.format((fieldInfo['fieldOriginToIsoDist_mm']-fieldInfo['VaccumExitToISODist_mm'])/10), file=rtplan_H)

    #write master for field
    for fieldInfo in fieldsInfo:
        print('pbmaster: {:d} ; '.format(fieldInfo['BeamNumber']), file=rtplan_H, end='')
        print('particle={:s};\t'.format(fieldInfo['RadiationType'].lower()), file=rtplan_H, end='')
        print('Xsec={:s};\t'.format(Xsec), file=rtplan_H, end='')
        print('emittanceRefPlaneDistance={:.3f};\t'.format(fieldInfo['fieldOriginToIsoDist_mm']/10), file=rtplan_H, end='')
        print('columns=[P.x,P.y,P.z,v.x,v.y,v.z,T,pSpread,N,twissAlphaX,twissBetaX,emittanceX,twissAlphaY,twissBetaY,emittanceY] ', file=rtplan_H)

    # group fields, prepare gantry region, save the default regions FoR
    print('group: fields {:s}'.format(' '.join('field_{:d}'.format(fieldInfo['BeamNumber'])  for fieldInfo in fieldsInfo)), file=rtplan_H)
    print('region: gantry ; O = [0,0,0] ; L = [1,1,1] ; lAdaptiveSize=t ; material = vacuum', file=rtplan_H)
    print('set_parent: gantry fields NozzleGroup', file=rtplan_H)
    print('save_regions: 0', file=rtplan_H)

    print('##################################################', file=rtplan_H)
    print('###### Start of setup and delivery sequence ######', file=rtplan_H)
    print('##################################################', file=rtplan_H)
    for fieldInfo in fieldsInfo:
        print('###### setup sequence for field {:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)

        print('# translate nozzle regions by snaut position', file=rtplan_H)
        print('transform: NozzleGroup shift_by {:.4f} {:.4f} {:.4f}'.format(0,-fieldInfo['SnoutPosition']/10,0), file=rtplan_H)

        print('# rotate field and NozzleGroup by gantry angle', file=rtplan_H)
        print('transform: gantry rotate {:s} {:.4f}'.format('z',fieldInfo['GantryAngle_deg']), file=rtplan_H)

        print('# translate and rotate phantom by iso position and couch rotation', file=rtplan_H)
        print('transform: phantom shift_by {:+.4f} {:+.4f} {:+.4f}'.format(-fieldInfo['IsocenterPosition_mm'][0]/10,-fieldInfo['IsocenterPosition_mm'][1]/10,-fieldInfo['IsocenterPosition_mm'][2]/10), file=rtplan_H)
        print('transform: phantom rotate {:s} {:+.4f}'.format('y',-fieldInfo['CouchAngle_deg']), file=rtplan_H)

        print('# delivery sequences for slices in field {:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)
        print('deactivate: fields, NozzleGroup', file=rtplan_H)
        print('activate: field_{:d} {:s}'.format(fieldInfo['BeamNumber'],fieldInfo['RangeShifterID']), file=rtplan_H)
        print('deliver: field_{:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)

        print('# reset field, nozzle regions and phantom to default FoR', file=rtplan_H)
        print('restore: 0', file=rtplan_H)

    print('##################################################', file=rtplan_H)
    print('###### End of setup and delivery sequence ########', file=rtplan_H)
    print('##################################################', file=rtplan_H)

    spotNumber=0
    for fieldInfo in fieldsInfo:
        for sliceInfo in fieldInfo['slicesInfo']:
            for _,sliceSpot in sliceInfo['sliceSpots'].iterrows():
                spotNumber+=1
                print('pb: {:d}\t{:d}\t'.format(spotNumber,fieldInfo['BeamNumber']), file=rtplan_H, end='')
                print('{:+.10f}\t{:+.10f}\t{:+.10f}\t'.format(sliceSpot['FRED_Px'],sliceSpot['FRED_Py'],sliceSpot['FRED_Pz']), file=rtplan_H, end='')
                print('{:+.10f}\t{:+.10f}\t{:+.10f}\t'.format(sliceSpot['FRED_Vx'],sliceSpot['FRED_Vy'],sliceSpot['FRED_Vz']), file=rtplan_H, end='')
                print('{:.3f}\t'.format(sliceInfo['FRED_beamEnergy']), file=rtplan_H, end='')
                print('{:.5E}\t'.format(sliceInfo['FRED_pSpread']), file=rtplan_H, end='')
                print('{:.10E}\t'.format(sliceSpot['FRED_protonNumber']), file=rtplan_H, end='')
                print('{:+.10E}\t{:+.10E}\t{:+.10E}\t'.format(sliceInfo['FRED_alphaX'],sliceInfo['FRED_betaX_cm'],sliceInfo['FRED_epsilonX_cm']), file=rtplan_H, end='')
                print('{:+.10E}\t{:+.10E}\t{:+.10E}\t'.format(sliceInfo['FRED_alphaY'],sliceInfo['FRED_betaY_cm'],sliceInfo['FRED_epsilonY_cm']), file=rtplan_H)
    if displayInfo:
        print('# RT plan saved to {:s}'.format(rtplanfileName))
    rtplan_H.close()

def getPlan(RTplanFileName, fieldToSave=None, FREDrtpalnFileName='current', defaultModelsFileName='default', beamModelInterpolationMethod='slinear', displayInfo=False):
    import os, re
    # version of getPlan
    version='3.4'
    if displayInfo:
        print('############################################################')
        print('# Importer of RTPLAN dicom for Fred at CCB v.{:s}'.format(version))
        print('# Implementation works for fred version 3.0.z2 or higher')
        print('############################################################')
    # get default beam model
    if defaultModelsFileName=='default':
        defaultModelsFileName='/home/shared/fred/beamModel/fred/defaultBeamModels/CCB_defaultModels.txt'
    else:
        if not os.path.isfile(defaultModelsFileName):
            print('Could not load file with default beam models (file {:s} does not exist)'.format(defaultModelsFileName))
            exit(1)

    # get gantry name
    dicomRT=loadDicomRT(RTplanFileName, displayInfo=False)
    for ifield in range(dicomRT.FractionGroupSequence[0].NumberOfBeams):
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='TREATMENT':
            TreatmentMachineName=dicomRT.IonBeamSequence[ifield].TreatmentMachineName
    if not TreatmentMachineName in ['GTR3','GTR4']:
        print('Room name \'{:s}\' not recognised'.format(TreatmentMachineName))
        exit(1)

    # get beam model, RS model and material definition file names
    with open(defaultModelsFileName) as f:
        for num, line in enumerate(f, 1):
            beamModelFileName_re = re.search('{:s}beamModelFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if beamModelFileName_re:
                beamModelFileName=beamModelFileName_re.group(1)
            RSModelFileName_re = re.search('{:s}RSModelFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if RSModelFileName_re:
                RSModelFileName=RSModelFileName_re.group(1)
            materialDefinitionFileName_re = re.search('{:s}materialDefinitionFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if materialDefinitionFileName_re:
                materialDefinitionFileName=materialDefinitionFileName_re.group(1)
    # build Plan
    planInfo, fieldsInfo=buildPLAN(RTplanFileName, beamModelFileName, RSModelFileName, materialDefinitionFileName, displayInfo=displayInfo)
    # find fred rtplan file name
    if FREDrtpalnFileName=='current':
        FREDrtpalnFileName=os.path.join(os.getcwd(),'rtplan.inp')
    # write plan
    writeRtplan(FREDrtpalnFileName, planInfo, fieldsInfo, fieldToSave=fieldToSave, getPlanVersion=version, displayInfo=displayInfo)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import accounts.models

class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0004_remove_userdetail_birthday'),
    ]

    operations = [
        migrations.AddField(
            model_name='userdetail',
            name='profile_image',
            field=models.ImageField(null=True, blank=True, upload_to=accounts.models.get_image_path),
            preserve_default=True,
        ),
    ]


resolution = 1000                 # pixels/um
dpml = 1.0                      # PML thickness
sz = 6 + 2*dpml
cell_size = mp.Vector3(z=sz)
pml_layers = [mp.PML(dpml)]
d = 0.5 # thickness = 500 nm
epsi = 12
geometry = [mp.Block(mp.Vector3(mp.inf,mp.inf,d), center=mp.Vector3(z=0),
                     material= mp.Medium(epsilon=epsi))]
dimensions = 1


""" Find, detect, and make available timeseries drivers implementations

Timeseries drivers must inherit from the Abstract Base Class
"AbstractTimeSeriesDriver" to be detected.
"""
import importlib
import os
import pkgutil
import sys

from ..logger import logger

class BrokenModule(object):
    """ Timeseries driver module "%s" is broken.

    Reason:
    %s

    Please see TSTools Github repository for more information or to report an
    issue:

    TSTools
    """
    def __init__(self, module, message):
        self.__doc__ = self.__doc__ % (module, message)

class TSManager(object):
    """ Timeseries Manager

    Finds and stores references to available timeseries
    """
    # Loaded timeseries
    ts = None

    def __init__(self, location=None):
        # Location of timeseires modules
        self.plugin_dir = []
        # All available timeseries
        self.ts_drivers = []

        if location and os.path.isdir(location):
            self.plugin_dir.append(location)

        file_location = os.path.join(os.path.dirname(__file__), 'drivers')
        self.plugin_dir.append('./' if file_location == '' else file_location)

        self.find_timeseries()

    def find_timeseries(self):
        """ Try to find timeseries classes """
        try:
            from . import timeseries
        except ImportError:
            logger.critical('Could not import "timeseries". Check your path')
            raise
        else:
            logger.debug('Found "timeseries" module')

        broken = []

        # Use pkgutil to search for timeseries
        logger.debug('Module name: {n}'.format(n=__name__))
        for loader, modname, ispkg in pkgutil.walk_packages(self.plugin_dir):
            full_path = '%s.drivers.%s' % (__name__.rsplit('.', 1)[0], modname)
            try:
                importlib.import_module(full_path)
            except ImportError as e:
                logger.error('Cannot import %s: %s' % (modname, e.message))
                broken_module = BrokenModule(modname,
                                             e.args[0] if e.args else
                                             'Unknown import error')
                broken_module.description = 'Broken: %s' % modname
                broken.append(broken_module)
            except:
                logger.error('Cannot import %s: %s' %
                             (modname, sys.exc_info()[0]))
                raise

        self.ts_drivers = timeseries.AbstractTimeSeriesDriver.__subclasses__()
        for tsd in self.ts_drivers:
            logger.info('Found driver: {tsd}'.format(tsd=tsd))

        # Find even more descendents
        for subclass in self.ts_drivers:
            self.recursive_find_subclass(subclass)

        self.ts_drivers.extend(broken)

    def recursive_find_subclass(self, subclass):
        """ Search subclass for descendents """

        sub_subclasses = subclass.__subclasses__()

        for sub_subclass in sub_subclasses:
            if sub_subclass not in self.ts_drivers:
                self.ts_drivers.append(sub_subclass)
                logger.info('Found driver: {tsd}'.format(tsd=sub_subclass))
            self.recursive_find_subclass(sub_subclass)

# Store timeseries manager
tsm = TSManager()
logger.debug('Found {i} TS data models'.format(i=len(tsm.ts_drivers)))


from __future__ import print_function
from tensorflow.contrib import rnn
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
import numpy as np
import random

config1 = tf.ConfigProto()
config1.gpu_options.allow_growth = True

# Load "X" (the neural network's training and testing inputs)

def load_X(X_signals_paths_emg, X_signals_paths_imu, y_path_emg, y_path_imu):
    X_signals_emg = []
    X_signals_imu = []
    X_signals = []
    move_data = []

    # 1. change to list in X_signals_XXX
    # emg-----------------------
    for signal_type_path in X_signals_paths_emg:
        file = open(signal_type_path, 'r')
        # Read dataset from disk, dealing with text files' syntax
        move_data.append(
            [np.array(serie, dtype=np.float32) for serie in [
                row.strip().split(' ') for row in file
            ]]
        )
        for serie in move_data[0]:
            serie = serie.tolist()
            X_signals_emg.append(serie)
        #
        file.close()

    # imu-----------------------
    move_data = []
    for signal_type_path in X_signals_paths_imu:
        file = open(signal_type_path, 'r')
        # Read dataset from disk, dealing with text files' syntax
        move_data.append(
            np.array(serie, dtype=np.float32) for serie in [
                row.strip().split(' ') for row in file
            ]
        )
        for serie in move_data[0]:
            serie = serie.tolist()
            X_signals_imu.append(serie)
        #
        file.close()

    # 2. pedding emg block according to imu y
    # make the emg table
    y_emg_table=[]
    y_emg=[]
    file = open(y_path_emg, 'r')
    y_emg.extend(row.strip().split(' ')[0] for row in file)
    file.close()
    _serie=y_emg[0]
    repeat_num=1
    num=1
    for serie in y_emg:
        if num!=1:
            if serie==_serie:
                repeat_num=repeat_num+1
                if y_emg_table.__len__()>1:
                    if (y_emg_table[y_emg_table.__len__()-1][2]+repeat_num == y_emg.__len__()-1):
                        y_emg_table.append([_serie, y_emg_table[y_emg_table.__len__() - 1][2] + 1,y_emg_table[y_emg_table.__len__() - 1][2] + repeat_num])
            else:
                if y_emg_table.__len__()==0:
                    y_emg_table.append([_serie, 0, repeat_num-1])
                else:
                    y_emg_table.append([_serie,y_emg_table[y_emg_table.__len__()-1][2]+1,y_emg_table[y_emg_table.__len__()-1][2]+repeat_num])
                _serie=serie
                repeat_num=1
        else:
            num=num+1
    # make the imu table
    y_imu_table = []
    y_imu = []
    file = open(y_path_imu, 'r')
    y_imu.extend(row.strip().split(' ')[0] for row in file)
    file.close()
    _serie = y_imu[0]
    repeat_num = 1
    num = 1
    for serie in y_imu:
        if num != 1:
            if serie == _serie:
                repeat_num = repeat_num + 1
                if y_imu_table.__len__() > 1:
                    if (y_imu_table[y_imu_table.__len__() - 1][2] + repeat_num == y_imu.__len__() - 1):
                        y_imu_table.append([_serie, y_imu_table[y_imu_table.__len__() - 1][2] + 1,
                                            y_imu_table[y_imu_table.__len__() - 1][2] + repeat_num])
            else:
                if y_imu_table.__len__() == 0:
                    y_imu_table.append([_serie, 0, repeat_num - 1])
                else:
                    y_imu_table.append([_serie, y_imu_table[y_imu_table.__len__() - 1][2] + 1,
                                        y_imu_table[y_imu_table.__len__() - 1][2] + repeat_num])
                _serie = serie
                repeat_num = 1
        else:
            num = num + 1
    #extend according to table
    if y_emg_table.__len__()!=y_imu_table.__len__():
        print("emg.table_num!=imu.table_num!")
        exit(0)
    for num in range(y_emg_table.__len__()):
        if y_imu_table[num][0]!=y_emg_table[num][0]:
            print("emg.table is not according to imu.table!")
            exit(0)
        diff_num=y_imu_table[num][2]-y_emg_table[num][2]
        if diff_num>0:
            extend_move=X_signals_emg[y_emg_table[num][2]]
            place_num=y_emg_table[num][2]+1
            for i in range(diff_num):
                X_signals_emg.insert(place_num,extend_move)
                place_num=place_num+1

            y_emg_table[num][2]=y_emg_table[num][2]+diff_num
            for i in range(num+1,y_emg_table.__len__()):
                y_emg_table[i][2] = y_emg_table[i][2] + diff_num
                y_emg_table[i][1] = y_emg_table[i][1] + diff_num
        elif diff_num<0:
            diff_num=-diff_num
            extend_move = X_signals_imu[y_imu_table[num][2]]
            place_num = y_imu_table[num][2] + 1
            for i in range(diff_num):
                X_signals_imu.insert(place_num, extend_move)
                place_num = place_num + 1

            y_imu_table[num][2] = y_imu_table[num][2] + diff_num
            for i in range(num + 1, y_imu_table.__len__()):
                y_imu_table[i][2] = y_imu_table[i][2] + diff_num
                y_imu_table[i][1] = y_imu_table[i][1] + diff_num

    print(
        "Number of emg and imu block is " + X_signals_emg.__len__().__str__() + " " + X_signals_imu.__len__().__str__())

    # 3. rewirte the responding y
    y=[]
    for i in range(y_emg_table.__len__()):
        y.extend([y_emg_table[i][0]] for k in range(y_emg_table[i][2]-y_emg_table[i][1]+1))

    # 4. 512 * x check
    for i in range(X_signals_emg.__len__()):
        if len(X_signals_emg[i])>256*16:
            for j in range(y_emg_table.__len__()):
                if y[j]==y[i] and len(X_signals_emg[j])<256*16:
                    X_signals_emg[i]=X_signals_emg[j]
                    break
        if len(X_signals_emg[i]) < 256 * 16:
            X_signals_emg[i].extend([0 for k in range(256*16-len(X_signals_emg[i]))])

    for i in range(X_signals_imu.__len__()):
        if len(X_signals_imu[i]) > 256 * 8:
            for j in range(y_imu_table.__len__()):
                if y[j] == y[i] and len(X_signals_imu[j]) < 256 * 8:
                    X_signals_imu[i] = X_signals_imu[j]
                    break
        if len(X_signals_imu[i]) < 256 * 8:
            X_signals_imu[i].extend([0 for k in range(256 * 8 - len(X_signals_imu[i]))])

    # 5. connection:16-8
    for i in range(y.__len__()):
        serie = []
        for j in range(256):
            serie.extend([X_signals_emg[i][k] for k in range(j * 16,(j+1) * 16)])
            serie.extend([X_signals_imu[i][k] for k in range(j * 8, (j + 1) * 8)])
        X_signals.append(serie)

    # 6. change 1*x*512*24 to 24*x*512
    _X_signals=[]
    for i in range(24):
        _serie_every_24=[]
        for serie in X_signals:
            _serie_every_move=[]
            for j in range(256):
                _serie_every_move.extend([serie[i+j*24]])
            _serie_every_24.append(_serie_every_move)
        _X_signals.append(_serie_every_24)

    return np.transpose(np.array(np.array(_X_signals,dtype=np.float32)), (1, 2, 0)), np.array(y,dtype=np.float32)

def data_segmentation(X,y):
    X_range=range(0,len(X))
    resultList = random.sample(X_range,int(len(X)*2/10))
    X_test = []
    X_train = []
    y_test = []
    y_train = []
    for i in X_range:
        if (i in resultList):
            X[i]
            X_test.append(X[i])
            y_test.append(y[i])
        else:
            X_train.append(X[i])
            y_train.append(y[i])
    return np.array(X_train,dtype=np.float32),np.array(y_train,dtype=np.float32),np.array(X_test,dtype=np.float32),np.array(y_test,dtype=np.float32)

class Config(object):
    """
    define a class to store parameters,
    the input should be feature mat of training and testing

    Note: it would be more interesting to use a HyperOpt search space:
    https://github.com/hyperopt/hyperopt
    """

    def __init__(self, X_train, X_test):
        # Input data
        self.n_layers = 2  # number of layers
        self.train_count = len(X_train)  # 7352 training series
        self.test_data_count = len(X_test)  # 2947 testing series
        self.n_steps = len(X_train[0])  # 128 time_steps per series

        # Training
        self.learning_rate = 0.0025
        # 0.0025
        self.lambda_loss_amount = 0.0015
        self.training_epochs = 1000
        self.batch_size = 150
        self.output_keep_prob = 0.5

        # LSTM structure
        self.n_inputs = len(X_train[0][0])
        self.n_hidden = 64  # nb of neurons inside the neural network
        self.n_classes = 10  # Final output classes

        # define weights name
        with tf.name_scope('weights'):
            self.W = {
                    'hidden': tf.Variable(tf.random_normal([self.n_inputs, self.n_hidden]), name = 'W'),
                    'output': tf.Variable(tf.random_normal([2*self.n_hidden, self.n_classes]),name = 'W')
            }
            tf.summary.histogram('output_layer_weights', self.W['hidden'])
            tf.summary.histogram('output_layer_weights_1', self.W['output'])
        # define biase
        with tf.name_scope('Wx_plus_b'):
                self.biases = {
                        'hidden': tf.Variable(tf.random_normal([self.n_hidden], mean=1.0)),
                        'output': tf.Variable(tf.random_normal([self.n_classes]))
                }
                tf.summary.histogram('output_layer_biases', self.biases['hidden'])
                tf.summary.histogram('output_layer_weights_1', self.W['output'])

def LSTM_Network(_X, config):

    # (NOTE: This step could be greatly optimised by shaping the dataset once
    # input shape: (batch_size, n_steps, n_input)
    _X = tf.transpose(_X, [1, 0, 2])  # permute n_steps and batch_size
    # Reshape to prepare input to hidden activation
    _X = tf.reshape(_X, [-1, config.n_inputs])
    # new shape: (n_steps*batch_size, n_input)

    # Linear activation
    _X = tf.nn.relu(tf.matmul(_X, config.W['hidden']) + config.biases['hidden'])
    # Split data because rnn cell needs a list of inputs for the RNN inner loop
    _X = tf.split(_X, config.n_steps, 0)
    # new shape: n_steps * (batch_size, n_hidden)

#    # Define two stacked LSTM cells (two recurrent layers deep) with tensorflow
#    lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(config.n_hidden, forget_bias=1.0, state_is_tuple=True)
#    lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(config.n_hidden, forget_bias=1.0, state_is_tuple=True)

    def lstm_cell(config):
        cell = rnn.LSTMCell(config.n_hidden)
        with tf.name_scope('lstm_dropout'):
            return rnn.DropoutWrapper(cell, output_keep_prob=config.output_keep_prob)
        # _X = tf.unstack(_X, config.n_steps, 1)

        # attn_cell = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=keep_prob)
        # 实现多层 LSTM
        # [attn_cell() for _ in range(n_layers)]
        # enc_cells = []
        # for i in range(0, self.n_layers):
        #     enc_cells.append(lstm_cell())
        # with tf.name_scope('lstm_cells_layers'):
        #     mlstm_cell = tf.contrib.rnn.MultiRNNCell(enc_cells, state_is_tuple=True)

    with tf.name_scope("cell_1"):
        lstm_cell_1 = tf.contrib.rnn.MultiRNNCell([lstm_cell(config) for _ in range(config.n_layers)], state_is_tuple = True)

    # Backward direction cell
    with tf.name_scope("cell_2"):
        lstm_cell_2 = tf.contrib.rnn.MultiRNNCell([lstm_cell(config) for _ in range(config.n_layers)], state_is_tuple = True)

        # # 全零初始化 state
        # _init_state = lstm_cell.zero_state(config.batch_size, dtype=tf.float32)
        # # dynamic_rnn 运行网络
        # outputs, states = tf.nn.dynamic_rnn(lstm_cell, _X, initial_state=_init_state, dtype=tf.float32,
        #                                     time_major=False)
        # # 输出
        # # return tf.matmul(outputs[:,-1,:], Weights) + biases
        # return tf.nn.softmax(tf.matmul(outputs[:, -1, :], config.W) + config.biases)

#    lstm_cell_1 = tf.contrib.rnn.MultiRNNCell(config.n_hidden, forget_bias=1.0, state_is_tuple=True)
#    # Backward direction cell
#    lstm_cell_2 = tf.contrib.rnn.MultiRNNCell(config.n_hidden, forget_bias=1.0, state_is_tuple=True)

#    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell_1, lstm_cell_2] * config.n_layers, state_is_tuple=True)
    # Get LSTM cell output
#    outputs, states = rnn.static_bidirectional_rnn(lstm_cell_1,lstm_cell_2, _X, dtype=tf.float32)

    try:
        outputs, _, _ = rnn.static_bidirectional_rnn(lstm_cell_1, lstm_cell_2, _X,
                                              dtype=tf.float32)
    except Exception: # Old TensorFlow version only returns outputs not states
        outputs = rnn.static_bidirectional_rnn(lstm_cell_1, lstm_cell_2, _X,
                                        dtype=tf.float32)
    # Get last time step's output feature for a "many to one" style classifier,
    # as in the image describing RNNs at the top of this page
#    lstm_last_output = outputs[-1]

    # Linear activation
    return tf.matmul(outputs[-1], config.W['output']) + config.biases['output']

def one_hot(y_):
    """
    Function to encode output labels from number indexes.

    E.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]
    """
    y_ = y_.reshape(len(y_))
    n_values = int(np.max(y_)) + 1
    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS

def main():
    # -----------------------------
    # Step 1: load and prepare data
    # -----------------------------

    # Those are separate normalised input features for the neural network
#    INPUT_SIGNAL_TYPES = [
#        "",
#    ]

#    DATA_PATH = "data/"
    DATASET_PATH = "data/"
    print("\n" + "Dataset is now located at: " + DATASET_PATH)
    TRAIN = "train/"
    TEST = "test/"

    X_train_signals_paths_emg = [
        DATASET_PATH + TRAIN + "x_data/emg/x_emg.txt"
    ]
    X_train_signals_paths_imu = [
        DATASET_PATH + TRAIN + "x_data/imu/x_imu.txt"
    ]
    X_test_signals_paths_emg = [
        DATASET_PATH + TEST + "x_data/emg/x_emg.txt"
    ]
    X_test_signals_paths_imu = [
        DATASET_PATH + TEST + "x_data/imu/x_imu.txt"
    ]

    y_train_path_emg = DATASET_PATH + TRAIN + "y_emg.txt"
    y_train_path_imu = DATASET_PATH + TRAIN + "y_imu.txt"
    y_test_path_emg = DATASET_PATH + TEST + "y_emg.txt"
    y_test_path_imu = DATASET_PATH + TEST + "y_imu.txt"

    X_all, y_all = load_X(X_train_signals_paths_emg, X_train_signals_paths_imu,y_train_path_emg,y_train_path_imu)
    X_train, y_train, X_test, y_test = data_segmentation(X_all,y_all)
    y_train = one_hot(y_train)
    y_test = one_hot(y_test)

    # ---------------
    # --------------------
    # Step 2: define parameters for model
    # -----------------------------------

    config = Config(X_train, X_test)
    print("Some useful info to get an insight on dataset's shape and "
          "normalisation:")
    print("features shape, labels shape, each features mean, each features standard deviation")
    print(X_test.shape, y_test.shape,
          np.mean(X_test), np.std(X_test))
    print("the dataset is therefore properly normalised, as expected.")

    # ------------------------------------------------------
    # Step 3: Let's get serious and build the neural network
    # ------------------------------------------------------
    with tf.name_scope('inputs'):
        X = tf.placeholder(tf.float32, [None, config.n_steps, config.n_inputs], name='x_input')
        Y = tf.placeholder(tf.float32, [None, config.n_classes], name='y_input')
        tf.output_keep_prob = tf.placeholder(tf.float32, name='keep_prob_input')  # 保持多少不被 dropout
        tf.batch_size = tf.placeholder(tf.int32, [], name='batch_size_input')  # 批大小

    with tf.name_scope('output_layer'):
        pred_Y = LSTM_Network(X, config)
        tf.summary.histogram('outputs', pred_Y)
    # Loss,optimizer,evaluation
    l2 = config.lambda_loss_amount * \
         sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())
    # Softmax loss and L2
    with tf.name_scope('loss'):
        cost = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=pred_Y)) + l2
        tf.summary.scalar('loss', cost)

    with tf.name_scope('train'):
        optimizer = tf.train.AdamOptimizer(
            learning_rate=config.learning_rate).minimize(cost)
        # tf.summary.scalar('optimizer', optimizer)

    # with tf.name_scope('optimizer'):
    #     correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1))
    # with tf.name_scope('accuracy'):
    #     accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32))
    #     tf.summary.scalar('accuracy', accuracy)

    # correct_pred = tf.equal(tf.argmax(pred_Y, 1), tf.argmax(Y, 1))
    # accuarcy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
    with tf.name_scope('accuracy'):
        accuracy = tf.metrics.accuracy(labels=tf.argmax(Y, axis=1), predictions=tf.argmax(pred_Y, axis=1))[1]
        tf.summary.scalar('accuracy', accuracy)

    # --------------------------------------------
    # Step 4: Hooray, now train the neural network
    # --------------------------------------------

    # Note that log_device_placement can be turned ON but will cause console spam with RNNs.
    sess = tf.InteractiveSession(config=config1)
    merged = tf.summary.merge_all()
    #sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True, device_count={"CPU": 11}))
    # init = tf.global_variables_initializer()
    init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
    # sess.run(init)

    with tf.Session() as sess:
        sess.run(init)

        writer = tf.summary.FileWriter("logs-myo/", sess.graph)

        best_accuracy = 0.0
        saver = tf.train.Saver(max_to_keep=1)
        run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
        run_metadata = tf.RunMetadata()
        # Start training for each batch and loop epochs
        for i in range(config.training_epochs):
            for start, end in zip(range(0, config.train_count, config.batch_size),
                                  range(config.batch_size, config.train_count + 1, config.batch_size)):
                sess.run(optimizer, feed_dict={X: X_train[start:end],
                                               Y: y_train[start:end]})
                # if i % 50 == 0:
                #     result = sess.run(merged,
                #                       feed_dict={X: X_train[start:end],
                #                                  Y: y_train[start:end]})
                #     writer.add_summary(result, i)

            # Test completely at every epoch: calculate accuracy
            result, pred_out, accuracy_out, loss_out = sess.run(
                [merged, pred_Y, accuracy, cost],
                feed_dict={
                    X: X_test,
                    Y: y_test
                },
                options=run_options,
                run_metadata=run_metadata
            )
            writer.add_summary(result, i)

            print("traing iter: {},".format(i) +
                  " test accuracy : {},".format(accuracy_out) +
                  " loss : {}".format(loss_out))

            best_accuracy = max(best_accuracy, accuracy_out)
            if best_accuracy < accuracy_out:
                best_accuracy = accuracy_out
                saver.save(sess, "Model/model.ckpt")
        writer.close()

        print("")
        print("final test accuracy: {}".format(accuracy_out))
        print("best epoch's test accuracy: {}".format(best_accuracy))
        print("")

if __name__ == "__main__":
    main()


"""
Copyright 2017-present Airbnb, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import base64
from collections import defaultdict
import json
import os
import re
import time
import zlib

from mock import patch, MagicMock

from streamalert.alert_processor import main as alert_processor
from streamalert.alert_processor.helpers import compose_alert
from streamalert.alert_processor.outputs.output_base import OutputDispatcher
from streamalert.classifier import classifier
from streamalert.classifier.parsers import ParserBase
from streamalert.rules_engine import rules_engine
from streamalert.shared import rule
from streamalert.shared.logger import get_logger
from streamalert.shared.stats import RuleStatisticTracker
from streamalert.shared.lookup_tables.table import LookupTable
from streamalert_cli.helpers import check_credentials
from streamalert_cli.test import DEFAULT_TEST_FILES_DIRECTORY
from streamalert_cli.test.format import format_green, format_red, format_underline, format_yellow
from streamalert_cli.test.mocks import mock_lookup_table_results, mock_threat_intel_query_results
from streamalert_cli.test.results import TestEventFile, TestResult
from streamalert_cli.utils import CLICommand, generate_subparser, UniqueSetAction

LOGGER = get_logger(__name__)

class TestCommand(CLICommand):
    description = 'Perform various integration/functional tests'

    @classmethod
    def setup_subparser(cls, subparser):
        """Add the test subparser: manage.py test"""
        test_subparsers = subparser.add_subparsers(dest="test subcommand", required=True)

        cls._setup_test_classifier_subparser(test_subparsers)
        cls._setup_test_rules_subparser(test_subparsers)
        cls._setup_test_live_subparser(test_subparsers)

    @classmethod
    def _setup_test_classifier_subparser(cls, subparsers):
        """Add the test validation subparser: manage.py test classifier [options]"""
        test_validate_parser = generate_subparser(
            subparsers,
            'classifier',
            description='Validate defined log schemas using integration test files',
            subcommand=True
        )

        cls._add_default_test_args(test_validate_parser)

    @classmethod
    def _setup_test_rules_subparser(cls, subparsers):
        """Add the test rules subparser: manage.py test rules [options]"""
        test_rules_parser = generate_subparser(
            subparsers,
            'rules',
            description='Test rules using integration test files',
            subcommand=True
        )

        # Flag to run additional stats during testing
        test_rules_parser.add_argument(
            '-s',
            '--stats',
            action='store_true',
            help='Enable outputing of statistical information on rules that run'
        )

        # Validate the provided repitition value
        def _validate_repitition(val):
            """Make sure the input is between 1 and 1000"""
            err = ('Invalid repitition value [{}]. Must be an integer between 1 '
                   'and 1000').format(val)
            try:
                count = int(val)
            except TypeError:
                raise test_rules_parser.error(err)

            if not 1 <= count <= 1000:
                raise test_rules_parser.error(err)

            return count

        # flag to run these tests a given number of times
        test_rules_parser.add_argument(
            '-n',
            '--repeat',
            default=1,
            type=_validate_repitition,
            help='Number of times to repeat the tests, to be used as a form performance testing'
        )

        cls._add_default_test_args(test_rules_parser)

    @classmethod
    def _setup_test_live_subparser(cls, subparsers):
        """Add the test live subparser: manage.py test live [options]"""
        test_live_parser = generate_subparser(
            subparsers,
            'live',
            description=(
                'Run end-to-end tests that will attempt to send alerts to each rule\'s outputs'
            ),
            subcommand=True
        )

        cls._add_default_test_args(test_live_parser)

    @staticmethod
    def _add_default_test_args(test_parser):
        """Add the default arguments to the test parsers"""
        test_filter_group = test_parser.add_mutually_exclusive_group(required=False)

        # add the optional ability to test against a rule/set of rules
        test_filter_group.add_argument(
            '-f',
            '--test-files',
            dest='files',
            metavar='FILENAMES',
            nargs='+',
            help='One or more file to test, separated by spaces',
            action=UniqueSetAction,
            default=set()
        )

        # add the optional ability to test against a rule/set of rules
        test_filter_group.add_argument(
            '-r',
            '--test-rules',
            dest='rules',
            nargs='+',
            help='One or more rule to test, separated by spaces',
            action=UniqueSetAction,
            default=set()
        )

        # add the optional ability to change the test files directory
        test_parser.add_argument(
            '-d',
            '--files-dir',
            help='Path to directory containing test files',
            default=DEFAULT_TEST_FILES_DIRECTORY
        )

        # Add the optional ability to log verbosely or use quite logging for tests
        verbose_group = test_parser.add_mutually_exclusive_group(required=False)

        verbose_group.add_argument(
            '-v',
            '--verbose',
            action='store_true',
            help='Output additional information during testing'
        )

        verbose_group.add_argument(
            '-q',
            '--quiet',
            action='store_true',
            help='Suppress output for passing tests, only logging if there is a failure'
        )

    @classmethod
    def handler(cls, options, config):
        """Handler for starting the test framework

        Args:
            options (argparse.Namespace): Parsed arguments
            config (CLIConfig): Loaded StreamAlert config

        Returns:
            bool: False if errors occurred, True otherwise
        """
        result = True
        opts = vars(options)
        repeat = opts.get('repeat', 1)
        for i in range(repeat):
            if repeat != 1:
                print('\nRepetition #', i+1)
            result = result and TestRunner(options, config).run()

        if opts.get('stats'):
            print(RuleStatisticTracker.statistics_info())
        return result

class TestRunner:
    """TestRunner to handle running various tests"""

    class Types:
        """Simple types enum for test types"""
        CLASSIFY = 'classifier'
        RULES = 'rules'
        LIVE = 'live'

    def __init__(self, options, config):
        self._config = config
        self._options = options
        self._type = options.subcommand
        self._files = options.files
        self._rules = options.rules
        self._files_dir = os.path.join(options.files_dir, '')  # ensure theres a trailing slash
        self._verbose = options.verbose
        self._quiet = options.quiet
        self._s3_mocker = patch('streamalert.classifier.payload.s3.boto3.resource').start()
        self._errors = defaultdict(list)  # cache errors to be logged at the endpoint
        self._tested_rules = set()
        self._threat_intel_mock = mock_threat_intel_query_results()
        self._lookup_tables_mock = mock_lookup_table_results()
        self._passed = 0
        self._failed = 0
        prefix = self._config['global']['account']['prefix']
        env = {
            'CLUSTER': 'prod',
            'STREAMALERT_PREFIX': prefix,
            'AWS_ACCOUNT_ID': self._config['global']['account']['aws_account_id'],
            'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
        }

        if 'stats' in options and options.stats:
            env['STREAMALERT_TRACK_RULE_STATS'] = '1'

        patch.dict(
            os.environ,
            env
        ).start()

    @staticmethod
    def _run_classification(record):
        """Create a fresh classifier and classify the record, returning the result"""
        with patch.object(classifier, 'SQSClient'), patch.object(classifier, 'FirehoseClient'):
            _classifier = classifier.Classifier()
            return _classifier.run(records=[record])

    def _run_rules_engine(self, record):
        """Create a fresh rules engine and process the record, returning the result"""
        with patch.object(rules_engine.ThreatIntel, '_query') as ti_mock, \
             patch.object(rules_engine, 'AlertForwarder'), \
             patch.object(rules_engine, 'RuleTable') as rule_table, \
             patch('rules.helpers.base.random_bool', return_value=True):

            # Emptying out the rule table will force all rules to be unstaged, which causes
            # non-required outputs to get properly populated on the Alerts that are generated
            # when running the Rules Engine.
            rule_table.return_value = False
            ti_mock.side_effect = self._threat_intel_mock

            _rules_engine = rules_engine.RulesEngine()

            self._install_lookup_tables_mocks(_rules_engine)

            return _rules_engine.run(records=record)

    def _install_lookup_tables_mocks(self, rules_engine_instance):
        """
        Extremely gnarly, extremely questionable manner to install mocking data into our tables.
        The reason this exists at all is to support the secret features of table scanning S3-backed
        tables, which isn't a "normally" available feature but is required for some pre-existing
        StreamAlert users.
        """
        from streamalert.shared.lookup_tables.drivers import EphemeralDriver

        dummy_configuration = {}
        mock_data = self._lookup_tables_mock

        # pylint: disable=protected-access
        for table_name in rules_engine_instance._lookup_tables._tables.keys():
            driver = EphemeralDriver(dummy_configuration)
            driver._cache = mock_data.get(table_name, {})
            ephemeral_table = LookupTable(table_name, driver, dummy_configuration)

            rules_engine_instance._lookup_tables._tables[table_name] = ephemeral_table

    @staticmethod
    def _run_alerting(record):
        """Create a fresh alerts processor and send the alert(s), returning the result"""
        with patch.object(alert_processor, 'AlertTable'):
            alert_proc = alert_processor.AlertProcessor()

            return alert_proc.run(event=record.dynamo_record())

    def _check_prereqs(self):
        if self._type == self.Types.LIVE:
            return check_credentials()

        return True

    def _finalize(self):
        summary = [
            format_underline('\nSummary:\n'),
            'Total Tests: {}'.format(self._passed + self._failed),
            format_green('Pass: {}'.format(self._passed)) if self._passed else 'Pass: 0',
            format_red('Fail: {}\n'.format(self._failed)) if self._failed else 'Fail: 0\n',
        ]

        print('\n'.join(summary))

        for path in sorted(self._errors):
            for error in self._errors[path]:
                message = '({}) {}'.format(path, error) if path != 'error' else error
                LOGGER.error(message)

        # If rule are being tested and no filtering is being performed, log any untested rules
        if self._testing_rules and not self._is_filtered:
            all_rules = set(rule.Rule.rule_names()) - rule.Rule.disabled_rules()
            untested_rules = sorted(all_rules.difference(self._tested_rules))
            if not untested_rules:
                return
            print(format_yellow('No test events configured for the following rules:'))
            for rule_name in untested_rules:
                print(format_yellow(rule_name))

    @property
    def _is_filtered(self):
        return bool(self._files or self._rules)

    @property
    def _testing_rules(self):
        return self._type in {self.Types.RULES, self.Types.LIVE}

    def _contains_filtered_rules(self, event):
        if not self._rules:
            return True

        expected_rules = set(event.get('trigger_rules', [])) - rule.Rule.disabled_rules()
        return bool(expected_rules.intersection(self._rules))

    def run(self):
        """Run the tests"""
        if not self._check_prereqs():
            return

        print('\nRunning tests for files found in: {}'.format(self._files_dir))

        for event_file in self._get_test_files():
            test_event = TestEventFile(event_file.replace(self._files_dir, ''))
            # Iterate over the individual test events in the file
            for idx, original_event, event in self._load_test_file(event_file):
                if not event:
                    continue

                if not self._contains_filtered_rules(original_event):
                    continue

                resource = original_event['source']

                for cluster_name, cluster_value in self._config['clusters'].items():
                    for service in cluster_value['data_sources'].values():
                        if resource in service:
                            os.environ['CLUSTER'] = cluster_name
                            break

                classifier_result = self._run_classification(event)

                test_result = TestResult(
                    idx,
                    original_event,
                    classifier_result[0] if classifier_result else False,
                    with_rules=self._testing_rules,
                    verbose=self._verbose
                )

                test_event.add_result(test_result)

                self._tested_rules.update(test_result.expected_rules)

                if not test_result:
                    continue

                if original_event.get('validate_schema_only'):
                    continue  # Do not run rules on events that are only for validation

                if self._type in {self.Types.RULES, self.Types.LIVE}:
                    alerts = self._run_rules_engine(classifier_result[0].sqs_messages)
                    test_result.alerts = alerts

                    if not original_event.get('skip_publishers'):
                        for alert in alerts:
                            publication_results = self._run_publishers(alert)
                            test_result.set_publication_results(publication_results)

                    if self._type == self.Types.LIVE:
                        for alert in alerts:
                            alert_result = self._run_alerting(alert)
                            test_result.add_live_test_result(alert.rule_name, alert_result)

            self._passed += test_event.passed
            self._failed += test_event.failed

            # It is possible for a test_event to have no results,
            # so only print it if it does and if quiet mode is no being used
            # Quite mode is overridden if not all of the events passed
            if test_event and not (self._quiet and test_event.all_passed):
                print(test_event)

        self._finalize()

        return self._failed == 0

    @staticmethod
    def _run_publishers(alert):
        """Runs publishers for all currently configured outputs on the given alert

        Args:
            - alert (Alert): The alert

        Returns:
            dict: A dict keyed by output:descriptor strings, mapped to nested dicts.
                  The nested dicts have 2 keys:
                  - publication (dict): The dict publication
                  - success (bool): True if the publishing finished, False if it errored.
        """
        configured_outputs = alert.outputs

        results = {}
        for configured_output in configured_outputs:
            [output_name, descriptor] = configured_output.split(':')

            try:
                output = MagicMock(spec=OutputDispatcher, __service__=output_name)
                results[configured_output] = {
                    'publication': compose_alert(alert, output, descriptor),
                    'success': True,
                }
            except (RuntimeError, TypeError, NameError) as err:
                results[configured_output] = {
                    'success': False,
                    'error': err,
                }
        return results

    def _get_test_files(self):
        """Helper to get rule files to be tested

        Yields:
            str: Path to test event file
        """
        files_filter = {
            os.path.splitext(name)[0] for name in self._files
        } if self._files else set()

        filtered = bool(files_filter)
        for root, _, test_event_files in os.walk(self._files_dir):
            for event_file in sorted(test_event_files):
                basename = os.path.splitext(event_file)[0]
                full_path = os.path.join(root, event_file)
                if not filtered or basename in files_filter:
                    yield full_path
                    if filtered:
                        files_filter.remove(basename)  # Remove this from the filter

        # Log any errors for filtered items that do not exist
        for basename in files_filter:
            self._append_error('No test event file found with base name \'{}\''.format(basename))

    def _setup_s3_mock(self, data):
        self._s3_mocker.return_value.Bucket.return_value.download_fileobj = (
            lambda k, d: d.write(json.dumps(data).encode())
        )

    def _append_error(self, error, path=None, idx=None):
        key = 'error'
        if path:
            key = os.path.split(path)[1]
        key = key if not idx else '{}:{}'.format(key, idx)
        self._errors[key].append(error)

    def _load_test_file(self, path):
        """Helper to json load the contents of a file with some error handling

        Test files should be formatted as:

        [
            {
                "data": {},
                "description": "...",
                "...": "..."
            }
        ]

        Args:
            path (str): Relative path to file on disk

        Returns:
            dict: Loaded JSON from test event file
        """
        with open(path, 'r') as test_event_file:
            try:
                data = json.load(test_event_file)
            except (ValueError, TypeError):
                self._append_error('Test event file is not valid JSON', path=path)
                return

            if not isinstance(data, list):
                self._append_error('Test event file is improperly formatted', path=path)
                return

            for idx, event in enumerate(data):
                valid, record = self._format_test_record(event)
                if not valid:
                    self._append_error(record, path=path, idx=idx)
                    continue
                yield idx, event, record

    def _format_test_record(self, test_event):
        """Create a properly formatted Kinesis, S3, or SNS record.

        Supports a dictionary or string based data record.  Reads in
        event templates from the tests/integration/templates folder.

        Args:
            test_event (dict): Test event metadata dict with the following structure:
                data|override_record - string or dict of the raw data
                description - a string describing the test that is being performed
                trigger - bool of if the record should produce an alert
                source - which stream/s3 bucket originated the data
                service - which aws service originated the data
                compress (optional) - if the payload needs to be gzip compressed or not

        Returns:
            dict: in the format of the specific service
        """
        valid, error = self._validate_test_event(test_event)
        if not valid:
            return False, error

        self._apply_helpers(test_event)
        self._apply_defaults(test_event)

        data = test_event['data']
        if isinstance(data, dict):
            data = json.dumps(data)
        elif not isinstance(data, str):
            return False, 'Invalid data type: {}'.format(type(data))

        if test_event['service'] not in {'s3', 'kinesis', 'sns', 'streamalert_app'}:
            return False, 'Unsupported service: {}'.format(test_event['service'])

        # Get a formatted record for this particular service
        return True, self._apply_service_template(
            test_event['service'],
            test_event['source'],
            data,
            test_event.get('compress', False)
        )

    def _apply_service_template(self, service, source, data, compress=False):
        """Provides a pre-configured template that reflects incoming payload from a service

        Args:
            service (str): The service for the payload template

        Returns:
            dict: Template of the payload for the given service
        """
        if service == 's3':
            # Assign the s3 mock for this data
            self._setup_s3_mock(data)
            return {
                'eventVersion': '2.0',
                'eventTime': '1970-01-01T00:00:00.000Z',
                'requestParameters': {
                    'sourceIPAddress': '127.0.0.1'
                },
                's3': {
                    'configurationId': ',,,',
                    'object': {
                        'eTag': '...',
                        'sequencer': '...',
                        'key': 'test_object_key',
                        'size': len(data)
                    },
                    'bucket': {
                        'arn': 'arn:aws:s3:::{}'.format(source),
                        'name': source,
                        'ownerIdentity': {
                            'principalId': 'EXAMPLE'
                        }
                    },
                    's3SchemaVersion': '1.0'
                },
                'responseElements': {
                    'x-amz-id-2': (
                        'EXAMPLE123/foo/bar'
                    ),
                    'x-amz-request-id': '...'
                },
                'awsRegion': 'us-east-1',
                'eventName': 'ObjectCreated:Put',
                'userIdentity': {
                    'principalId': 'EXAMPLE'
                },
                'eventSource': 'aws:s3'
            }

        if service == 'kinesis':
            if compress:
                data = zlib.compress(data)

            kinesis_data = base64.b64encode(data.encode())

            return {
                'eventID': '...',
                'eventVersion': '1.0',
                'kinesis': {
                    'approximateArrivalTimestamp': 1428537600,
                    'partitionKey': 'partitionKey-3',
                    'data': kinesis_data,
                    'kinesisSchemaVersion': '1.0',
                    'sequenceNumber': ',,,'
                },
                'invokeIdentityArn': 'arn:aws:iam::EXAMPLE',
                'eventName': 'aws:kinesis:record',
                'eventSourceARN': 'arn:aws:kinesis:us-east-1:123456789012:stream/{}'.format(
                    source
                ),
                'eventSource': 'aws:kinesis',
                'awsRegion': 'us-east-1'
            }

        if service == 'sns':
            return {
                'EventVersion': '1.0',
                'EventSubscriptionArn': 'arn:aws:sns:us-east-1:123456789012:{}'.format(source),
                'EventSource': 'aws:sns',
                'Sns': {
                    'SignatureVersion': '1',
                    'Timestamp': '1970-01-01T00:00:00.000Z',
                    'Signature': 'EXAMPLE',
                    'SigningCertUrl': 'EXAMPLE',
                    'MessageId': '95df01b4-ee98-5cb9-9903-4c221d41eb5e',
                    'Message': data,
                    'MessageAttributes': {
                        'Test': {
                            'Type': 'String',
                            'Value': 'TestString'
                        }
                    },
                    'Type': 'Notification',
                    'UnsubscribeUrl': '...',
                    'TopicArn': 'arn:aws:sns:us-east-1:123456789012:{}'.format(source),
                    'Subject': '...'
                }
            }

        if service == 'streamalert_app':
            return {'streamalert_app': source, 'logs': [data]}

    @staticmethod
    def _validate_test_event(test_event):
        """Check if the test event contains the required keys

        Args:
            test_event (dict): The loaded test event from json

        Returns:
            bool: True if the proper keys are present
        """
        required_keys = {'description', 'log', 'service', 'source'}

        test_event_keys = set(test_event)
        if not required_keys.issubset(test_event_keys):
            req_key_diff = required_keys.difference(test_event_keys)
            missing_keys = ', '.join('\'{}\''.format(key) for key in req_key_diff)
            return False, 'Missing required key(s) in test event: {}'.format(missing_keys)

        acceptable_data_keys = {'data', 'override_record'}
        if not test_event_keys & acceptable_data_keys:
            return False, 'Test event must contain either \'data\' or \'override_record\''

        optional_keys = {'compress', 'trigger_rules', 'validate_schema_only'}

        key_diff = test_event_keys.difference(required_keys | optional_keys | acceptable_data_keys)

        # Log a warning if there are extra keys declared in the test log
        if key_diff:
            extra_keys = ', '.join('\'{}\''.format(key) for key in key_diff)
            LOGGER.warning('Additional unnecessary keys in test event: %s', extra_keys)

        return True, None

    def _apply_defaults(self, test_event):
        """Apply default values to the given test event

        Args:
            test_event (dict): The loaded test event
        """
        if 'override_record' not in test_event:
            return

        event_log = self._config['logs'].get(test_event['log'])

        configuration = event_log.get('configuration', {})
        schema = configuration.get('envelope_keys', event_log['schema'])

        # Add apply default values based on the declared schema
        default_test_event = {
            key: ParserBase.default_optional_values(value)
            for key, value in schema.items()
        }

        # Overwrite the fields included in the 'override_record' field,
        # and update the test event with a full 'data' key
        default_test_event.update(test_event['override_record'])
        test_event['data'] = default_test_event

    @staticmethod
    def _apply_helpers(test_record):
        """Detect and apply helper functions to test event data

        Helpers are declared in test fixtures via the following keyword:
        "<helpers:helper_name>"

        Supported helper functions:
            last_hour: return the current epoch time minus 60 seconds to pass the
                       last_hour rule helper.

        Args:
            test_record (dict): loaded fixture file JSON as a dict.
        """
        # declare all helper functions here, they should always return a string
        record_helpers = {
            'last_hour': lambda: str(int(time.time()) - 60)
        }
        helper_regex = re.compile(r'<helper:(?P<helper>\w+)>')

        def _find_and_apply_helpers(test_record):
            """Apply any helpers to the passed in test_record"""
            for key, value in test_record.items():
                if isinstance(value, str):
                    test_record[key] = re.sub(
                        helper_regex,
                        lambda match: record_helpers[match.group('helper')](),
                        value
                    )
                elif isinstance(value, dict):
                    _find_and_apply_helpers(test_record[key])

        _find_and_apply_helpers(test_record)


import tensorflow as tf
import numpy as np
import pickle
import cv2
from PIL import Image
import imutils

class Data(object):
    def __init__(self, data_dir, height, width, batch_size):
        self.data_dir = data_dir
        self.height = height
        self.width = width
        self.batch_size = batch_size

    def get_rot_data_iterator(self, images, labels):
        images = tf.cast(images, dtype=tf.float32)
        dataset = tf.data.Dataset.from_tensor_slices((images, labels))
        dataset = dataset.shuffle(buffer_size=10000, reshuffle_each_iteration=True)
        dataset = dataset.batch(self.batch_size)
        dataset = dataset.repeat()
        #dataset = dataset.map(self.preprocess, num_parallel_calls=2)
        dataset = dataset.prefetch(self.batch_size)
        iterator = dataset.make_initializable_iterator()
        print(tf.compat.v1.data.get_output_shapes(dataset))
        print(tf.compat.v1.data.get_output_types(dataset))
        return iterator

    def get_training_data(self):
        images = []
        labels = []
        for i in range(1, 6):
            data = self._get_next_batch_from_file(i)
            images.extend(list(self.convert_images(data[b"data"])))
            labels.extend(list(data[b"labels"]))
        return np.array(images), tf.keras.utils.to_categorical(np.array(labels))

    def convert_images(self, raw_images):
        images = raw_images / 255.0
        images = raw_images.reshape([-1, 3, self.height, self.width])
        images = images.transpose([0, 2, 3, 1])
        return images

    def _get_next_batch_from_file(self, batch_number):
        data = self._unpickle_data(self.data_dir + self._get_batch_name(batch_number))
        return data

    def _get_batch_name(self, number):
        return "data_batch_{0}".format(number)

    def _unpickle_data(self, filepath):
        with open(filepath, 'rb') as data:
            dict = pickle.load(data, encoding='bytes')
        return dict

    def get_test_data(self):
         data = self._unpickle_data(self.data_dir + "test_batch")
         return data[b"data"], tf.keras.utils.to_categorical(data[b"labels"])

    def preprocess(self, images, labels):
        rot_labels = []
        rot_images = []
        rotations = [90, 180, 270]
        images = tf.cast(images, dtype=tf.float32)
        for image in images:
            rot_labels.append(0)
            rot_images.append(image)
            for i, angle in enumerate(rotations, 1):
                rotated = imutils.rotate_bound(image, angle)
                rot_images.append(rotated)
                rot_labels.append(i)

        return np.array(rot_images), tf.keras.utils.to_categorical(np.array(rot_labels))

    @staticmethod
    def print_image_to_screen(data):
        """
        Used for debugging purposes.
        """
        img = Image.fromarray(data, 'RGB')
        img.show()

    @staticmethod
    def get_image(image_path):
        return

if __name__ == "__main__":
    DATA_DIR = "./data/cifar-10-batches-py/"
    data_obj = Data(DATA_DIR, 32, 32, 5000)
    x, y = data_obj.get_training_data()
    xr, yr = data_obj.preprocess(x, y)


# encoding: UTF-8
import json
import traceback
from datetime import datetime, date
import pandas as pd
from pandas import DataFrame
from vnpy.app.cta_strategy.backtesting import BacktestingEngine
from vnpy.app.cta_strategy.strategies.boll_channel_strategy import BollChannelStrategy
from vnpy.app.cta_strategy.strategies.turtle_signal_strategy import TurtleSignalStrategy
from vnpy.app.cta_strategy.strategies.double_ma_strategy import DoubleMaStrategy

class BatchCTABackTest:
	"""
	提供批量CTA策略回测，输出结果到excel或pdf，和CTA策略批量优化，输出结果到excel或pdf，
	"""

	def __init__(self, vtSymbolconfig="vtSymbol.json", exportpath=".\\"):
		"""
		加载配置路径
		"""
		config = open(vtSymbolconfig)
		self.setting = json.load(config)
		self.exportpath = exportpath

	def addParameters(self, engine, vt_symbol: str, startDate, endDate, interval="1m", capital=1_000_000):
		"""
		从vtSymbol.json文档读取品种的交易属性，比如费率，交易每跳，比率，滑点
		"""
		if vt_symbol in self.setting:
			engine.set_parameters(
				vt_symbol=vt_symbol,
				interval=interval,
				start=startDate,
				end=endDate,
				rate=self.setting[vt_symbol]["rate"],
				slippage=self.setting[vt_symbol]["slippage"],
				size=self.setting[vt_symbol]["size"],
				pricetick=self.setting[vt_symbol]["pricetick"],
				capital=capital
			)
		else:
			print("symbol %s hasn't be maintained in config file" % vt_symbol)
		return engine

	def runBatchTest(self, strategy_setting, startDate, endDate, portfolio):
		"""
		进行回测
		"""
		resultDf = DataFrame()
		dfportfolio = None
		for strategy_name, strategy_config in strategy_setting.items():
			engine = BacktestingEngine()
			vt_symbol = strategy_config["vt_symbol"]
			engine = self.addParameters(engine, vt_symbol, startDate, endDate)
			if type(strategy_config["setting"]) is str:
				print(strategy_config["setting"])
				engine.add_strategy(
					eval(strategy_config["class_name"]),
					json.loads(strategy_config["setting"], )
				)
			else:
				engine.add_strategy(
					eval(strategy_config["class_name"]),
					strategy_config["setting"]
				)
			engine.load_data()
			engine.run_backtesting()
			df = engine.calculate_result()
			if portfolio == True:
				if dfportfolio is None:
					dfportfolio = df
				else:
					dfportfolio = dfportfolio + df
			resultDict = engine.calculate_statistics(df, False)
			resultDict["class_name"] = strategy_config["class_name"]
			resultDict["setting"] = strategy_config["setting"]
			resultDict["vt_symbol"] = strategy_config["vt_symbol"]
			resultDf = resultDf.append(resultDict, ignore_index=True)

		if portfolio == True:
			# dfportfolio = dfportfolio.dropna()
			engine = BacktestingEngine()
			engine.calculate_statistics(dfportfolio)
			engine.show_chart(dfportfolio)
		return resultDf

	def runBatchTestJson(self, jsonpath="ctaStrategy.json", startDate=datetime(2019, 7, 1),
	                     endDate=datetime(2020, 1, 1), exporpath=None, portfolio=True):
		"""
		从ctaStrategy.json去读交易策略和参数，进行回测
		"""
		with open(jsonpath, mode="r", encoding="UTF-8") as f:
			strategy_setting = json.load(f)
		resultDf = self.runBatchTest(strategy_setting, startDate, endDate, portfolio)
		self.ResultExcel(resultDf, exporpath)
		return strategy_setting

	def runBatchTestExcecl(self, path="ctaStrategy.xls", startDate=datetime(2019, 7, 1),
	                       endDate=datetime(2020, 1, 1), exporpath=None, portfolio=False):
		"""
		从ctaStrategy.excel去读交易策略和参数，进行回测
		"""
		df = pd.read_excel(path)
		strategy_setting = df.to_dict(orient='index')
		resultDf = self.runBatchTest(strategy_setting, startDate, endDate, portfolio)
		self.ResultExcel(resultDf, exporpath)
		return strategy_setting

	def ResultExcel(self, result, export=None):
		"""
		输出交易结果到excel
		"""
		if export != None:
			exportpath = export
		else:
			exportpath = self.exportpath
		try:
			path = exportpath + "CTABatch" + str(date.today()) + "v0.xls"
			result.to_excel(path, index=False)
			print("CTA Batch result is export to %s" % path)
		except:
			print(traceback.format_exc())

		return None

if __name__ == '__main__':
	bts = BatchCTABackTest()
	bts.runBatchTestJson()


# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/gaogaotiantian/codesnap/blob/master/NOTICE.txt

import sys
from . import CodeSnap

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: python -m codesnap your_script_to_run.py [args]")
    try:
        f = sys.argv[1]
        code_string = open(f).read()
    except FileNotFoundError:
        print("No such file as {}".format(f))
        exit(1)
    sys.argv.pop(0)
    snap = CodeSnap()
    snap.start()
    exec(code_string)
    snap.stop()
    snap.save()