import json
import traceback
from collections import OrderedDict, defaultdict

import numpy as np
import pandas as pd
from scipy import stats

import rrcf
from django.core.serializers.json import DjangoJSONEncoder
from django.utils import timezone
from django_pandas.io import read_frame
from django.core.exceptions import ObjectDoesNotExist

from datacollection.models import Event, URL, CustomSession, Player
from dataprocessing.models import Task
from kimchi.celery import app
from shadowspect.models import Level, Replay
from dashboard.views import create_player_map, get_completed_puzzles_map, get_puzzles_dict
from dataprocessing.elo import adaptedData, run
from dataprocessing.misconceptions import sequenceWithinPuzzlesForMisconceptions, tagMisconceptions

difficultyMapping = ['Sandbox~0.000001','1. One Box~0.000002', '2. Separated Boxes~0.111127', '3. Rotate a Pyramid~0.083447', '4. Match Silhouettes~0.061887', '5. Removing Objects~0.106021', '6. Stretch a Ramp~0.107035', '7. Max 2 Boxes~0.078039', '8. Combine 2 Ramps~0.068608', '9. Scaling Round Objects~0.128647',
               'Square Cross-Sections~0.199714', 'Bird Fez~0.156674', 'Pi Henge~0.067346', '45-Degree Rotations~0.096715',  'Pyramids are Strange~0.179600', 'Boxes Obscure Spheres~0.266198', 'Object Limits~0.257177', 'Not Bird~0.260197', 'Angled Silhouette~0.147673',
               'Warm Up~0.183971','Tetromino~0.226869', 'Stranger Shapes~0.283971', 'Sugar Cones~0.085909', 'Tall and Small~0.266869', 'Ramp Up and Can It~0.206271', 'More Than Meets Your Eye~0.192319', 'Unnecessary~0.76', 'Zzz~0.234035', 'Bull Market~0.358579', 'Few Clues~0.324041', 'Orange Dance~0.647731', 'Bear Market~1.000000']

typeMapping = ['Sandbox~SAND', '1. One Box~Tutorial', '2. Separated Boxes~Tutorial', '3. Rotate a Pyramid~Tutorial', '4. Match Silhouettes~Tutorial', '5. Removing Objects~Tutorial', '6. Stretch a Ramp~Tutorial', '7. Max 2 Boxes~Tutorial', '8. Combine 2 Ramps~Tutorial', '9. Scaling Round Objects~Tutorial',
               'Square Cross-Sections~Easy Puzzles', 'Bird Fez~Easy Puzzles', 'Pi Henge~Easy Puzzles', '45-Degree Rotations~Easy Puzzles',  'Pyramids are Strange~Easy Puzzles', 'Boxes Obscure Spheres~Easy Puzzles', 'Object Limits~Easy Puzzles', 'Not Bird~Easy Puzzles', 'Angled Silhouette~Easy Puzzles',
               'Warm Up~Hard Puzzles','Tetromino~Hard Puzzles', 'Stranger Shapes~Hard Puzzles', 'Sugar Cones~Hard Puzzles', 'Tall and Small~Hard Puzzles', 'Ramp Up and Can It~Hard Puzzles', 'More Than Meets Your Eye~Hard Puzzles', 'Unnecessary~Hard Puzzles', 'Zzz~Hard Puzzles', 'Bull Market~Hard Puzzles', 'Few Clues~Hard Puzzles', 'Orange Dance~Hard Puzzles', 'Bear Market~Hard Puzzles']

difficultyPuzzles = dict()

for puzzle in difficultyMapping:
    desc = puzzle.split("~")
    difficultyPuzzles[desc[0]] = float(desc[1])

tutorialPuzzles = []

for puzzle in typeMapping:
    desc = puzzle.split("~")
    if(desc[1] == 'Tutorial'):
        tutorialPuzzles.append(desc[0])

advancedPuzzles = []

for puzzle in typeMapping:
    desc = puzzle.split("~")
    if(desc[1] == 'Hard Puzzles'):
        advancedPuzzles.append(desc[0])

intermediatePuzzles = []

for puzzle in typeMapping:
    desc = puzzle.split("~")
    if(desc[1] == 'Easy Puzzles'):
        intermediatePuzzles.append(desc[0])

allPuzzles = []
for puzzle in typeMapping:
    desc = puzzle.split("~")
    allPuzzles.append(desc[0])

@app.task
def process_task(task, *args):
    # this fixes an extraneous comma in tuples
    if len(args) == 1:
        args = args[0]
    # and this fixes when there are no args
    if len(args) == 0:
        task_sig = task.s()
    else:
        task_sig = task.s(args)
    taskname = str(task_sig)
    print("processing task", taskname)
    task_db, created = Task.objects.get_or_create(signature=taskname)
    task_db.state = "starting"
    task_db.save(update_fields=['state'])
    try:
        result = task_sig.apply_async()
        task_db.state = "processing"
        task_db.save(update_fields=['state'])
        task_db.result = result.get()
        task_db.state = "done"
        task_db.time_ended = timezone.now()
        task_db.errors = ""
    except Exception as exc:
        tb = traceback.format_exception(etype=type(exc), value=exc, tb=exc.__traceback__)
        task_db.errors = tb
        task_db.state = "error"
        print(tb)
    task_db.save()
    return task_db

# of particular interest
all_data_collection_urls = ['ginnymason', 'chadsalyer', 'kristinknowlton', 'lori day', 'leja', 'leja2', 'debbiepoull',
                            'juliamorgan']

@app.task
def computeFunnelByPuzzle(group='all'):
    if group == 'all':
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')
    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in
                           dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]

    # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[
        ((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']

    # userFunnelDict key: (group~user~puzzle), json values: started, create_shape, submitted, completed
    userFunnelDict = dict()

    for user in dataEvents['group_user_id'].unique():

        user_events = dataEvents[dataEvents['group_user_id'] == user]
        user_puzzle_key = None

        for enum, event in user_events.iterrows():

            if (event['type'] in ['ws-start_level', 'ws-puzzle_started']):
                user_puzzle_key = event['group'] + '~' + event['user'] + '~' + json.loads(event['data'])['task_id']
                if (user_puzzle_key not in userFunnelDict.keys()):
                    userFunnelDict[user_puzzle_key] = json.loads(
                        '{"started": 0, "create_shape": 0, "submitted": 0, "completed": 0}')

            if (event['type'] == 'ws-puzzle_started'):
                userFunnelDict[user_puzzle_key]['started'] += 1
            elif (event['type'] == 'ws-create_shape'):
                userFunnelDict[user_puzzle_key]['create_shape'] += 1
            elif (event['type'] == 'ws-check_solution'):
                userFunnelDict[user_puzzle_key]['submitted'] += 1
            elif (event['type'] == 'ws-puzzle_complete'):
                userFunnelDict[user_puzzle_key]['completed'] += 1

    userFunnelList = []
    for key in userFunnelDict.keys():
        key_split = key.split('~')
        userFunnelList.append([key_split[0], key_split[1], key_split[2], userFunnelDict[key]])

    userFunnelByPuzzle = pd.DataFrame(userFunnelList, columns=['group', 'user', 'task_id', 'funnel'])

    return userFunnelByPuzzle.to_json()

@app.task
def sequenceBetweenPuzzles(group='all'):
    if group == 'all' :
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]
    # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']

    # filtering to only take the group passed as argument
    if(group != 'all'):
        dataEvents = dataEvents[dataEvents['group'].isin(group)]

       # Data Cleaning
    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

    userPuzzleDict = {}

    for user in dataEvents['group_user_id'].unique():
            #Select rows
            user_events = dataEvents[dataEvents['group_user_id'] == user]
            user_events_na_dropped = user_events.dropna()
            for enum, event in user_events_na_dropped.iterrows():
                user_key = event['user']
                if(user_key not in userPuzzleDict.keys()):
                    userPuzzleDict[user_key] = {}
                    numPuzzles = 1
                if(event['type'] == 'ws-start_level'):
                    activePuzzle = json.loads(event['data'])['task_id']
                    if (activePuzzle == 'Sandbox'):
                        continue
                    secondKey = str(numPuzzles) + '~' + event['session']
                    if (userPuzzleDict[user_key].get(secondKey) == None):
                        userPuzzleDict[user_key][secondKey] = dict()
                    if (userPuzzleDict[user_key][secondKey].get(activePuzzle) == None):
                        userPuzzleDict[user_key][secondKey] = {activePuzzle : 'started'}
                elif(event['type'] == 'ws-puzzle_started'):
                    if (activePuzzle == 'Sandbox' or userPuzzleDict[user_key][secondKey][activePuzzle] in ['started','shape_created', 'submitted', 'completed']):
                        continue
                    userPuzzleDict[user_key][secondKey] = {activePuzzle : 'started'}
                elif(event['type'] == 'ws-create_shape'):
                    if (activePuzzle == 'Sandbox' or userPuzzleDict[user_key][secondKey][activePuzzle] in ['shape_created', 'submitted', 'completed']):
                        continue
                    userPuzzleDict[user_key][secondKey] = {activePuzzle : 'shape_created'}
                elif(event['type'] == 'ws-check_solution'):
                    if (activePuzzle == 'Sandbox' or userPuzzleDict[user_key][secondKey][activePuzzle] in ['submitted', 'completed']):
                        continue
                    userPuzzleDict[user_key][secondKey] = {activePuzzle :'submitted'}
                elif(event['type'] == 'ws-puzzle_complete'):
                    if (activePuzzle == 'Sandbox'):
                        continue
                    userPuzzleDict[user_key][secondKey] = {activePuzzle :'completed'}
                elif(event['type'] in ['ws-exit_to_menu', 'ws-disconnect', 'ws-login_user']):
                    if (activePuzzle == 'Sandbox'):
                        continue
                    numPuzzles +=1

    userSessionList = []
    for key in userPuzzleDict.keys():
        for sequence in userPuzzleDict[key].keys():
                key_split = sequence.split('~')
                userSessionList.append([key, key_split[1], int(key_split[0]), userPuzzleDict[key][sequence]])

    userSequence = pd.DataFrame(userSessionList, columns=['user', 'session', 'sequence', 'task_id'])
    #Recalculate sequence
    mod = []
    for user in userSequence['user'].unique():
            previousAttempt = 1
            n_attempt = 1
            individualDf = userSequence[userSequence['user'] == user]
            for enum, event in individualDf.iterrows():
                if (event['sequence'] != previousAttempt):
                    n_attempt += 1
                previousAttempt = event['sequence']
                event['sequence'] = n_attempt
                mod.append(event)
    modDf = pd.DataFrame(mod, columns=['user', 'session', 'sequence', 'task_id'])
    return modDf.to_json()

@app.task
def sequenceWithinPuzzles(group='all'):
    if group == 'all':
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    tutorialList = ['1. One Box', '2. Separated Boxes', '3. Rotate a Pyramid', '4. Match Silhouettes',
                    '5. Removing Objects', '6. Stretch a Ramp', '7. Max 2 Boxes', '8. Combine 2 Ramps',
                    '9. Scaling Round Objects', 'Sandbox']
    # Remove SandBox and tutorial levels.
    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in
                           dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]
    # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[
        ((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']
    # filtering to only take the group passed as argument
    if (group != 'all'):
        dataEvents = dataEvents[dataEvents['group'].isin(group)]
    # Data Cleaning
    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

    newDataEvents = []
    # Select puzzle and actions
    notSelectedEvents = ['ws-mode_change', 'ws-click_nothing', 'ws-click_disabled', 'ws-select_shape',
                         'ws-deselect_shape', 'ws-paint', 'ws-palette_change', 'ws-toggle_paint_display',
                         'ws-toggle_snapshot_display', 'ws-create_user', 'ws-redo_action', 'ws-undo_action',
                         'ws-restart_puzzle', 'ws-puzzle_started']
    # Selected puzzles
    selectedPuzzles = ['Square Cross-Sections', 'Bird Fez', 'Pi Henge', '45-Degree Rotations', 'Pyramids are Strange',
                       'Boxes Obscure Spheres', 'Object Limits', 'Warm Up', 'Angled Silhouette',
                       'Sugar Cones', 'Stranger Shapes', 'Tall and Small', 'Ramp Up and Can It',
                       'More Than Meets Your Eye', 'Not Bird', 'Unnecesary', 'Zzz', 'Bull Market', 'Few Clues',
                       'Orange Dance', 'Bear Market']

    eventsWithMetaData = ['ws-create_shape', 'ws-delete_shape', 'ws-rotate_shape', 'ws-scale_shape', 'ws-move_shape']

    for user in dataEvents['group_user_id'].unique():
        # Select rows
        user_events = dataEvents[dataEvents['group_user_id'] == user]
        user_events_na_dropped = user_events.dropna()
        activePuzzle = None
        nAttempt = 1
        prevCheck = False
        prevEvent = None
        figureDict = dict()
        for enum, event in user_events_na_dropped.iterrows():
            # Ignore event
            if (prevCheck == True):
                if (event['type'] == 'ws-puzzle_complete'):
                    prevEvent['metadata']['correct'] = True
                    newDataEvents.append(prevEvent)
                    prevCheck = False
                    prevEvent = None
                    continue
                else:
                    prevEvent['metadata']['correct'] = False
                newDataEvents.append(prevEvent)
                prevCheck = False
                prevEvent = None

            if (event['type'] in notSelectedEvents):
                continue

            elif (event['type'] == 'ws-start_level'):
                activePuzzle = json.loads(event['data'])['task_id']
                event['task_id'] = activePuzzle

            elif (event['type'] == 'ws-create_shape'):
                event['task_id'] = activePuzzle
                if (event['task_id'] in selectedPuzzles):
                    event['n_attempt'] = nAttempt
                    shape_id = json.loads(event['data'])['objSerialization']
                    shape_type = json.loads(event['data'])['shapeType']
                    figureDict[shape_id] = shape_type
                    event['metadata'] = dict()
                    event['metadata']['shape_id'] = shape_id
                    event['metadata']['shape_type'] = shape_type
                    newDataEvents.append(event)

            elif (event['type'] == 'ws-delete_shape' or event['type'] == 'ws-move_shape'):
                event['task_id'] = activePuzzle
                if (event['task_id'] in selectedPuzzles):
                    event['n_attempt'] = nAttempt
                    if (event['type'] == 'ws-delete_shape'):
                        idList = json.loads(event['data'])['deletedShapes']
                    elif (event['type'] == 'ws-move_shape'):
                        idList = json.loads(event['data'])['selectedObjects']
                    for shapeId in idList:
                        shape_id = shapeId
                        shape_type = figureDict[shape_id]
                        event['metadata'] = dict()
                        event['metadata']['shape_id'] = shape_id
                        event['metadata']['shape_type'] = shape_type
                        newDataEvents.append(event)

            elif (event['type'] == 'ws-rotate_shape' or event['type'] == 'ws-scale_shape'):
                event['task_id'] = activePuzzle
                if (event['task_id'] in selectedPuzzles):
                    event['n_attempt'] = nAttempt
                    shape_id = json.loads(event['data'])['selectedObject']
                    shape_type = figureDict[shape_id]
                    event['metadata'] = dict()
                    event['metadata']['shape_id'] = shape_id
                    event['metadata']['shape_type'] = shape_type
                    newDataEvents.append(event)

            elif ((event['type'] in ['ws-exit_to_menu', 'ws-login_user']) and (activePuzzle in selectedPuzzles)):
                figureDict.clear()
                nAttempt += 1

            else:
                event['task_id'] = activePuzzle
                if (event['task_id'] in selectedPuzzles):
                    event['n_attempt'] = nAttempt
                    event['metadata'] = dict()
                    if (event['type'] == 'ws-check_solution'):
                        prevCheck = True
                        prevEvent = event
                    else:
                        newDataEvents.append(event)

    taskDf = pd.DataFrame(newDataEvents,
                          columns=['id', 'time', 'group_user_id', 'task_id', 'n_attempt', 'type', 'metadata'])

    data = taskDf

    listEvent = ['ws-rotate_view', 'ws-rotate_shape', 'ws-undo_action', 'ws-move_shape', 'ws-snapshot',
                 'ws-scale_shape']

    dataConvert2 = []
    for user in data['group_user_id'].unique():
        individualDf = data[data['group_user_id'] == user]
        # Current action set
        currentAction = []
        # String with action types
        actionString = ""
        actualEvent = 'None'
        for enum, event in individualDf.iterrows():
            key = event['group_user_id']
            key_split = key.split('~')
            event['group_id'] = key_split[0]
            event['user'] = key_split[1]
            actualEvent = event['type']
            eq = True
            for a in currentAction:
                if (a['type'] != actualEvent):
                    # Ver si podemos compactar
                    eq = False

            if (eq == False):
                igual = True
                prev = ""
                for a2 in currentAction:
                    if (a2['type'] != prev):
                        if (prev == ""):
                            igual = True
                        else:
                            igual = False
                    prev = a2['type']
                if ((igual == True) and (prev in listEvent)):
                    add = currentAction[0]
                    # add['type'] = add['type'] + 'x' + str(len(currentAction))
                    add['n_times'] = len(currentAction)
                    dataConvert2.append(add)
                    currentAction.clear()
                    currentAction.append(event)
                else:  # igual != True
                    for a in currentAction:
                        a['n_times'] = 1
                        dataConvert2.append(a)
                    currentAction.clear()
                    currentAction.append(event)
            else:  # eq = True
                if (event['type'] not in listEvent):
                    currentAction.append(event)
                    for a in currentAction:
                        a['n_times'] = 1
                        dataConvert2.append(a)
                    currentAction.clear()

                else:
                    if (len(currentAction) > 0):
                        if (currentAction[0]['type'] in eventsWithMetaData):
                            # Event with metadata, check if it is the same shape_id
                            if (currentAction[0]['metadata']['shape_id'] == event['metadata']['shape_id']):
                                currentAction.append(event)
                            else:
                                add = currentAction[0]
                                # add['type'] = add['type'] + 'x' + str(len(currentAction))
                                add['n_times'] = len(currentAction)
                                dataConvert2.append(add)
                                currentAction.clear()
                                currentAction.append(event)
                        # Event without metaData, just concatenate.
                        else:
                            currentAction.append(event)

                    elif (len(currentAction) == 0):
                        currentAction.append(event)

        # Add last elems
        # We must check if last elems can be also replaced.
        final = ""
        if (len(currentAction) > 0):
            igual2 = True
            prev = ""
            for a2 in currentAction:
                if (a2['type'] != prev):
                    if (prev == ""):
                        igual2 = True
                    else:
                        igual2 = False
                prev = a2['type']
            if ((igual == True) and (prev in listEvent)):
                add = currentAction[0]
                # add['type'] = add['type'] + 'x' + str(len(currentAction))
                add['n_times'] = len(currentAction)
                dataConvert2.append(add)
                currentAction.clear()
                currentAction.append(event)
            else:  # igual != True
                for a in currentAction:
                    a['n_times'] = 1
                    dataConvert2.append(a)
                currentAction.clear()
                currentAction.append(event)

    # Create dataframe from list
    # consecutiveDf = pd.DataFrame(dataConvert2, columns=['id', 'time', 'group_user_id', 'task_id', 'n_attempt', 'type', 'metadata'])
    data = pd.DataFrame(dataConvert2,
                        columns=['group_id', 'user', 'task_id', 'n_attempt', 'type', 'n_times', 'metadata'])

    # Recalculate n_attempt
    mod = []
    for user in data['user'].unique():
        previousAttempt = 1
        n_attempt = 1
        individualDf = data[data['user'] == user]
        for enum, event in individualDf.iterrows():
            if (event['n_attempt'] != previousAttempt):
                n_attempt += 1
            previousAttempt = event['n_attempt']
            event['n_attempt'] = n_attempt
            mod.append(event)
    modDf = pd.DataFrame(mod, columns=['group_id', 'user', 'task_id', 'n_attempt', 'type', 'n_times', 'metadata'])
    return modDf.to_json()

@app.task
def computeLevelsOfActivity(group='all'):
    ### DATA COLLECTION AND INITIAL PROCESSING
    if group == 'all':
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

    #iterates in the groups and users of the data
    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['task_id'] = [json.loads(x)['task_id'] if 'task_id' in json.loads(x).keys() else '' for x in dataEvents['data']]

    # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']
    dataEvents['group_user_task_id'] = dataEvents['group'] + '~' + dataEvents['user']+'~'+dataEvents['task_id']

    # filtering to only take the group passed as argument
    if(group != 'all'):
        dataEvents = dataEvents[dataEvents['group'].isin(group)]

    # the data is grouped by the necessary variables
    activity_by_user = dataEvents.groupby(['group_user_id','group', 'user','group_user_task_id','task_id']).agg({'id':'count',
                                             'type':'nunique'}).reset_index().rename(columns={'id':'events',
                                                                                'type':'different_events'})

    #indicate the index variable
    activity_by_user.index = activity_by_user['group_user_task_id'].values

    typeEvents = ['ws-snapshot','ws-paint', 'ws-rotate_view','ws-move_shape','ws-rotate_shape' ,'ws-scale_shape','ws-create_shape','ws-delete_shape','ws-undo_action','ws-redo_action', 'ws-check_solution']

    #initialize the metrics
    activity_by_user['timeTotal'] = np.nan
    activity_by_user['inactive_time'] = np.nan
    activity_by_user['event'] = np.nan
    activity_by_user['different_events'] = np.nan
    activity_by_user['active_time'] = np.nan
    for event in typeEvents:
        activity_by_user[event] = 0

    #initialize the data structures
    userFunnelDict = dict()
    puzzleEvents = dict()
    eventsDiff = []
    eventsDiff_puzzle = dict()
    timePuzzle = dict()
    globalTypesEvents = dict()
    eventInitial = dict()
    totalTime = dict()

    for user in dataEvents['group_user_id'].unique():

        # Computing active time
        previousEvent = None
        theresHoldActivity = 60 # np.percentile(allDifferences, 98) is 10 seconds
        limit = 3600
        activeTime = []

        user_events = dataEvents[dataEvents['group_user_id'] == user]
        user_puzzle_key = None

        for enum, event in user_events.iterrows():

            # If it is the first event
            if(previousEvent is None):
                previousEvent = event
                continue

            if(event['type'] in ['ws-start_level', 'ws-puzzle_started']):

                #create id: group+user+task_id
                user_puzzle_key = event['group'] + '~' + event['user'] + '~' + json.loads(event['data'])['task_id']

                if(user_puzzle_key not in totalTime.keys()):
                    totalTime[user_puzzle_key] = 0

                delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                if((delta_seconds < limit)):
                    totalTime[user_puzzle_key] += delta_seconds
                previousEvent = event

                # initialize if the id is new
                if(user_puzzle_key not in puzzleEvents.keys()):
                    #totalTime[user_puzzle_key] = 0
                    puzzleEvents[user_puzzle_key]= 1
                    eventsDiff_puzzle[user_puzzle_key] = []
                    eventsDiff_puzzle[user_puzzle_key].append(event['type'])
                    timePuzzle[user_puzzle_key] = 0
                    globalTypesEvents[user_puzzle_key] = dict()
                    for ev in typeEvents:
                        globalTypesEvents[user_puzzle_key][ev]= 0

            # the event is not final event
            if(event['type'] not in ['ws-exit_to_menu', 'ws-puzzle_complete', 'ws-create_user', 'ws-login_user']):
                    puzzleEvents[user_puzzle_key] += 1

                    #add the event type
                    eventsDiff_puzzle[user_puzzle_key].append(event['type'])

                    #calculate the duration of the event
                    delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                    if((delta_seconds < limit)):
                        totalTime[user_puzzle_key] += delta_seconds
                    if((delta_seconds < theresHoldActivity)):
                        timePuzzle[user_puzzle_key] += delta_seconds

                    previousEvent = event

                    #update event counters by type
                    if(event['type'] in typeEvents):
                        globalTypesEvents[user_puzzle_key][event['type']] +=1

            # the puzzle ends
            if(event['type'] in ['ws-exit_to_menu', 'ws-puzzle_complete']):

                    puzzleEvents[user_puzzle_key] += 1

                    #add the event type
                    eventsDiff_puzzle[user_puzzle_key].append(event['type'])

                    #calculate the duration of the event
                    delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                    if((delta_seconds < limit)):
                        totalTime[user_puzzle_key] += delta_seconds
                    if((delta_seconds < theresHoldActivity)):
                        timePuzzle[user_puzzle_key] += delta_seconds

                    previousEvent = event

    # add the data by group_user_task_id
    for i in dataEvents['group_user_task_id'].unique():
        key_split = i.split('~')
        if(key_split[2] != ''):
            activity_by_user.at[i, 'timeTotal'] = totalTime[i]
            activity_by_user.at[i, 'inactive_time'] = totalTime[i]-timePuzzle[i]
            activity_by_user.at[i, 'event'] = puzzleEvents[i]
            activity_by_user.at[i, 'different_events'] = len(set(eventsDiff_puzzle[i]))
            activity_by_user.at[i, 'active_time'] = timePuzzle[i]
            for event in typeEvents:
                activity_by_user.at[i, event] = globalTypesEvents[i][event]

    #delete row with NaN
    activity_by_user.dropna(inplace=True)
    #delete group_user_task_id column
    activity_by_user.drop(columns=['group_user_task_id'], inplace=True)

    #data output preparation
    activity_by_user = pd.melt(activity_by_user, id_vars=['group', 'user','task_id'],
        value_vars=['timeTotal','inactive_time','event','different_events', 'active_time','ws-snapshot','ws-paint','ws-rotate_view','ws-rotate_shape','ws-move_shape','ws-scale_shape','ws-create_shape','ws-delete_shape','ws-undo_action','ws-redo_action','ws-check_solution'],
        var_name='metric', value_name='value')

    ### MERGING ROWS CORRESPONDING TO THE SAME USER
    activity_dict = activity_by_user.to_dict()
    max_index = len(activity_dict['group'])
    merged_activity = {"all": {"no_normalization": {}}}
    player_map = {}

    for url in urls:
        player_map[url.name] = {v: k for k, v in create_player_map(url.name).items()}

    for i in range(max_index):
        user = player_map.get(activity_dict['group'][i]).get(activity_dict['user'][i])
        if user == None:
            continue

        metric = activity_dict['metric'][i]
        task_id = activity_dict['task_id'][i]

        if task_id not in merged_activity:
            merged_activity[task_id] = {"no_normalization": defaultdict(dict)}
        merged_activity[task_id]["no_normalization"][user][metric] = float(activity_dict['value'][i])

        if user not in merged_activity["all"]["no_normalization"]:
            merged_activity["all"]["no_normalization"][user] = defaultdict(float)

        if metric != "different_events":
            # if metric == "timeTotal":
            #     print("here " + str(activity_dict['value'][i]) + "\n")
            merged_activity["all"]["no_normalization"][user][metric] += float(activity_dict['value'][i])
        else:
            # if metric == "timeTotal":
            #     print("here bad\n")
            merged_activity["all"]["no_normalization"][user][metric] = max(merged_activity["all"]["no_normalization"][user][metric], float(activity_dict['value'][i]))

    ### GENERATING STATISTICS
    completed_puzzles_map = {}
    for url in urls:
        completed_puzzles_map.update(get_completed_puzzles_map(url.name))
        metric_keys = list(list(merged_activity.values())[0]["no_normalization"].values())[0].keys()

    for task in merged_activity.keys():
        statistics = {}
        completed_statistics = {}

        values = {}
        completed_values = {}

        for key in metric_keys:
            statistics[key] = {
                'min': float("inf"),
                'max': float("-inf"),
                'median': 0,
                'mean': 0,
                'stdev': 0
            }
            completed_statistics[key] = {
                'min': float("inf"),
                'max': float("-inf"),
                'median': 0,
                'mean': 0,
                'stdev': 0
            }
            values[key] = []
            completed_values[key] = []

        users = merged_activity[task]["no_normalization"]
        items = users.items()

        for student, value in items:
            # if value['ws-create_shape'] == 0:
            #     continue
            # TODO: why is timeTotal sum incorrect?
            if task == "all":
                value["timeTotal"] = value["inactive_time"] + value["active_time"]
            if student in completed_puzzles_map and task in completed_puzzles_map[student]:
                for key in value.keys():
                    values[key].append(value[key])
                    completed_values[key].append(value[key])

                    if statistics[key]['min'] > value[key]:
                        statistics[key]['min'] = value[key]
                    if statistics[key]['max'] < value[key]:
                        statistics[key]['max'] = value[key]

                    if completed_statistics[key]['min'] > value[key]:
                        completed_statistics[key]['min'] = value[key]
                    if completed_statistics[key]['max'] < value[key]:
                        completed_statistics[key]['max'] = value[key]
            else:
                for key in value.keys():
                    values[key].append(value[key])

                    if statistics[key]['min'] > value[key]:
                        statistics[key]['min'] = value[key]
                    if statistics[key]['max'] < value[key]:
                        statistics[key]['max'] = value[key]

        for key in metric_keys:
            statistics[key]['median'] = np.median(values[key])
            statistics[key]['mean'] = np.mean(values[key])
            statistics[key]['stdev'] = np.std(values[key])

            completed_statistics[key]['median'] = np.median(completed_values[key])
            completed_statistics[key]['mean'] = np.mean(completed_values[key])
            completed_statistics[key]['stdev'] = np.std(completed_values[key])

        merged_activity[task]['no_normalization']['all_stats'] = statistics
        no_completed_stats = completed_statistics["event"]["min"] == float("inf") if max_index else True
        merged_activity[task]['no_normalization']['completed_stats'] = None if no_completed_stats else completed_statistics

        ### CALCULATING NORMALIZED VALUES
        merged_activity[task]["minmax_normalization"] = {"all_stats": {}, "completed_stats": None if no_completed_stats else {}}
        merged_activity[task]["standard_normalization"] = {"all_stats": {}, "completed_stats": None if no_completed_stats else {}}

        for student, value in items:
            if (student == "all_stats" or student == "completed_stats"):
                continue

            merged_activity[task]["minmax_normalization"]['all_stats'][student] = {}
            merged_activity[task]["standard_normalization"]['all_stats'][student] = {}

            if not no_completed_stats:
                merged_activity[task]["minmax_normalization"]['completed_stats'][student] = {}
                merged_activity[task]["standard_normalization"]['completed_stats'][student] = {}

            for key, key_val in value.items():
                min_val = merged_activity[task]['no_normalization']['all_stats'][key]['min']
                max_val = merged_activity[task]['no_normalization']['all_stats'][key]['max']
                stdev_val = merged_activity[task]['no_normalization']['all_stats'][key]['stdev']
                mean_val = merged_activity[task]['no_normalization']['all_stats'][key]['mean']

                merged_activity[task]["minmax_normalization"]['all_stats'][student][key] = (key_val - min_val) / (max_val - min_val) if max_val - min_val != 0 else 0
                merged_activity[task]["standard_normalization"]['all_stats'][student][key] = ((key_val - mean_val) / stdev_val) if stdev_val != 0 else 0

                if no_completed_stats:
                    continue

                min_val = merged_activity[task]['no_normalization']['completed_stats'][key]['min']
                max_val = merged_activity[task]['no_normalization']['completed_stats'][key]['max']
                stdev_val = merged_activity[task]['no_normalization']['completed_stats'][key]['stdev']
                mean_val = merged_activity[task]['no_normalization']['completed_stats'][key]['mean']

                merged_activity[task]["minmax_normalization"]['completed_stats'][student][key] = (key_val - min_val) / (max_val - min_val) if max_val - min_val != 0 else 0
                merged_activity[task]["standard_normalization"]['completed_stats'][student][key] = ((key_val - mean_val) / stdev_val) if stdev_val != 0 else 0

    return json.dumps(merged_activity)

@app.task
def computeLevelsOfActivityOutliers(group='all'):
    url = URL.objects.get(name__in=group)
    puzzles = get_puzzles_dict(url.name)["puzzles"]
    task = list(Task.objects.filter(signature__contains="computeLevelsOfActivity(['"+url.name+"']").values_list("result", flat=True))[0]
    task_dict = json.loads(task)
    if len(task_dict.keys()) <= 1:
        return {}
    np.random.seed(0)

    outlier_vals = {}
    outlier_vals_puzzles = {}
    outlier_metrics = []
    outlier_metrics_students = []
    outlier_metrics_puzzles = []

    for puzzle_list in puzzles.values():
        for puzzle in puzzle_list:
            if not (puzzle in task_dict):
                continue
            data = task_dict[puzzle]["minmax_normalization"]["all_stats"]
            outlier_metrics.extend(data.values())
            outlier_metrics_students.extend(data.keys())
            outlier_metrics_puzzles.extend([puzzle] * len(data.keys()))

    n = len(outlier_metrics)
    d = len(list(outlier_metrics[0].keys()))

    # Generate data
    X = np.zeros((n, d))
    for i in range(n):
        X[i,...] = np.array(list(outlier_metrics[i].values()))

    num_trees = 100 if n > 100 else n
    tree_size = 256 if n > 256 else n
    sample_size_range = (n // tree_size, tree_size)

    # Construct forest
    forest = []
    while len(forest) < num_trees:
        # Select random subsets of points uniformly
        ixs = np.random.choice(n, size=sample_size_range,
                            replace=False)
        # Add sampled trees to forest
        trees = [rrcf.RCTree(X[ix], index_labels=ix)
                for ix in ixs]
        forest.extend(trees)

    # Compute average CoDisp
    avg_codisp = pd.Series(0.0, index=np.arange(n))
    index = np.zeros(n)
    for tree in forest:
        codisp = pd.Series({leaf : tree.codisp(leaf)
                        for leaf in tree.leaves})
        avg_codisp[codisp.index] += codisp
        np.add.at(index, codisp.index.values, 1)
    avg_codisp /= index

    threshold = avg_codisp.quantile(0.99)
    print(threshold)

    for i in range(len(outlier_metrics)):
        student = outlier_metrics_students[i]
        puzzle = outlier_metrics_puzzles[i]
        if student not in outlier_vals:
            outlier_vals[student] = []
            outlier_vals_puzzles[student] = []
        outlier_vals[student].append(avg_codisp[i])
        outlier_vals_puzzles[student].append(puzzle)

    outlier_result = {}
    for student in outlier_vals:
        indices = np.where(outlier_vals[student] > threshold)
        if len(indices[0]) > 0:
            outlier_result[student] = list(np.array(outlier_vals_puzzles[student])[indices])

    return json.dumps(outlier_result)

@app.task
def computePersistence(group = 'all'):
    if group == 'all' :
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

        #iterates in the groups and users of the data
    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['task_id'] = [json.loads(x)['task_id'] if 'task_id' in json.loads(x).keys() else '' for x in dataEvents['data']]

        # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']
    dataEvents['group_user_task_id'] = dataEvents['group'] + '~' + dataEvents['user']+'~'+dataEvents['task_id']

        # filtering to only take the group passed as argument
    #if(group != 'all'):
    #    dataEvents = dataEvents[dataEvents['group'].isin(group)]

        # the data is grouped by the necessary variables
    activity_by_user = dataEvents.groupby(['group_user_id']).agg({'id':'count',
                                                 'type':'nunique'}).reset_index().rename(columns={'id':'events',
                                                                                'type':'different_events'})

           # Data Cleaning
        #dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

    typeEvents = ['ws-snapshot','ws-paint', 'ws-rotate_view','ws-move_shape','ws-rotate_shape' ,'ws-scale_shape','ws-create_shape','ws-delete_shape','ws-undo_action','ws-redo_action', 'ws-check_solution']
    manipulationTypeEvents = ['ws-move_shape','ws-rotate_shape' ,'ws-scale_shape','ws-create_shape','ws-delete_shape']

        #initialize the metrics
    activity_by_user['completed'] = np.nan
    activity_by_user['active_time'] = np.nan
    activity_by_user['n_events'] = np.nan
    activity_by_user['timestamp'] = np.nan

    for event in typeEvents:
        activity_by_user[event] = 0

    #initialize the data structures
    puzzleEvents = dict()
    timePuzzle = dict()
    globalTypesEvents = dict()
    n_attempts = dict()
    completados = dict()
    timestamp = dict()

    percentilAtt = dict()
    percentilTime = dict()

    percentilAttValue = 90
    percentilTimeValue = 90

    breaksPuzzle = dict()
    cumAttempts = OrderedDict()
    puzzleAttempts = dict()
    userCumAttempts = OrderedDict()
    puzzleCumAttempts = dict()
    prevReg = dict()
    actualAtt = 0
    prevAtt = 0
    idComplete = dict()
    contParc = dict()
    orden = []
    ids = []
    attemptsAux = dict()

    contCheckSol = dict()

    manipulationEvents = dict()
    userManipulationEvents = dict()
    contManipulation = 0
    timeFirstCheck = dict()
    timeSubExit = dict()
    timeCheckActual = dict()
    timeBetweenSub = dict()

    for user in dataEvents['group_user_id'].unique():

            # Computing active time
        previousEvent = None
        theresHoldActivity = 60 # np.percentile(allDifferences, 98) is 10 seconds
        activeTime = []

        user_events = dataEvents[dataEvents['group_user_id'] == user]
        user_puzzle_key = None
        userParc = None
        task_id = None
        initialTime = None
        prev_id = 1

        for enum, event in user_events.iterrows():

                # If it is the first event
                if(previousEvent is None):
                    previousEvent = event
                    continue

                if( event['type'] in ['ws-start_level'] ):

                    #create id: group+user+task_id
                    task_id = json.loads(event['data'])['task_id']
                    if (task_id =="Sandbox"):
                        continue

                    if(user_puzzle_key not in timeSubExit.keys()):
                        timeSubExit[user_puzzle_key] = str(0)
                        timeBetweenSub[user_puzzle_key] = str(0)

                    if(event['user'] not in userCumAttempts.keys()):
                        userCumAttempts[event['user']] = 0
                        actualAtt = 0
                        attemptsAux[event['user']] = dict()
                        timeCheckActual[event['user']] = 0

                    if(event['user'] not in userManipulationEvents.keys()):
                        #print("Se inicializa con ", event['user'])
                        userManipulationEvents[event['user']] = 0

                    #if(user_puzzle_key not in manipulationEvents.keys()):
                    #    manipulationEvents[user_puzzle_key] = 0
                    #    contManipulation = 0

                    if(task_id not in attemptsAux[event['user']].keys()): attemptsAux[event['user']][task_id]=0

                    user_puzzle_key = event['group'] + '~' + event['user'] + '~' + task_id# + '~' + str(n_attempts[prev_id])
                    if(user_puzzle_key not in prevReg.keys()):

                        prevReg[user_puzzle_key] = 1
                        user_puzzle_key = event['group'] + '~' + event['user'] + '~' + task_id + '~' + '1'
                        n_attempts[user_puzzle_key] = 1
                        attemptsAux[event['user']][task_id] = n_attempts[user_puzzle_key]

                    else:

                        user_puzzle_key = event['group'] + '~' + event['user'] + '~' + task_id + '~' + str(attemptsAux[event['user']][task_id])
                        n_attempts[user_puzzle_key] = attemptsAux[event['user']][task_id]

                    key_split = user_puzzle_key.split('~')
                    userParc = key_split[1]

                    if(user_puzzle_key not in idComplete.keys()): idComplete[user_puzzle_key] = 0

                    if(task_id not in attemptsAux[userParc].keys()): attemptsAux[userParc][task_id]=0
                    if(user_puzzle_key not in cumAttempts.keys()):
                        cumAttempts[user_puzzle_key] = 1

                    # initialize if the id is new
                    if(user_puzzle_key not in puzzleEvents.keys()):

                        breaksPuzzle[user_puzzle_key] = 0
                        timestamp[user_puzzle_key] = 0
                        percentilAtt[user_puzzle_key] = percentilAttValue
                        percentilTime[user_puzzle_key] = percentilTimeValue
                        completados[user_puzzle_key] = 0
                        puzzleEvents[user_puzzle_key]= 1
                        timePuzzle[user_puzzle_key] = 0
                        contCheckSol[user_puzzle_key] = 0
                        manipulationEvents[user_puzzle_key] = 0
                        timeFirstCheck[user_puzzle_key] = 0

                        globalTypesEvents[user_puzzle_key] = dict()
                        for ev in typeEvents:
                            globalTypesEvents[user_puzzle_key][ev]= 0

                    #timestamp
                    if(event['type'] in 'ws-start_level'):

                        timestamp[user_puzzle_key] = event['time']
                        initialTime = timestamp[user_puzzle_key]

                # the event is not final event
                if(event['type'] not in ['ws-exit_to_menu' , 'ws-disconnect', 'ws-create_user', 'ws-login_user']):
                        if (task_id == "Sandbox"):
                            continue

                        if(event['type'] in ['ws-puzzle_complete']): completados[user_puzzle_key] = 1

                        puzzleEvents[user_puzzle_key] += 1

                        #calculate the duration of the event
                        delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                        if((delta_seconds < theresHoldActivity)):
                            timePuzzle[user_puzzle_key] += delta_seconds

                        #breaks
                        if((delta_seconds > 15)):
                            breaksPuzzle[user_puzzle_key] += 1

                        previousEvent = event

                        #update event counters by type
                        if(event['type'] in typeEvents):
                            globalTypesEvents[user_puzzle_key][event['type']] +=1

                        if(globalTypesEvents[user_puzzle_key]['ws-check_solution'] == 1): timeFirstCheck[user_puzzle_key] = event['time']

                        if(event['type'] in manipulationTypeEvents):
                            manipulationEvents[user_puzzle_key] +=1

                        if(event['type'] == 'ws-check_solution'):
                            timeCheckActual[event['user']] = event['time']
                            contCheckSol[user_puzzle_key] +=1

                # the puzzle ends
                if(event['type'] in ['ws-exit_to_menu', 'ws-disconnect']):
                        if (task_id == "Sandbox"):
                            continue

                        idComplete[user_puzzle_key] = 1
                        puzzleEvents[user_puzzle_key] += 1

                        if(completados[user_puzzle_key] == 0 and globalTypesEvents[user_puzzle_key]['ws-check_solution'] > 0):
                            timeSubExit[user_puzzle_key] = str(round((event['time'] - timeFirstCheck[user_puzzle_key]).total_seconds(), 2))
                        else: timeSubExit[user_puzzle_key] = 'NA'

                        if(globalTypesEvents[user_puzzle_key]['ws-check_solution'] == 0): timeBetweenSub[user_puzzle_key] = 'NA'
                        else: timeBetweenSub[user_puzzle_key] = str(round(((timeCheckActual[event['user']] - timestamp[user_puzzle_key]) /globalTypesEvents[user_puzzle_key]['ws-check_solution']).total_seconds(), 2))

                        #calculate the duration of the event
                        delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                        if((delta_seconds < theresHoldActivity)):
                            timePuzzle[user_puzzle_key] += delta_seconds

                        #breaks
                        if((delta_seconds > 15)):
                            breaksPuzzle[user_puzzle_key] += 1

                        previousEvent = event

                        userCumAttempts[userParc] +=1
                        n_attempts[user_puzzle_key] +=1
                        actualAtt+=1
                        cumAttempts[user_puzzle_key] = actualAtt
                        attemptsAux[userParc][task_id] = n_attempts[user_puzzle_key]

                        #manipulationEvents[user_puzzle_key] = userManipulationEvents[event['user']]

                        ###########

    userTime = dict()
    userAtt = dict()
    userEvent = dict()

    for i in puzzleEvents.keys():
        if(idComplete[i]==0):
            continue
        key_split = i.split('~')
        if(key_split[1] not in userTime.keys()):
            userTime[i] = 0
            userAtt[i] = 0
            userEvent[i] = 0

        if(key_split[2] != ''):

            if(key_split[2] in allPuzzles): userAtt[i] = contCheckSol[i]

            if(key_split[2] in allPuzzles): userTime[i] = timePuzzle[i]

            if(key_split[2] in allPuzzles): userEvent[i] = puzzleEvents[i]

    puzzleTime = dict()
    puzzleAtt = dict()
    puzzleEvent = dict()

    for i in userTime.keys():
        #for puzzle in userTime[user]:
        key_split = i.split('~')
        if(key_split[2] not in puzzleTime.keys()):
            puzzleTime[key_split[2]] = []
            puzzleAtt[key_split[2]] = []
            puzzleEvent[key_split[2]] = []

        puzzleTime[key_split[2]].append(userTime[i])
        puzzleAtt[key_split[2]].append(userAtt[i])
        puzzleEvent[key_split[2]].append(userEvent[i])

    persistent = dict()
    percentileActiveTime = dict()
    percentileAtt = dict()
    percentileEvent = dict()
    percentileComposite = dict()
    averagePercentileComposite = dict()
    averagePercentilePartial = dict()

    difficultyNumber = dict()

    contNonPer = dict()
    totalNonPer = dict()
    contRapid = dict()
    totalRapid = dict()
    contUnpro = dict()
    totalUnpro = dict()
    contProduct = dict()
    totalProduct=dict()
    contNoBehavior = dict()
    totalNoBehavior = dict()
    persistantCumPerc = dict()

    cumDifficulty = dict()
    cumUserPercentage = dict()

    diffPercentage = dict()

    compositeUser = dict()

    for i in puzzleEvents.keys():
        if(idComplete[i]==0):
            continue
        key_split = i.split('~')
        if(key_split[2] != ''):

            #difficulty
            difficultyNumber[i] = difficultyPuzzles[key_split[2]]
            if(i not in diffPercentage.keys()): diffPercentage[i] = 0

            if(key_split[1] not in contNonPer.keys()):
                contNonPer[key_split[1]] = 0
                totalNonPer[key_split[1]] =0
                contRapid[key_split[1]]=0
                totalRapid[key_split[1]]=0
                contUnpro[key_split[1]]=0
                totalUnpro[key_split[1]]=0
                contProduct[key_split[1]]=0
                totalProduct[key_split[1]]=0
                contNoBehavior[key_split[1]]=0
                totalNoBehavior[key_split[1]]=0
                cumDifficulty[key_split[1]]=0
                compositeUser[key_split[1]]= []
                cumUserPercentage[key_split[1]]=0

            percentileActiveTime[i] = stats.percentileofscore(puzzleTime[key_split[2]], userTime[i])
            percentileAtt[i] = stats.percentileofscore(puzzleAtt[key_split[2]], userAtt[i], kind='weak')
            percentileEvent[i] = stats.percentileofscore(puzzleEvent[key_split[2]], userEvent[i], kind='weak')
            percentileComposite[i] = (stats.percentileofscore(puzzleTime[key_split[2]], userTime[i], kind='weak') + stats.percentileofscore(puzzleAtt[key_split[2]], userAtt[i], kind='weak') + stats.percentileofscore(puzzleEvent[key_split[2]], userEvent[i], kind='weak')) / 3
            compositeUser[key_split[1]].append(percentileComposite[i])

            cumDifficulty[key_split[1]] = cumDifficulty[key_split[1]] + difficultyPuzzles[key_split[2]]
            cumUserPercentage[key_split[1]] = cumUserPercentage[key_split[1]] + (difficultyPuzzles[key_split[2]] * percentileComposite[i])
            diffPercentage[i] = cumUserPercentage[key_split[1]] / cumDifficulty[key_split[1]]

            if(key_split[1] not in averagePercentilePartial.keys()): averagePercentilePartial[key_split[1]]=0
            if(i not in averagePercentileComposite.keys()): averagePercentileComposite[i]=0

            if(cumAttempts[i] == 0): averagePercentileComposite[i] = averagePercentileComposite[i]
            else:

                averagePercentilePartial[key_split[1]] = averagePercentilePartial[key_split[1]] + percentileComposite[i]
                averagePercentileComposite[i] = averagePercentilePartial[key_split[1]] / cumAttempts[i]

            if(i not in persistent.keys()):
                persistent[i] = ''

            if(percentileComposite[i] < 25 and completados[i] == 0):
                persistent[i] = 'NON_PERSISTANT'
                contNonPer[key_split[1]] +=1

            if(percentileComposite[i] < 25 and completados[i] == 1):
                persistent[i] = 'RAPID_SOLVER'
                contRapid[key_split[1]]+=1

            if(percentileComposite[i] > 75 and completados[i] == 1):
                persistent[i] = 'PRODUCTIVE_PERSISTANCE'
                contProduct[key_split[1]]+=1

            if(percentileComposite[i] > 75 and completados[i] == 0):
                persistent[i] = 'UNPRODUCTIVE_PERSISTANCE'
                contUnpro[key_split[1]]+=1

            if(percentileComposite[i] >= 25 and percentileComposite[i] <= 75):
                persistent[i] = 'NO_BEHAVIOR'
                contNoBehavior[key_split[1]]+=1

            if(contNonPer[key_split[1]] == 0 or cumAttempts[i]==0):
                totalNonPer[key_split[1]] =0
            else:
                totalNonPer[key_split[1]] = 100 * (contNonPer[key_split[1]] / cumAttempts[i])

            if(contRapid[key_split[1]] == 0 or cumAttempts[i]==0): totalRapid[key_split[1]] =0
            else: totalRapid[key_split[1]] = 100 * (contRapid[key_split[1]] / cumAttempts[i])

            if(contProduct[key_split[1]] == 0 or cumAttempts[i]==0): contProduct[key_split[1]] =0
            else: totalProduct[key_split[1]] = 100 * (contProduct[key_split[1]] / cumAttempts[i])

            if(contUnpro[key_split[1]] == 0 or cumAttempts[i]==0): contUnpro[key_split[1]] =0
            else: totalUnpro[key_split[1]] = 100 * (contUnpro[key_split[1]] / cumAttempts[i])

            if(contNoBehavior[key_split[1]] == 0 or cumAttempts[i]==0): contNoBehavior[key_split[1]] =0
            else: totalNoBehavior[key_split[1]] = 100 * (contNoBehavior[key_split[1]] / cumAttempts[i])

            persistantCumPerc[i] = json.dumps({"NON_PERSISTANT ": round(totalNonPer[key_split[1]],2), "RAPID_SOLVER": round(totalRapid[key_split[1]],2), "PRODUCTIVE_PERSISTANCE": round(totalProduct[key_split[1]],2), "UNPRODUCTIVE_PERSISTANCE": round(totalUnpro[key_split[1]],2), "NO_BEHAVIOR": round(totalNoBehavior[key_split[1]],2) })

    resultPart = 0
    for i in puzzleEvents.keys():
        if(idComplete[i]==0):
            continue
        key_split = i.split('~')
        if(key_split[2] != '' and key_split[1] != '' and i != ''):
            activity_by_user.at[i, 'user'] = key_split[1]
            activity_by_user.at[i, 'group'] = key_split[0]
            activity_by_user.at[i, 'task_id'] = key_split[2]
            activity_by_user.at[i, 'n_events'] = puzzleEvents[i]
            activity_by_user.at[i, 'active_time'] = round(timePuzzle[i],2)
            activity_by_user.at[i, 'percentileAtt'] = round(percentileAtt[i],2)
            activity_by_user.at[i, 'percentileActiveTime'] = round(percentileActiveTime[i],2)
            activity_by_user.at[i, 'percentileEvents'] = round(percentileEvent[i],2)
            activity_by_user.at[i, 'percentileComposite'] = round(percentileComposite[i],2)
            activity_by_user.at[i, 'completed'] = completados[i]
            activity_by_user.at[i, 'puzzle_difficulty'] = difficultyNumber[i]
            activity_by_user.at[i, 'cum_global_puzzle_attempts'] = cumAttempts[i]
            activity_by_user.at[i, 'cum_this_puzzle_attempt'] = key_split[3]
            activity_by_user.at[i, 'cum_avg_perc_composite'] = round(averagePercentileComposite[i],2)
            activity_by_user.at[i, 'cum_avg_persistence'] = persistantCumPerc[i]
            activity_by_user.at[i, 'timestamp'] = timestamp[i]
            activity_by_user.at[i, 'persistence'] = persistent[i]
            activity_by_user.at[i, 'n_breaks'] = breaksPuzzle[i]
            activity_by_user.at[i, 'n_manipulation_events'] = manipulationEvents[i]
            activity_by_user.at[i, 'cum_weighted_difficulty_perc_composite'] = round(diffPercentage[i],2)
            resultPart = stats.percentileofscore(compositeUser[key_split[1]], percentileComposite[i])
            activity_by_user.at[i, 'percentileCompositeAcrossAttempts'] = round(resultPart,2)
            if(resultPart >= 75): activity_by_user.at[i, 'persistenceAcrossAttempts'] = 'MORE_PERSISTANCE_THAN_NORMAL'
            if(resultPart < 75 and resultPart>25): activity_by_user.at[i, 'persistenceAcrossAttempts'] = 'NORMAL_PERSISTANCE'
            if(resultPart <= 25): activity_by_user.at[i, 'persistenceAcrossAttempts'] = 'LESS_PERSISTANCE_THAN_NORMAL'
            activity_by_user.at[i, 'time_failed_submission_exit'] = timeSubExit[i]
            activity_by_user.at[i, 'avg_time_between_submissions'] = timeBetweenSub[i]
            activity_by_user.at[i, 'n_check_solution'] = globalTypesEvents[i]['ws-check_solution']
            activity_by_user.at[i, 'n_snapshot'] = globalTypesEvents[i]['ws-snapshot']
            activity_by_user.at[i, 'n_rotate_view'] = globalTypesEvents[i]['ws-rotate_view']

    #delete row with NaN
    activity_by_user.dropna(subset = ['user'], inplace=True)

    #data output preparation
    activity_by_user = pd.DataFrame(activity_by_user, columns=['group', 'user','task_id','puzzle_difficulty' ,'completed','timestamp', 'active_time','percentileActiveTime','n_events','percentileEvents', 'n_check_solution','percentileAtt','percentileComposite' ,'persistence','n_breaks','n_snapshot','n_rotate_view','n_manipulation_events','time_failed_submission_exit','avg_time_between_submissions','cum_weighted_difficulty_perc_composite','percentileCompositeAcrossAttempts','persistenceAcrossAttempts','cum_global_puzzle_attempts','cum_this_puzzle_attempt','cum_avg_perc_composite', 'cum_avg_persistence'])

    return activity_by_user.to_json()

# TODO: where is the best place to put this?
@app.task
def getPuzzleDifficulty(group = 'all'):
    return json.dumps(difficultyPuzzles)

@app.task
def computePersistenceByPuzzle(group = 'all'):
    if group == 'all' :
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    dataEvents['time'] = pd.to_datetime(dataEvents['time'])
    dataEvents = dataEvents.sort_values('time')

    #iterates in the groups and users of the data
    dataEvents['group'] = [json.loads(x)['group'] if 'group' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['user'] = [json.loads(x)['user'] if 'user' in json.loads(x).keys() else '' for x in dataEvents['data']]
    dataEvents['task_id'] = [json.loads(x)['task_id'] if 'task_id' in json.loads(x).keys() else '' for x in dataEvents['data']]

    # removing those rows where we dont have a group and a user that is not guest
    dataEvents = dataEvents[((dataEvents['group'] != '') & (dataEvents['user'] != '') & (dataEvents['user'] != 'guest'))]
    dataEvents['group_user_id'] = dataEvents['group'] + '~' + dataEvents['user']
    dataEvents['group_user_task_id'] = dataEvents['group'] + '~' + dataEvents['user']+'~'+dataEvents['task_id']

    # the data is grouped by the necessary variables
    activity_by_user = dataEvents.groupby(['group_user_id']).agg({'id':'count',
                                                 'type':'nunique'}).reset_index().rename(columns={'id':'events',
                                                                                'type':'different_events'})

    dataEvents = dataEvents.sort_values('time')

    # Events type structures
    typeEvents = ['ws-snapshot','ws-paint', 'ws-rotate_view','ws-move_shape','ws-rotate_shape' ,'ws-scale_shape','ws-create_shape','ws-delete_shape','ws-undo_action','ws-redo_action', 'ws-check_solution']
    manipulationTypeEvents = ['ws-move_shape','ws-rotate_shape' ,'ws-scale_shape','ws-create_shape','ws-delete_shape']

    # Initialize type events structure
    for event in typeEvents:
        activity_by_user[event] = 0

    # Initialize the data structures
    puzzleEvents = dict()
    timePuzzle = dict()
    globalTypesEvents = dict()
    n_attempts = dict()
    completed = dict()
    timestamp = dict()

    breaksPuzzle = dict()
    contCheckSol = dict()

    manipulationEvents = dict()
    timeFirstCheck = dict()
    timeSubExit = dict()
    timeCheckActual = dict()
    timeBetweenSub = dict()

    avgTime_start_check = dict()
    differentDays = dict()
    lastDay = dict()

    checkSolProd = dict()
    timePuzzleProd = dict()
    puzzleEventsProd = dict()

    categoryPuzz = dict()
    userDissconect = dict()

    for user in dataEvents['group_user_id'].unique():

        # Computing active time
        previousEvent = None
        theresHoldActivity = 60 # np.percentile(allDifferences, 98) is 10 seconds
        activeTime = []

        user_events = dataEvents[dataEvents['group_user_id'] == user]
        user_puzzle_key = None
        #userParc = None
        task_id = None
        initialTime = None
        #prev_id = 1

        for enum, event in user_events.iterrows():

            # If it is the first event
            if(previousEvent is None):
                previousEvent = event
                continue

            if( event['type'] in ['ws-start_level', 'ws-puzzle_started'] ):

                task_id = json.loads(event['data'])['task_id']
                if(task_id == "Sandbox"): continue

                # ID
                user_puzzle_key = event['group'] + '~' + event['user'] + '~' + task_id
                key_split = user_puzzle_key.split('~')

                # Continue if the user has completed the puzzle
                if(user_puzzle_key not in completed.keys()): completed[user_puzzle_key] = 0
                if(completed[user_puzzle_key] == 1): continue

                # Initialize data structures
                if(event['type'] == 'ws-puzzle_started'):
                    if(user_puzzle_key not in n_attempts.keys()):
                        n_attempts[user_puzzle_key] = 0
                    #else: n_attempts[user_puzzle_key] +=1
                if(user_puzzle_key not in timeSubExit.keys()):
                    timeSubExit[user_puzzle_key] = str(0)
                    timeBetweenSub[user_puzzle_key] = str(0)

                # initialize if the id is new
                if(user_puzzle_key not in puzzleEvents.keys()):

                    breaksPuzzle[user_puzzle_key] = 0
                    categoryPuzz[user_puzzle_key] = ''
                    puzzleEvents[user_puzzle_key]= 1
                    timePuzzle[user_puzzle_key] = 0
                    contCheckSol[user_puzzle_key] = 0
                    manipulationEvents[user_puzzle_key] = 0
                    timeFirstCheck[user_puzzle_key] = 0
                    userDissconect[user_puzzle_key] = 0

                    globalTypesEvents[user_puzzle_key] = dict()
                    for ev in typeEvents:
                        globalTypesEvents[user_puzzle_key][ev]= 0

                # Category puzzle
                if(task_id in tutorialPuzzles):
                    categoryPuzz[user_puzzle_key] = 'Tutorial'
                elif(task_id in advancedPuzzles):
                    categoryPuzz[user_puzzle_key] = 'Advanced'
                else: categoryPuzz[user_puzzle_key] = 'Intermediate'

                #timestamp
                if(event['type'] in 'ws-puzzle_started'):
                    timestamp[user_puzzle_key] = event['time']
                    initialTime = timestamp[user_puzzle_key]

            # the event is not final event
            if(event['type'] not in ['ws-exit_to_menu' , 'ws-puzzle_complete', 'ws-disconnect', 'ws-create_user', 'ws-login_user']):
                if(user_puzzle_key in completed.keys() and completed[user_puzzle_key] != 1):

                    #Different days
                    date = str(event['time']).split('-')
                    day = date[2].split(" ")
                    if(event['type'] in ['ws-check_solution', 'w-snapshot']):
                        if(user not in lastDay.keys()): lastDay[user] = str(0)
                        if(lastDay[user] != date[1]+'~'+day[0]):
                            lastDay[user] = date[1]+'~'+day[0]
                            if(user not in differentDays.keys()): differentDays[user] = 1
                            else: differentDays[user] += 1

                    # Cont the event
                    puzzleEvents[user_puzzle_key] += 1

                    #Calculate the duration of the event
                    delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                    if((delta_seconds < theresHoldActivity)):
                        timePuzzle[user_puzzle_key] += delta_seconds

                    #Breaks
                    if((delta_seconds > 15)):
                        breaksPuzzle[user_puzzle_key] += 1

                    previousEvent = event

                    #Update event counters by type
                    if(event['type'] in typeEvents):
                        globalTypesEvents[user_puzzle_key][event['type']] +=1

                    # Time the first check
                    if(globalTypesEvents[user_puzzle_key]['ws-check_solution'] == 1): timeFirstCheck[user_puzzle_key] = event['time']

                    # Update the manipulation events counter
                    if(event['type'] in manipulationTypeEvents):
                        manipulationEvents[user_puzzle_key] +=1

                    # Time check solution
                    if(event['type'] == 'ws-check_solution'):
                        timeCheckActual[user_puzzle_key] = event['time']
                        contCheckSol[user_puzzle_key] +=1
                        if(user_puzzle_key not in avgTime_start_check.keys()): avgTime_start_check[user_puzzle_key] = 0
                        else: avgTime_start_check[user_puzzle_key] += (timeCheckActual[user_puzzle_key] - timestamp[user_puzzle_key]).total_seconds()

            # the puzzle ends
            if(event['type'] in ['ws-exit_to_menu', 'ws-puzzle_complete', 'ws-disconnect'] ):
                if(user_puzzle_key in completed.keys() and completed[user_puzzle_key] != 1):
                    n_attempts[user_puzzle_key] +=1
                    puzzleEvents[user_puzzle_key] += 1
                    userDissconect[user_puzzle_key] = 1

                    # To complete events, time and attempts
                    if(event['type'] in ['ws-puzzle_complete']):
                        sep = user_puzzle_key.split('~')
                        if(sep[2] not in 'Sandbox'):
                            checkSolProd[user_puzzle_key] = contCheckSol[user_puzzle_key]
                            timePuzzleProd[user_puzzle_key] = timePuzzle[user_puzzle_key]
                            puzzleEventsProd[user_puzzle_key] = puzzleEvents[user_puzzle_key]

                        completed[user_puzzle_key] = 1

                    # Calculate average time between submits and time between submit and exit
                    if(completed[user_puzzle_key] == 0 and globalTypesEvents[user_puzzle_key]['ws-check_solution'] > 0):
                        timeSubExit[user_puzzle_key] = str(round((event['time'] - timeFirstCheck[user_puzzle_key]).total_seconds(), 2))
                    else: timeSubExit[user_puzzle_key] = 'NA'

                    if((globalTypesEvents[user_puzzle_key]['ws-check_solution'] == 0) or (avgTime_start_check[user_puzzle_key]==0)): timeBetweenSub[user_puzzle_key] = 'NA'
                    else: timeBetweenSub[user_puzzle_key] = str(round(avgTime_start_check[user_puzzle_key] /globalTypesEvents[user_puzzle_key]['ws-check_solution'], 2))

                    #Calculate the duration of the event
                    delta_seconds = (event['time'] - previousEvent['time']).total_seconds()
                    if((delta_seconds < theresHoldActivity)):
                        timePuzzle[user_puzzle_key] += delta_seconds

                    #Breaks
                    if((delta_seconds > 15)):
                        breaksPuzzle[user_puzzle_key] += 1

                    previousEvent = event

    userTime = dict()
    userAtt = dict()
    userEvent = dict()

    userTimeProd = dict()
    userAttProd = dict()
    userEventProd = dict()

    # Save attempts, time and events productive and inproductive
    for i in puzzleEvents.keys():

        if(userDissconect[i] != 1):
            n_attempts[i] += 1

        key_split = i.split('~')
        if(key_split[1] not in userTime.keys()):
            userTime[i] = 0
            userAtt[i] = 0
            userEvent[i] = 0

        if(key_split[2] != ''):

            if(key_split[2] in allPuzzles):
                userAtt[i] = contCheckSol[i]

                if(i in checkSolProd.keys()):
                    userAttProd[i] = checkSolProd[i]

            if(key_split[2] in allPuzzles):

                userTime[i] = timePuzzle[i]
                if(i in timePuzzleProd.keys()):
                    userTimeProd[i] = timePuzzleProd[i]

            if(key_split[2] in allPuzzles):

                userEvent[i] = puzzleEvents[i]
                if(i in puzzleEventsProd.keys()):
                    userEventProd[i] = puzzleEventsProd[i]

    puzzleTime = dict()
    puzzleAtt = dict()
    puzzleEvent = dict()

    puzzleTimeProd = dict()
    puzzleAttProd = dict()
    puzzleEventProd = dict()

    # Save the attempts, events and time per puzzle for the distribution (productive and unproductive)
    for i in userTime.keys():

        key_split = i.split('~')
        if(key_split[2] not in allPuzzles): continue
        if(key_split[2] not in puzzleTime.keys()):
            puzzleTime[key_split[2]] = []
            puzzleAtt[key_split[2]] = []
            puzzleEvent[key_split[2]] = []

        if(key_split[2] not in puzzleTimeProd.keys()):
            puzzleTimeProd[key_split[2]] = []
            puzzleAttProd[key_split[2]] = []
            puzzleEventProd[key_split[2]] = []

        puzzleTime[key_split[2]].append(userTime[i])
        puzzleAtt[key_split[2]].append(userAtt[i])
        puzzleEvent[key_split[2]].append(userEvent[i])

        if(i in userTimeProd.keys()):
            puzzleTimeProd[key_split[2]].append(userTimeProd[i])
            puzzleAttProd[key_split[2]].append(userAttProd[i])
            puzzleEventProd[key_split[2]].append(userEventProd[i])

    difficultyNumber = dict()
    persistent = dict()

    percentileActiveTime = dict()
    percentileAtt = dict()
    percentileEvent = dict()

    percentileActiveTimeProd = dict()
    percentileAttProd = dict()
    percentileEventProd = dict()

    percentileCompositeProd = dict()
    averagePercentileComposite = dict()
    averagePercentilePartial = dict()

    compositeUserProd = dict()

    for i in puzzleEvents.keys():

        key_split = i.split('~')
        if(key_split[2] not in ['', 'Sandbox']):

            # Difficulty puzzle
            difficultyNumber[i] = difficultyPuzzles[key_split[2]]

            if(key_split[1] not in compositeUserProd.keys()):
                compositeUserProd[key_split[1]]= []

            # General percentile
            percentileActiveTime[i] = stats.percentileofscore(puzzleTime[key_split[2]], userTime[i])
            percentileAtt[i] = stats.percentileofscore(puzzleAtt[key_split[2]], userAtt[i], kind='weak')
            percentileEvent[i] = stats.percentileofscore(puzzleEvent[key_split[2]], userEvent[i], kind='weak')

            # Productive percentile
            percentileActiveTimeProd[i] = stats.percentileofscore(puzzleTimeProd[key_split[2]], userTime[i])
            percentileAttProd[i] = stats.percentileofscore(puzzleAttProd[key_split[2]], userAtt[i], kind='weak')
            percentileEventProd[i] = stats.percentileofscore(puzzleEventProd[key_split[2]], userEvent[i], kind='weak')
            #percentileCompositeProd[i] = (stats.percentileofscore(puzzleTimeProd[key_split[2]], userTime[i], kind='weak') + stats.percentileofscore(puzzleEventProd[key_split[2]], userEvent[i], kind='weak')) / 2
            percentileCompositeProd[i] = (percentileActiveTimeProd[i] + percentileEventProd[i]) / 2
            #percentileCompositeProd[i] = (stats.percentileofscore(puzzleTimeProd[key_split[2]], userTime[i], kind='weak') + stats.percentileofscore(puzzleAttProd[key_split[2]], userAtt[i], kind='weak') + stats.percentileofscore(puzzleEventProd[key_split[2]], userEvent[i], kind='weak')) / 3
            compositeUserProd[key_split[1]].append(percentileCompositeProd[i])

            # Initialize persistent structure
            if(i not in persistent.keys()):
                persistent[i] = ''

            # Persistent labels
            if(percentileCompositeProd[i] < 5 and completed[i] == 0):
                persistent[i] = 'NON_PERSISTANT'

            if(percentileCompositeProd[i] < 25 and completed[i] == 1):
                persistent[i] = 'RAPID_SOLVER'

            if(percentileCompositeProd[i] > 75 and completed[i] == 1):
                persistent[i] = 'PRODUCTIVE_PERSISTANCE'

            if(percentileCompositeProd[i] > 90 and completed[i] == 0):
                persistent[i] = 'UNPRODUCTIVE_PERSISTANCE'

            if(persistent[i] == ''):
                persistent[i] = 'NO_BEHAVIOR'

    resultPart = 0
    for i in puzzleEvents.keys():

        key_split = i.split('~')
        if(key_split[2] not in ['', 'Sandbox'] and key_split[1] != '' and i != ''):
            activity_by_user.at[i, 'user'] = key_split[1]
            activity_by_user.at[i, 'group'] = key_split[0]
            activity_by_user.at[i, 'task_id'] = key_split[2]
            activity_by_user.at[i, 'n_events'] = puzzleEvents[i]
            activity_by_user.at[i, 'active_time'] = round(timePuzzle[i],2)
            activity_by_user.at[i, 'percentileAtt'] = round(percentileAttProd[i],2)
            activity_by_user.at[i, 'percentileActiveTime'] = round(percentileActiveTimeProd[i],2)
            activity_by_user.at[i, 'percentileEvents'] = round(percentileEventProd[i],2)
            activity_by_user.at[i, 'percentileComposite'] = round(percentileCompositeProd[i],2)
            activity_by_user.at[i, 'completed'] = completed[i]
            activity_by_user.at[i, 'puzzle_difficulty'] = difficultyNumber[i]
            activity_by_user.at[i, 'puzzle_category'] = categoryPuzz[i]
            activity_by_user.at[i, 'n_attempts'] = n_attempts[i]
            activity_by_user.at[i, 'timestamp'] = timestamp[i]
            activity_by_user.at[i, 'persistence'] = persistent[i]
            activity_by_user.at[i, 'n_breaks'] = breaksPuzzle[i]
            activity_by_user.at[i, 'n_manipulation_events'] = manipulationEvents[i]
            activity_by_user.at[i, 'time_failed_submission_exit'] = timeSubExit[i]
            activity_by_user.at[i, 'avg_time_between_submissions'] = timeBetweenSub[i]
            activity_by_user.at[i, 'n_check_solution'] = globalTypesEvents[i]['ws-check_solution']
            activity_by_user.at[i, 'n_snapshot'] = globalTypesEvents[i]['ws-snapshot']
            activity_by_user.at[i, 'n_rotate_view'] = globalTypesEvents[i]['ws-rotate_view']

    #delete row with NaN
    activity_by_user.dropna(subset = ['user'], inplace=True)

    #data output preparation
    activity_by_user = pd.DataFrame(activity_by_user, columns=['group', 'user','task_id','puzzle_difficulty' ,'puzzle_category','n_attempts','completed','timestamp', 'active_time','percentileActiveTime','n_events','percentileEvents', 'n_check_solution','percentileAtt','percentileComposite' ,'persistence','n_breaks','n_snapshot','n_rotate_view','n_manipulation_events','time_failed_submission_exit','avg_time_between_submissions',])

    return activity_by_user.to_json()

@app.task
def computeELO(group = 'all'):
    if group == 'all' :
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    totalData, train_set, test_set = adaptedData(dataEvents, group=group)
    competency_ELO, difficulty_ELO = run(1.8, 0.05, 'standard', totalData, train_set, test_set)
    return "{\"difficulty_ELO\": " + difficulty_ELO.to_json() + ", \"competency_ELO\": " + competency_ELO.to_json() + "}"

@app.task
def computeMisconceptions(group='all'):
    if group == 'all' :
        toFilter = all_data_collection_urls
    else:
        toFilter = group

    urls = URL.objects.filter(name__in=toFilter)
    sessions = CustomSession.objects.filter(url__in=urls)
    qs = Event.objects.filter(session__in=sessions)
    dataEvents = read_frame(qs)

    df1 = sequenceWithinPuzzlesForMisconceptions(dataEvents)
    df2 = tagMisconceptions(df1)
    return df2.to_json()

# @app.task
# def generate_metadata_and_run_tasks():
#     urls = URL.objects.all()
#     all_data_collection_urls = []
#     interesting_urls = ['ginnymason', 'chadsalyer', 'kristinknowlton', 'lori day', 'leja', 'leja2', 'debbiepoull',
#                         'juliamorgan']
#     for url in urls:
#         all_data_collection_urls.append(str(url.name))
#         task = process_task(computeFunnelByPuzzle, [url.name])
#         task.input_urls.add(url)
#         task.save()
#         task = process_task(sequenceBetweenPuzzles, [url.name])
#         task.input_urls.add(url)
#         task.save()
#         task = process_task(computeLevelsOfActivity, [url.name])
#         task.input_urls.add(url)
#         task.save()
#
#     interesting_task = process_task(computeFunnelByPuzzle, interesting_urls)
#     for url in interesting_urls:
#         interesting_task.input_urls.add(URL.objects.get(name=url))
#     interesting_task.save()
#     interesting_task = process_task(sequenceBetweenPuzzles, interesting_urls)
#     for url in interesting_urls:
#         interesting_task.input_urls.add(URL.objects.get(name=url))
#     interesting_task.save()
#     interesting_task = process_task(computeLevelsOfActivity, interesting_urls)
#     for url in interesting_urls:
#         interesting_task.input_urls.add(URL.objects.get(name=url))
#     interesting_task.save()
#
#     all_task = process_task(computeFunnelByPuzzle, [all_data_collection_urls])
#     for url in urls:
#         all_task.input_urls.add(url)
#     all_task.save()
#     all_task = process_task(sequenceBetweenPuzzles, [all_data_collection_urls])
#     for url in urls:
#         all_task.input_urls.add(url)
#     all_task.save()
#     all_task = process_task(computeLevelsOfActivity, [all_data_collection_urls])
#     for url in urls:
#         all_task.input_urls.add(url)
#     all_task.save()

@app.task
def msg(arg):
    print(arg)

@app.task
def process_tasks_for_flagged_urls():
    # tasks = [computeFunnelByPuzzle, sequenceBetweenPuzzles, computeLevelsOfActivity]
    tasks = [computeLevelsOfActivityOutliers, sequenceBetweenPuzzles, computePersistence, computeLevelsOfActivity,
             computeFunnelByPuzzle, computePersistenceByPuzzle, computeMisconceptions] # not incl. computeELO
    print('Checking for URLs to process')
    urls = URL.objects.filter(process=True)
    for url in urls:
        print("URL " + url.name + " flagged for processing")
        for task in tasks:
            sig = str(task.s([url.name]))
            try:
                state = Task.objects.get(signature=sig).state
            except ObjectDoesNotExist:
                state = ""
            if state != "starting" or state != "processing":
                try:
                    result = process_task(task, [url.name])
                    print("task finished with state: " + result.state)
                except:
                    print("FAILED TASK")
            else:
                print("TASKS ALREADY IN QUEUE")
        url.process = False
        url.save()

# @app.task
def sync_url_events(url_name=None):
    levels = Level.objects.using('production').all()
    for level in levels.iterator():
        level.save(using='default')
    url = URL.objects.using('production').get(name=url_name)
    players = Player.objects.using('production').filter(url=url)
    sessions = CustomSession.objects.none()
    events = Event.objects.none()
    for player in players.iterator():
        print("saving player: " + str(event.pk))
        player.save(using='default')
        sessions = sessions | CustomSession.objects.filter(player=player)
    for session in sessions.iterator():
        print("saving session: " + str(event.pk))
        session.save(using='default')
        events = events | Event.objects.using('production').filter(session=session)
    for event in events.iterator():
        print("saving event: " + str(event.pk))
        event.save(using='default')

@app.task
def event_waterfall():
    last_event = Event.objects.using('default').last()
    new_events_production = Event.objects.using('production').filter(pk__gt=last_event.pk)
    # new_events_production = Event.objects.all()
    print("Checking for new events beyond dev event ID: " + str(last_event.pk))
    if new_events_production.count() > 0:
        print("Events found!")
        for event in new_events_production.iterator():
            event_session = event.session
            if not CustomSession.objects.using('default').filter(session_key=event_session.pk).exists():
                event_player = event_session.player
                event_url = event_session.url
                if not URL.objects.using('default').filter(url=event_url).exists():
                    event_url.save(using='default')
                    print('created url: ' + str(event_url.name))
                if not Player.objects.using('default').filter(url=event_url).filter(name=event_player.name).exists():
                    event_player.save(using='default')
                    print('created player: ' + str(event_player.name))
                event_session.save(using='default')
                print('created session: ' + str(event_session.key) )
            event.save(using='default')
            print('created event: ' + str(event.pk))

@app.on_after_finalize.connect
def schedule_tasks(sender, **kwargs):
    # Tries to auto_process_tasks every 10 seconds.
    sender.add_periodic_task(60.0, process_tasks_for_flagged_urls.s(), name="processed_flagged_urls")
    sender.add_periodic_task(60.0, event_waterfall.s(), name="event_waterfall")

@app.task
def generate_all_replays():
    urls = URL.objects.all()
    for url in urls:
        players = Player.objects.filter(url=url)
        for player in players:
            text = ""
            player_events = Event.objects.none()
            for session in player.customsession_set.all():
                session_events = Event.objects.filter(session=session)
                # Here | is the set union operator, not bitwise OR. Thanks, Django.
                player_events = player_events | session_events
            start_events = []
            end_events = []
            for event in player_events.values():
                text += (str(event) + '\n')
                if 'ws-start_level' in event['type']:
                    start_events.append(event['id'])
                if 'ws-exit_to_menu' in event['type'] or 'ws-disconnect' in event['type']:
                    end_events.append(event['id'])
            # print(start_events)
            events = start_events + end_events
            events.sort()
            if events is not None:
                for index, event in enumerate(events):
                    if index < len(events) - 1:
                        if events[index] in start_events and events[index + 1] in end_events:
                            print(events[index])
                            replay_start_time = Event.objects.get(pk=events[index]).time
                            generic_replay = {"events": [], }
                            level_name = 'no level name found'
                            for event in player_events.values():
                                if events[index] <= event['id'] <= events[index + 1]:
                                    generic_replay["events"].append(event)
                                    # Check to see if there's a level name in the event
                                    data_json = json.loads(event['data'])
                                    if 'task_id' in data_json:
                                        level_name = data_json['task_id']
                            event_range = [events[index], events[index + 1]]
                            print(event_range)
                            replay_obj, created = Replay.objects.get_or_create(
                                event_range=event_range,
                                player=player,
                                url=url,
                                level=Level.objects.get(filename=level_name)
                            )
                            replay_obj.replay_start_time = replay_start_time
                            replay_obj.replay = json.dumps(generic_replay, cls=DjangoJSONEncoder)
                            if not created:
                                replay_obj.last_updated = timezone.now()
                            replay_obj.save()


#import necessary modules
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import datetime
import random

#load chrome_driver
driver = webdriver.Chrome(r'C:\Users\HP\Downloads\chromedriver_win32\chromedriver.exe')

driver.get("https://web.whatsapp.com/")

#Wait for the user to scan the QR-code
wait = WebDriverWait(driver, 600)

good_morning_messages=["Good Morning! :)","Wake up..its late :D","Have a nice day!!","Heyy there.."]

done=0
wish_morning=0

#Schedule time to send the message
scheduled_time=str(input('Enter time at which you want to send the message(HH:MM): '))
h=int(scheduled_time.split(":")[0])
m=int(scheduled_time.split(":")[1])

#Specify group/contacts to whom we want to send the message
target = str(input('Enter name of person/group you want to send message to:'))

#Keep the bot running
while True:

    #Identify time
    cur_time=datetime.datetime.now()
    time=cur_time.time()
    cur_hour=time.hour
    cur_min=time.minute

    #Wish "Good Morning" at specified time
    if wish_morning==0 and cur_hour==h and cur_min==m:

        #Select random greeting
        string = good_morning_messages[random.randrange(0,100)%4]

        #Identify correct text input panel (search bar)
        x_arg = '//span[contains(@title, '+ '"' +target + '"'+ ')]'
        group_title = wait.until(EC.presence_of_element_located((
                By.XPATH, x_arg)))
        group_title.click()

        #Identify correct text input panel (chat box)
        inp_xpath = '//div[@class="_2S1VP copyable-text selectable-text"]'
        input_box = wait.until(EC.presence_of_element_located((
                By.XPATH, inp_xpath)))

        #Send message, and set flag to indicate done for day
        input_box.send_keys("Hi " + target + ", " + string + Keys.ENTER)
        wish_morning=1
        print("Message sent at",str(h)+":"+str(m))

    #Reset flag
    if cur_hour!=h or cur_min!=m:
        wishmorning=0


from flask_cors import CORS
from flask import request, Flask
from celery.result import AsyncResult
from json import dumps
import tasks

app = Flask(__name__)
app.config.update(dict(SECRET_KEY='your_secret_key', CSRF_ENABLED=True, ))
cors = CORS(app, resources={r"/*": {"origins": "*"}})

@app.route('/search_person', methods=['GET', 'POST'])
def search_person_api():
    linkedin_user_name = request.args.get('linkedin_user_name', None)
    res = tasks.search_person.delay(linkedin_user_name)
    AsyncResult(res, app=tasks.celery)

    return dumps({"status":linkedin_user_name})

@app.route('/search_company', methods=['GET', 'POST'])
def search_company_api():
    company_name = request.args.get('company_name', None)
    res = tasks.search_company.delay(company_name)
    AsyncResult(res, app=tasks.celery)

    return dumps({"status":company_name})

@app.route('/update_data', methods=['GET', 'POST'])
def update_data_api():
    linkedin_user_name = request.args.get('linkedin_user_name', None)
    twitter_user_name = request.args.get('twitter_user_name', None)
    instagram_user_name = request.args.get('instagram_user_name', None)

    res = tasks.update_data.delay(linkedin_user_name,
                                  twitter_user_name,
                                  instagram_user_name)
    AsyncResult(res, app=tasks.celery)

    return dumps({"update":linkedin_user_name})

if __name__ == '__main__':
    app.run()


# 基本库
import random
import tensorflow
import matplotlib.pyplot as plt
from model import *
from tensorflow.keras.utils import to_categorical
import os
from helper import check_feature

check_feature()

def set_seeds(seed=666):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    tensorflow.random.set_seed(seed)

set_seeds()

# load feature
temp = np.array(np.load('./temp.npy', allow_pickle=True))
data = temp.transpose()
# feature_mel = np.array(np.load('./mfcc.npy'))
feature_mel_frame = np.array(np.load('./melframe.npy', allow_pickle=True))

# 获取特征
X = np.vstack(feature_mel_frame)
# 获取标签
Y = np.array(data[:, 1])

print('X的特征尺寸是：', X.shape)
print('Y的特征尺寸是：', Y.shape)

# Y = to_categorical(Y)
# print(X.shape)
# print(Y.shape)
#
# # split the training data and test data
# X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=1, stratify=Y)
# print('训练集的大小', len(X_train))
# print('测试集的大小', len(X_test))
# X_train = X_train.reshape(-1, 16, 8, 1)
# X_test = X_test.reshape(-1, 16, 8, 1)

# k 折
nfold = 5
kf = KFold(n_splits=nfold, shuffle=True, random_state=2020)
input_dim = (128 * 116, 1)
i = 0
history = []
for train_index, valid_index in kf.split(X, Y):
    print('\nFold {}'.format(i + 1))
    train_x, val_x = X[train_index], X[valid_index]
    train_y, val_y = Y[train_index], Y[valid_index]
    train_x = train_x.reshape(-1, 128 * 116, 1)
    val_x = val_x.reshape(-1, 128 * 116, 1)
    train_y = to_categorical(train_y)
    val_y = to_categorical(val_y)
    model = SwishNet(input_dim, 20)
    h = model.fit(train_x, train_y, epochs=600, batch_size=100, validation_data=(val_x, val_y))
    history.append(h)
    i += 1

# plot the accuracy and val_accuracy
plt.figure()
plt.plot(history[-1].history['accuracy'])
plt.plot(history[-1].history['val_accuracy'])
plt.legend()
plt.show()


import numpy as np
import scipy.signal as sg
from matplotlib import pyplot as plt

# load file
data = np.load('FilterRecording2020_5_15_20_54.npz')['signals'] # get the signals outta there!

# impulse response occurs between 4300 and 4500
fftChunk = data[4300:4500]

fftChunk = sg.detrend(fftChunk) # detrend removes the mean + any linear trend over the interval (makes the endpoints 0)
# asssume the signal ~0 at the endpoints, no need to window

# I use rfft bc the signal is real. Normally when you take an fft of N time domain sample you get N samples back in the
# frequency domain, however half of the samples will be negative frequencies (dont think about it too hard) and half
# positive. The points in the frequency domain will be equally spread out between -Fs/2 and Fs/2 (Fs=sampling frequency)
# (this is also where the nyquist limit comes from :) ) When the signal is purely real, the negative and positive frequencies
# are symetric, so you only need to calculate half

freqDomain = np.abs(np.fft.rfft(fftChunk)) # take the absolute value, fft returns complex values but we want the real magnitude

# With only the real half of the frequencies, we have N // 2 + 1 points spread out between 0 and Fs/2
# original signal is 200 points, so the fft will have 100 + 1 points

Fs = 40 # 40 hz sampling frequency

frequencies = np.arange(101) * Fs / (2 * 100) # scales the range 0-100 to 0 to Fs/2

# plot frequency and time domains

time = np.arange(200) / Fs # time in seconds of our chunk

f, (ax1, ax2) = plt.subplots(2,1)
ax1.plot(time,fftChunk)           # 0 centered time domain signal
ax2.plot(frequencies, freqDomain) # frequency domain view of the same signal
plt.show()


import pandas as pd
import matplotlib.pyplot as plt

df = pd.read_csv("./data/일별평균대기오염도_2022.csv", encoding="cp949")

print(df.info())

print(df.columns)

name_age_col = df.iloc[:, [0, 1]]
print(name_age_col)

sub_df = df.loc[(df['미세먼지농도(㎍/㎥)'] >= 30) & (df['오존농도(ppm)'] >= 0.001)]
print(sub_df)

sub_df = df.loc[df['미세먼지농도(㎍/㎥)'] >= 30][['측정일시', '측정소명', '미세먼지농도(㎍/㎥)']]
print(sub_df)

sub_df_5 = df[df['미세먼지농도(㎍/㎥)'] > 30][['측정일시', '측정소명', '미세먼지농도(㎍/㎥)']]
print(sub_df_5)

# 필요한 필드 추출
df = df[['측정일시', '측정소명', '미세먼지농도(㎍/㎥)', '초미세먼지농도(㎍/㎥)']]

# 결측치 처리
df = df.dropna()

# 측정일시 컬럼의 데이터 타입을 datetime으로 변경
df['측정일시'] = pd.to_datetime(df['측정일시'], format='%Y%m%d')

print(df.head(20))

# 지역별 미세먼지와 초미세먼지 농도 평균 계산
df_area = df.groupby('측정소명').mean(numeric_only=True).head(10)

# 그래프 그리기
plt.bar(df_area.index, df_area['미세먼지농도(㎍/㎥)'], label='PM10')
plt.bar(df_area.index, df_area['초미세먼지농도(㎍/㎥)'], label='PM2.5')
plt.legend()
plt.xlabel('Area')
plt.ylabel('Concentration')
plt.title('Air Pollution by Area')
plt.show()

# 그래프 그리기
plt.boxplot([df['미세먼지농도(㎍/㎥)'], df['초미세먼지농도(㎍/㎥)']])
plt.xticks([1,2],['PM10', 'PM2.5'])
plt.ylabel('Concentration')
plt.title('Air Pollution Boxplot')
plt.show()

import pandas as pd

# 데이터 파일 경로
data_path = './data/일별평균대기오염도_2022.csv'

# CSV 파일 읽기
df = pd.read_csv(data_path, encoding='cp949')

# 필요한 필드 추출
df = df[['측정일시', '측정소명', '미세먼지농도(㎍/㎥)', '초미세먼지농도(㎍/㎥)']]

# 결측치 처리
df = df.dropna()

# 측정일시 컬럼의 데이터 타입을 datetime으로 변경
df['측정일시'] = pd.to_datetime(df['측정일시'], format='%Y%m%d')

# 이상치를 모아서 result.csv 파일로 저장
q1 = df['미세먼지농도(㎍/㎥)'].quantile(0.25)
q3 = df['미세먼지농도(㎍/㎥)'].quantile(0.75)
iqr = q3 - q1
outlier_df = df[(df['미세먼지농도(㎍/㎥)'] < q1 - 1.5 * iqr) | (df['미세먼지농도(㎍/㎥)'] > q3 + 1.5 * iqr)]
outlier_df.to_csv('result.csv', index=False, encoding='utf-8-sig')


import logging
import boto3
from botocore.exceptions import ClientError

def create_bucket(bucket_name):
    try:
        s3_client = boto3.client('s3')
        s3_client.create_bucket(Bucket=bucket_name)
        print("Bucket successfully created!")
    except ClientError as e:
        logging.error(e)
        print('Bucket already exists or could not be created.')

if __name__ == '__main__':
    bucket_name = 'bucket-for-audio-12345'
    create_bucket(bucket_name)