# Libraries and Core Files
import time
import FFX_Logs
#import FFX_core
import FFX_Screen
import FFX_Battle
import FFX_Xbox
FFXC = FFX_Xbox.FFXC

#Gamestate, "none" for new game, or set to a specific section to start from the first save.
#See the if statement tree below to determine starting position for Gamestate.
#Gamestate = "Sin"
#StepCounter = 3
#Gamestate = "Djose"
#StepCounter = 1
Gamestate = "none"
StepCounter = 1

#Game length. Full is the same as any%, short is about 35 minutes with memory manip.
#gameLength = "short"
gameLength = "full"
forceBlitzWin = True
autoEggHunt = True
print("Game type will be: ", gameLength)

#--------------------------------------------------------------------------------------
tasFinalHit = 0 #TAS-only, final hit will be this character number
#--------------------------------------------------------------------------------------

#Other variables
speedCount = 0
strengthCount = 0
endGameVersion = 0
gems = 0 #Set to 2 if loading in after Evrae Altana with two gems
earlyTidusGrid = False
if forceBlitzWin == True:
    blitzWin = True
else:
    blitzWin = False

#Main functions
def reportGamestate():

    global Gamestate
    global StepCounter
    logText = "Gamestate: " + Gamestate + " : StepCounter: " + str(StepCounter)
    FFX_Logs.writeLog(logText + "\n")
    FFX_Screen.clearMouse(0)

#Main
print("FFX automation starting")
FFX_Logs.nextFile()
FFX_Logs.nextStats()
print("Please launch the game now.")
#time.sleep(5)
#print("Now attempting to activate FFX window")
reportGamestate()

#Press Start until the main menu comes up
#---------- MAKE SURE THIS IS ON FOR A FRESH RUN --------------------
while not FFX_Screen.PixelTest(1076,552,(157, 159, 157)):
    FFXC.set_value('BtnStart', 1)
    time.sleep(0.1)
    FFXC.set_value('BtnStart', 0)

print("Game start screen")
FFX_Screen.clearMouse(0)

#Initiate memory reading, after we know the game is open.
import FFX_memory
FFX_memory.start()

#Next, check if we are loading to a save file
if Gamestate != "none" :
    FFX_Logs.writeLog("Loading to a specific gamestate.\n")
    startTime = FFX_Logs.timeStamp()
    FFX_Logs.writeStats("Start time:")
    FFX_Logs.writeStats(str(startTime))
    reportGamestate()
    import FFX_LoadGame

    if Gamestate == "Baaj" and StepCounter == 1:
        FFX_LoadGame.LoadBaaj()
    if Gamestate == "Baaj" and StepCounter == 5:
        FFX_LoadGame.LoadFirst()
    if Gamestate == "Besaid" and StepCounter == 1 : #Save pop-up after falling off of Rikku's boat
        FFX_LoadGame.loadOffset(45)
    if Gamestate == "Besaid" and StepCounter == 2 : #Crusader's lodge before trials start
        FFX_LoadGame.BesaidTrials()
    if Gamestate == "Besaid" and StepCounter == 3 : #Crusader's lodge after "Enough, Wakka!"
        FFX_LoadGame.loadOffset(25)
    if Gamestate == "Boat1" : #Besaid beach before boarding SS Liki ( nice alliteration :D )
        FFX_LoadGame.Boat1()
    if Gamestate == "Kilika" and StepCounter == 1: #Just after entering the woods
        FFX_LoadGame.loadOffset(22)
    if Gamestate == "Kilika" and StepCounter == 3: #Temple save sphere before scene with Donna
        FFX_LoadGame.KilikaTrials()
    if Gamestate == "Luca" and StepCounter == 1: # Approaching Luca via boat
        FFX_LoadGame.loadOffset(37)
    if Gamestate == "Luca" and StepCounter == 3: # after Oblitzerator, before Blitzball
        FFX_LoadGame.loadOffset(9)
    if Gamestate == "Luca" and StepCounter == 5: # After Blitzball, before battles.
        FFX_LoadGame.loadOffsetBattle(7)
        earlyHaste = 1
    #if Gamestate == "Luca" and StepCounter == 6: #After the talk with Auron
    #    FFX_LoadGame.loadPostBlitz()
    if Gamestate == "Miihen" and StepCounter == 1: #After the talk with Auron
        FFX_LoadGame.LoadMiihenStart()
    if Gamestate == "MRR" and StepCounter == 1: #Mi'ihen North after meeting Seymour
        FFX_LoadGame.LoadMRR()
    if Gamestate == "MRR" and StepCounter == 2: #Just before the last lift to the battle site
        FFX_LoadGame.LoadMRR2()
    if Gamestate == "Djose" and StepCounter == 1: # Aftermath, after talking to Seymour and then Auron
        FFX_LoadGame.AfterGui()
    if Gamestate == "Djose" and StepCounter == 2: #Just before the Djose temple
        FFX_LoadGame.djoseTemple()
    if Gamestate == "Moonflow" and StepCounter == 2: #North bank, before Rikku
        FFX_LoadGame.moonflow2()
    if Gamestate == "Guadosalam" and StepCounter == 2: #After the Farplane
        FFX_LoadGame.loadGuadoSkip()
    if Gamestate == "Macalania" and StepCounter == 1: #1 = south, 2 = north
        FFX_LoadGame.loadOffset(6)
    if Gamestate == "Macalania" and StepCounter == 2: #1 = south, 2 = north
        FFX_LoadGame.loadOffset(7)
    if Gamestate == "Macalania" and StepCounter == 3: #between Spherimorph and Crawler. Move to lake
        FFX_LoadGame.loadMacLake()
    if Gamestate == "Macalania" and StepCounter == 4: #Right before Jyscal skip
        FFX_LoadGame.loadMacTemple()
    if Gamestate == "Macalania" and StepCounter == 5: #After Seymour, before trials
        FFX_LoadGame.loadMacTemple2()
    if Gamestate == "Macalania" and StepCounter == 6: #Outside temple, before escaping.
        FFX_LoadGame.loadOffset(15)
        time.sleep(0.5)
        FFXC.set_value('AxisLy', 1)
        time.sleep(1.5)
        FFXC.set_value('AxisLy', 0)
        time.sleep(0.5)
    if Gamestate == "Macalania" and StepCounter == 7: #Before Wendigo
        FFX_LoadGame.loadWendigo()
    if Gamestate == "Home" and StepCounter == 1:
        FFX_LoadGame.loadOffset(99)
    if Gamestate == "Home" and StepCounter == 2:
        FFX_LoadGame.loadOffset(4)
    if Gamestate == "rescueYuna" and StepCounter == 1: # Airship, before pathing to the deck
        FFX_LoadGame.loadRescue()
    if Gamestate == "rescueYuna" and StepCounter == 2: # Bevelle trials
        FFX_LoadGame.loadBahamut()
    if Gamestate == "rescueYuna" and StepCounter == 5: # Highbridge before Seymour Natus
        FFX_LoadGame.loadOffset(4)
    if Gamestate == "Gagazet" and StepCounter == 1: # Just before Calm Lands
        FFX_LoadGame.loadCalm()
    if Gamestate == "Gagazet" and StepCounter == 2: # Gagazet gates
        FFX_LoadGame.loadGagaGates()
    if Gamestate == "Gagazet" and StepCounter == 3: # Just before Seymour Flux
        FFX_LoadGame.LoadNeutral()
    if Gamestate == "Gagazet" and StepCounter == 4: # After the dream
        FFX_LoadGame.loadGagazetDream()
    if Gamestate == "Zanarkand" and StepCounter == 1: # Intro scene revisited
        FFX_LoadGame.zanEntrance()
    if Gamestate == "Zanarkand" and StepCounter == 2: # Just before the trials.
        FFX_LoadGame.zanTrials()
    if Gamestate == "Zanarkand" and StepCounter == 4: # After Sanctuary Keeper
        FFX_LoadGame.loadOffset(1)
    if Gamestate == "Sin" and StepCounter == 2: #Save sphere on the Highbridge before talking to Shedinja
        FFX_LoadGame.loadOffset(22)
    if Gamestate == "Sin" and StepCounter == 3: #Before "inside sin" pathing
        FFX_LoadGame.loadOffset(2)

#Movement files
import FFX_DreamZan
if gameLength != "short":
    import FFX_Baaj
    import FFX_Besaid1
    import FFX_Besaid2
    import FFX_Boats
    import FFX_Kilika
    import FFX_Luca
    import FFX_Blitz
    import FFX_Miihen
    import FFX_MRR
    import FFX_Djose
    import FFX_Moonflow
    import FFX_Guadosalam
    import FFX_ThunderPlains
    import FFX_mWoods
    import FFX_mTemple
    import FFX_home
    import FFX_rescueYuna
    import FFX_Gagazet
    import FFX_Zanarkand
import FFX_Sin

try:

    #Start of the game, start of Dream Zanarkand section
    if Gamestate == "none" and StepCounter == 1:
        reportGamestate()
        Gamestate = "DreamZan"
        time.sleep(0.5)
        FFX_DreamZan.NewGame(gameLength)
        startTime = FFX_Logs.timeStamp()
        FFX_Logs.writeStats("Start time:")
        FFX_Logs.writeStats(str(startTime))
        print("Timer starts now.")
        FFX_DreamZan.listenStory()
        #Start of the game, up through Sinspawn Ammes fight
        StepCounter = 2

    if Gamestate == "DreamZan" and StepCounter == 2:
        reportGamestate()
        FFX_Battle.Ammes()
        #Finishes Sinspawn Ammes fight

        StepCounter = 3
        reportGamestate()

    if Gamestate == "DreamZan" and StepCounter == 3:
        FFX_DreamZan.AfterAmmes()

        #Sin drops us near Baaj temple.
        StepCounter = 4
        reportGamestate()

    if Gamestate == "DreamZan" and StepCounter == 4:
        if gameLength == "short":
            FFX_DreamZan.SwimToJecht_shortGame()
            Gamestate = "shortGame"
            StepCounter = 1
        else:
            FFX_DreamZan.SwimToJecht()
            Gamestate = "Baaj"
            StepCounter = 1

    if Gamestate == "shortGame" and StepCounter == 1:
        #Only for the short game. Run these pieces then skip down to the last Sin section.
        #If not a short game, continue as regular to the Baaj sections.
        import FFX_cheater_cheese
        FFX_cheater_cheese.BaajEntrance()
        FFX_cheater_cheese.sphereGrid()
        FFX_cheater_cheese.items()
        FFX_cheater_cheese.BackToSin()
        FFX_Sin.insideSin(gameLength, autoEggHunt)
        FFX_Battle.BFA_TASonly(tasFinalHit)
        Gamestate = "gameOver"
        stepCounter = 999

    if Gamestate == "Baaj" and StepCounter == 1 :
        reportGamestate()
        print ("Starting Baaj temple section")
        FFX_Baaj.Entrance()
        StepCounter = 2

    if Gamestate == "Baaj" and StepCounter == 2 :
        reportGamestate()
        FFX_Baaj.Baaj_puzzle()
        StepCounter = 3
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Baaj" and StepCounter == 3 :
        FFX_Baaj.Klikk_fight()
        StepCounter = 4
        reportGamestate()

    if Gamestate == "Baaj" and StepCounter == 4:
        #Klikk fight done. Now to wait for the Al Bhed ship.
        print("Al Bhed boat part 1")
        FFX_Baaj.ABboat1()
        StepCounter = 5

    if Gamestate == "Baaj" and StepCounter == 5:
        reportGamestate()
        FFX_Baaj.ABswimming1()
        StepCounter = 6
        reportGamestate()

    if Gamestate == "Baaj" and StepCounter == 6:
        print("Underwater Airship section")
        FFX_Baaj.ABswimming2()
        Gamestate = "Besaid"
        StepCounter = 1
        reportGamestate()

    if Gamestate == "Besaid" and StepCounter == 1 :
        reportGamestate()
        FFX_Besaid1.Beach()
        FFX_Besaid1.swimming1()
        FFX_Besaid1.enteringVillage()
        StepCounter = 2
        reportGamestate()

    if Gamestate == "Besaid" and StepCounter == 2 :
        FFX_Besaid2.trials()
        FFX_Besaid2.aeonAndSleep()
        StepCounter = 3
        reportGamestate()

    if Gamestate == "Besaid" and StepCounter == 3 :
        earlyTidusGrid = FFX_Besaid2.leaving()
        FFX_Besaid2.waterfalls()

        Gamestate = "Boat1"
        StepCounter = 1
        reportGamestate()

    if Gamestate == "Boat1" :
        reportGamestate()
        FFX_Boats.ssLiki(earlyTidusGrid)
        FFX_Kilika.arrival()
        Gamestate = "Kilika"

    if Gamestate == "Kilika" and StepCounter == 1 :
        reportGamestate()
        FFX_Kilika.forest1()
        FFX_Kilika.forest2()
        reportGamestate()
        #speedCount += FFX_Kilika.forest2()
        #print ("Speed spheres: ",speedCount)
        StepCounter = 2
        FFX_Kilika.Geneaux()
        StepCounter = 3

    if Gamestate == "Kilika" and StepCounter == 3 :
        reportGamestate()
        FFX_Kilika.trials()
        StepCounter = 4

    if Gamestate == "Kilika" and StepCounter == 4 :
        reportGamestate()
        FFX_Kilika.forest3()
        StepCounter = 5

    if Gamestate == "Kilika" and StepCounter == 5 :
        reportGamestate()
        FFX_Kilika.departure()
        StepCounter = 1
        Gamestate = "Boat2"
        FFX_Logs.nextFile()

    if Gamestate == "Boat2" :
        reportGamestate()
        FFX_Boats.ssWinno()
        Gamestate = "Luca"

    if Gamestate == "Luca" and StepCounter == 1:
        reportGamestate()
        FFX_Luca.arrival()
        FFX_Luca.followYuna()
        StepCounter = 2

    if Gamestate == "Luca" and StepCounter == 2:
        reportGamestate()
        earlyHaste = FFX_Luca.preBlitz()
        StepCounter = 3

    if Gamestate == "Luca" and StepCounter == 3:
        reportGamestate()
        FFX_Luca.blitzStart()
        StepCounter = 4

    if Gamestate == "Luca" and StepCounter == 4:
        reportGamestate()
        blitzWin = FFX_Blitz.blitzMain(forceBlitzWin)
        FFX_Screen.awaitSave()
        StepCounter = 5

    if Gamestate == "Luca" and StepCounter == 5:
        reportGamestate()
        FFX_Luca.afterBlitz(earlyHaste)
        StepCounter = 1
        Gamestate = "Miihen"
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Miihen" and StepCounter == 1:
        reportGamestate()
        selfDestruct = FFX_Miihen.arrival()
        FFX_Miihen.midPoint()
        print("End of Miihen mid point section.")
        FFX_Miihen.lowRoad(selfDestruct)
        StepCounter = 2

    if Gamestate == "Miihen" and StepCounter == 2:
        reportGamestate()
        FFX_Miihen.wrapUp()
        StepCounter = 1
        Gamestate = "MRR"
        FFX_Logs.nextFile()

    if Gamestate == "MRR" and StepCounter == 1:
        reportGamestate()
        wakkaLateMenu = FFX_MRR.arrival()
        FFX_MRR.mainPath(wakkaLateMenu)
        StepCounter = 2
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "MRR" and StepCounter == 2:
        reportGamestate()
        success = FFX_MRR.battleSite()
        if success == 2:
            Gamestate = "Error"
            reportGamestate()
            time.sleep(90)
        if success == 1:
            Gamestate = "Djose"
            StepCounter = 1
        FFX_Logs.nextFile()

    if Gamestate == "Djose" and StepCounter == 1:
        reportGamestate()
        FFX_Djose.path()
        StepCounter = 2

    if Gamestate == "Djose" and StepCounter == 2:
        reportGamestate()
        FFX_Djose.temple()
        FFX_Djose.trials()
        StepCounter = 3

    if Gamestate == "Djose" and StepCounter == 3:
        reportGamestate()
        FFX_Djose.leavingDjose()
        StepCounter = 1
        Gamestate = "Moonflow"

    if Gamestate == "Moonflow" and StepCounter == 1:
        reportGamestate()
        FFX_Moonflow.arrival()
        FFX_Moonflow.southBank()
        StepCounter = 2

    if Gamestate == "Moonflow" and StepCounter == 2:
        reportGamestate()
        FFX_Moonflow.northBank()
        StepCounter = 1
        Gamestate = "Guadosalam"

    if Gamestate == "Guadosalam" and StepCounter == 1:
        reportGamestate()
        FFX_Guadosalam.arrival()
        FFX_Guadosalam.afterSpeech()
        StepCounter = 2

    if Gamestate == "Guadosalam" and StepCounter == 2:
        reportGamestate()
        FFX_Guadosalam.guadoSkip()
        StepCounter = 1
        Gamestate = "ThunderPlains"
        FFX_Logs.nextFile()

    if Gamestate == "ThunderPlains" and StepCounter == 1:
        status = [False,False,False,False]
        reportGamestate()
        status = FFX_ThunderPlains.southPathing(blitzWin)
        StepCounter = 2

    if Gamestate == "ThunderPlains" and StepCounter == 2:
        FFX_ThunderPlains.agency(blitzWin)
        StepCounter = 3

    if Gamestate == "ThunderPlains" and StepCounter == 3:
        status = FFX_ThunderPlains.northPathing(status)
        rikkucharged = status[0]
        Gamestate = "Macalania"
        StepCounter = 1

    if Gamestate == "Macalania" and StepCounter == 1:
        reportGamestate()
        FFX_mWoods.arrival(rikkucharged)
        StepCounter = 2

    if Gamestate == "Macalania" and StepCounter == 2:
        reportGamestate()
        FFX_mWoods.lakeRoad()
        StepCounter = 3

    if Gamestate == "Macalania" and StepCounter == 3:
        reportGamestate()
        FFX_mWoods.lake()
        FFX_mWoods.afterCrawler()
        StepCounter = 4
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Macalania" and StepCounter == 4:
        reportGamestate()
        FFX_mTemple.arrival(blitzWin)
        StepCounter = 5

    if Gamestate == "Macalania" and StepCounter == 5:
        reportGamestate()
        FFX_mTemple.trials()
        #Gamestate = "manualBreak" # Used for testing only.
        StepCounter = 6

    if Gamestate == "Macalania" and StepCounter == 6:
        reportGamestate()
        FFX_mTemple.escape()
        StepCounter = 7
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Macalania" and StepCounter == 7:
        FFX_mTemple.underLake()
        StepCounter = 1
        Gamestate = "Home"
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Home" and StepCounter == 1:
        reportGamestate()
        FFX_home.desert1()
        StepCounter = 2
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Home" and StepCounter == 2:
        reportGamestate()
        FFX_home.findSummoners(blitzWin)
        FFX_rescueYuna.preEvrae()
        StepCounter = 1
        Gamestate = "rescueYuna"
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "rescueYuna" and StepCounter == 1:
        reportGamestate()
        FFX_rescueYuna.Evrae()
        FFX_rescueYuna.guards()
        StepCounter = 2
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "rescueYuna" and StepCounter == 2:
        reportGamestate()
        FFX_rescueYuna.trials()
        StepCounter = 3

    if Gamestate == "rescueYuna" and StepCounter == 3:
        reportGamestate()
        FFX_rescueYuna.ViaPurifico()
        StepCounter = 4

    if Gamestate == "rescueYuna" and StepCounter == 4:
        reportGamestate()
        FFX_rescueYuna.evraeAltana()
        #Gamestate = "manualBreak" # Used for testing only.
        StepCounter = 5

    if Gamestate == "rescueYuna" and StepCounter == 5:
        reportGamestate()
        FFX_rescueYuna.seymourNatus(blitzWin)
        StepCounter = 1
        Gamestate = "Gagazet"
        FFX_Logs.nextFile()
        #Gamestate = "manualBreak" # Used for testing only.

    if Gamestate == "Gagazet" and StepCounter == 1:
        reportGamestate()
        FFX_Gagazet.calmLands(blitzWin)
        FFX_Gagazet.defenderX()
        StepCounter = 2

    if Gamestate == "Gagazet" and StepCounter == 2:
        reportGamestate()
        endGameVersion = FFX_Gagazet.gagazetGates(blitzWin)
        FFX_Gagazet.afterFlux()
        StepCounter = 3

    #In case we're loading mid game...
    if endGameVersion == 0:
        endGameVersion = 4
        # 1 = two Return spheres, two Friend spheres
        # 2 = two Return spheres, two Friend spheres
        # 3 = Game over (four Friend spheres)
        # 4 = four Return spheres

    if Gamestate == "Gagazet" and StepCounter == 3:
        reportGamestate()
        FFX_Gagazet.dream()
        StepCounter = 4

    if Gamestate == "Gagazet" and StepCounter == 4:
        reportGamestate()
        FFX_Gagazet.cave()
        FFX_Gagazet.wrapUp()
        StepCounter = 1
        Gamestate = "Zanarkand"

    if Gamestate == "Zanarkand" and StepCounter == 1:
        reportGamestate()
        FFX_Zanarkand.arrival()
        StepCounter = 2

    if Gamestate == "Zanarkand" and StepCounter == 2:
        reportGamestate()
        FFX_Zanarkand.trials()
        StepCounter = 3

    if Gamestate == "Zanarkand" and StepCounter == 3:
        reportGamestate()
        if endGameVersion == 0:
            endGameVersion = 1 #Used during segmented testing.
            # 4 == four Return spheres
            # 3 == four Friend spheres
            # 1 or 2 == two of each.
        FFX_Zanarkand.sanctuaryKeeper(endGameVersion)
        StepCounter = 4

    if Gamestate == "Zanarkand" and StepCounter == 4:
        reportGamestate()
        FFX_Zanarkand.yunalesca()
        StepCounter = 1
        Gamestate = "Sin"

    if Gamestate == "Sin" and StepCounter == 1:
        reportGamestate()
        FFX_Sin.makingPlans()
        StepCounter = 2

    print("Mark1")
    if Gamestate == "Sin" and StepCounter == 2:
        reportGamestate()
        FFX_Sin.Shedinja()
        #FFX_Sin.auronWeap()
        FFX_Sin.facingSin()
        StepCounter = 3

    print("Mark2")
    if Gamestate == "Sin" and StepCounter == 3:
        reportGamestate()
        FFX_Sin.insideSin(gameLength, autoEggHunt)
        FFX_Battle.BFA()
        StepCounter = 4
    print("Mark3")

    #print("Waiting for Yu Yevon to die.")
    #time.sleep(6)
    print("Time! The game is now over.")

except Exception as errMsg:
    print("--------------------------------------------------")
    print("Something went wrong during the run. Error:")
    print(errMsg)
    print("--------------------------------------------------")
    time.sleep(20)
    FFXC.set_value('AxisLx',0)
    FFXC.set_value('AxisLy',0)
    FFXC.set_value('BtnB',0)
    FFXC.set_value('BtnA',0)
    FFXC.set_value('BtnX',0)
    FFXC.set_value('BtnY',0)
    FFXC.set_value('Dpad', 0)
    FFXC.set_value('BtnShoulderL', 0)
    FFXC.set_value('BtnShoulderR', 0)

endTime = FFX_Logs.timeStamp()
FFX_Logs.writeStats("End time:")
FFX_Logs.writeStats(str(endTime))

totalTime = endTime - startTime
FFX_Logs.writeStats("Total time:")
FFX_Logs.writeStats(str(totalTime))
print("The game duration was: ", str(totalTime))
time.sleep(10)

FFX_memory.end()
try:
    FFX_Screen.clickImage("stop_recording.JPG")
except:
    print("Could not stop recording.")

print("Automation complete. Unplugging controller.")
import Reset_Controller
print("Unplugging complete. Shutting it down! Have a great day!")


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.9.1+dev
#   kernelspec:
#     display_name: Python [conda env:generic_expression] *
#     language: python
#     name: conda-env-generic_expression-py
# ---

# # Identify generic genes and pathways
#
# Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These generic genes and subsequent pathways are not necessarily specific to the biological process being studied but instead represent a more systematic change.
#
# This notebook identifies generic genes and pathways and then evaluates if those identified are consistent with published findings.
#
# **Steps to identify generic genes:**
# 1. Simulates N gene expression experiments using [ponyo](https://github.com/ajlee21/ponyo)
# 2. Perform DE analysis to get association statistics for each gene
# 3. For each gene, aggregate statsitics across all simulated experiments
# 4. Rank genes based on this aggregated statistic (i.e. log fold change, or p-value)
#
#
# **Steps to identify generic gene sets (pathways):**
# 1. Using the same simulated experiments from above, perform GSEA analysis. This analysis will determine whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list of genes, where genes are ranked by log fold change, indicating a correlation with change in expression.
# 2. For each gene set (pathway), aggregate statistics across all simulated experiments
# 3. Rank gene sets based on this aggregated statistic
#
# **Evaluation:**
# * We want to compare the ranking of genes identified using the above method with the ranking found from [Crow et. al.](https://www.pnas.org/content/pnas/116/13/6491.full.pdf), which identified a set of genes as generic based on how frequently they were found to be DE across 600 experiments
# * We want to compare the ranking of pathways identified using the above method with the ranking based on the [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) data, where ranking was determined based on the fraction of 442 experiments a pathway was found to be enriched
# * This comparison will validate our method being used as a way to automatically identify generic genes and pathways.

# +
# %load_ext autoreload
# %load_ext rpy2.ipython
# %autoreload 2

import os
import sys
import pandas as pd
import numpy as np
import pickle
import glob
import scipy.stats as ss
from keras.models import load_model
from rpy2.robjects import pandas2ri
from ponyo import utils
from generic_expression_patterns_modules import process, stats, ranking

pandas2ri.activate()

np.random.seed(123)

# +
# Read in config variables
base_dir = os.path.abspath(os.path.join(os.getcwd(), "../"))

config_filename = os.path.abspath(
    os.path.join(base_dir, "configs", "config_human_cancer.tsv")
)

params = utils.read_config(config_filename)

# +
# Load params
local_dir = params["local_dir"]
dataset_name = params["dataset_name"]
NN_architecture = params["NN_architecture"]
num_runs = params["num_simulated"]
project_id = params["project_id"]
metadata_col_id = params["metadata_colname"]
mapped_template_filename = params["mapped_template_filename"]
processed_template_filename = params["processed_template_filename"]
normalized_compendium_filename = params["normalized_compendium_filename"]
scaler_filename = params["scaler_filename"]
col_to_rank_genes = params["rank_genes_by"]
col_to_rank_pathways = params["rank_pathways_by"]
statistic = params["gsea_statistic"]
logFC_name = params["DE_logFC_name"]
pvalue_name = params["DE_pvalue_name"]

# Load metadata file with grouping assignments for samples
sample_id_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_process_samples.tsv"
)

# Load metadata file with grouping assignments for samples
grp_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_groups.tsv"
)

# Load pickled file
with open(scaler_filename, "rb") as scaler_fh:
    scaler = pickle.load(scaler_fh)

# Percentile threshold to identify generic genes
percentile_threshold = 80.0

# +
# Output files
gene_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_gene_summary_{project_id}.tsv"
)

pathway_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_pathway_summary_{project_id}.tsv"
)

# -

# ## Need to customize code from ponyo
#
# The current simulation-related function in ponyo, `get_sample_ids` assumes that the user is using one of two different metadata files (one associated with the pseudomonas compendium and another associated with recount2). The compendium dataset we are using here has a slightly different format for their metadata file.
#
# Here we are temporarily writing our own function customized for this Powers et. al. dataset. But we will be updating ponyo to allow for different metadata files in the future. Issue in ponyo is [here](https://github.com/greenelab/ponyo/issues/18)

def get_sample_ids(experiment_id, dataset_name, sample_id_colname):
    """
    Returns sample ids (found in gene expression df) associated with
    a given list of experiment ids (found in the metadata)

    Arguments
    ----------
    experiment_ids_file: str
        File containing all cleaned experiment ids

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    sample_id_colname: str
        Column header that contains sample id that maps expression data
        and metadata

    """
    # Read in metadata
    metadata = pd.read_csv(metadata_filename, header=0)
    metadata.set_index("gse", inplace=True)

    selected_metadata = metadata.loc[experiment_id]
    sample_ids = list(selected_metadata[sample_id_colname])

    return sample_ids

def shift_template_experiment_with_metadatafile(
    normalized_data_file,
    selected_experiment_id,
    sample_id_colname,
    NN_architecture,
    dataset_name,
    scaler,
    local_dir,
    base_dir,
    run,
    metadata_filename,
):
    """
    Generate new simulated experiment using the selected_experiment_id as a template
    experiment using the same workflow as `simulate_by_latent_transform`

    This will return a file with a single simulated experiment following the workflow mentioned.
    This function can be run multiple times to generate multiple simulated experiments from a
    single selected_experiment_id.

    Arguments
    ----------
    normalized_data_file: str
        File containing normalized gene expression data

        ------------------------------| PA0001 | PA0002 |...
        05_PA14000-4-2_5-10-07_S2.CEL | 0.8533 | 0.7252 |...
        54375-4-05.CEL                | 0.7789 | 0.7678 |...
        ...                           | ...    | ...    |...

    selected_experiment_id: str
        Experiment id selected as template

    sample_id_colname: str
        Column header that contains sample id that maps expression data and metadata

    NN_architecture: str
        Name of neural network architecture to use.
        Format 'NN_<intermediate layer>_<latent layer>'

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    scaler: minmax model
        Model used to transform data into a different range

    local_dir: str
        Parent directory on local machine to store intermediate results

    base_dir: str
        Root directory containing analysis subdirectories

    run: int
        Simulation run

    Returns
    --------
    simulated_data_file: str
        File containing simulated gene expression data

    """

    # Files
    NN_dir = os.path.join(base_dir, dataset_name, "models", NN_architecture)
    latent_dim = NN_architecture.split("_")[-1]

    model_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_model.h5"))[0]

    weights_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_weights.h5"))[0]

    model_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_model.h5"))[0]

    weights_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_weights.h5"))[0]

    # Load saved models
    loaded_model = load_model(model_encoder_file, compile=False)
    loaded_decode_model = load_model(model_decoder_file, compile=False)

    loaded_model.load_weights(weights_encoder_file)
    loaded_decode_model.load_weights(weights_decoder_file)

    # Read data
    normalized_data = pd.read_csv(normalized_data_file, header=0, sep="\t", index_col=0)

    # Get corresponding sample ids
    sample_ids = get_sample_ids(
        selected_experiment_id, metadata_filename, sample_id_colname
    )

    # Gene expression data for selected samples
    selected_data_df = normalized_data.loc[sample_ids]

    # Encode selected experiment into latent space
    data_encoded = loaded_model.predict_on_batch(selected_data_df)
    data_encoded_df = pd.DataFrame(data_encoded, index=selected_data_df.index)

    # Get centroid of original data
    centroid = data_encoded_df.mean(axis=0)

    # Add individual vectors(centroid, sample point) to new_centroid

    # Encode original gene expression data into latent space
    data_encoded_all = loaded_model.predict_on_batch(normalized_data)
    data_encoded_all_df = pd.DataFrame(data_encoded_all, index=normalized_data.index)

    data_encoded_all_df.head()

    # Find a new location in the latent space by sampling from the latent space
    encoded_means = data_encoded_all_df.mean(axis=0)
    encoded_stds = data_encoded_all_df.std(axis=0)

    latent_dim = int(latent_dim)
    new_centroid = np.zeros(latent_dim)

    for j in range(latent_dim):
        new_centroid[j] = np.random.normal(encoded_means[j], encoded_stds[j])

    shift_vec_df = new_centroid - centroid
    # print(shift_vec_df)

    simulated_data_encoded_df = data_encoded_df.apply(
        lambda x: x + shift_vec_df, axis=1
    )

    # Decode simulated data into raw gene space
    simulated_data_decoded = loaded_decode_model.predict_on_batch(
        simulated_data_encoded_df
    )

    simulated_data_decoded_df = pd.DataFrame(
        simulated_data_decoded,
        index=simulated_data_encoded_df.index,
        columns=selected_data_df.columns,
    )

    # Un-normalize the data in order to run DE analysis downstream
    simulated_data_scaled = scaler.inverse_transform(simulated_data_decoded_df)

    simulated_data_scaled_df = pd.DataFrame(
        simulated_data_scaled,
        columns=simulated_data_decoded_df.columns,
        index=simulated_data_decoded_df.index,
    )

    # Save template data for visualization validation
    test_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "template_normalized_data_" + selected_experiment_id + "_test.txt",
    )

    selected_data_df.to_csv(test_file, float_format="%.3f", sep="\t")

    # Save
    out_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "selected_simulated_data_" + selected_experiment_id + "_" + str(run) + ".txt",
    )

    simulated_data_scaled_df.to_csv(out_file, float_format="%.3f", sep="\t")

    out_encoded_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_encoded_data_{selected_experiment_id}_{run}.txt",
    )

    simulated_data_encoded_df.to_csv(out_encoded_file, float_format="%.3f", sep="\t")

# ### Simulate experiments using selected template experiment
#
# Workflow:
# 1. Get the gene expression data for the selected template experiment
# 2. Encode this experiment into a latent space using the trained VAE model
# 3. Linearly shift the encoded template experiment in the latent space
# 4. Decode the samples. This results in a new experiment
# 5. Repeat steps 1-4 to get multiple simulated experiments

# Load metadata file with grouping assignments for samples
metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "all_experiments_sample_annotations.csv"
)

# Simulate multiple experiments
# This step creates the following files in "<local_dir>/pseudo_experiment/" directory:
#   - selected_simulated_data_SRP012656_<n>.txt
#   - selected_simulated_encoded_data_SRP012656_<n>.txt
#   - template_normalized_data_SRP012656_test.txt
# in which "<n>" is an integer in the range of [0, num_runs-1]
os.makedirs(os.path.join(local_dir, "pseudo_experiment"), exist_ok=True)
for run_id in range(num_runs):
    shift_template_experiment_with_metadatafile(
        normalized_compendium_filename,
        project_id,
        metadata_col_id,
        NN_architecture,
        dataset_name,
        scaler,
        local_dir,
        base_dir,
        run_id,
        metadata_filename,
    )

# ### Process template and simulated experiments
#
# * Remove samples not required for comparison. Since this experiment contains multiple conditions (i.e. estradiol vs EtOH at 12, 24, and 48 hrs are each considered a different comparison) being tested, we will only include those samples within the same condition.
# * Make sure ordering of samples matches metadata for proper comparison

# +
if not os.path.exists(sample_id_metadata_filename):
    sample_id_metadata_filename = None

stats.process_samples_for_limma(
    mapped_template_filename,
    grp_metadata_filename,
    processed_template_filename,
    sample_id_metadata_filename,
)

for i in range(num_runs):
    simulated_filename = os.path.join(
        local_dir, "pseudo_experiment", f"selected_simulated_data_{project_id}_{i}.txt"
    )
    out_simulated_filename = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_data_{project_id}_{i}_processed.txt",
    )
    stats.process_samples_for_limma(
        simulated_filename,
        grp_metadata_filename,
        out_simulated_filename,
        sample_id_metadata_filename,
    )
# -

# ### Differential expression analysis
#
# The gene expression dataset is array-based so we will use Limma in this case

# Create subdirectory: "<local_dir>/DE_stats/"
os.makedirs(os.path.join(local_dir, "DE_stats"), exist_ok=True)

# + magic_args="-i metadata_filename -i project_id -i processed_template_filename -i local_dir -i base_dir" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))
#
# # File created: "<local_dir>/DE_stats/DE_stats_template_data_SRP012656_real.txt"
# get_DE_stats_limma(metadata_filename,
#                    project_id,
#                    processed_template_filename,
#                    "template",
#                    local_dir,
#                    "real")

# +
# Check number of DEGs
template_DE_stats_filename = os.path.join(
    local_dir, "DE_stats", f"DE_stats_template_data_{project_id}_real.txt"
)

template_DE_stats = pd.read_csv(
    template_DE_stats_filename, sep="\t", header=0, index_col=0
)

selected = template_DE_stats[
    (template_DE_stats["adj.P.Val"] < 0.05) & (abs(template_DE_stats["logFC"]) > 1)
]
print(selected.shape)

# + magic_args="-i metadata_filename -i project_id -i base_dir -i local_dir -i num_runs" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))
#
# # Files created: "<local_dir>/DE_stats/DE_stats_simulated_data_SRP012656_<n>.txt"
# for (i in 0:(num_runs-1)){
#     simulated_data_filename <- paste(local_dir,
#                                      "pseudo_experiment/selected_simulated_data_",
#                                      project_id,
#                                      "_",
#                                      i,
#                                      ".txt",
#                                      sep = "")
#
#     get_DE_stats_limma(metadata_filename,
#                        project_id,
#                        simulated_data_filename,
#                        "simulated",
#                        local_dir,
#                        i)
# }
# -

# ### Rank genes

analysis_type = "DE"
template_DE_stats, simulated_DE_summary_stats = ranking.process_and_rank_genes_pathways(
    template_DE_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_genes,
    logFC_name,
    pvalue_name,
)

# ### Gene summary table

# +
summary_gene_ranks = ranking.generate_summary_table(
    template_DE_stats_filename,
    template_DE_stats,
    simulated_DE_summary_stats,
    col_to_rank_genes,
    local_dir,
    "gene",
    params,
)

summary_gene_ranks.head()
# -

# Check if there is an NaN values, there should not be
summary_gene_ranks.isna().any()

# Create `gene_summary_fielname`
summary_gene_ranks.to_csv(gene_summary_filename, sep="\t")

# ### Compare gene ranking
# Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These *generic genes* are not necessarily specific to the biological process being studied but instead represent a more systematic change.
#
# We want to compare the ability to detect these generic genes using our method vs those found by [Crow et. al. publication](https://www.pnas.org/content/pnas/116/13/6491.full.pdf). Their genes are ranked 0 = not commonly DE; 1 = commonly DE. Genes were ranked by the number differentially expressed gene sets a gene appeared in across 600 experiments.

# +
# Get generic genes identified by Crow et. al.
DE_prior_filename = params["reference_gene_filename"]
ref_gene_col = params["reference_gene_name_col"]
ref_rank_col = params["reference_rank_col"]

figure_filename = f"gene_ranking_{col_to_rank_genes}.svg"

corr, shared_ranking = ranking.compare_gene_ranking(
    summary_gene_ranks, DE_prior_filename, ref_gene_col, ref_rank_col, figure_filename
)
# -

# Hypergeometric test:
# Given N number of genes with K common genes in Crow et al.
# SOPHIE identifies n genes as being common
# What is the probability that k of the genes identified by SOPHIE
# are also common in Crow et al.? What is the probability of drawing
# k or more concordant genes?
num_Crow_genes = shared_ranking.shape[0]
num_generic_Crow_genes = shared_ranking.query(f"{ref_rank_col}>=80.0").shape[0]
num_generic_SOPHIE_genes = shared_ranking[
    shared_ranking["Percentile (simulated)"] >= percentile_threshold
].shape[0]
num_concordant_generic_genes = shared_ranking[
    (shared_ranking[ref_rank_col] >= percentile_threshold)
    & (shared_ranking["Percentile (simulated)"] >= percentile_threshold)
].shape[0]

print(num_Crow_genes)
print(num_generic_Crow_genes)
print(num_generic_SOPHIE_genes)
print(num_concordant_generic_genes)

p = ss.hypergeom.sf(
    num_concordant_generic_genes,
    num_Crow_genes,
    num_generic_Crow_genes,
    num_generic_SOPHIE_genes,
)
print(p)

# **Takeaway:**
# * Previously we compared gene ranks obtained from (recount2)-trained VAE model vs gene ranks obtained from manual curation using Crow et. al data. This [PR](https://github.com/ajlee21/generic-expression-patterns/blob/807377d76f63b6282c62255d7b160feb8585e0e2/human_analysis/2_identify_generic_genes_pathways.ipynb) shows that the correlation of gene ranks are very consistent.
#
# * Here we are comparing gene ranks obtained from a (Powers et. al.)-trained VAE model vs gene ranks obtained from manual curation using Crow et. al. Based on this correlation plot there is a high correlation between those very high and low ranked genes -- high correlation at the extremes but there is a lot of noise in the middle.

# ### GSEA
# **Goal:** To detect modest but coordinated changes in prespecified sets of related genes (i.e. those genes in the same pathway or share the same GO term).
#
# 1. Rank all genes using DE association statistics.
# 2. An enrichment score (ES) is defined as the maximum distance from the middle of the ranked list. Thus, the enrichment score indicates whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list (indicating a correlation with change in expression).
# 3. Estimate the statistical significance of the ES by a phenotypic-based permutation test in order to produce a null distribution for the ES (i.e. scores based on permuted phenotype)

# Create "<local_dir>/GSEA_stats/" subdirectory
os.makedirs(os.path.join(local_dir, "GSA_stats"), exist_ok=True)

# Load pathway data
hallmark_DB_filename = params["pathway_DB_filename"]

# + magic_args="-i base_dir -i template_DE_stats_filename -i hallmark_DB_filename -i statistic -i local_dir -o template_enriched_pathways" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))
#
# out_filename <- paste(local_dir,
#                      "GSA_stats/GSEA_stats_template_data_",
#                      project_id,
#                      "_real.txt",
#                      sep = "")
#
# template_enriched_pathways <- find_enriched_pathways(template_DE_stats_filename, hallmark_DB_filename, statistic)
# template_enriched_pathways <- as.data.frame(template_enriched_pathways[1:7])
#
# write.table(template_enriched_pathways, file = out_filename, row.names = F, sep = "\t")
# -

print(template_enriched_pathways.shape)
template_enriched_pathways[template_enriched_pathways["padj"] < 0.05].sort_values(
    by="padj"
)

# **Quick check:** Looks like enriched pathways are consistent with estradiol being estrogen hormone treatment.

# + magic_args="-i project_id -i local_dir -i hallmark_DB_filename -i num_runs -i statistic -i base_dir" language="R"
#
# source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))
#
# # New files created: "<local_dir>/GSEA_stats/GSEA_stats_simulated_data_<project_id>_<n>.txt"
# for (i in 0:(num_runs-1)) {
#     simulated_DE_stats_filename <- paste(local_dir,
#                                      "DE_stats/DE_stats_simulated_data_",
#                                      project_id,
#                                      "_",
#                                      i,
#                                      ".txt",
#                                      sep = "")
#
#     out_filename <- paste(local_dir,
#                      "GSA_stats/GSEA_stats_simulated_data_",
#                      project_id,
#                      "_",
#                      i,
#                      ".txt",
#                      sep = "")
#
#     enriched_pathways <- find_enriched_pathways(simulated_DE_stats_filename, hallmark_DB_filename, statistic)
#
#     write.table(as.data.frame(enriched_pathways[1:7]), file = out_filename, row.names = F, sep = "\t")
# }
# -

# ### Rank pathways

analysis_type = "GSA"
template_GSEA_stats_filename = os.path.join(
    local_dir, "GSA_stats", f"GSEA_stats_template_data_{project_id}_real.txt"
)
(
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
) = ranking.process_and_rank_genes_pathways(
    template_GSEA_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_pathways,
    logFC_name,
    pvalue_name,
    "GSEA",
)

# ### Pathway summary table

# +
# Create intermediate file: "<local_dir>/gene_summary_table_<col_to_rank_pathways>.tsv"
summary_pathway_ranks = ranking.generate_summary_table(
    template_GSEA_stats_filename,
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
    col_to_rank_pathways,
    local_dir,
    "pathway",
    params,
)

summary_pathway_ranks.sort_values(by="Rank (simulated)", ascending=False).head(10)
# -

# Create `pathway_summary_filename`
summary_pathway_ranks.to_csv(pathway_summary_filename, sep="\t")

# ### Compare pathway ranking

# Studies have found that there are some pathways (gene sets) that are more likely to be significantly enriched in DEGs across a wide range of experimental designs. These generic pathways are not necessarily specific to the biological process being studied but instead represents a more systematic change.
#
# We want to compare the ability to detect these generic pathways using our method vs those found by [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) publication.  We will use the `Hallmarks_qvalues_GSEAPreranked.csv` file from https://www.synapse.org/#!Synapse:syn11806255 as a reference. The file contains the q-value (adjusted p-value) for the test: given the enrichment score (ES) of the experiment is significant compared to the null distribution of enrichment scores, where the null set is generated from permuted gene sets. For each gene set (pathway) they calculate the q-value using this test.
#
#
# To get a `reference ranking`, we calculate the fraction of experiments that a given pathway was significant (q-value <0.05) and use this rank pathways. `Our ranking` is to rank pathways based on the median q-value across the simulated experiments. We can then compare `our ranking` versus the `reference ranking.`

# Load Powers et. al. results file
powers_rank_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "Hallmarks_qvalues_GSEAPreranked.csv"
)

# Read Powers et. al. data
# This file contains qvalue results for hallmark pathways across ~400 experiments
powers_rank_df = pd.read_csv(powers_rank_filename, header=0, index_col=0)
powers_rank_df.drop(["Category"], axis=1, inplace=True)
print(powers_rank_df.shape)
powers_rank_df.head()

# +
# Count the number of experiments where a given pathway was found to be enriched (qvalue < 0.05)
total_num_experiments = powers_rank_df.shape[1]
frac_enriched_pathways = (powers_rank_df < 0.05).sum(axis=1) / total_num_experiments

# Rank pathways from 0-50, 50 indicating that the pathways was frequently enriched
pathway_ranks = frac_enriched_pathways.rank()

powers_rank_stats_df = pd.DataFrame(
    data={
        "Fraction enriched": frac_enriched_pathways.values,
        "Powers Rank": pathway_ranks.values,
    },
    index=powers_rank_df.index,
)
powers_rank_stats_df.sort_values(by="Powers Rank", ascending=False).head()

# +
# Save reference file for input into comparison
powers_rank_processed_filename = os.path.join(
    base_dir,
    dataset_name,
    "data",
    "metadata",
    "Hallmarks_qvalues_GSEAPreranked_processed.tsv",
)

powers_rank_stats_df.to_csv(
    powers_rank_processed_filename,
    sep="\t",
)

# +
figure_filename = f"pathway_ranking_{col_to_rank_pathways}.svg"

ranking.compare_pathway_ranking(
    summary_pathway_ranks, powers_rank_processed_filename, figure_filename
)
# -

# * Our method ranked pathways using median adjusted p-value score across simulated experiments.
# * Powers et. al. ranked pathways based on the fraction of experiments they had adjusted p-value < 0.05.
#
# **Takeaway:**
# * Previously we compared pathway ranks obtained from (recount2)-trained VAE model vs pathway ranking based on manual curation using Powers et. al. This [PR](https://github.com/ajlee21/generic-expression-patterns/blob/807377d76f63b6282c62255d7b160feb8585e0e2/human_analysis/2_identify_generic_genes_pathways.ipynb) shows that there was no correlation.
#
# * Here we validated that our analysis pipeline is working correctly by comparing pathway ranks obtained from a (Powers et. al.)-trained VAE model vs pathway ranking based on manual curation using Powers et. al datasets. We expect to see a high correlation between pathway ranks given that we are using the same training dataset. Indeed that is what we find

# **Conclusion:**
#
# * We find relatively similar generic genes using our simulation approach (i.e. VAE model trained on a cancer-specific dataset, Powers et. al.) compared to generic genes found from real general experiments from Crow et. al. These generic genes are not *that* context-specific at the extremes.
#
# * We found very different generic pathways training using our simulation approach trained on a general dataset (recount2) compared to generic pathways found from real cancer-specific experiments from Powers et. al. See [analysis](../human_general_analysis/2_identify_generic_genes_pathways.ipynb). But we get very similar generic pathways using our simulation approach trained on a cancer-specific dataset (Powers et. al.) compared with generic pathways found from cancer-specific dataset (Powers et. al.). This indicates that generic pathways are more context specific.
#
# * Why would the context matter more for pathways as opposed to genes? One way to think about this is using this figure from a recent [preprint](https://www.biorxiv.org/content/10.1101/2020.07.30.228296v1).Information flows from a stimulation that activates proteins within pathways and these proteins regulate gene expression. Say we have a context specific signal, that changes the TF within some pathways, this eventually trickles down to changes in gene expression. So if we think about flow of information, measuring pathway activity (or pathway enrichment, etc) will be more sensitive to our context compared to measuring DE in individual genes. Since the genes are regulated as a group, you'd see coordinated changes in expression that are correlated with your condition but looking at the expression of individual genes you wouldn’t necessarily see this correlation with condition.


#!/usr/bin/python

from numpy import *
import unicodedata

dico_file = open("liste_francais.txt", 'r')

content_dic = dico_file.read()
liste_mots = content_dic.split()

dico_file.close()

new_dico_file = open("modif_liste_fr.txt", 'w')

for mot in liste_mots:
	mot1 = unicodedata.normalize('NFD', mot).encode('ASCII', 'ignore')
	mot2 = mot1.lower()
	new_dico_file.write(mot2.decode('utf-8') + ",\n")

new_dico_file.close()

dic_test_file = open("modif_liste_fr.txt", 'r')

content_test = dic_test_file.read()

dic_test_file.close()
test_liste =  content_test.split(',\n')
test_liste.pop(-1)
print(test_liste[-1])


# -----------------------------------------------------------------------------
# BSD 3-Clause License
#
# Copyright (c) 2017-2020, Science and Technology Facilities Council.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# -----------------------------------------------------------------------------
# Authors R. W. Ford, A. R. Porter and S. Siso, STFC Daresbury Lab
# -----------------------------------------------------------------------------

''' This module implements the PSyclone Dynamo 0.1 API by specialising the
    required base classes (PSy, Invokes, Invoke, InvokeSchedule, Loop, Kern,
    Arguments and Argument). '''

from __future__ import absolute_import
from psyclone.configuration import Config
from psyclone.psyir.nodes import Loop, Literal, Reference, ArrayReference, \
    Schedule
from psyclone.psyir.symbols import DataSymbol, INTEGER_TYPE
from psyclone.psyGen import PSy, Invokes, Invoke, InvokeSchedule, \
    CodedKern, Arguments, Argument, GenerationError
from psyclone.parse.kernel import KernelType, Descriptor
from psyclone.parse.utils import ParseError

class DynDescriptor(Descriptor):
    '''This class captures the dynamo0.1 api metadata found in an argument
    descriptor within the kernel metadata.

    :param str access: An access descriptor describing how the \
    argument is used in the kernel
    :param str funcspace: A description of the function space that \
    this argument is assumed to be on.
    :param str stencil: The type of stencil access performed by the \
    kernel on this argument. Currently the value is limited to 'fe'.
    :param str basis: Whether or not basis information is required for \
    this field. The values can be '.true.' or '.false.'.
    :param str diff_basis: Whether or not basis information is \
    required for this field. The value can be '.true.' or '.false.'.
    :param str gauss_quad: Whether or not gaussian quadrature \
    information is required for this field. The value can be '.true.' \
    or '.false.'.

    '''
    def __init__(self, access, funcspace, stencil, basis, diff_basis,
                 gauss_quad):
        Descriptor.__init__(self, access, funcspace, stencil)
        self._basis = basis
        self._diff_basis = diff_basis
        self._gauss_quad = gauss_quad

    @property
    def basis(self):
        '''Return whether a basis function is required or not.

        :returns: '.true.' if a basis function is required and \
        '.false.' if not.
        :rtype: str

        '''
        return self._basis

    @property
    def diff_basis(self):
        '''Return whether a differential basis function is required or not.

        :returns: '.true.' if a differential basis function is \
        required and '.false.' if not.
        :rtype: str

        '''
        return self._diff_basis

    @property
    def gauss_quad(self):
        '''Return whether gaussian quadrature is required or not.

        :returns: '.true.' if a gaussian quadrature is required and \
        '.false.' if not.
        :rtype: str

        '''
        return self._gauss_quad

class DynKernelType(KernelType):
    '''This class captures the dynamo0.1 api kernel metadata by extracting
    it from the supplied AST.

    :param ast: fparser1 AST for the parsed gocean0.1 kernel \
    meta-data.
    :type ast: :py:class:`fparser.one.block_statements.BeginSource`
    :param name: name of the Fortran derived type describing the \
    kernel. This is an optional argument which defaults to `None`
    :type name: str or NoneType.

    '''

    def __init__(self, ast, name=None):
        KernelType.__init__(self, ast, name=name)
        self._arg_descriptors = []
        for init in self._inits:
            if init.name != 'arg_type':
                raise ParseError(
                    "dynamo0p1.py:DynKernelType:__init__: Each meta_arg "
                    "value must be of type 'arg_type' for the "
                    "dynamo0.1 api, but found '{0}'.".format(init.name))
            access = init.args[0].name
            funcspace = init.args[1].name
            stencil = init.args[2].name
            x1 = init.args[3].name
            x2 = init.args[4].name
            x3 = init.args[5].name
            self._arg_descriptors.append(DynDescriptor(access, funcspace,
                                                       stencil, x1, x2, x3))

class DynamoPSy(PSy):
    '''
    The Dynamo specific PSy class. This creates a Dynamo specific
    invokes object (which controls all the required invocation calls).
    Also overrides the PSy gen method so that we generate dynamo
    specific PSy module code.

    :param invoke_info: An object containing the required invocation \
                        information for code optimisation and generation.
    :type invoke_info: :py:class:`psyclone.parse.FileInfo`
    '''
    def __init__(self, invoke_info):
        PSy.__init__(self, invoke_info)
        self._invokes = DynamoInvokes(invoke_info.calls, self)

    @property
    def gen(self):
        '''
        Generate PSy code for the Dynamo0.1 api.

        :rtype: ast

        '''
        from psyclone.f2pygen import ModuleGen, UseGen

        # create an empty PSy layer module
        psy_module = ModuleGen(self.name)
        # include the lfric module
        lfric_use = UseGen(psy_module, name="lfric")
        psy_module.add(lfric_use)
        # add all invoke specific information
        self.invokes.gen_code(psy_module)
        return psy_module.root

class DynamoInvokes(Invokes):
    ''' The Dynamo specific invokes class. This passes the Dynamo specific
        invoke class to the base class so it creates the one we require. '''
    def __init__(self, alg_calls, psy):
        self._0_to_n = DynInvoke(None, None, None)  # for pyreverse
        Invokes.__init__(self, alg_calls, DynInvoke, psy)

class DynInvoke(Invoke):
    ''' The Dynamo specific invoke class. This passes the Dynamo specific
        schedule class to the base class so it creates the one we require.
        Also overrides the gen_code method so that we generate dynamo
        specific invocation code. '''
    def __init__(self, alg_invocation, idx, invokes):
        self._schedule = DynInvokeSchedule('name', None)  # for pyreverse
        Invoke.__init__(self, alg_invocation, idx, DynInvokeSchedule, invokes)

    def gen_code(self, parent):
        ''' Generates Dynamo specific invocation code (the subroutine called
            by the associated invoke call in the algorithm layer). This
            consists of the PSy invocation subroutine and the declaration of
            its arguments.'''
        from psyclone.f2pygen import SubroutineGen, TypeDeclGen
        # create the subroutine
        invoke_sub = SubroutineGen(parent, name=self.name,
                                   args=self.psy_unique_var_names)
        self.schedule.gen_code(invoke_sub)
        parent.add(invoke_sub)
        # add the subroutine argument declarations
        my_typedecl = TypeDeclGen(invoke_sub, datatype="field_type",
                                  entity_decls=self.psy_unique_var_names,
                                  intent="inout")
        invoke_sub.add(my_typedecl)

class DynInvokeSchedule(InvokeSchedule):
    ''' The Dynamo specific InvokeSchedule sub-class. This passes the Dynamo
        specific loop and infrastructure classes to the base class so it
        creates the ones we require. '''
    def __init__(self, name, arg, reserved_names=None, parent=None):
        InvokeSchedule.__init__(self, name, DynKernCallFactory,
                                DynBuiltInCallFactory, arg, reserved_names,
                                parent=parent)

class DynLoop(Loop):
    ''' The Dynamo specific Loop class. This passes the Dynamo specific
        loop information to the base class so it creates the one we require.
        Creates Dynamo specific loop bounds when the code is being generated.
    '''
    def __init__(self, parent=None, loop_type=""):
        Loop.__init__(self, parent=parent,
                      valid_loop_types=["", "colours", "colour"])
        self.loop_type = loop_type

        # Work out the variable name from  the loop type
        if self._loop_type == "colours":
            tag = "colours_loop_idx"
            suggested_name = "colour"
        elif self._loop_type == "colour":
            tag = "colour_loop_idx"
            suggested_name = "cell"
        else:
            tag = "cell_loop_idx"
            suggested_name = "cell"

        symtab = self.scope.symbol_table
        try:
            self.variable = symtab.lookup_with_tag(tag)
        except KeyError:
            self.variable = symtab.new_symbol(
                suggested_name, tag, symbol_type=DataSymbol,
                datatype=INTEGER_TYPE)

        # Pre-initialise the Loop children  # TODO: See issue #440
        self.addchild(Literal("NOT_INITIALISED", INTEGER_TYPE,
                              parent=self))  # start
        self.addchild(Literal("NOT_INITIALISED", INTEGER_TYPE,
                              parent=self))  # stop
        self.addchild(Literal("1", INTEGER_TYPE, parent=self))  # step
        self.addchild(Schedule(parent=self))  # loop body

    def load(self, kern):
        ''' Load the state of this Loop using the supplied Kernel
        object. This method is provided so that we can individually
        construct Loop objects for a given kernel call. '''
        self._field = kern.arguments.iteration_space_arg()
        self._field_name = self._field.name
        self._field_space = self._field.function_space

    def gen_code(self, parent):
        ''' Work out the appropriate loop bounds and then call the base
            class to generate the code '''
        self.start_expr = Literal("1", INTEGER_TYPE, parent=self)
        if self._loop_type == "colours":
            self.stop_expr = Reference(DataSymbol("ncolour", INTEGER_TYPE),
                                       parent=self)
        elif self._loop_type == "colour":
            self.stop_expr = ArrayReference(DataSymbol("ncp_ncolour",
                                                       INTEGER_TYPE),
                                            parent=self)
            self.stop_expr.addchild(
                Reference(DataSymbol("colour", INTEGER_TYPE)),
                parent=self.stop_expr)
        else:
            # This is a hack as the name is not a valid DataSymbol, it
            # is a call to a type-bound function.
            self.stop_expr = Reference(
                DataSymbol(self.field_name+"%get_ncell()", INTEGER_TYPE),
                parent=self)
        Loop.gen_code(self, parent)

class DynBuiltInCallFactory(object):
    ''' A Dynamo 0.1 specific Built-In call factory. No built-in
        calls are supported in Dynamo 0.1 so we do nothing. '''
    @staticmethod
    def create(call, parent=None):
        ''' Creates a specific built-in call. Currently does
        nothing '''
        return None

class DynKernCallFactory(object):
    ''' A Dynamo 0.1 specific kernel call factory. '''
    @staticmethod
    def create(call, parent=None):
        # Loop over cells
        cloop = DynLoop(parent=parent)

        # The kernel itself
        kern = DynKern()
        kern.load(call, cloop.loop_body)

        # Add the kernel as a child of the loop
        cloop.loop_body.addchild(kern)

        # Set-up the loop now we have the kernel object
        cloop.load(kern)

        # Return the outermost loop
        return cloop

class DynKern(CodedKern):
    ''' Stores information about Dynamo Kernels as specified by the Kernel
        metadata. Uses this information to generate appropriate PSy layer
        code for the Kernel instance. '''
    def __init__(self):
        if False:
            self._arguments = DynKernelArguments(None, None)  # for pyreverse

    def load(self, call, parent):
        '''
        Load this DynKern object with state pulled from the call object.

        :param call: details of the Algorithm-layer call of this Kernel.
        :type call: :py:class:`psyclone.parse.algorithm.KernelCall`
        :param parent: parent of this kernel node in the PSyIR.
        :type parent: :py:class:`psyclone.dynamo0p1.DynLoop` or NoneType.

        '''
        super(DynKern, self).__init__(DynKernelArguments, call, parent)

    def local_vars(self):
        return ["cell", "map"]

    def gen_code(self, parent):
        ''' Generates dynamo version 0.1 specific psy code for a call to
            the dynamo kernel instance. '''
        from psyclone.f2pygen import CallGen, DeclGen, AssignGen, UseGen

        # TODO: we simply choose the first field as the lookup for the moment
        field_name = self.arguments.args[0].name

        # add a dofmap lookup using first field.
        # TODO: This needs to be generalised to work for multiple dofmaps
        parent.add(CallGen(parent, field_name+"%vspace%get_cell_dofmap",
                           ["cell", "map"]))
        parent.add(DeclGen(parent, datatype="integer",
                           entity_decls=["cell"]))
        parent.add(DeclGen(parent, datatype="integer", pointer=True,
                           entity_decls=["map(:)"]))

        # create the argument list on the fly so we can also create
        # appropriate variables and lookups
        arglist = []
        arglist.append("nlayers")
        arglist.append("ndf")
        arglist.append("map")

        found_gauss_quad = False
        gauss_quad_arg = None
        for arg in self._arguments.args:
            if arg.requires_basis:
                basis_name = arg.function_space+"_basis_"+arg.name
                arglist.append(basis_name)
                new_parent, position = parent.start_parent_loop()
                new_parent.add(CallGen(new_parent,
                                       field_name+"%vspace%get_basis",
                                       [basis_name]),
                               position=["before",
                                         position])
                parent.add(DeclGen(parent, datatype="real", kind="dp",
                                   pointer=True,
                                   entity_decls=[basis_name+"(:,:,:,:,:)"]))
            if arg.requires_diff_basis:
                raise GenerationError("differential basis has not yet "
                                      "been coded")
            if arg.requires_gauss_quad:
                if found_gauss_quad:
                    raise GenerationError("found more than one gaussian "
                                          "quadrature in this kernel")
                found_gauss_quad = True
                gauss_quad_arg = arg
            dataref = "%data"
            arglist.append(arg.name+dataref)

        if found_gauss_quad:
            gq_name = "gaussian_quadrature"
            arglist.append(gauss_quad_arg.name+"%"+gq_name)

        # generate the kernel call and associated use statement
        parent.add(CallGen(parent, self._name, arglist))
        if not self.module_inline:
            parent.add(UseGen(parent, name=self._module_name,
                              only=True, funcnames=[self._name]))

        # declare and initialise the number of layers and the number
        # of degrees of freedom. Needs to be generalised.
        parent.add(DeclGen(parent, datatype="integer",
                           entity_decls=["nlayers", "ndf"]))
        new_parent, position = parent.start_parent_loop()
        new_parent.add(AssignGen(new_parent, lhs="nlayers",
                                 rhs=field_name+"%get_nlayers()"),
                       position=["before", position])
        new_parent.add(AssignGen(new_parent, lhs="ndf",
                                 rhs=field_name+"%vspace%get_ndf()"),
                       position=["before", position])

class DynKernelArguments(Arguments):
    ''' Provides information about Dynamo kernel call arguments collectively,
        as specified by the kernel argument metadata. This class currently
        adds no additional functionality to its base class other than
        ensuring that initialisation is performed correctly. '''
    def __init__(self, call, parent_call):
        if False:
            self._0_to_n = DynKernelArgument(None, None, None)  # for pyreverse
        Arguments.__init__(self, parent_call)
        self._args = []
        for (idx, arg) in enumerate(call.ktype.arg_descriptors):
            self._args.append(DynKernelArgument(arg, call.args[idx],
                                                parent_call))
        self._dofs = []

    @property
    def dofs(self):
        ''' Currently required for invoke base class although this makes no
            sense for dynamo. Need to refactor the Invoke class and remove the
            need for this dofs property (#279). '''
        return self._dofs

class DynKernelArgument(Argument):
    ''' Provides information about individual Dynamo kernel call arguments
        as specified by the kernel argument metadata. '''
    def __init__(self, arg, arg_info, call):
        self._arg = arg
        api_config = Config.get().api_conf("dynamo0.1")
        access_mapping = api_config.get_access_mapping()
        Argument.__init__(self, call, arg_info, access_mapping[arg.access])

    @property
    def function_space(self):
        ''' Returns the expected finite element function space for this
            argument as specified by the kernel argument metadata.'''
        return self._arg.function_space

    @property
    def requires_basis(self):
        ''' Returns true if the metadata for this argument specifies that
            its basis function values should be passed into the routine. '''
        if self._arg.basis.lower() == ".true.":
            return True
        if self._arg.basis.lower() == ".false.":
            return False
        raise GenerationError("error: basis is not set to .true. or .false.")

    @property
    def requires_diff_basis(self):
        ''' Returns true if the metadata for this argument specifies that
            its differential basis function values should be passed into
            the routine. '''
        if self._arg.diff_basis.lower() == ".true.":
            return True
        if self._arg.diff_basis.lower() == ".false.":
            return False
        raise GenerationError("error: diff_basis is not set to .true. "
                              "or .false.")

    @property
    def requires_gauss_quad(self):
        ''' Returns true if the metadata for this argument specifies that
            its gausian quadrature values should be passed into the
            routine. '''
        if self._arg.gauss_quad.lower() == ".true.":
            return True
        if self._arg.gauss_quad.lower() == ".false.":
            return False
        raise GenerationError("error: gaussian quadrature is not set to "
                              ".true. or .false.")


class ParsedNewsContent:

    def __init__(self, title, desc, author, category, image_url, website_name, timestamp):
        self.title = title
        self.desc = desc
        self.author = author
        self.category = category
        self.image_url = image_url
        self.website_name = website_name
        self.timestamp = timestamp


from flask import Flask, render_template, request
app = Flask(__name__)

@app.route('/')
@app.route('/"ever"')
def home(name=None):
    return render_template('index.html', name=name)

@app.route('/saludo', methods=['GET', 'POST'])
def saludo():
    nombre = request.form.get('nombre')
    edad = int(request.form.get('edad'))

    return render_template('saludo.html', name=nombre, age=edad)

app.run(debug=True)


import os
import time
import pytest
import socket

from knit import Knit
from knit.exceptions import HDFSConfigException, KnitException

def check_docker():
    """check if inside docker container"""
    return os.path.exists('/.dockerenv')

inside_docker = check_docker

def wait_for_status(k, status, timeout=30):
    cur_status = k.runtime_status()
    while cur_status != status and timeout > 0:
        print("Current status is {0}, waiting for {1}".format(cur_status, status))

        time.sleep(2)
        timeout -= 2
        cur_status = k.runtime_status()

    return timeout > 0

def wait_for_containers(k, running_containers, timeout=30):
    cur_running_containers = k.status()['app']['runningContainers']
    while cur_running_containers != running_containers and timeout > 0:
        print("Current number of containers is {0}, waiting for {1}".format(cur_running_containers, running_containers))

        time.sleep(2)
        timeout -= 2
        cur_running_containers = k.status()['app']['runningContainers']

    return timeout > 0

@pytest.yield_fixture
def k():
    knitter = Knit(nn_port=8020, rm_port=8088)
    yield knitter

def test_port():
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=90000, rm_port=90000)
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=90000, rm_port=8088)
    with pytest.raises(HDFSConfigException):
        Knit(nn_port=9000, rm_port=5000)

    if inside_docker:
        # should pass without incident
        Knit(nn_port=8020, rm_port=8088)

def test_hostname(k):
    with pytest.raises(HDFSConfigException):
        Knit(nn="foobarbiz")
    with pytest.raises(HDFSConfigException):
        Knit(rm="foobarbiz")

    if inside_docker:
        # should pass without incident
        Knit(nn="localhost")
        k = Knit(autodetect=True)
        str(k) == 'Knit<NN=localhost:8020;RM=localhost:8088>'

def test_argument_parsing(k):
    cmd = "sleep 10"
    with pytest.raises(KnitException):
        k.start(cmd, files='a,b,c')

    with pytest.raises(KnitException):
        k.start(cmd, memory='128')

def test_cmd(k):
    cmd = "python -c 'import socket; print(socket.gethostname()*2)'"
    k.start(cmd, memory=128)

    if not k.wait_for_completion(30):
        k.kill()

    hostname = socket.gethostname() * 2
    logs = k.logs(shell=True)
    print(logs)

    assert hostname in logs, logs

def test_multiple_containers(k):
    cmd = "sleep 30"
    k.start(cmd, num_containers=2)

    wait_for_status(k, 'RUNNING')

    got_containers = wait_for_containers(k, 3)

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

    if not got_containers:
        logs = k.logs(shell=True)
        print(logs)

    assert got_containers

def test_add_remove_containers(k):
    cmd = "sleep 60"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    got_containers = wait_for_containers(k, 2)

    containers = k.get_containers()
    assert len(containers) == 2

    k.add_containers(num_containers=1)

    got_more_containers = wait_for_containers(k, 3)

    containers = k.get_containers()
    assert len(containers) == 3
    k.remove_containers(containers[1])

    got_more_containers = wait_for_containers(k, 2)
    containers = k.get_containers()
    assert len(containers) == 2

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

    if not (got_containers and got_more_containers):
        logs = k.logs(shell=True)
        print(logs)

    assert got_containers and got_more_containers

def test_memory(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=2, memory=300)

    wait_for_status(k, 'RUNNING')

    time.sleep(2)
    status = k.status()

    # not exactly sure on getting an exact number
    # 300*2+128(AM)
    assert status['app']['allocatedMB'] >= 728, status['app']['allocatedMB']

    # wait for job to finish
    if not k.wait_for_completion(30):
        k.kill()

def test_cmd_w_conda_env(k):
    env_zip = k.create_env(env_name='dev', packages=['python=2.6'], remove=True)
    cmd = "$PYTHON_BIN -c 'import sys; print(sys.version_info); import random; print(str(random.random()))'"
    k.start(cmd, env=env_zip)

    if not k.wait_for_completion(30):
        k.kill()

    logs = k.logs(shell=True)
    assert "(2, 6, 9, 'final', 0)" in logs, logs

cur_dir = os.path.dirname(__file__)
txt_file = os.path.join(cur_dir, 'files', 'upload_file.txt')
py_file = os.path.join(cur_dir, 'files', 'read_uploaded.py')

def test_file_uploading(k):
    cmd = 'python ./read_uploaded.py'
    k.start(cmd, files=[txt_file, py_file])

    if not k.wait_for_completion(30):
        k.kill()

    logs = k.logs(shell=True)
    assert "rambling on" in logs, logs

def test_kill_status(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.kill()

    status = k.runtime_status()
    assert status == 'KILLED'

def test_yarn_kill_status(k):
    cmd = "sleep 10"
    app_id = k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.yarn_api.kill(app_id)

    status = k.runtime_status()
    assert status == 'KILLED'

def test_logs(k):
    cmd = "sleep 10"
    k.start(cmd, num_containers=1)

    wait_for_status(k, 'RUNNING')

    assert k.logs()

    if not k.wait_for_completion(30):
        k.kill()

# temporarily removing test until vCore handling is better resolved in the core
# def test_vcores(k):
#     cmd = "sleep 10"
#     appId = k.start(cmd, num_containers=1, memory=300, virtual_cores=2)
#
#     time.sleep(2)
#     status = k.status(appId)
#
#     while status['app']['state'] != 'RUNNING':
#         status = k.status(appId)
#         time.sleep(2)
#
#     time.sleep(2)
#     status = k.status(appId)
#
#     assert status['app']['allocatedVCores'] == 3
#


import numpy as np

import openmc
import openmc.opencg_compatible as opencg_compatible
import infermc.beavrs

####################   User-specified Simulation Parameters  ###################

batches = 1000
inactive = 100
particles = 10000000

#########   Exporting to OpenMC geometry.xml and materials.xml Files  ##########

# Write all BEAVRS materials to materials.xml file
infermc.beavrs.make_iso_in_lab()
infermc.beavrs.write_materials_file()

# Extract fuel assembly of interest from BEAVRS model
fuel_assembly = infermc.beavrs.find_assembly('Fuel 3.1% enr instr no BAs')
openmc_geometry = opencg_compatible.get_openmc_geometry(fuel_assembly)
openmc_geometry.export_to_xml()

##################   Exporting to OpenMC settings.xml File  ###################

# Construct uniform initial source distribution over fissionable zones
lower_left = fuel_assembly.bounds[:3]
upper_right = fuel_assembly.bounds[3:]
source = openmc.source.Source(space=openmc.stats.Box(lower_left, upper_right))
source.space.only_fissionable = True

settings_file = openmc.Settings()
settings_file.batches = batches
settings_file.inactive = inactive
settings_file.particles = particles
settings_file.ptables = True
settings_file.output = {'tallies': False}
settings_file.source = source
settings_file.sourcepoint_write = False

settings_file.entropy_dimension = [17,17,1]
settings_file.entropy_upper_right = upper_right
settings_file.entropy_lower_left = lower_left

settings_file.export_to_xml()

##################   Exporting to OpenMC plots.xml File  ######################

# Initialize the BEAVRS color mapping scheme
b = infermc.beavrs.beavrs
b.write_openmc_plots()

# Complete assembly
plot1 = openmc.Plot(plot_id=1)
bounds = fuel_assembly.bounds
plot1.width = [fuel_assembly.max_x-fuel_assembly.min_x,
              fuel_assembly.max_y-fuel_assembly.min_y]
plot1.origin = [bounds[0] + (bounds[3] - bounds[0]) / 2.,
                bounds[1] + (bounds[4] - bounds[1]) / 2.,
                bounds[2] + (bounds[5] - bounds[2]) / 2.]
plot1.color = 'mat'
plot1.filename = 'assembly'
plot1.col_spec = b.plots.colspec_mat
plot1.pixels = [2000, 2000]

# Zoom in of fuel pin
plot2 = openmc.Plot(plot_id=2)
bounds = fuel_assembly.bounds
plot2.width = [(fuel_assembly.max_x-fuel_assembly.min_x) / 17.,
               (fuel_assembly.max_y-fuel_assembly.min_y) / 17.]
plot2.origin = [bounds[0] + (bounds[3] - bounds[0]) / 2. + (bounds[3] - bounds[0]) / 17. * 8.,
                bounds[1] + (bounds[4] - bounds[1]) / 2. + (bounds[4] - bounds[1]) / 17. * 8.,
                bounds[2] + (bounds[5] - bounds[2]) / 2.]
plot2.color = 'mat'
plot2.filename = 'fuel-pin'
plot2.col_spec = b.plots.colspec_mat

# Zoom in of instrument tube
plot3 = openmc.Plot(plot_id=3)
bounds = fuel_assembly.bounds
plot3.width = [(fuel_assembly.max_x-fuel_assembly.min_x) / 17.,
               (fuel_assembly.max_y-fuel_assembly.min_y) / 17.]
plot3.origin = [bounds[0] + (bounds[3] - bounds[0]) / 2.,
                bounds[1] + (bounds[4] - bounds[1]) / 2.,
                bounds[2] + (bounds[5] - bounds[2]) / 2.]
plot3.color = 'mat'
plot3.filename = 'instr-tube'
plot3.col_spec = b.plots.colspec_mat

# Zoom in of guide tube
plot4 = openmc.Plot(plot_id=4)
bounds = fuel_assembly.bounds
plot4.width = [(fuel_assembly.max_x-fuel_assembly.min_x) / 17.,
               (fuel_assembly.max_y-fuel_assembly.min_y) / 17.]
plot4.origin = [bounds[0] + (bounds[3] - bounds[0]) / 2. + (bounds[3] - bounds[0]) / 17. * 5.,
                bounds[1] + (bounds[4] - bounds[1]) / 2. + (bounds[4] - bounds[1]) / 17. * 5.,
                bounds[2] + (bounds[5] - bounds[2]) / 2.]
plot4.color = 'mat'
plot4.filename = 'guide-tube'
plot4.col_spec = b.plots.colspec_mat

plot_file = openmc.Plots([plot1, plot2, plot3, plot4])
plot_file.export_to_xml()

###################  Create Mesh Tallies for Verification  ####################

# Instantiate a tally Mesh
mesh = openmc.Mesh(name='assembly mesh')
mesh.type = 'regular'
mesh.dimension = [17, 17, 1]
mesh.lower_left = lower_left
mesh.width = (np.array(upper_right) - np.array(lower_left))
mesh.width[:2] /= 17

# Instantiate tally Filter
mesh_filter = openmc.Filter()
mesh_filter.mesh = mesh

# Instantiate energy-integrated fission rate mesh Tally
fission_rates = openmc.Tally(name='fission rates')
fission_rates.filters = [mesh_filter]
fission_rates.scores = ['fission']

# Instantiate energy-wise U-238 capture rate mesh Tally
capture_rates = openmc.Tally(name='u-238 capture')
capture_rates.filters = [mesh_filter]
capture_rates.nuclides = ['U238']
capture_rates.scores = ['absorption', 'fission']

# Create a "tallies.xml" file for the mesh tallies
tallies_file = openmc.Tallies([fission_rates, capture_rates])
tallies_file.export_to_xml()


import os
from binance.client import Client
import pandas as pd
from datetime import datetime
import logging

class BinanceDS():
    FILENAME = ''
    API = ''
    API_SECRET = ''
    COIN_CONTEXT = ''
    PERIOD = 3
    MINUTES = 60000
    PAIR_LIST = ['BTCUSDT','ETHUSDT', 'EOSUSDT', 'XRPUSDT']
    X,y = '', ''

    def __init__(self, filename, coin_context):
        self.logger = logging.getLogger(__name__)
        with open('binance_keys.txt') as f:
            keys = f.read()
            keys = keys.split(',')
            BinanceDS.API, BinanceDS.API_SECRET = keys[0], keys[1]
        self.client = Client(BinanceDS.API, BinanceDS.API_SECRET)
        self.feature_cols = []
        BinanceDS.FILENAME, BinanceDS.COIN_CONTEXT = filename, coin_context
        self.logger.info("Creating new dataset")
        print("Creating new dataset")
        print("Getting data from {} minutes ago ".format(BinanceDS.MINUTES))
        self.logger.info("Creating new dataset")
        self.dataset = pd.DataFrame()
        for pair in BinanceDS.PAIR_LIST:
            self.get_price_info(pair)
        self.get_class()
        print('Saving dataset to csv file with filename {}'.format(BinanceDS.FILENAME))
        self.dataset.to_csv(BinanceDS.FILENAME)

    def get_class(self):
        print("Getting target classification information for {}".format(BinanceDS.COIN_CONTEXT))
        self.dataset['future'] = self.dataset[BinanceDS.COIN_CONTEXT+'_close'].shift(-BinanceDS.PERIOD)
        self.dataset['target'] = list(map(self.build_classification, self.dataset[BinanceDS.COIN_CONTEXT+'_close'],  self.dataset['future']))

    def get_price_info(self, coin_pair):
        df = pd.DataFrame()
        coin = coin_pair[:3]
        print("Getting pricing information for {}".format(coin))
        price_data = self.client.get_historical_klines(coin_pair, Client.KLINE_INTERVAL_1MINUTE, "{} minutes ago GMT".format(BinanceDS.MINUTES))
        time = [entry[0] for entry in price_data]
        df['date'] = [datetime.fromtimestamp(int(x)/1000).strftime('%d-%H-%M') for x in time]
        df['{}_close'.format(coin)] = [entry[4] for entry in price_data]
        df['{}_volume'.format(coin)] = [entry[5] for entry in price_data]
        df.set_index('date', inplace=True)
        if self.dataset.empty:
            self.dataset = df
        else:
            self.dataset = pd.concat([self.dataset, df], axis=1, join='inner')
        print("Pricing information for {} is complete.".format(coin))

    def build_classification(self, current, future):
        if float(future) > float(current):
            return 1
        else:
            return 0


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 20 00:18:52 2019

@author: a
"""

import cv2
import os
import numpy as np
import matplotlib.pyplot as plt

img = cv2.imread("images/source/m1.JPG")
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

lower_white = np.array([0,0,51])
lower_white = np.array([0,0,20])
upper_white = np.array([255,25,66])

# 白以外にマスク
mask_white = cv2.inRange(hsv, lower_white, upper_white)
res_white = cv2.bitwise_and(img,img, mask= mask_white)

#res_white = img

res_white = cv2.resize(res_white ,dsize=None, fx=0.5, fy=0.5)

#(np.where(a < 4, True, Falseのときのvalue)

whit = np.where(res_white == 0, 255 ,res_white)

cv2.imshow("hi",whit)
cv2.waitKey(0)
cv2.destroyAllWindows()


import gc
from concurrent.futures import Executor, ThreadPoolExecutor

import pytest
from asphalt.core.context import Context
from sqlalchemy import create_engine
from sqlalchemy.engine.base import Engine
from sqlalchemy.orm.session import sessionmaker, Session
from sqlalchemy.pool import NullPool

from asphalt.sqlalchemy.component import SQLAlchemyComponent

@pytest.fixture
def connection():
    engine = create_engine('sqlite:///:memory:')
    connection = engine.connect()
    connection.execute('CREATE TABLE foo (id INTEGER PRIMARY KEY)')
    yield connection
    connection.close()
    engine.dispose()

@pytest.fixture
def executor():
    pool = ThreadPoolExecutor(1)
    yield pool
    pool.shutdown()

@pytest.mark.asyncio
@pytest.mark.parametrize(('poolclass'), [(None), ('sqlalchemy.pool:StaticPool')])
async def test_component_start(poolclass):
    """Test that the component creates all the expected resources."""
    component = SQLAlchemyComponent(url='sqlite:///:memory:', poolclass=poolclass)
    async with Context() as ctx:
        await component.start(ctx)

        engine = ctx.require_resource(Engine)
        ctx.require_resource(sessionmaker)
        assert ctx.sql is ctx.require_resource(Session)
        assert ctx.sql.bind is engine

@pytest.mark.asyncio
async def test_multiple_engines():
    component = SQLAlchemyComponent(engines={'db1': {}, 'db2': {}}, url='sqlite:///:memory:')
    async with Context() as ctx:
        await component.start(ctx)

        engine1 = ctx.require_resource(Engine, 'db1')
        engine2 = ctx.require_resource(Engine, 'db2')
        assert ctx.db1.bind is engine1
        assert ctx.db2.bind is engine2

@pytest.mark.parametrize('asynchronous', [False, True], ids=['sync', 'async'])
@pytest.mark.asyncio
async def test_ready_callback(asynchronous):
    def ready_callback(engine, factory):
        nonlocal engine2, factory2
        engine2 = engine
        factory2 = factory

    async def ready_callback_async(engine, factory):
        nonlocal engine2, factory2
        engine2 = engine
        factory2 = factory

    engine2 = factory2 = None
    callback = ready_callback_async if asynchronous else ready_callback
    component = SQLAlchemyComponent(url='sqlite:///:memory:', ready_callback=callback)
    async with Context() as ctx:
        await component.start(ctx)

        engine = ctx.require_resource(Engine)
        factory = ctx.require_resource(sessionmaker)
        assert engine is engine2
        assert factory is factory2

@pytest.mark.asyncio
async def test_bind():
    """Test that a Connection can be passed as "bind" in place of "url"."""
    engine = create_engine('sqlite:///:memory:')
    connection = engine.connect()
    component = SQLAlchemyComponent(bind=connection)
    async with Context() as ctx:
        await component.start(ctx)

        assert ctx.require_resource(Engine) is engine
        assert ctx.sql.bind is connection

def test_no_url_or_bind():
    exc = pytest.raises(TypeError, SQLAlchemyComponent)
    exc.match('both "url" and "bind" cannot be None')

@pytest.mark.parametrize('raise_exception', [False, True])
@pytest.mark.parametrize('commit_executor', [None, 'default', 'instance'],
                         ids=['none', 'default', 'instance'])
@pytest.mark.asyncio
async def test_finish_commit(raise_exception, executor, commit_executor, tmpdir):
    """
    Tests that the session is automatically committed if and only if the context was not exited
    with an exception.

    """
    db_path = tmpdir.join('test.db')
    engine = create_engine('sqlite:///%s' % db_path, poolclass=NullPool)
    engine.execute('CREATE TABLE foo (id INTEGER PRIMARY KEY)')

    component = SQLAlchemyComponent(
        url={'drivername': 'sqlite', 'database': str(db_path)},
        commit_executor=executor if commit_executor == 'instance' else commit_executor)
    ctx = Context()
    ctx.add_resource(executor, types=[Executor])
    await component.start(ctx)
    ctx.sql.execute('INSERT INTO foo (id) VALUES(3)')
    await ctx.close(Exception('dummy') if raise_exception else None)

    rows = engine.execute('SELECT * FROM foo').fetchall()
    assert len(rows) == (0 if raise_exception else 1)

@pytest.mark.asyncio
async def test_memory_leak():
    """Test that creating a session in a context does not leak memory."""
    component = SQLAlchemyComponent(url='sqlite:///:memory:')
    async with Context() as ctx:
        await component.start(ctx)
        assert isinstance(ctx.sql, Session)

    del ctx
    gc.collect()  # needed on PyPy
    assert next((x for x in gc.get_objects() if isinstance(x, Context)), None) is None


import os

from fabric.api import local, env, execute, task, cd, run
from fabric.decorators import runs_once

@runs_once
def pre_deploy(branch=None):
    """
    Make sure that ``local.deploy.prep`` is only run
    once when the deploy command is run on multiple
    hosts.
    """

    execute('local.deploy.prep', branch=branch, hosts=[env.host_string])

@task(hosts=[])
def deploy(branch=None):
    """
    Deploy this project.

    Internally calls local.deploy.prep once and then
    ``local.deploy.do`` for each host.

    Takes an optional branch argument that can be used
    to deploy a branch other than master.
    """

    if not env.get('deploy_ready', False):
        pre_deploy(branch=branch)
        env.deploy_ready = True
    execute('local.deploy.do', branch=branch, hosts=[env.host_string])

@task(hosts=[])
def migrate():
    """
    Database migration using south
    """

    manage_py = os.path.join(env.git_working_dir, 'project', 'manage.py')
    python_exe = os.path.join(env.git_working_dir, 'env', 'bin', 'python')

    run('%s %s migrate --all' %(python_exe, manage_py))


from fabric.api import *
from StringIO import StringIO
import yaml

def start():
    run('JAVA_HOME=~/fab/java nohup ~/fab/cassandra/bin/cassandra')

def stop():
    with settings(warn_only=True):
        run('pkill -9 -f "java.*org.apache.*.CassandraDaemon"')

def update(revision):
    # make sure cassandra is dead

    with settings(warn_only=True):
        run('pgrep -f cassa | xargs kill')

    topIO = StringIO()
    get('~/fab/cassandra/conf/cassandra-topology.properties', topIO)
    topIO.seek(0)
    topology = topIO.read()

    rackdcIO = StringIO()
    get('~/fab/cassandra/conf/cassandra-rackdc.properties', rackdcIO)
    rackdcIO.seek(0)
    rackdc = rackdcIO.read()

    yamlIO = StringIO()
    get('~/fab/cassandra/conf/cassandra.yaml', yamlIO)
    yamlIO.seek(0)
    config = yaml.load(yamlIO.read())

    git_checkout_status = run('test -d ~/fab/cassandra.git', quiet=True)
    if git_checkout_status.return_code > 0:
        run('git init --bare ~/fab/cassandra.git')
        for name,url in git_repos:
            run('git --git-dir=$HOME/fab/cassandra.git remote add {name} {url}'.format(name=name, url=url), quiet=True)
        for name,url in reversed(git_repos):
            run('git --git-dir=$HOME/fab/cassandra.git fetch {name}'.format(name=name))

    run('rm -rf ~/fab/cassandra')

    # Find the SHA for the revision requested:
    git_id = run('git --git-dir=$HOME/fab/cassandra.git rev-parse {revision}'.format(revision=revision)).strip()

    # Build Cassandra Checkout revision/tag:
    run('mkdir ~/fab/cassandra')
    run('git --git-dir=$HOME/fab/cassandra.git archive %s | tar x -C ~/fab/cassandra' % revision)
    run('echo -e \'%s\\n%s\\n\' > ~/fab/cassandra/0.GIT_REVISION.txt' % (revision, git_id))
    run('JAVA_HOME=~/fab/java ~/fab/ant/bin/ant -f ~/fab/cassandra/build.xml clean')
    run('JAVA_HOME=~/fab/java ~/fab/ant/bin/ant -f ~/fab/cassandra/build.xml')

    # Save config:
    conf_file = StringIO()
    conf_file.write(yaml.safe_dump(config, encoding='utf-8', allow_unicode=True))
    conf_file.seek(0)
    put(conf_file, '~/fab/cassandra/conf/cassandra.yaml')

    topology_file = StringIO()
    topology_file.write(topology)
    topology_file.seek(0)
    put(topology_file, '~/fab/cassandra/conf/cassandra-topology.properties')

    rackdc_file = StringIO()
    rackdc_file.write(rackdc)
    rackdc_file.seek(0)
    put(rackdc_file, '~/fab/cassandra/conf/cassandra-rackdc.properties')

def get_log(address):
    path = get(remote_path='~/fab/cassandra/logs/system.log', local_path='~/fab/'+ address + '.log')
    return path[0]