import pygame
import neat
import time
import os
import random
import math
import pickle
pygame.font.init()

#create a game of asteroids that learns using neat.
#The player can rotate, shoot forward, thrust forward and keep that velocity, there is no break either. The player should be able to see in an appropriate amount of directions around themselves (8 maybe?)
    #this might be done by creating a ray type object come out from the player and see if it collides with anything and then return the distance from the player to that collison
#the asteroids should spawn in 3 sizes, small medium and large, when they are shot they break into 2 of the smaller denominations, the small ones just die. They spawn offscreen and move in a random direction and velocity.
#if anything goes off screen they should loop back around
    #this may be hard so maybe just kill anything off screen

#set our screen size and font
WIN_WIDTH = 800
WIN_HEIGHT = 800
STAT_FONT = pygame.font.SysFont("comicsans", 50)
#set game speed, making this number higher will let generations go faster
GAME_SPEED = 1
#constant to convert degrees to radians
D_TO_R = math.pi / 180
#theres a better way to track gens globally
gen = 0

#get the slope of a line
def get_slope(line):
    xdif = line[1][1] - line[0][1]
    ydif = line[1][0] - line[0][0]
    if ydif == 0:
        return 9999
    return xdif / ydif

#get the y intercept of a line given the line
def get_y_int(line):
    return line[0][1] - get_slope(line) * line[0][0]

#check if a number is between two other numbers regardless of sign
def is_between_points(middle, end1, end2):
    if middle < end1 and middle > end2 or middle > end1 and middle < end2:
        return True

#decide if a point is on a line
def is_on_line(point, line):
    if is_between_points(point[0], line[1][0], line[0][0]):
        if is_between_points(point[1], line[1][1], line[0][1]):
            return True
    return False

#given two lines use y=mx+b to find if they intersect
def find_intersection(line1, line2):
    m1 = get_slope(line1)
    m2 = get_slope(line2)
    b1 = get_y_int(line1)
    b2 = get_y_int(line2)
    if m1 == m2:
        return False
    x = (b2 - b1) / (m1-m2)
    y = m1* x + b1
    if is_on_line((x, y), line1) and is_on_line((x, y), line2):
        return x, y
    return False

#get the distance between two points
def get_distance(point1, point2):
        xdif = (point1[0] - point2[0])**2
        ydif = (point1[1] - point2[1])**2
        return int(math.sqrt(xdif + ydif))

class Ship:
    def __init__(self):
        self.x = WIN_WIDTH // 2
        self.y = WIN_HEIGHT // 2
        self.MAX_SPEED = 7
        self.size = 20
        self.rotation = 0
        self.rad_rotation = 0
        self.speed = 0
        self.horizontal_speed = 0
        self.vertical_speed = 0
        self.is_thrust = False

    def thrust(self):
        self.horizontal_speed += math.cos(self.rotation * D_TO_R) / 2
        self.vertical_speed += math.sin(self.rotation * D_TO_R) / 2
        self.stay_under_max_speed()
        self.speed = math.sqrt(self.horizontal_speed**2 + self.vertical_speed**2)
        self.is_thrust = True

    def stay_under_max_speed(self):
        if self.horizontal_speed > self.MAX_SPEED:
            self.horizontal_speed = self.MAX_SPEED
        if self.vertical_speed > self.MAX_SPEED:
            self.vertical_speed = self.MAX_SPEED

    def glide(self):
        if not self.is_thrust and self.horizontal_speed != 0:
            self.horizontal_speed -= self.horizontal_speed * 0.02
        if not self.is_thrust and self.vertical_speed != 0:
            self.vertical_speed -= self.vertical_speed * 0.02

    def stay_on_screen(self):
        if self.x > WIN_WIDTH:
            self.x = 0
        elif self.x < 0:
            self.x = WIN_WIDTH
        if self.y > WIN_HEIGHT:
            self.y = 0
        elif self.y < 0:
            self.y = WIN_HEIGHT

    def rotate_left(self):
        self.rotation -= 8
        self.rad_rotation = math.radians(self.rotation)

    def rotate_right(self):
        self.rotation += 8
        self.rad_rotation = math.radians(self.rotation)

    def move(self):
        self.glide()
        self.x += self.horizontal_speed
        self.y += self.vertical_speed
        self.stay_on_screen()

    def shoot(self):
        return Bullet(self.x, self.y, self.rotation)

    def get_ship_coordinates(self):
        const0 = (self.size * math.sqrt(130) / 12)
        const1 = math.atan(7 / 9)
        coord1 = (int(self.x - const0 * math.cos(const1 + self.rad_rotation)), int(self.y - const0 * math.sin(const1 + self.rad_rotation)))
        coord2 = (int(self.x - const0 * math.cos(const1 - self.rad_rotation)), int(self.y + const0 * math.sin(const1 - self.rad_rotation)))
        coord3 = (int(self.x + self.size * math.cos(self.rad_rotation)), int(self.y + self.size * math.sin(self.rad_rotation)))
        return coord1, coord2, coord3

    def draw(self, win):
        coord1, coord2, coord3 = self.get_ship_coordinates()
        pygame.draw.lines(win, (255,255,255), True, (coord1,coord2, coord3))
        if self.is_thrust:
            self.is_thrust = False
            coord1 = (int(self.x - (self.size+10) * math.cos(self.rad_rotation)), int(self.y - (self.size+10) * math.sin(self.rad_rotation)))
            coord2 = (int(self.x - ((self.size+10) * 0.5 * math.cos(self.rad_rotation + math.pi / 6))), int(self.y - ((self.size+10) * 0.5 * math.sin(self.rad_rotation + math.pi / 6))))
            pygame.draw.line(win, (255,255,255), coord1, coord2)
            coord1 = (int(self.x - (self.size+10) * math.cos(-self.rad_rotation)), int(self.y + (self.size+10) * math.sin(-self.rad_rotation)))
            coord2 = (int(self.x - ((self.size+10) * 0.5 * math.cos(-self.rad_rotation + math.pi / 6))), int(self.y + ((self.size+10) * 0.5 * math.sin(-self.rad_rotation + math.pi / 6))))
            pygame.draw.line(win, (255,255,255), coord1, coord2)

class Bullet:
    def __init__(self, x, y, direction):
        self.x = x
        self.y = y
        self.VEL = 80
        self.direction = direction
        self.life_time = 10
        self.alive = True

    def move(self):
        self.life_time -= 1
        self.x += self.VEL * math.cos(self.direction * D_TO_R)
        self.y += self.VEL * math.sin(self.direction * D_TO_R)
        if self.life_time < 0:
            self.alive = False
        return not self.alive

    def draw(self, win):
        pygame.draw.circle(win, (255,255,255), (int(self.x), int(self.y)), 2)

class Asteroid:
    def __init__(self, x = -1, y = -1, size = -1, direction = -1):
        if size != -1:
            self.size = size
        else:
            self.size = random.randint(0,2)
            if self.size == 0:
                self.size = 20
            elif self.size == 1:
                self.size = 40
            elif self.size == 2:
                self.size = 80
        if x == -1:
            self.x = 0
        else:
            self.x = x
        if y == -1:
            self.y = random.randrange(0, WIN_HEIGHT)
        else:
            self.y = y
        if direction == -1:
            self.direction = random.randrange(0, 360) * D_TO_R
        else:
            self.direction = direction * D_TO_R
        self.speed = random.randrange(3, 7)
        self.rotation = 0
        self.lines = []

        # Make random asteroid sprites
        full_circle = random.uniform(45,55)
        self.vertices = []
        while full_circle < 360:
            self.vertices.append([self.size, full_circle])
            dist = self.size * 0.9
            full_circle += random.uniform(45, 55)

        for v in range(len(self.vertices)):
            if v == len(self.vertices) - 1:
                next_v = self.vertices[0]
            else:
                next_v = self.vertices[v + 1]
            this_v = self.vertices[v]
            line = ((int(self.x + this_v[0] * math.cos(this_v[1] * D_TO_R)), int(self.y + this_v[0] * math.sin(this_v[1] * D_TO_R))),
            (int(self.x + next_v[0] * math.cos(next_v[1] * D_TO_R)), int(self.y + next_v[0] * math.sin(next_v[1] * D_TO_R))))
            self.lines.append(line)

    def move(self):
        self.x += self.speed / 1.5 * math.cos(self.direction)
        self.y += self.speed / 1.5 * math.sin(self.direction)
        self.rotate()
        if self.x > WIN_WIDTH + self.size:
            self.x = -self.size
        elif self.x < -self.size:
            self.x = WIN_WIDTH + self.size
        if self.y > WIN_HEIGHT + self.size:
            self.y = -self.size
        elif self.y < -self.size:
            self.y = WIN_HEIGHT + self.size

    def rotate(self):
        self.rotation += 2

    def draw(self, win):
        for v in range(len(self.vertices)):
            if v == len(self.vertices) - 1:
                next_v = self.vertices[0]
            else:
                next_v = self.vertices[v + 1]
            this_v = self.vertices[v]
            x1 = (self.x + this_v[0] * math.cos(this_v[1] * D_TO_R)) - self.x
            y1 = (self.y + this_v[0] * math.sin(this_v[1] * D_TO_R)) - self.y
            x2 = (self.x + next_v[0] * math.cos(next_v[1] * D_TO_R)) - self.x
            y2 = (self.y + next_v[0] * math.sin(next_v[1] * D_TO_R)) - self.y
            x11 = (x1 * math.cos(self.rotation * D_TO_R) - y1 * math.sin(self.rotation * D_TO_R)) + self.x
            y11 = (y1 * math.cos(self.rotation * D_TO_R) + x1 * math.sin(self.rotation * D_TO_R)) + self.y
            x22 = (x2 * math.cos(self.rotation * D_TO_R) - y2 * math.sin(self.rotation * D_TO_R)) + self.x
            y22 = (y2 * math.cos(self.rotation * D_TO_R) + x2 * math.sin(self.rotation * D_TO_R)) + self.y
            line = ((int(x11), int(y11)), (int(x22), int(y22)))
            self.lines[v] = line
            pygame.draw.line(win, (255,255,255), line[0], line[1])

class Game:
    def __init__(self, win):
        self.ship = Ship()
        self.asteroids = []
        self.bullets = []
        self.distances = []
        self.sight_lines = []
        self.stationary_count = 200
        self.score = 0
        self.shoot_cooldown = 0
        self.win = win
        self.hit_asteroid = False
        self.bullet_missed = False
        self.num_bullets_missed = 0
        for i in range(4):
            self.asteroids.append(Asteroid())
        self.asteroids.append(Asteroid(WIN_WIDTH / 2, WIN_HEIGHT / 4, 40, 90))

    def move(self):
        self.ship.move()
        self.sight_lines = (
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos(self.ship.rotation * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin(self.ship.rotation * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation + 45) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation + 45) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation - 45) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation - 45) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation + 90) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation + 90) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation - 90) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation - 90) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation + 135) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation + 135) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation - 135) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation - 135) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation + 180) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation + 180) * D_TO_R) * WIN_HEIGHT * 2))),
            ((int(self.ship.x), int(self.ship.y)), (int(self.ship.x + math.cos((self.ship.rotation - 180) * D_TO_R) * WIN_WIDTH * 2), int(self.ship.y + math.sin((self.ship.rotation - 180) * D_TO_R) * WIN_HEIGHT * 2)))
        )
        self.distances = self.get_sight()
        self.shoot_cooldown -= 1
        self.hit_asteroid = self.check_collisions(self.win)
        self.bullet_missed = False
        if len(self.asteroids) < 5:
            flag = random.randint(0,1)
            if flag == 0:
                self.attack_player()
            else:
                self.asteroids.append(Asteroid())
        if self.ship.speed != 0:
            self.stationary_count -= 1
        else:
            self.stationary_count -= 2
        if self.stationary_count < 1:
            self.stationary_count = 200
            if len(self.asteroids) < 12:
                self.attack_player()
        for asteroid in self.asteroids:
            asteroid.move()
        for bullet in self.bullets:
            self.bullet_missed = bullet.move()
            if self.bullet_missed:
                self.bullets.remove(bullet)
                break

    def shoot(self):
        if self.shoot_cooldown <= 0:
            self.bullets.append(self.ship.shoot())
            self.shoot_cooldown = 40

    def is_colliding(self, x, y, xTo, yTo, size):
        if x > xTo - size and x < xTo + size and y > yTo - size and y < yTo + size:
            return True
        return False

    def check_collisions(self, win):
        for b in self.bullets:
            for a in self.asteroids:
                if self.is_colliding(b.x, b.y, a.x, a.y, a.size):
                    if a.size != 20:
                        self.asteroids.append(Asteroid(a.x, a.y, a.size / 2, a.direction - random.randrange(5, 20)))
                        self.asteroids.append(Asteroid(a.x, a.y, a.size / 2, a.direction + random.randrange(5, 20)))
                    self.asteroids.remove(a)
                    self.bullets.remove(b)
                    self.score += 1
                    return True
        return False

    def get_sight(self):
        s_lines = self.sight_lines
        distances = []
        for sl in self.sight_lines:
            closest = 10000
            for a in self.asteroids:
                for al in a.lines:
                    coord = find_intersection(sl, al)
                    if coord:
                        dis = get_distance((self.ship.x, self.ship.y), coord)
                        if dis < closest:
                            closest = dis
            distances.append(closest)
        return distances

    def attack_player(self):
        s = self.ship
        for i in range(random.randint(2,4)):
            temp = 0
            while abs(temp) < 200:
                x = self.ship.x + random.randint(-300, 300)
                temp = self.ship.x - x
            temp = 0
            while abs(temp) < 200:
                y = self.ship.y + random.randint(-300, 300)
                temp = self.ship.y - y
            angle = math.atan2(self.ship.x - x, y - self.ship.y) * 180 / math.pi + 270
            self.asteroids.append(Asteroid(x,y, -1, angle))

def draw_window(win, ship, bullets, asteroids, score, gen):
    win.fill((0,0,0))
    for asteroid in asteroids:
        asteroid.draw(win)
    for bullet in bullets:
        bullet.draw(win)
    ship.draw(win)
    text = STAT_FONT.render("Score: " + str(score), 1, (255,255,255))
    win.blit(text, (int(WIN_WIDTH / 2 - text.get_width() / 2), 150))
    text = STAT_FONT.render("Generation: " + str(gen), 1, (255,255,255))
    win.blit(text, (int(WIN_WIDTH / 2 - text.get_width() / 2), 100))
    pygame.display.update()

def neat_main(genomes, config):
    global gen
    gen += 1
    nets = []
    ge = []
    games = []
    best_fitness = 0
    win = pygame.display.set_mode((WIN_WIDTH, WIN_HEIGHT))
    clock = pygame.time.Clock()

    for _,g in genomes:
        net = neat.nn.FeedForwardNetwork.create(g, config)
        nets.append(net)
        games.append(Game(win))
        g.fitness = 0
        ge.append(g)
    best_genome = ge[0]

    run = True
    while run:
        clock.tick(30 * GAME_SPEED)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
                pygame.quit()
                quit()

        if len(games) < 1:
            run = False
            break

        for x, game in enumerate(games):
            ship = game.ship
            game.move()

            if x == 0:
                s = getattr(game, 'ship')
                b = game.bullets
                a = game.asteroids
                sc = game.score
                draw_window(win, s, b, a, sc, gen)

            outputs = nets[x].activate((game.distances[0], game.distances[1], game.distances[2], game.distances[3], game.distances[4], game.distances[5], game.distances[6], game.distances[7]))
            if outputs[0] > 0.8:
                ship.rotate_left()
            if outputs[1] > 0.8:
                ship.rotate_right()
            if outputs[2] > 0.8:
                ge[x].fitness -= .009
                ship.thrust()
            if outputs[3] > 0.75:
                game.shoot()

            if game.hit_asteroid:
                ge[x].fitness += 2
            if game.bullet_missed:
                ge[x].fitness -= 1
            ge[x].fitness += 0.01
            for a in game.asteroids:
                if game.is_colliding(ship.x, ship.y, a.x, a.y, a.size):
                    ge[x].fitness -= 5
                    if ge[x].fitness > best_fitness:
                        best_fitness = ge[x].fitness
                        best_genome = ge[x]
                    games.pop(x)
                    nets.pop(x)
                    ge.pop(x)
                    break
    #create or open a pickle file to write to named f and save our linear data in it
    name = "pickles/best_genome_gen_" + str(gen) + ".p"
    pickle.dump(best_genome, open(name, "wb"))

def replay_main(genomes, config):
    global gen
    gen += 1
    nets = []
    ge = []
    games = []
    best_fitness = 0
    win = pygame.display.set_mode((WIN_WIDTH, WIN_HEIGHT))
    clock = pygame.time.Clock()

    for _,g in genomes:
        net = neat.nn.FeedForwardNetwork.create(g, config)
        nets.append(net)
        games.append(Game(win))
        g.fitness = 0
        ge.append(g)
    best_genome = ge[0]

    run = True
    while run:
        clock.tick(30 * GAME_SPEED)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
                pygame.quit()
                quit()

        if len(games) < 1:
            run = False
            break

        for x, game in enumerate(games):
            ship = game.ship
            game.move()

            if x == 0:
                s = getattr(game, 'ship')
                b = game.bullets
                a = game.asteroids
                sc = game.score
                draw_window(win, s, b, a, sc, gen)
            outputs = nets[x].activate((game.distances[0], game.distances[1], game.distances[2], game.distances[3], game.distances[4], game.distances[5], game.distances[6], game.distances[7]))
            if outputs[0] > 0.8:
                ship.rotate_left()
            if outputs[1] > 0.8:
                ship.rotate_right()
            if outputs[2] > 0.8:
                ship.thrust()
                ge[x].fitness -= (0.005 + game.ship.speed * 0.001)
            if outputs[3] > 0.75:
                game.shoot()

            if game.hit_asteroid:
                ge[x].fitness += 2
            if game.bullet_missed:
                ge[x].fitness -= 1
            ge[x].fitness += .01
            for a in game.asteroids:
                if game.is_colliding(ship.x, ship.y, a.x, a.y, a.size):
                    if ge[x].fitness > best_fitness:
                        best_fitness = ge[x].fitness
                        best_genome = ge[x]
                    print(best_fitness)
                    games.pop(x)
                    nets.pop(x)
                    ge.pop(x)
                    break

def player_main():
    win = pygame.display.set_mode((WIN_WIDTH, WIN_HEIGHT))
    clock = pygame.time.Clock()
    global gen
    gen = "Player"
    game = Game(win)

    run = True
    while run:
        clock.tick(30 * GAME_SPEED)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
                pygame.quit()
                quit()
        #check for user input
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_SPACE:
                    game.shoot()

        pressed = pygame.key.get_pressed()
        if pressed[pygame.K_w]:
            game.ship.thrust()
        if pressed[pygame.K_a]:
            game.ship.rotate_left()
        if pressed[pygame.K_d]:
            game.ship.rotate_right()

        game.move()
        draw_window(win, game.ship, game.bullets, game.asteroids, game.score, gen)

        for a in game.asteroids:
            if game.is_colliding(game.ship.x, game.ship.y, a.x, a.y, a.size):
                run = False
                break

def load_pickles(config_path):
    file_paths = []
    for root, directories, files in os.walk("pickles"):
        for filename in files:
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)
    for path in file_paths:
        replay_genome(config_path, path)

def replay_genome(config_path, pickle_path):
    config = neat.config.Config(neat.DefaultGenome, neat.DefaultReproduction, neat.DefaultSpeciesSet, neat.DefaultStagnation, config_path)
    with open(pickle_path, "rb") as f:
        genome = pickle.load(f)
    genomes = [(1,genome)]
    replay_main(genomes, config)

def run(config_path):
    config = neat.config.Config(neat.DefaultGenome, neat.DefaultReproduction, neat.DefaultSpeciesSet, neat.DefaultStagnation, config_path)
    p = neat.Population(config)
    p.add_reporter(neat.StdOutReporter(True))
    stats = neat.StatisticsReporter()
    p.add_reporter(stats)
    winner = p.run(neat_main,500)

if __name__ == "__main__":
    flag = True
    while flag:
        print("Would you like to...")
        print("Run NEAT (1)")
        print("Replay the best of each last recorded generation? (2)")
        print("Play the game yourself? (3)")
        x = int(input())
        if x == 1 or x == 2 or x == 3:
            flag = False
    if x == 3:
        player_main()
    input
    local_dir = os.path.dirname(__file__)
    config_path = os.path.join(local_dir, "config-feedforward.txt")
    if x == 1:
        run(config_path)
    elif x == 2:
        load_pickles(config_path)


import discord

# Credentials:
token = "Bot Secret Token"
owner_id = "Admin Discord ID"

# Proxy setup:
class Proxy:
    def __init__(self):
        self.endpoint = None

proxy = Proxy()

# Start client:
client = discord.Client()

# --------------------------------------------------------------------------- #
# On Message:
# --------------------------------------------------------------------------- #
@client.event
async def on_message(message):
    # Ignore self and bot messages:
    if message.author == client.user or message.author.bot:
        return

    # Update endpoint channel:
    if str(message.channel.type) != "private" and message.content.startswith("!spawn"):
        if proxy.endpoint == None:
            proxy.endpoint = message.channel
            await message.add_reaction("\u2611")
        else:
            print("{0} tried to spawn your proxy at [{1} | {2}]".format(message.author.name, message.guild, message.channel))

    # Command Center:
    if str(message.channel.type) == "private" and str(message.author.id) == owner_id:
        if message.content.startswith("!say"):
            tmp = message.content.split("!say ")[-1]
            await proxy.endpoint.trigger_typing()
            await proxy.endpoint.send(tmp)

# --------------------------------------------------------------------------- #
# Run
# --------------------------------------------------------------------------- #
client.run(token)


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelBinarizer

train = pd.read_csv("/Users/fadouabadr/Library/Mobile Documents/com~apple~CloudDocs/OBJECTIF CDI/IT stuff/PYTHON/Titanic/Datas/Train.csv")
test = pd.read_csv("/Users/fadouabadr/Library/Mobile Documents/com~apple~CloudDocs/OBJECTIF CDI/IT stuff/PYTHON/Titanic/Datas/Test.csv")

train_copy = train.copy()
test_copy = test.copy()

survived_train = train['Survived'].copy() # Serues object
PassengerId_test = test['PassengerId']

# ***************************************************************************************************
# ***************************************************************************************************
#print first 3 rows of train and test data
train.head(3)
test.head(3)

#print the total number of columns in the train data
print("The total number of rows in the training data is", format(train.shape[0]))
print("the total number of rows in the test data is", format(test.shape[0]))
print(60*'-')
print("The total number of columns in the training data is", format(train.shape[1]))
print("the total number of columns in the test data is", format(test.shape[1]))

print("The different attributes in the train set are", format(train.columns))

#let's remove the PassengerId/Cabin attribute from the test and train set as they are literally useless for our models
train = train.drop(columns=['PassengerId', 'Cabin'], axis=1)
test = test.drop(columns=['PassengerId','Cabin'], axis=1)
# ***************************************************************************************************
# ***************************************************************************************************

# *************************************             ******************************************
# ==================================== DATA ANALYSIS ==========================================
# *************************************             ******************************************
print((train.groupby(['Sex','Survived']).Survived.count() * 100) / train.groupby('Sex').Survived.count())

print((train.groupby(['Pclass','Survived']).Survived.count() * 100) / train.groupby('Pclass').Survived.count())

print((train.groupby(['Embarked','Survived']).Survived.count() * 100) / train.groupby('Embarked').Survived.count())

print(train.groupby(by=["Survived"]).mean()["Age"])

#let's use the info() method to get a better view of our train set
info_train = train.info()
info_test = test.info()

# ************************************                                 ******************************************
# ==================================== DEALING WITH NAN/MISSING VALUES ==========================================
# ************************************                                 ******************************************

datas = [train,test]

#train dataset : 177 values are missing in the 'Age' attribute, 2 are missing in the 'Embarked' attribute
#test dataset : 86 values are missing in the 'Age' attrivute, 1 is missing in the 'Fare' attribute
#Let's visualise the number of null-values/missing values in the training set using matplotlib's bar() method
for data in datas:
    print(data.isna().sum()) #here isna and isnull is the same (nan values)
    plt.figure(figsize=(10,3))
    plt.bar(data.columns, data.isnull().sum())
    plt.xticks(rotation=90)
    plt.xlabel("Attributes"); plt.ylabel("# of missing values in the dataset")
    print(20 * '*')

print('Survived/Dead\n', train['Survived'].value_counts())
print(50*'-')
#print('Class\n', train["Pclass"].value_counts())
print(50*'-')
print('Sex\n', train["Sex"].value_counts())
print(50*'-')
print('Siblings/Spouses\n', train['SibSp'].value_counts())
print(50*'-')
print('Parents/Children\n', train['Parch'].value_counts())
print(50*'-')
print('Embarked\n', train['Embarked'].value_counts())

# ***************************************************************************************************
# ***************************************************************************************************
#*******************************************************************************************************
#The 'Pclass' attribute does not have any missing values

#----------------------------------------------------- Plot ----------------------------------------------------------------------
# Visualising whether Pclass affects the survival rate or not
plt.figure(figsize=(10,5))
sns.countplot(x = 'Pclass', hue = 'Survived', data = train, palette = 'Paired')
plt.show()
# We notice that the 3rd class is the class that had the most deadly rate followed by the 2dn class and the 1st.
# This will be confirmed when looking to the Correlation matrix

#Let's plot several conditional relationship related the the Pclass attributes
grid = sns.FacetGrid(train, col='Survived', row='Pclass', palette='Paired')
grid.map_dataframe(sns.histplot, x='Age')
plt.legend()
plt.show()

fg = sns.FacetGrid(train, hue='Pclass', aspect=3)
fg.map(sns.kdeplot, 'Age', shade=True)
plt.legend()
fg.set(xlim=(0, 80))

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
encoder = LabelBinarizer()
train_pclass = encoder.fit_transform(train["Pclass"])
test_pclass = encoder.fit_transform(test["Pclass"])

#use of the classes_ attribute to see which index corresponds to which class
#Rq: do not confuse the class (which is an indication of the socio economical background) and the class (which is .....)
print('The different classes in the Pclass attribute are', encoder.classes_) #class 0 = 1st class / class 1 = 2nd Class / class 2 = 3rd Class

#concatenate with the new dummies variables and delete the 'Embarked' attribute
#Rq: the train_pclass output is an array and we have to convert it to a df
train.drop(columns=["Pclass"], axis=1, inplace=True)
test.drop(columns=["Pclass"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_pclass)], axis=1)
test = pd.concat([test, pd.DataFrame(test_pclass)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'1st class',1:'2nd class',2:'3rd class'})
test = test.rename(columns={0:'1st class',1:'2nd class',2:'3rd class'})
#*******************************************************************************************************

############################################## SEX ATTRIBUTE #######################################
#The 'Sex' attrivute does not have any missing value or nan value
plt.figure(figsize=(10,5))
sns.countplot(x = 'Sex', hue = 'Survived', data = train, palette = 'Paired')
plt.show()

#Let's plot several conditional relationship related the the Pclass attributes
grid = sns.FacetGrid(train, col='Survived', row='Sex', palette='Paired')
grid.map_dataframe(sns.histplot, x='Age')
plt.show()

fg = sns.FacetGrid(train, hue="Sex", aspect=3)
fg.map(sns.kdeplot, "Age", shade=True)
plt.legend()
fg.set(xlim=(0, 80))

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
# Let's convert the numerical attributes 'male' and 'female' into indicator variables using the LabelBinarizer function
encoder = LabelBinarizer()
train_sex = encoder.fit_transform(train["Sex"])
test_sex = encoder.fit_transform(test["Sex"])

#drop the 'Sex' attrivute for both the data and the test set
#concatenate the new variables into our training and test datasets
#Rq: we have only added the column female because this columns gives us all the necessary information we need
train.drop(columns=["Sex"], axis=1, inplace=True)
test.drop(columns=["Sex"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_sex)], axis=1)
test = pd.concat([test, pd.DataFrame(test_sex)], axis=1)

#rename the columns 'female' into 'Sex' #female = 1, male = 0
train = train.rename(columns={0:'female=1/male=0'})
test = test.rename(columns={0:'female=1/male=0'})

##############################################  AGE ATTRIBUTE #######################################
#the 'Age' attribute has 714 non-null values which gives us 177 null/nan values

#plotting histogram of the Age
plt.hist(train['Age'], bins= 10)
plt.xlabel("Age"); plt.ylabel("Frequence")
plt.show()

#breakdown by age and sex

fg = sns.FacetGrid(train_copy, col='Sex', row='Pclass', hue='Pclass', height=2.5, aspect=2.5)
fg.map(sns.kdeplot, 'Age', shade=True)
fg.map(sns.rugplot, 'Age')
sns.despine(left=True)
fg.set(xlim=(0, 80))

#let's get a better view of statistical properties of the 'Age' attrivute in the train dataset
#and fill the 'nan' values for 'Age' with the mean
for data in datas:
    print(round(data['Age'].describe(),3))
    print(30*'-')

#Let's compute the missing values by replacing them by the median.
train['Age'].fillna(train['Age'].median(), inplace=True)
test['Age'].fillna(test['Age'].median(), inplace=True)

##############################################  FARE ATTRIBUTE #######################################
#there is 1 missing value in the test dataset for the 'Fare' attribute.

#plotting histogram of the Fare price pa
f, axes = plt.subplots(1,1, figsize=(10,5))
g1 = sns.histplot(x=train.Fare, color='red', ax= axes)
plt.title("Fare price paid by passengers")
plt.xlabel("Fare"); plt.ylabel("Frequence")
plt.show()

fg = sns.FacetGrid(train, hue="Fare", aspect=3)
fg.map(sns.kdeplot, "Age", shade=True)
plt.legend()
fg.set(xlim=(0, 80))

fg = sns.FacetGrid(train_copy, hue="Sex", aspect=3)
fg.map(sns.kdeplot, "Fare", shade=True)
plt.legend()
fg.set(xlim=(-100, 100))

fg = sns.FacetGrid(train_copy, col='Survived', height=2.5, aspect=2.5)
fg.map(sns.kdeplot, 'Fare', shade=True)
fg.map(sns.rugplot, 'Fare')
sns.despine(left=True)
fg.set(xlim=(-100, 100))

'''
#creating of a sublist of Fare price paid by people that died/survived
Fare_Died = []
Fare_Survived = []
for i in range(0,len(train)):
    if train.Survived[i] == 0: #did not survived
        Fare_Died.append(train.Fare[i])
    else:
        Fare_Survived.append(train.Fare[i])

f, axes = plt.subplots(nrows=1, ncols=1, figsize=(20,10))
g1 = sns.histplot(Fare_Died, color="orange",ax = axes)
plt.title("Fare distribution for the people who did not survive")
plt.show()

f, axes = plt.subplots(1,1, figsize = (20, 10))
g1 = sns.histplot(Fare_Survived, color="blue",ax = axes)
plt.title("Fare distribution for the people who survived")
plt.show()
'''

#since there is 1 missing value in the 'Fare' attrovute in the test dataset, we simply fillna by the mean
test["Fare"].fillna(test['Fare'].mean(), inplace = True)

'''
# here, we want to break it up into discrete bins by breaking down the data into several ranges
# let's breal k up our Fare features into multiple thresolds (the Q1, Q2, Q3)
train['Fare'] = np.digitize(train["Fare"], bins=[7.910400,14.454200,31.000000], right=True)
test['Fare'] = np.digitize(test["Fare"], bins=[7.895800,14.454200, 31.500000], right=True)

#use of the LabelBinarizer function that gives numerical attributes to text values
encoder = LabelBinarizer()
train_fare = encoder.fit_transform(train["Fare"])
test_fare = encoder.fit_transform(test["Fare"])

#use of the classes_ attrivute to see which index is which class
print(encoder.classes_)

train = pd.concat([train, pd.DataFrame(train_fare)], axis=1)
train = train.drop(columns=["Fare"], axis=1)
test = pd.concat([test, pd.DataFrame(test_fare)], axis=1)
test = test.drop(columns=["Fare"], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'Fare range 1',1:'Fare range 2',2:'Fare range 3',3:'Fare range 4'})
test = test.rename(columns={0:'Fare range 1',1:'Fare range 2',2:'Fare range 3',3:'Fare range 4'})
'''

##############################################  EMBARKED ATTRIBUTE #######################################
sns.countplot(y='Embarked', hue='Survived', data=train, orient='h',palette='Set3')
plt.show()

#from datavisualisation, we can see that Southhampton is the most frequent embarked port
#so, filling the missing values with 'S' (2 missing values in the train dataset)
train['Embarked'].fillna('S', inplace=True)
test['Embarked'].fillna('S', inplace=True)

fg = sns.FacetGrid(train_copy, col='Survived', row='Embarked', height=2.5, aspect=2.5, palette='Set2')
fg.map(sns.kdeplot, 'Fare', shade=True)
fg.map(sns.rugplot, 'Fare')
sns.despine(left=True)
fg.set(xlim=(-100, 100))

grid = sns.FacetGrid(train_copy, col='Survived', row='Embarked', palette='Set2')
grid.map(sns.barplot, 'Sex', 'Fare', ci=None)
grid.add_legend()
plt.show()

#use of the LabelBinarizer function that gives numerical attributes to text values
# here C, S, Q
encoder = LabelBinarizer()
train_embarked = encoder.fit_transform(train["Embarked"]) #print the classes (C = classe 0, Q = Classe 1, S = Classe 2)
test_embarked = encoder.fit_transform(test["Embarked"])

#use of the classes_ attrivute to see which index is which class
print('The different classes in the Pclass attribute are', encoder.classes_) #class 0 = C / class 1 = Q / class 2 = S

#concatenate with the new dummies variables and delete the 'Embarked' attribute
#Rq: the train_embarked output is an array and we have to convert it to a df
train.drop(columns=["Embarked"], axis=1, inplace=True)
test.drop(columns=["Embarked"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_embarked)], axis=1)
test = pd.concat([test, pd.DataFrame(test_embarked)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'C',1:'Q',2:'S'})
test = test.rename(columns={0:'C',1:'Q',2:'S'})

##############################################  SIBSP & PARCH ATTRIBUTES #######################################
#When looking at the correlation matrix, we see that the corr betwee SibSp and parch = 41%
# so we  create a new attribute 'alone'

def accompanied(x):
        if (x['SibSp'] + x['Parch']) > 0:
            return 1
        else:
            return 0

##for data in datas:
#    data['family'] = data.apply(fam, axis=1)
train['Accompanied'] = train.apply(accompanied, axis = 1)
test['Accompanied'] = test.apply(accompanied, axis = 1)

#drop both the columns Parch and Sibsp
train.drop(columns=['Parch','SibSp'], axis=1, inplace=True)
test.drop(columns=['Parch','SibSp'], axis=1, inplace=True)

#let's plot the histogram of the new attribute vs Survived/Not survuved
plt.figure(figsize=(10,6))
sns.countplot(x='Accompanied', hue='Survived', data = train, palette="YlOrBr")
plt.show()
#those having accompaned have better chance to survive according to the data

##############################################  Name ATTRIBUTES #######################################
# ************************************                                          ******************************************
# ==================================== CREATING NEW FEATURE TITLE EXTRACTING FROM ==========================================
#                                              EXISTING FEATURE NAME
# ************************************                                          ************************************
#we create a 'Title' feature which contains the title of each passenger and drop the Name column

print(train['Name'].str.extract(' ([A-Za-z]+)\.', expand=False).unique().size)
print(test['Name'].str.extract(' ([A-Za-z]+)\.', expand=False).unique().size)

train['Title'] = train['Name'].str.extract(' ([A-Za-z]+)\.', expand=False)
test['Title'] = test['Name'].str.extract(' ([A-Za-z]+)\.', expand=False)
train.drop(columns=['Name'], axis=1, inplace=True)
test.drop(columns=['Name'], axis=1, inplace=True)

#number of titles in the train set
#for data in datas:
#    print(data.Title.value_counts()) #Mr / Miss / Mrs / Master / Dr
#    print(30*'-')

print('All the Titles are: \n', set(train.Title) | set(test.Title))

#Let's replace least occuring (1 time) title in the dataset with rare
least_occuring =  ['Countess', 'Capt', 'Dona', 'Dr', 'Ms', 'Major', 'Mme',
                   'Jonkheer', 'Col', 'Don', 'Lady', 'Sir', 'Mlle', 'Rev']

train['Title'] = train['Title'].replace(least_occuring, "rare")
test['Title'] = test['Title'].replace(least_occuring, "rare")

#Let's take a look at our Titles
print(train["Title"].value_counts(), test["Title"].value_counts())

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
encoder = LabelBinarizer() #class 0 = Master / Class 1 = Miss / class 2 = Mr / Class 3 = Mrs / Class 4 = rare
train_title = encoder.fit_transform(train['Title'])
test_title = encoder.fit_transform(test['Title'])
print('The different classes in the Pclass attribute are', encoder.classes_)

#concatenate with the new dummies variables and delete the 'Title' attribute
train.drop(columns=["Title"], axis=1, inplace=True)
test.drop(columns=["Title"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_title)], axis=1)
test = pd.concat([test, pd.DataFrame(test_title)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'Master',1:'Miss',2:'Mr',3:'Mrs',4:'rare'})
test = test.rename(columns={0:'Master',1:'Miss',2:'Mr',3:'Mrs',4:'rare'})

##############################################  Ticket ATTRIBUTES #######################################
#I don't see any accurate relation between this variable and the others, so we'll delete it for the moment
train.drop(columns=['Ticket'], axis=1, inplace=True)
test.drop(columns=['Ticket'], axis=1, inplace=True)

########################################## CHECKING FOR MISSING VALUES ##########################
#no missing values in both the datasets
print(train.info())
print(train.isna().sum())
print(test.isna().sum())

# *************************************                         ******************************************
# ==================================== LOOKING FOR CORRELATIONS ==========================================
# *************************************                         ******************************************

#Pearson Correlation matrix
plt.figure(figsize=(15,10))
corr = train.corr()
sns.heatmap(corr, annot=True)
plt.show()

# look at how much each attrivute correlates with the "survived" instance
print(corr["Survived"].sort_values(ascending=False))
# there is a positive linear correlation between the number of person survived and the Sex (female in this case)
# there is a negative linear correlation between the number of person that survived and the Class

# ************************************  FEATURE SCALING WITH STANDARDIZATION                                   ******************************************
# ====================================  ==========================================
from sklearn.preprocessing import StandardScaler

standard_scaler = StandardScaler()

train.drop(['Survived'], axis=1, inplace=True)
X_train = standard_scaler.fit_transform(train)
y_train = survived_train
X_test = standard_scaler.fit_transform(test)

###############################################################################################
from sklearn.linear_model import SGDClassifier, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.ensemble import VotingClassifier

classifiers = ['LogisticRegression','SGDClassifier','DecisionTreeClassifier',
               'RandomForestClassifier', 'AdaBoostClassifier', 'KNeighborsClassifier']

models = [LogisticRegression(), SGDClassifier(random_state=42), DecisionTreeClassifier(random_state=42),
          RandomForestClassifier(), AdaBoostClassifier(DecisionTreeClassifier(random_state=42)), KNeighborsClassifier()]

#voting_clf = VotingClassifier(estimators=[('lr', LogisticRegression()), ('sgd', SGDClassifier(random_state=42)),
#                                                                 ('dtc', DecisionTreeClassifier(random_state=42)),
#                                                                 ('rfc', RandomForestClassifier())], voting='hard')

#from sklearn.metrics import accuracy_score

#for clf in (LogisticRegression(), SGDClassifier(random_state=42), DecisionTreeClassifier(random_state=42), RandomForestClassifier(), voting_clf):
#    clf.fit(X_train, y_train)
#    y_pred = clf.predict(X_train)
#    print(clf.__class__.__name__, accuracy_score(y_train, y_pred))

#Let's use cross_val_score() method to evaluate our model using K-fold cross-validation
from sklearn.model_selection import cross_val_score, StratifiedKFold, cross_validate
################################################################################################

##########################################################################################
#Use of cross_validate function that allows us to use multiple scores / It is best using this instead of what is up
j =0
results = {}
cross_validate_ = {}
for i in models:
    model = i
    scoring = ['accuracy','precision','f1']
    cross_validate_[classifiers[j]] = cross_validate(model, X_train, y_train, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1), scoring=scoring, n_jobs=-1)
    results[j] = pd.DataFrame(cross_validate_[classifiers[j]])
    j += 1

j =0
#test_precision_LG, test_precision_SGD, test_precision_DTC, test_precision_RFC, test_precision_ADA, test_precision_KNN = []
for i in models:
    test_precision_ = {}
    model = i
    for index in range(0,6):
        test_precision_[classifiers[j]] = pd.DataFrame(results[0]['test_precision']).T
    j += 1

test_precision_LG = pd.DataFrame(results[0]['test_precision']).T
test_precision_SGD = pd.DataFrame(results[1]['test_precision']).T
test_precision_DTC = pd.DataFrame(results[2]['test_precision']).T
test_precision_RFC = pd.DataFrame(results[3]['test_precision']).T
test_precision_ADA = pd.DataFrame(results[4]['test_precision']).T
test_precision_KNN = pd.DataFrame(results[5]['test_precision']).T
test_precision_all = pd.concat([test_precision_LG, test_precision_SGD, test_precision_DTC, test_precision_RFC, test_precision_ADA, test_precision_KNN], axis=0)
test_precision_all.index = classifiers
test_precision_mean = test_precision_all.mean(axis=1)
test_precision_all= pd.concat([test_precision_all, test_precision_mean], axis=1)
test_precision_all.columns = ['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5', 'Fold 6', 'Fold 7', 'Fold 8', 'Fold 9', 'Fold 10','Mean']

test_accuracy_LG = pd.DataFrame(results[0]['test_accuracy']).T
test_accuracy_SGD = pd.DataFrame(results[1]['test_accuracy']).T
test_accuracy_DTC = pd.DataFrame(results[2]['test_accuracy']).T
test_accuracy_RFC = pd.DataFrame(results[3]['test_accuracy']).T
test_accuracy_ADA = pd.DataFrame(results[4]['test_accuracy']).T
test_accuracy_KNN = pd.DataFrame(results[5]['test_accuracy']).T
test_accuracy_all = pd.concat([test_accuracy_LG, test_accuracy_SGD, test_accuracy_DTC, test_accuracy_RFC, test_accuracy_ADA, test_accuracy_KNN], axis=0)
test_accuracy_all.index = classifiers
test_accuracy_mean = test_accuracy_all.mean(axis=1)
test_accuracy_all= pd.concat([test_accuracy_all, test_accuracy_mean], axis=1)
test_accuracy_all.columns = ['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5', 'Fold 6', 'Fold 7', 'Fold 8', 'Fold 9', 'Fold 10','Mean']

##########################################################################################

##########################################################################################
# A much better way to evaluate the performance is to look at the confusion matrix
from sklearn.model_selection import cross_val_predict #performs K folds cross validation but instead of returning the evaluation scores, it returns the predictions made on each test fold
from sklearn.metrics import confusion_matrix, roc_auc_score

j =0
cross_val_predictions = {}
conf_matrix = {}
roc_auc_scores = {}
for i in models:
    model = i
    cross_val_predictions[classifiers[j]] = cross_val_predict(model, X_train, y_train, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1))
    conf_matrix[classifiers[j]] = confusion_matrix(y_train, cross_val_predictions[classifiers[j]])
    plt.show()
    j += 1

##########################################################################################

# ************************************  FINE TUNE YOUR MODEL ******************************************
# ====================================  Grid Search ==========================================
#let search for the best possible combinations of hyperparameters values for RandomFirestClassifier
from sklearn.model_selection import GridSearchCV
import numpy as np

param_grid = {'n_estimators': [int(x) for x in np.linspace(200, 2000, num = 10)], 'max_features': ['auto', 'sqrt'],
              'max_depth' : [int(x) for x in np.linspace(10, 110, num = 11)],
              'criterion' :['gini', 'entropy']}

grid_search = GridSearchCV(RandomForestClassifier(), param_grid=param_grid, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1))

best_model = grid_search.fit(X_train, y_train)
print(best_model.best_params_)

classifier_best = RandomForestClassifier(random_state=42, n_estimators=300, criterion='entropy',max_depth=6, max_features='sqrt')
classifier_best.fit(X_train, y_train)

X_test_pred = classifier_best.predict(X_test)
final_results = pd.DataFrame({'PassengerId': PassengerId_test, 'Survived': X_test_pred})
final_results


import unittest
from dateparser import future_datetime as future, Especial
from datetime import datetime

class TestFutureDate(unittest.TestCase):

    def test_especial_later(self):
        base_date = datetime(2023,10,27,11,30,0)
        self.assertEqual(future(especial=Especial.LATER, base_date=base_date), datetime(2023,10,27,15,0,0),
            "It should be 4 hours later without minutes and seconds")

    def test_especial_weekend(self):
        # laborable day
        base_date = datetime(2023,10,25,11,30,0) # Wednesday
        self.assertEqual(future(base_date=base_date, especial=Especial.WEEKEND), datetime(2023,10,28,8,0), "Should be next Saturday")

        # Saturday
        base_date = datetime(2023,10,28,11,30,0) # Saturday
        self.assertEqual(future(base_date=base_date, especial=Especial.WEEKEND), datetime(2023,11,4,8,0), "Should be Saturday next week")

    def test_tonight(self):
        # morning
        base_date = datetime(2023,10,25,11,30,0) # morning
        self.assertEqual(future(base_date=base_date, especial=Especial.TONIGHT), datetime(2023,10,25,20,0), "Should be today at 20:00")

        # night
        base_date = datetime(2023,10,25,20,30,0) # night
        self.assertEqual(future(base_date=base_date, especial=Especial.TONIGHT), None, "Should be None because it is already tonight")

    def test_just_wednesday(self):
        WEDNESDAY = 2
        # it's Wednesday, so returns 7 days ahead
        base_date = datetime(2023,10,25)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,1,8,0), "Should be next Wednesday")

        # it's Friday, so returns 5 days ahead
        base_date = datetime(2023,11,3)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,8,8,0), "Should be next Wednesday")

        # it's Monday, so returns 2 days ahead
        base_date = datetime(2023,11,13)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,15,8,0), "Should be next Wednesday")

    def test_just_18th(self):
        # it's 18th, so returns 18th next month
        base_date = datetime(2023,10,18,9)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,11,18,8,0), "Should be 18th next month")

        # it's 19th, so returns 18th next month
        base_date = datetime(2023,10,19)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,11,18,8,0), "Should be 18th next month")

        # it's 17th, so returns 17th same month
        base_date = datetime(2023,10,17)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,10,18,8,0), "Should be 18th same month")

    def test_edge_cases(self):
        # test 31st in June 5th, should return 31st July
        base_date = datetime(2023,6,5)
        self.assertEqual(future(base_date=base_date, day_number=31), datetime(2023,7,31,8,0), "Should be 31st July")

        # test 8am in January 31st 9 am, should return 8am on February 1st
        base_date = datetime(2023,1,31,9)
        self.assertEqual(future(base_date=base_date, hour=8), datetime(2023,2,1,8,0), "Should be 8am in February 1st")

        # test 8am in Dicember 31st 9 am, should return 8am on January 1st
        base_date = datetime(2023,12,31,9)
        self.assertEqual(future(base_date=base_date, hour=8), datetime(2024,1,1,8,0), "Should be 8am in January 1st")

        # test February 29th in April 2023, should return February 29th in 2024
        base_date = datetime(2023,4,1,9)
        self.assertEqual(future(base_date=base_date, month=2, day_number=29), datetime(2024,2,29,8,0), "Should be February 29th 2024")

        # test February 29th in April 2022, should return February 29th in 2023
        base_date = datetime(2022,4,1,9)
        self.assertEqual(future(base_date=base_date, month=2, day_number=29), datetime(2023,3,1,8,0), "Should be March 1st 2023")

    def test_just_april(self):
        APRIL = 4
        # it's June, so returns April next year (2024-04-01 08:00)
        base_date = datetime(2023,6,5)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2024,4,1,8,0), "Should be April next year")

        # it's April, so returns April next year
        base_date = datetime(2023,4,10)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2024,4,1,8), "Should be April next year")

        # it's February, so returns April this year (2023-04-01 08:00)
        base_date = datetime(2023,2,15)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2023,4,1,8,0), "Should be April same year")

    def test_just_2023(self):
        # it's 2023, so returns None
        base_date = datetime(2023,5,1)
        self.assertEqual(future(base_date=base_date, year=2023), None, "Should be None (same year)")

        # it's 2022, so returns 2023-01-01 08:00
        base_date = datetime(2022,6,5)
        self.assertEqual(future(base_date=base_date, year=2023), datetime(2023,1,1,8,0), "Should be first day of 2023")

        # it's 2024, so returns None
        base_date = datetime(2024,6,5)
        self.assertEqual(future(base_date=base_date, year=2023), None, "Should be None (previous year)")

    def test_2nd_may(self):
        # it's 2nd May 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,5,2,9)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # it's 1st May 2023, so returns 2023-05-02 08:00
        base_date = datetime(2023,5,1)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2023,5,2,8,0), "Should be 2nd May 2023")

        # it's 3rd May 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,5,3)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # it's 15th April 2023, so returns 2023-05-02 08:00
        base_date = datetime(2023,4,15)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2023,5,2,8,0), "Should be 2nd May 2023")

        # it's 10th June 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,6,10)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # try with hour
        base_date = datetime(2023,6,10,8)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5, hour=10, minute=30), datetime(2024,5,2,10,30), "Should be 2nd May 2024 10:30")

    def test_april_2023(self):
        # # it's February 2023, so returns 2023-04-01 08:00
        base_date = datetime(2023,2,1)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), datetime(2023,4,1,8,0), "Should be April 2023")

        # # it's April 2023, so returns None
        base_date = datetime(2023,4,1,8)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), None, "Should be None (same month)")

        # # it's June 2023, so returns None
        base_date = datetime(2023,6,1)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), None, "Should be None (previous month)")

        # try with hour 11:15
        base_date = datetime(2023,2,1,8,0)
        self.assertEqual(future(base_date=base_date, month=4, year=2023, hour=11, minute=15), datetime(2023,4,1,11,15), "Should be April 1st 2023 11:15")

    def test_16th_2024(self):
        # it's 16th Jan 2024, so returns next month
        base_date = datetime(2024,1,16, 9)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,2,16,8,0), "Should be next month")

        # it's 18th Jan 2024, so returns 16th Feb 2024
        base_date = datetime(2024,1,18)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,2,16,8,0), "Should be 16th next month")

        # it's 1st January 2023 so returns 2024-01-16 08:00
        base_date = datetime(2023,1,1)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,1,16,8,0), "Should be 16th January 2024")

        # try 4 pm
        base_date = datetime(2023,1,1,8)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024, hour=16), datetime(2024,1,16,16,0), "Should be 16th January 2024 16:00")

    def test_april_15th_2023(self):
        # it's April 1st 2023 so returns 2023-04-15 08:00
        # print("First case")
        base_date = datetime(2023,1,4)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), datetime(2023,4,15,8,0), "Should be April 15th 2023")

        # print("Second case")
        # it's April 15th 2023 so returns None
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), None, "Should be None (same day)")

        # print("Third case")
        # it's April 30th 2023 so returns None
        base_date = datetime(2023,4,30)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), None, "Should be None (previous day)")

    def test_april_15th_2023_9am(self):
        # it's April 15th 2023 8am so returns 2023-04-15 09:00
        base_date = datetime(2023,4,15,8)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=9), datetime(2023,4,15,9,0), "Should be April 15th 2023 9am")

        # it's April 15th 2023 9am so returns None
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=9), None, "Should be None (same date and time)")

        # it's April 15th 2023 10am so returns None
        base_date = datetime(2023,4,15,10)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=10), None, "Should be None (previous time)")

    def test_just_9am(self):
        # it's 8am so returns today 9am
        base_date = datetime(2023,4,15,8)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,15,9,0), "Should be today 9am")

        # it's 9am so returns tomorrow 9am
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,16,9,0), "Should be tomorrow 9am")

        # it's 10am so returns tomorrow 9am
        base_date = datetime(2023,4,15,10)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,16,9,0), "Should be tomorrow 9am")

    def test_tuesday_april(self):
        # last Tuesday in April 2023 is April 25th
        # it's Wednesday after last Tuesday in April, so returns first Tuesday in April next year
        base_date = datetime(2023,4,26,8)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2024,4,2,8,0), "Should be first Tuesday in April next year")

        # it's March same year, so returns first Tuesday in April this year
        base_date = datetime(2023,1,15)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2023,4,4,8,0), "Should be first Tuesday in April this year")

        # it's May same year, so returns first Tuesday in April next year
        base_date = datetime(2023,5,26)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2024,4,2,8,0), "Should be first Tuesday in April next year")

    def test_monday_month_starts_on_monday(self):
        base_date = datetime(2023,11,6,8)
        self.assertEqual(future(base_date=base_date, weekday=0, month=1), datetime(2024,1,1,8,0,0), "Should be first Monday in January 2024, which is 1st")

    def test_thursday_2024(self):
        # it's Friday after last Thursday in 2024, so returns None
        base_date = datetime(2024,12,27)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024), None, "Should be None")

        # it's Thursday same year, so returns next Thursday 08:00
        base_date = datetime(2024,12,12,11)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024, hour=8), datetime(2024,12,19,11), "Should be next Thursday")

        # it's Friday same year, so returns next Thursday 08:00
        base_date = datetime(2024,12,13)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024), datetime(2024,12,19), "Should be next Thursday")

    def test_wednesday_2nd(self):
        # it's April 2nd 2023, first 2nd Wedneesday is in August
        base_date = datetime(2023,4,2,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2023,8,2,8,0), "Should be first 2nd that is Wednesday since next month")

        # it's Tuesday 1st, so returns tomorrow
        base_date = datetime(2022,11,1,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2022,11,2,8,0), "Should be next day")

        # it's 3rd, so returns first 2nd that is Wednesnday since next month
        base_date = datetime(2023,9,3,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2024,10,2,8,0), "Should be first 2nd that is Wednesday since next month")

    def test_thursday_3rd_january(self):
        # it's 3rd January 2023, first 3rd Thursday is in February
        base_date = datetime(2023,1,3,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2030,1,3,8,0), "Should be first 3rd that is Thursday since next year")

        # it's 1st January 2019, so returns 3rd January 209
        base_date = datetime(2019,1,1,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2019,1,3,8,0), "Should be 3rd January 2019")

        # it's 4th January 2030, so returns first 3rd that is Thursday since next month
        base_date = datetime(2030,1,4,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2036,1,3,8,0), "Should be first 3rd that is Thursday since next month")

    def test_friday_4th_2024(self):
        # it's 4th January 2024, first 4th Friday is in October
        base_date = datetime(2024,1,4,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), datetime(2024,10,4,8,0), "Should be first 4th that is Friday since next month")

        # it's 3rd October 2024, so returns 4th October 2024
        base_date = datetime(2024,10,3,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), datetime(2024,10,4,8,0), "Should be 4th October 2024")

        # it's 5th October 2024, so returns None
        base_date = datetime(2024,10,5,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), None, "There is no 4th Friday in 2024 after October")

    def test_saturday_march_2024(self):
        # it's 5th January 2024, first Saturday March is March 2nd
        base_date = datetime(2024,1,5,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=5), datetime(2024,3,2,8,0), "Should be first Saturday in March 2024")

        # it's 4th March 2024, so returns 9th March 2024
        base_date = datetime(2024,3,4,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=5), datetime(2024,3,9,9,0), "Should be second Saturday in March 2024")

    def test_friday_march_2024(self):
        # it's Saturday 30th March 2024, so returns None
        base_date = datetime(2024,3,30,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=4), None, "There is no Saturday in March after March 30th 2024")

    def test_just_q2(self):
        # it's q1_2023, so returns 2023-04-01
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2023,4,1,8), "Should be April 1st 2023")

        # it's q2_2023, so returns 2024-04-01 08:00
        base_date = datetime(2023,4,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2024,4,1,8), "Should be April 1st 2024")

        # it's q4_2023, so returns 2024-04-01 08:00
        base_date = datetime(2023,12,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2024,4,1,8), "Should be April 1st 2024")

    def test_q1_2023(self):
        # it's q1_2023, so returns None
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), None, "Should be None")

        # it's q3_2022, so returns 2023-01-01 08:00
        base_date = datetime(2022,12,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), datetime(2023,1,1,8), "Should be January 1st 2023")

        # it's q4_2023, so returns None
        base_date = datetime(2023,10,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), None, "Should be None")

    def test_q2_9am(self):
        # it's q1_2023, so returns 2023-04-01
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2023,4,1,9), "Should be April 1st 2023 09am")

        # it's q2_2023, so returns 2024-04-01 09:00
        base_date = datetime(2023,4,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2024,4,1,9), "Should be April 1st 2024 09am")

        # it's q4_2023, so returns 2024-04-01 09:00
        base_date = datetime(2023,12,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2024,4,1,9), "Should be April 1st 2024")

    def test_inconsistent_weekday(self):
        # ask for Friday, October 26th 2023, 26th is Thursday so returns Thursday, October 26th 2023
        base_date = datetime(2023,2,26,7)
        self.assertEqual(future(base_date=base_date, month=10, day_number=26, year=2023, weekday=4), None, "Should be None because 2023-26-10 is Thursday not Friday")

        base_date = datetime(2023,1,26,7)
        self.assertEqual(future(base_date=base_date, month=10, day_number=26, year=2023, weekday=3), datetime(2023,10,26,8), "Should be 2023-26-10 because it is Thursday")

    def test_in_200_seconds(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, seconds=200), datetime(2023,4,1,8,18,20), "Should be 200 seconds later")

    def test_in_90_minutes(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, minutes=90), datetime(2023,4,1,9,45,0), "Should be 90 minutes later")

    def test_in_3_hours(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, hours=3), datetime(2023,4,1,11,15,0), "Should be 3 hours later")

    def test_in_4_days(self):
        base_date = datetime(2023,4,1,16,25)
        self.assertEqual(future(base_date=base_date, days=4), datetime(2023,4,5,16,25,0), "Should be 4 days later")

        # in 4 days 9 am
        self.assertEqual(future(base_date=base_date, days=4, hour=9, minute=0, second=0), datetime(2023,4,5,9,0,0), "Should be 4 days later 9 am")

    def test_in_5_weeks(self):
        base_date = datetime(2023,4,1,17,0)
        self.assertEqual(future(base_date=base_date, weeks=5), datetime(2023,5,6,17,0,0), "Should be 5 weeks later")

    def test_in_6_months(self):
        base_date = datetime(2023,7,2,18,10)
        self.assertEqual(future(base_date=base_date, months=6), datetime(2024,1,2,18,10,0), "Should be 6 months later")

    def test_in_1_year(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, years=1), datetime(2024,8,3,19,5,0), "Should be 1 year later")

    def test_in_1_month_and_3_days(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, months=1, days=3), datetime(2023,9,6,19,5,0), "Should be 1 month and 3 days later")

    def test_in_3_weeks_and_2_days(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, weeks=3, days=2), datetime(2023,8,26,19,5,0), "Should be 23 days later")

    def test_in_5_hours_and_40_minutes(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, hours=5, minutes=40), datetime(2023,8,4,0,45,0), "Should be 5 hours and 40 minutes later")

    def test_in_2_years_and_5_months(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, years=2, months=5), datetime(2026,1,3,19,5,0), "Should be 28 months later")

if __name__ == '__main__':
    unittest.main()


#!/opt/software/occigen/tools/python/2.7.12/intel/17.0/bin/python

from scipy.ndimage import gaussian_filter
import numpy as np
import sys
#import tensorflow as tf
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
import time
import logging
import os
import argparse
import matplotlib.pyplot as plt

DATAPATH='/scratch/cnt0028/ias1717/SHARED/RD12_data/dmc/MISS03/DATA/PBR_JMD'
OUTNAME='545_FSL'
OUTNAME='857_FSL'

boloref=[sys.argv[1]]
TFLEARN=DATAPATH+'/'+'FSL_545_INITMODEL'

dostep=int(sys.argv[2])
ostep=dostep
TFLEARN=DATAPATH+'/'+'FSL_%s_S%d_INITMODEL'%(boloref[0],ostep)
print(TFLEARN)

do857=True
doDef=False
docor=False
dovar=False
doadd=False
doplot=False

if len(sys.argv)>3:
    doplot=True

lamptab=[27.5,2.0,20.0,18.0]

if sys.argv[1]=='857-1':
    lamp=27.5
    positivity=1.0
    #======================================================================
    #=
    #=    REGLAGE THRESHOLD 857-1
    #=
    #======================================================================
    thresh=0.0007
if sys.argv[1]=='857-2':
    positivity=1.0
    lamp=2.0
    thresh=0.02
if sys.argv[1]=='857-3':
    positivity=-1.0
    lamp=20.0
    thresh=0.0012
if sys.argv[1]=='857-4':
    positivity=1.0
    lamp=8.0
    thresh=0.002

if do857==False:
    res=np.load(DATAPATH+'/INITCODE_%s_ALL.npy'%(OUTNAME))
    nx,ny,nz=res.shape
    res=np.append(np.load(DATAPATH+'/INITCODE_%s_ALL.npy'%(OUTNAME)),np.load(DATAPATH+'/INITCODE1_%s_ALL.npy'%(OUTNAME))).reshape(2*nx,ny,nz)
    hres=np.append(np.load(DATAPATH+'/INITCODE_%s_HALL.npy'%(OUTNAME)),np.load(DATAPATH+'/INITCODE1_%s_HALL.npy'%(OUTNAME))).reshape(2*nx,ny,nz)
else:
    #res=np.load(DATAPATH+'/INITCODE_%s_ALL.npy'%(OUTNAME))[0:4,:,:]
    #hres=np.load(DATAPATH+'/INITCODE_%s_HALL.npy'%(OUTNAME))[0:4,:,:]
    if dostep>0:
        ires=np.load(DATAPATH+'/ERRCODE_%s_%s_ALL.npy'%(OUTNAME,sys.argv[1]))
        ihres=np.load(DATAPATH+'/ERRCODE_%s_%s_HALL.npy'%(OUTNAME,sys.argv[1]))

        nx,ny,nz=ires.shape
        bolo=['857-1','857-2','857-3','857-4']
        tmp=np.load('TMPFSL.npy')
        res=np.zeros([5,ny,nz])
        hres=np.zeros([5,ny,nz])
        res[0,:,:]=ires[0,:,:]
        hres[0,:,:]=ihres[0,:,:]
        if doplot==True:
            tmp=np.load('TMPFSL.npy')
            plt.subplot(1,5,1)
            plt.imshow(tmp[0,:,:]+tmp[1,:,:]+tmp[2,:,:]+tmp[3,:,:],cmap='jet')

        for i in range(4):
            res2=np.load(DATAPATH+'/ERRCODE_%s_%s_ALL.npy'%(OUTNAME,bolo[i]))
            hres2=np.load(DATAPATH+'/ERRCODE_%s_%s_HALL.npy'%(OUTNAME,bolo[i]))
            if dostep==1:
                mm=np.load('/export/home1/jmdeloui/DATA4SROLL4/VECT/%s_offsets_PROD_REP6_RD12_545GHz_DATA_CNN2D_TL_FSL_1_-32_DECOD_0.npy'%(bolo[i])).reshape(1024,512)
            else:
                mm=np.load('/export/home1/jmdeloui/DATA4SROLL4/VECT/%s_offsets_PROD_REP6_RD12_545GHz_DATA_CNN2D_TL_FSL_S%d_1_-32_DECOD_0.npy'%(bolo[i],dostep-1)).reshape(1024,512)

            aa=((mm*res2[0,:,:])*(hres2[0,:,:]>0)*(hres[0,:,:]>0)).sum()/((mm*mm)*(hres2[0,:,:]>0)*(hres[0,:,:]>0)).sum()
            print(aa)
            aa=1E-3
            template=2*((i!=2)-0.5)*(mm*aa-res2[0,:,:])
            template*=(abs(template)>0.01) #>0.01*abs(template).max())
            res[i+1,:,:]=(ires[0,:,:]+positivity*template)*(hres2[0,:,:]>0)*(hres[0,:,:]>0)
            hres[i+1,:,:]=ihres[0,:,:]*(hres2[0,:,:]>0)*(hres[0,:,:]>0)
            if doplot==True:
                plt.subplot(1,5,2+i)
                plt.imshow((res[i+1,:,:]-ires[0,:,:])*(hres2[0,:,:]>0)*(hres[0,:,:]>0),cmap='jet')
        if doplot==True:
            plt.show()
            exit(0)

    else:
        #======================================================================
        #=    ON NE PASSE QU'ICI
        #=    ON NE PASSE QU'ICI
        #=    ON NE PASSE QU'ICI
        #======================================================================
        res=np.load(DATAPATH+'/ERRCODE_%s_%s_ALL.npy'%(OUTNAME,sys.argv[1]))
        hres=np.load(DATAPATH+'/ERRCODE_%s_%s_HALL.npy'%(OUTNAME,sys.argv[1]))
        nx,ny,nz=res.shape

        if dostep<0:
            res=np.load(DATAPATH+'/ERRCODE_%s_%s_ALL.npy'%(OUTNAME,'857-2'))
            hres=np.load(DATAPATH+'/ERRCODE_%s_%s_HALL.npy'%(OUTNAME,'857-2'))
            nx,ny,nz=res.shape

            tmp=np.zeros([3,ny,nz])
            htmp=np.zeros([3,ny,nz])
            thresh=0.001
            for i in range(3):
                if i>=1:
                    ii=i+1
                else:
                    ii=i
                print(lamptab[ii])
                tmp[i,:,:]=(res[1+i,:,:])*(abs(res[0,:,:])>thresh)/lamptab[ii]
                htmp[i,:,:]=hres[1+i,:,:]
                plt.subplot(1,3,1+i)
                #=================================================================================================
                # SUPPRIME LES STRUCTURES NON LIEES AU FSL DANS LE MODEL 3-2
                if i==1:
                    vv=1E-3
                    oo=4E-5
                    ww=(np.repeat(np.exp(-vv*(np.arange(ny)-480)**2),nz)*np.tile(np.exp(-oo*(np.arange(nz)-50)**2),ny)).reshape(ny,nz)
                    ww+=(np.repeat(np.exp(-vv*(np.arange(ny)-30)**2),nz)*np.tile(np.exp(-oo*(np.arange(nz)-50)**2),ny)).reshape(ny,nz)
                    ww+=(np.repeat(np.exp(-vv*(np.arange(ny)-930)**2),nz)*np.tile(np.exp(-oo*(np.arange(nz)-50)**2),ny)).reshape(ny,nz)
                    ww+=(np.repeat(np.exp(-vv*(np.arange(ny)-670)**2),nz)*np.tile(np.exp(-oo*(np.arange(nz)-460)**2),ny)).reshape(ny,nz)
                    ww+=(np.repeat(np.exp(-vv*(np.arange(ny)-220)**2),nz)*np.tile(np.exp(-oo*(np.arange(nz)-460)**2),ny)).reshape(ny,nz)
                    tmp[i,:,:]=(1-ww)*tmp[1,:,:]

                plt.imshow(tmp[i,:,:],cmap='jet')
                plt.ylabel('res%d'%(i))
            plt.show()

            print('save the result')
            np.save('TMPFSL2.npy',tmp)
            np.save('TMPFSL2H.npy',htmp)
            exit(0)

        tmp=np.load('TMPFSL2.npy')
        htmp=np.load('TMPFSL2H.npy')
        for i in range(3):
            res[1+i,:,:]=res[0,:,:]+tmp[i,:,:]
            hres[1+i,:,:]=hres[0,:,:]*(htmp[i,:,:]>0)
        dobolo2=False
        if dobolo2==True:
            #============================================================================
            # ONLY LEARN DIFF WITH BOLOMETER 2
            #============================================================================
            if sys.argv[1]=='857-1':
                for k in range(3):
                    res[1+k:,:]=res[0,:,:]+tmp[0,:,:]*(3*k-1.0)
                    hres[1+k,:,:]=hres[0,:,:]*(htmp[0,:,:]>0)
            if sys.argv[1]=='857-2':
                nx,ny,nz=res.shape
                for k in range(3):
                    res[1+k,:,:]=res[0,:,:]+np.random.randn(ny,nz)*0.1*res[0,:,:].std()
            if sys.argv[1]=='857-3':
                for k in range(3):
                    res[1+k,:,:]=res[0,:,:]+tmp[1,:,:]*(3*k-1.0)
                    hres[1+k,:,:]=hres[0,:,:]*(htmp[1,:,:]>0)
            if sys.argv[1]=='857-4':
                for k in range(3):
                    res[1+k,:,:]=res[0,:,:]-tmp[2,:,:]*(3*k-1.0)
                    hres[1+k,:,:]=hres[0,:,:]*(htmp[2,:,:]>0)

        if doplot==True:
            plt.subplot(1,4,1)
            plt.imshow(res[0,:,:],cmap='jet',vmin=-0.1*res[0,:,:].max(),vmax=res[0,:,:].max())
            for i in range(3):
                plt.subplot(1,4,2+i)
                plt.imshow(res[1+i,:,:]-res[0,:,:],cmap='jet',vmin=-0.1*res[0,:,:].max(),vmax=0.1*res[0,:,:].max())
            plt.show()
            exit(0)

    res=res.astype('float32')
    hres=hres.astype('float32')

nx,ny,nz=res.shape

if doadd==True:
    temp=np.load(DATAPATH+'/ERRCODE_%s_ALL.npy'%(OUTNAME))
    htemp=np.load(DATAPATH+'/ERRCODE_%s_HALL.npy'%(OUTNAME))
    ncorr,ny,nz=temp.shape
    tmp  = np.zeros([(nx+ncorr*2),ny,nz])
    htmp = np.zeros([(nx+ncorr*2),ny,nz])

    for i in range(nx):
        tmp[i,:,:]=res[i,:,:]
        htmp[i,:,:]=hres[i,:,:]

    ttt=[0,2,3]

    for j in range(ncorr):
        tmp[nx+2*j,:,:]    = res[ttt[j],:,:]+temp[j,:,:]/8.0
        htmp[nx+2*j,:,:]   = hres[ttt[j],:,:]
        tmp[nx+2*j+1,:,:]  = res[ttt[j],:,:]-temp[j,:,:]/8.0
        htmp[nx+2*j+1,:,:] = hres[ttt[j],:,:]

    res=tmp.astype('float32')
    hres=htmp.astype('float32')

print(res.shape)
nx,ny,nz=res.shape
for i in range(nx-1):
    res[i+1,:,:]=res[i+1,:,:]/res[i+1,:,:].std()*res[i,:,:].std()

nd,ny,nx=res.shape

if dovar==True:
    nharm=1
    tmp=np.zeros([(1+4*nharm)*nd,ny,nx])
    htmp=np.zeros([(1+4*nharm)*nd,ny,nx])
    amp=0.2

    x = np.arange(nx)
    y = np.arange(ny)
    xx, yy = np.meshgrid(x, y)

    for i in range(nd):

        tmp[i*(1+4*nharm),:,:]=res[i,:,:]
        htmp[i*(1+4*nharm),:,:]=hres[i,:,:]
        plt.subplot(nd,1+4*nharm,1+i*(1+4*nharm))
        plt.imshow(tmp[i*(1+4*nharm),:,:],cmap='jet')
        for j in range(nharm):
            tmp[i*(1+4*nharm)+1+4*j,:,:]=res[i,:,:]*(1+amp*np.sin(xx/nx*2*np.pi*(1+j)))
            htmp[i*(1+4*nharm)+1+4*j,:,:]=hres[i,:,:]
            plt.subplot(nd,1+4*nharm,2+4*j+i*(1+4*nharm))
            plt.imshow(tmp[i*(1+4*nharm)+1+4*j,:,:],cmap='jet')
            tmp[i*(1+4*nharm)+1+4*j+1,:,:]=res[i,:,:]*(1-amp*np.sin(xx/nx*2*np.pi*(1+j)))
            htmp[i*(1+4*nharm)+1+4*j+1,:,:]=hres[i,:,:]
            plt.subplot(nd,1+4*nharm,3+4*j+i*(1+4*nharm))
            plt.imshow(tmp[i*(1+4*nharm)+1+4*j+1,:,:],cmap='jet')
            tmp[i*(1+4*nharm)+1+4*j+2,:,:]=res[i,:,:]*(1+amp*np.sin(yy/ny*2*np.pi*(1+j)))
            htmp[i*(1+4*nharm)+1+4*j+2,:,:]=hres[i,:,:]
            plt.subplot(nd,1+4*nharm,4+4*j+i*(1+4*nharm))
            plt.imshow(tmp[i*(1+4*nharm)+1+4*j+2,:,:],cmap='jet')
            tmp[i*(1+4*nharm)+1+4*j+3,:,:]=res[i,:,:]*(1-amp*np.sin(yy/ny*2*np.pi*(1+j)))
            htmp[i*(1+4*nharm)+1+4*j+3,:,:]=hres[i,:,:]
            plt.subplot(nd,1+4*nharm,5+4*j+i*(1+4*nharm))
            plt.imshow(tmp[i*(1+4*nharm)+1+4*j+3,:,:],cmap='jet')
    plt.show()
    res=tmp.astype('float32')
    hres=htmp.astype('float32')

nd,ny,nx=res.shape

if docor==True:
    ntest=4

    x0=341
    y0a=343
    y0b=793

    x1=90
    y1a=386
    y1b=835

    sx=0.0006
    sy=0.03

    tmp=np.zeros([(1+2*ntest)*nd,ny,nx])
    htmp=np.zeros([(1+2*ntest)*nd,ny,nx])

    x = np.arange(nx)
    y = np.arange(ny)
    xx, yy = np.meshgrid(x, y)
    for i in range(nd):
        dx=30*np.random.randn(ntest)
        dy=30*np.random.randn(ntest)

        tmp[i*(1+2*ntest),:,:]=res[i,:,:]
        htmp[i*(1+2*ntest),:,:]=hres[i,:,:]
        plt.subplot(nd,1+2*ntest,1+i*(1+2*ntest))
        plt.imshow(tmp[i*(1+2*ntest),:,:],cmap='jet')
        for j in range(ntest):
            tmp[i*(1+2*ntest)+1+2*j,:,:]=res[i,:,:]+0.01*(np.exp(-(sx*(xx-x0+dx[j])**2+sy*(yy-y0a+dy[j])**2))+np.exp(-(sx*(xx-x0+dx[j])**2+sy*(yy-y0b+dy[j])**2)))
            htmp[i*(1+2*ntest)+1+2*j,:,:]=hres[i,:,:]
            plt.subplot(nd,1+2*ntest,2+2*j+i*(1+2*ntest))
            plt.imshow(tmp[i*(1+2*ntest)+1+2*j,:,:],cmap='jet')
            tmp[i*(1+2*ntest)+1+2*j+1,:,:]=res[i,:,:]+0.01*(np.exp(-(sx*(xx-x1+dx[j])**2+sy*(yy-y1a+dy[j])**2))+np.exp(-(sx*(xx-x1+dx[j])**2+sy*(yy-y1b+dy[j])**2)))
            htmp[i*(1+2*ntest)+1+2*j+1,:,:]=hres[i,:,:]
            plt.subplot(nd,1+2*ntest,3+2*j+i*(1+2*ntest))
            plt.imshow(tmp[i*(1+2*ntest)+1+2*j+1,:,:],cmap='jet')
    plt.show()
    res=tmp.astype('float32')
    hres=htmp.astype('float32')

import scipy.special as sp

def calcshift(nx,sig=128.0,phase=0.0,amp=0.5):
    xx=np.ones([nx])+np.exp(-0.5*((np.arange(nx)-phase)/sig)**2)
    xx=np.cumsum(xx)
    xx-=xx.min()
    xx=nx*xx/xx.max()
    return(xx)

nd,ny,nx=res.shape
if doDef==True:
    tmp=np.zeros([3*nd,ny,nx])
    htmp=np.zeros([3*nd,ny,nx])

    import scipy.interpolate as interpolate

    x = np.arange(nx)
    y = np.arange(ny)
    xx, yy = np.meshgrid(x, y)
    for i in range(nd):
        f = interpolate.interp2d(x, y, res[i,:,:], kind='cubic')
        fh = interpolate.interp2d(x, y, hres[i,:,:], kind='cubic')
        tmp[i*3,:,:]=res[i,:,:]
        htmp[i*3,:,:]=hres[i,:,:]
        tmp[i*3+1,:,:]=f(calcshift(nx,phase=nx/4),y)
        htmp[i*3+1,:,:]=fh(calcshift(nx,phase=nx/4),y)
        tmp[i*3+2,:,:]=f(calcshift(nx,phase=3*nx/4),y)
        htmp[i*3+2,:,:]=fh(calcshift(nx,phase=3*nx/4),y)

    res=tmp.astype('float32')
    hres=htmp.astype('float32')

nx,ny,nz=res.shape
for i in range(nx-1):
    res[i+1,:,:]=res[i+1,:,:]/res[i+1,:,:].max()*res[i,:,:].max()

#================================================================================
#
#    MULTIPLY BY 1000 TO MAKE IT AT THE GOOD LEVEL (x1000)
#
#================================================================================

res=res*1000.
XNHIDDEN=2
YNHIDDEN=XNHIDDEN
NCOMP=16
SCALE=4
NHIDDEN=XNHIDDEN*YNHIDDEN*NCOMP
DEEPNESS=2*NCOMP
EVAL_FREQUENCY = 100
KERNELSZ = 7
SEED = 1234
DORELU = True
LEARNING_RATE= 0.03
DECAY_RATE=0.995
NUM_EPOCHS= 10000
BATCHSZ=nx
DOFULLYCONNECTED = True

if DOFULLYCONNECTED==False:
    NTOTHIDDEN=NHIDDEN//(XNHIDDEN*YNHIDDEN)
    NUM_DCONV_CHAN =[ NHIDDEN//(XNHIDDEN*YNHIDDEN), DEEPNESS, DEEPNESS, 1]
else:
    NTOTHIDDEN=1
    NUM_DCONV_CHAN =[ DEEPNESS, DEEPNESS, DEEPNESS, 1]
NDCONV=len(NUM_DCONV_CHAN)-1
NMAT,XIMAGE_SIZE,YIMAGE_SIZE=res.shape

if DOFULLYCONNECTED:
    fc2_weights = tf.Variable(0.1*np.random.randn(NTOTHIDDEN, XIMAGE_SIZE // (SCALE**NDCONV) * YIMAGE_SIZE // (SCALE**NDCONV) * NUM_DCONV_CHAN[0]).astype('float32'))
    fc2_biases = tf.Variable(0.1*np.random.randn(XIMAGE_SIZE // (SCALE**NDCONV) * YIMAGE_SIZE // (SCALE**NDCONV) * NUM_DCONV_CHAN[0]).astype('float32'))

dconv_weights={}
dconv_biases={}
nw=0
for i in range(NDCONV):
    dconv_weights[i] = tf.Variable(0.1*np.random.randn(KERNELSZ, KERNELSZ, NUM_DCONV_CHAN[i+1], NUM_DCONV_CHAN[i]).astype('float32'))
    dconv_biases[i] = tf.Variable(0.1*np.random.randn(NUM_DCONV_CHAN[i+1]).astype('float32'))

params = tf.Variable(np.zeros([NMAT,NTOTHIDDEN],dtype='float32'))

train_data = tf.placeholder(tf.float32,shape=(BATCHSZ,XIMAGE_SIZE,YIMAGE_SIZE,1))
hdata=tf.constant(hres.reshape(BATCHSZ,XIMAGE_SIZE,YIMAGE_SIZE,1))
run_param  = tf.placeholder(tf.float32,shape=(1,NTOTHIDDEN))

def lrelu(x, alpha=0.3):
  #return tf.nn.relu(x)
  return tf.maximum(x, tf.multiply(x, alpha))

def wdecod(parameter):
    # convert into image/bias
    #hidden = tf.matmul(parameter, fc1_weights) + fc1_biases
    #if DORELU==True:
    #    hidden = lrelu(hidden)
    print('NPAR PER IMAGE',int((parameter.get_shape().as_list())[1]))
    if DOFULLYCONNECTED==False:
        hidden = parameter
    else:
        hidden = tf.matmul(parameter, fc2_weights) + fc2_biases
        if DORELU==True:
            hidden = lrelu(hidden)
    #hidden = lrelu(hidden)
    NTIME=int((parameter.get_shape().as_list())[0])

    relu = tf.reshape(hidden,[NTIME, XIMAGE_SIZE // (SCALE**NDCONV) , YIMAGE_SIZE // (SCALE**NDCONV) , NUM_DCONV_CHAN[0]])
    #
    ## deconvolve : First layer
    # DeConvolution kernel
    for i in range(NDCONV):
        conv = tf.nn.conv2d_transpose(relu,dconv_weights[i],strides=[1, SCALE, SCALE, 1],padding='SAME',
                                      output_shape=[NTIME,XIMAGE_SIZE // (SCALE**(NDCONV-1-i)),YIMAGE_SIZE // (SCALE**(NDCONV-1-i)),
                                                    NUM_DCONV_CHAN[i+1]],name='dconv_%d'%(i))
        tmp=tf.nn.bias_add(conv, dconv_biases[i])
        print(tmp.get_shape().as_list(),dconv_weights[i].get_shape().as_list())

        if DORELU==True:
            if i!=NDCONV-1:
                relu = lrelu(tmp)
            else:
                relu = lrelu(tmp,alpha=0)
        else:
            relu=tmp

    relu=relu/100

    return relu

def model(parameter):

    modpar=wdecod(parameter)
    return modpar

lpar = params
logits = model(lpar)

synthe = model(run_param)

loss=tf.reduce_sum(hdata*tf.square((logits-train_data)))

numbatch = tf.Variable(0, dtype=tf.float32)

# Decay once per epoch, using an exponential schedule starting at 0.01.
learning_rate = tf.train.exponential_decay(
        LEARNING_RATE,       # Base learning rate.
        numbatch,            # Current index into the dataset.
        10,                  # Decay step.
        DECAY_RATE,          # Decay rate.
        staircase=True)

optimizer=tf.train.AdamOptimizer(learning_rate,0.9).minimize(loss,
                                                                           global_step=numbatch)
# Create a local session to run the training.
start_time = time.time()
print('Initialized 1')

import matplotlib.pyplot as plt

loss_log=np.zeros([NUM_EPOCHS//EVAL_FREQUENCY])

minloss=1E30
with tf.Session() as sess:
    # Run all the initializers to prepare the trainable parameters.
    tf.global_variables_initializer().run()
    print('Initialized 2')
    # Loop through training steps.

    for step in range(NUM_EPOCHS):

        feed_dict = {train_data: res.reshape(BATCHSZ,XIMAGE_SIZE,YIMAGE_SIZE,1)}

        sess.run(optimizer, feed_dict=feed_dict)
        # print some extra information once reach the evaluation frequency
        if step % EVAL_FREQUENCY == 0:
            # fetch some extra nodes' data
            l, lr = sess.run([loss, learning_rate],feed_dict=feed_dict)
            loss_log[step//EVAL_FREQUENCY]=l

            elapsed_time = time.time() - start_time
            start_time = time.time()

            print('Step %d[%5.2f], Mloss[par of sigma]: %7.4f, lrate: %8.5f DURATION %4.2f' % (step,step*BATCHSZ/(1.0*NMAT),np.sqrt(l/(BATCHSZ*XIMAGE_SIZE*YIMAGE_SIZE)), lr,elapsed_time))

    lf,lb =sess.run([fc2_weights,fc2_biases])
    np.save('%s_%d_fc2w.npy'%(TFLEARN,0),lf)
    np.save('%s_%d_fc2b.npy'%(TFLEARN,0),lb)

    for i in range(NDCONV):
        lw,lb =sess.run([dconv_weights[i],dconv_biases[i]])
        np.save('%s_%d_w_%d.npy'%(TFLEARN,0,i),lw)
        np.save('%s_%d_b_%d.npy'%(TFLEARN,0,i),lb)

    predictions,par=sess.run([logits,lpar], feed_dict=feed_dict)
    np.save('%s_DECODREBUILD.npy'%(TFLEARN),predictions.reshape(NMAT,XIMAGE_SIZE,YIMAGE_SIZE))
    alldata=predictions
    allparams=par
    for i in range(BATCHSZ):
        np.save('%s_%d_param.npy'%(TFLEARN,i),par[i,:].reshape(1,NTOTHIDDEN))
        minloss=l
        np.save('%s_step.npy'%(TFLEARN),ostep)
        print('MIN SAVE',l)
    print(allparams)


import folium
import pandas

data = pandas.read_csv("Volcanoes.txt")
lat = list(data["LAT"])
lon = list(data["LON"])
elev = list(data["ELEV"])
name = list(data["NAME"])

html = """
Volcano name:

%s

Height: %s m
"""

def volcano_color(el):
    if el < 1000:
        return 'blue'
    elif el < 2000:
        return 'orange'
    else:
        return 'red'

map = folium.Map(location=[40,-100], zoom_start=6, tiles="Stamen Terrain")
fgv = folium.FeatureGroup(name="Volcanoes")

for lt, ln, el, nm in zip(lat, lon, elev, name):
    iframe = folium.IFrame(html=html % (nm, nm, el), width=200, height=100)
    #fg.add_child(folium.Marker(location=[lt,ln], popup=str(el), icon=folium.Icon(color='red')))
    #fg.add_child(folium.Marker(location=[lt, ln], popup=folium.Popup(iframe), icon = folium.Icon(color = volcano_color(el))))
    fgv.add_child(folium.CircleMarker(location=[lt, ln], radius=5, popup=folium.Popup(iframe),
    fill_color = volcano_color(el), fill_opacity=0.8, color='gray'))

fgp = folium.FeatureGroup(name="Population")

fgp.add_child(folium.GeoJson(data=open('world.json', 'r', encoding='utf-8-sig').read(),
style_function = lambda x: {'fillColor' : 'green' if x['properties']['POP2005'] < 10000000
else 'orange' if x['properties']['POP2005'] < 20000000 else 'red'}))  #to form polygons

map.add_child(fgv)
map.add_child(fgp)
map.add_child(folium.LayerControl())
map.save("Map1.html")


import numpy as np
from lightfm import LightFM
from lightfm.datasets import fetch_movielens

data = fetch_movielens()

print(repr(data["train"]))
print(repr(data["test"]))

model = LightFM(loss="warp")
model.fit(data["train"], epochs=50, num_threads=3)

def rec_init(model, data, user_id):
    n_users, n_items = data["train"].shape
    for user in user_id:
        known_positive = data["item_labels"][data["train"].tocsr()[user].indices]
        scores = model.predict(user, np.arrange(n_items))
        top_items = data["item_labels"][np.argsort(scores)]


import os
from PIL import Image
from datetime import datetime

def gif_creat(directory_files):

    directory_files.sort(key=lambda x: int(x.split(".")[0]))
    frame_list = []

    for frame_number in directory_files:
        frame = Image.open(os.path.join('homer', frame_number))
        frame_list.append(frame)

    gif_name = F'gif_{datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.gif'

    frame_list[0].save(
        f'gif_save/{gif_name}',
        save_all=True,
        append_images=frame_list[1:],
        optimize=True,
        duration=100,
        loop=0
    )

    return gif_name


import pytest
import json
from pelandobot.scraper.promotion_scraper import PromotionScraper
from pelandobot.tgbot.utils.database import sync_db
from tests.variables_and_parameters import (
    TEST_POPULATE_DB_WITH_PROMOTIONS_EXPECTED as expected_promotions_id,
    TEST_GET_PROMOTION_TAGS_REGEX_PARAMS,
    TEST_GET_PROMOTION_INFO_PARAMS,
)

def test_get_params_com_first_query_true(scraper: PromotionScraper):
    target_params = {
        "operationName": "RecentOffersQuery",
        "variables": '{"limit":50}',
        "extensions": '{"persistedQuery":'
        '{"version":1,'
        '"sha256Hash":"38c288ba7f66706afcb33f3206b00229c0a86fb140bad72a262603e12b422802"}}',
    }
    params = scraper.get_params()
    assert params == target_params

def test_get_params_com_first_query_false(scraper: PromotionScraper):
    target_params = {
        "operationName": "RecentOffersQuery",
        "variables": '{"limit":25}',
        "extensions": '{"persistedQuery":'
        '{"version":1,'
        '"sha256Hash":"38c288ba7f66706afcb33f3206b00229c0a86fb140bad72a262603e12b422802"}}',
    }
    scraper.first_query = False
    params = scraper.get_params()
    assert params == target_params

def test_populate_db_with_promotions(scraper: PromotionScraper, prepare_db):

    with open("./tests/mock_data.json", "r") as json_file:
        mock_data = json.load(json_file)
    scraper.populate_db_with_promotions(mock_data)
    db_ids = sync_db.redis.lrange("promotions.id", 0, -1)
    assert db_ids == expected_promotions_id

@pytest.mark.parametrize(
    "test_input, expected_tags",
    TEST_GET_PROMOTION_TAGS_REGEX_PARAMS,
)
def test_get_promotion_tags_regex(scraper: PromotionScraper, test_input, expected_tags):
    tags = scraper.get_promotion_tags(test_input)
    assert tags == expected_tags

@pytest.mark.parametrize("test_input, expected_promotion_info", TEST_GET_PROMOTION_INFO_PARAMS)
def test_get_promotion_info(test_input, expected_promotion_info):
    scraper = PromotionScraper()
    promotion_info = scraper.get_promotion_info(test_input)
    assert promotion_info == expected_promotion_info

@pytest.fixture(scope="function")
def mock_data():
    return {
        "id": "45c2ba07-0e40-41a4-b7c1-5ec63ce39791",
        "title": '[AME R$2960]Smart TV 65" Crystal UHD 4K Samsung 65BU8000 '
        "(R$ 962,00 cashback pela ame)",
        "price": "3699.99",
        "url": "https://www.pelando.com.br/d/45c2ba07-0e40-41a4-b7c1-5ec63ce39791",
        "image": "https://api.pelando.com.br/media/f55db4fb-074c-4a31-b102-eaa3a982e88b?v=2",
        "tags": {
            "65",
            "samsung",
            "smart",
            "cashback",
            "pela",
            "tv",
            "ame",
            "4k",
            "96200",
            "65bu8000",
            "2960",
            "crystal",
            "uhd",
        },
    }

def test_push_promotions_to_db_exists_unsent_promotions(
    scraper: PromotionScraper, mock_data, prepare_db
):
    scraper.push_promotion_to_db(mock_data)
    assert sync_db.redis.exists("unsent.promotions.id") == 1

def test_push_promotions_to_db_exists_promotion_info_tags(
    scraper: PromotionScraper, mock_data, prepare_db
):
    scraper.push_promotion_to_db(mock_data)
    tags = sync_db.redis.smembers(f"promotion.{mock_data['id']}.tags")
    assert tags == mock_data["tags"]

def test_push_promotions_to_db_exists_promotion_info(scraper: PromotionScraper, mock_data, prepare_db):
    scraper.push_promotion_to_db(mock_data)
    promotion_info = sync_db.redis.hgetall(f"promotion.{mock_data['id']}.info")
    mock_data.pop("tags")
    mock_data.pop("id")
    assert promotion_info == mock_data


from __future__ import annotations
from abc import ABC, abstractmethod
import pygame

# abstract class for ui elements
class UIElement(ABC):
    elements: list[UIElement] = []  # list of all ui elements

    # update all elements
    @staticmethod
    def update_all() -> None:
        for element in UIElement.elements:
            if element._parent:
                continue
            element.__update()

    # initialize an element with a target surface, position, and dimension
    def __init__(
        self,
        screen: pygame.Surface,
        x: int,
        y: int,
        width: int,
        height: int,
        background_color: str = None,
    ) -> None:
        self._screen = screen
        self.x = x
        self.y = y
        self.__width = width
        self.__height = height
        self.background_color = background_color

        self._surface = pygame.Surface((self.__width, self.__height))
        self._rect = pygame.Rect(0, 0, self.__width, self.__height)
        self._children: list[UIElement] = []
        self._parent = None

        self.visible = True
        UIElement.elements.append(self)

    @property
    def width(self) -> int:
        return self.__width

    @width.setter
    def width(self, value: int) -> None:
        self.__width = value
        self._surface = pygame.Surface((self.__width, self.__height))
        self._rect = pygame.Rect(
            self._rect.x, self._rect.y, self.__width, self.__height
        )

    @property
    def height(self) -> int:
        return self.__height

    @height.setter
    def height(self, value: int) -> None:
        self.__height = value
        self._surface = pygame.Surface((self.__width, self.__height))
        self._rect = pygame.Rect(
            self._rect.x, self._rect.y, self.__width, self.__height
        )

    # get the rect in world space
    @property
    def world_rect(self) -> pygame.Rect:
        x, y = self.x, self.y
        if self._parent:
            parent_rect = self._parent.world_rect
            x += parent_rect.x
            y += parent_rect.y
        return pygame.Rect(x, y, self.__width, self.__height)

    @property
    def parent(self) -> UIElement:
        return self._parent

    @parent.setter
    def parent(self, value: UIElement) -> None:
        # if parent exists, remove it
        if self._parent:
            self._parent._children.remove(self)

        # set the new parent and add this to its child list
        self._parent = value
        if value:
            value._children.append(self)
            value._on_child_added()

    # get the active area in world space
    def get_active_area(self) -> pygame.Rect:
        area = self.world_rect
        if self._parent:
            area = area.clip(self._parent.get_active_area())
        return area

    # delete this element and all its children
    def delete(self) -> None:
        self.delete_all_children()
        self.background_color = None
        self._surface = None
        self._rect = None
        self.parent = None
        self.visible = False
        UIElement.elements.remove(self)

    # delete all children
    def delete_all_children(self) -> None:
        while self._children:
            self._children[-1].delete()

    # update the element and its children once per frame
    def __update(self) -> None:
        # if background color is not None, fill the surface
        if self.background_color:
            self._surface.fill(self.background_color)

        self._on_update()
        for child in self._children:
            child.__update()
        self.__draw()

    # draw the element to the screen
    def __draw(self) -> None:
        if not self.visible:
            return

        # draw on the screen
        surface = self._screen

        # if parent exists, draw on the parent's surface
        if self._parent:
            surface = self._parent._surface
        surface.blit(self._surface, (self.x, self.y), self._rect)

    # called on update
    @abstractmethod
    def _on_update(self) -> None:
        pass

    # called when a new child is added
    def _on_child_added(self) -> None:
        pass