#!/usr/bin/env python
# -*- coding: utf-8 -*-

import pygame
import time
import os
import sys
from pygame.locals import *

#Initialisation de Pygame

screen_width = 1000
screen_height = 600
screen = pygame.display.set_mode((screen_width,screen_height))
game_icon = pygame.image.load("pictures/icon_game.png").convert_alpha()
screen_icon = pygame.display.set_icon(game_icon)
pygame.display.set_caption('STEX JOURNEY')
clock = pygame.time.Clock()
pygame.init()

#Chargement des données du jeu

wallpaper = pygame.image.load("pictures/wallpaper.png").convert()
blur_wallpaper = pygame.image.load("pictures/blur_wallpaper.png").convert_alpha()
blur_menu1 = pygame.image.load("pictures/blur_menu_out.png").convert_alpha()
credits_screen = pygame.image.load("pictures/screen_credits.png").convert_alpha()
commands_screen = pygame.image.load("pictures/screen_commands.png").convert_alpha()
fondingame = pygame.image.load("pictures/fondingame.png").convert()
game_intro = pygame.image.load("pictures/game_intro.png").convert()
loading_1 = pygame.image.load("pictures/loading_1.png").convert()
loading_2 = pygame.image.load("pictures/loading_2.png").convert()
loading_3 = pygame.image.load("pictures/loading_3.png").convert()
first_house = pygame.image.load("pictures/first_house.png").convert()
second_house = pygame.image.load("pictures/second_house.png").convert()
up_house = pygame.image.load("pictures/up_house.png").convert()

luffy_d = pygame.image.load("pictures/avatar/luffy_d.png").convert_alpha()
luffy_d_blur = pygame.image.load("pictures/avatar/luffy_d_blur.png").convert_alpha()
luffy_d1 = pygame.image.load("pictures/avatar/luffy_d1.png").convert_alpha()
luffy_d2 = pygame.image.load("pictures/avatar/luffy_d2.png").convert_alpha()
luffy_d3 = pygame.image.load("pictures/avatar/luffy_d3.png").convert_alpha()
luffy_d4 = pygame.image.load("pictures/avatar/luffy_d4.png").convert_alpha()

luffy_u = pygame.image.load("pictures/avatar/luffy_u.png").convert_alpha()
luffy_u1 = pygame.image.load("pictures/avatar/luffy_u1.png").convert_alpha()
luffy_u2 = pygame.image.load("pictures/avatar/luffy_u2.png").convert_alpha()
luffy_u3 = pygame.image.load("pictures/avatar/luffy_u3.png").convert_alpha()
luffy_u4 = pygame.image.load("pictures/avatar/luffy_u4.png").convert_alpha()

luffy_r = pygame.image.load("pictures/avatar/luffy_r.png").convert_alpha()
luffy_r1 = pygame.image.load("pictures/avatar/luffy_r1.png").convert_alpha()
luffy_r2 = pygame.image.load("pictures/avatar/luffy_r2.png").convert_alpha()
luffy_r3 = pygame.image.load("pictures/avatar/luffy_r3.png").convert_alpha()
luffy_r4 = pygame.image.load("pictures/avatar/luffy_r4.png").convert_alpha()

luffy_l = pygame.image.load("pictures/avatar/luffy_l.png").convert_alpha()
luffy_l1 = pygame.image.load("pictures/avatar/luffy_l1.png").convert_alpha()
luffy_l2 = pygame.image.load("pictures/avatar/luffy_l2.png").convert_alpha()
luffy_l3 = pygame.image.load("pictures/avatar/luffy_l3.png").convert_alpha()
luffy_l4 = pygame.image.load("pictures/avatar/luffy_l4.png").convert_alpha()

villager = pygame.image.load("pictures/avatar/bot.png").convert_alpha()

mcdo = pygame.image.load("pictures/items/mcdo.png").convert_alpha()
mcdo2 = pygame.image.load("pictures/items/mcdo2.png").convert_alpha()
mcdo3 = pygame.image.load("pictures/items/mcdo3.png").convert_alpha()
mcdo4 = pygame.image.load("pictures/items/mcdo4.png").convert_alpha()

potion = pygame.image.load("pictures/items/potion.png").convert_alpha()
potion2 = pygame.image.load("pictures/items/potion2.png").convert_alpha()
potion3 = pygame.image.load("pictures/items/potion3.png").convert_alpha()
potion4 = pygame.image.load("pictures/items/potion4.png").convert_alpha()

shield = pygame.image.load("pictures/items/shield.png").convert_alpha()
shield2 = pygame.image.load("pictures/items/shield2.png").convert_alpha()
shield3 = pygame.image.load("pictures/items/shield3.png").convert_alpha()
shield4 = pygame.image.load("pictures/items/shield4.png").convert_alpha()

sword = pygame.image.load("pictures/items/sword.png").convert_alpha()
sword2 = pygame.image.load("pictures/items/sword2.png").convert_alpha()
sword3 = pygame.image.load("pictures/items/sword3.png").convert_alpha()
sword4 = pygame.image.load("pictures/items/sword4.png").convert_alpha()

nothing = pygame.image.load("pictures/nothing.png").convert_alpha()

game_title = pygame.image.load("buttons/home_title.png").convert_alpha()
play_1 = pygame.image.load("buttons/jouer_1.png").convert_alpha()
play_2 = pygame.image.load("buttons/jouer_2.png").convert_alpha()
settings_1 = pygame.image.load("buttons/options_1.png").convert_alpha()
settings_2 = pygame.image.load("buttons/options_2.png").convert_alpha()
commands_1 = pygame.image.load("buttons/commandes_1.png").convert_alpha()
commands_2 = pygame.image.load("buttons/commandes_2.png").convert_alpha()
credits_1 = pygame.image.load("buttons/credits_1.png").convert_alpha()
credits_2 = pygame.image.load("buttons/credits_2.png").convert_alpha()
back_button = pygame.image.load("buttons/retour.png").convert_alpha()
son_on = pygame.image.load("buttons/son_on.png").convert_alpha()
son_off = pygame.image.load("buttons/son_off.png").convert_alpha()

play_r = play_1.get_rect()
play_r.x, play_r.y = 0,240
settings_r = settings_1.get_rect()
settings_r.x, settings_r.y = 0,320
commands_r = commands_1.get_rect()
commands_r.x, commands_r.y = 0,405
credits_r = credits_1.get_rect()
credits_r.x, credits_r.y = 0,490
back_r = back_button.get_rect()
back_r.x, back_r.y = 925,25
son_r = son_on.get_rect()
son_r.x, son_r.y = 325,200

#Fontions

def LoadingMenu():
    screen_wallpaper = screen.blit(wallpaper,(0,-45))
    screen_title = screen.blit(game_title,(0,0))
    play_btn = screen.blit(play_1,(0,240))
    options_btn = screen.blit(settings_1,(0,320))
    commandes_btn = screen.blit(commands_1,(0,405))
    credits_btn = screen.blit(credits_1,(0,490))
    global CurrentMenu
    CurrentMenu = "LoadingMenu"
    print(CurrentMenu + ' loaded')

def Play_intro():
    screen_wallpaper = screen.blit(game_intro,(0,0))
    screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
    sword_display = screen.blit(sword,(254,128))
    pygame.mouse.set_visible(False)
    pygame.key.set_repeat(True)
    pygame.display.flip()
    global CurrentMenu
    CurrentMenu = "PlayMenu"
    print(CurrentMenu + 'loaded')

def Settings_on():
    screen_blur = screen.blit(blur_wallpaper,(0,-45))
    screen_title = screen.blit(game_title,(0,0))
    screen_back = screen.blit(back_button,(925,25))
    screen_sonon = screen.blit(son_on,(325,200))
    pygame.display.flip()
    global CurrentMenu
    CurrentMenu = "SettingsMenu_on"
    print(CurrentMenu + ' loaded')

def Settings_off():
    screen_blur = screen.blit(blur_wallpaper,(0,-45))
    screen_title = screen.blit(game_title,(0,0))
    screen_back = screen.blit(back_button,(925,25))
    screen_sonoff = screen.blit(son_off,(325,200))
    pygame.display.flip()
    global CurrentMenu
    CurrentMenu = "SettingsMenu_off"
    print(CurrentMenu + ' loaded')

def Commands():
    screen_blur = screen.blit(blur_wallpaper,(0,-45))
    screen_commands = screen.blit(commands_screen,(0,0))
    screen_title = screen.blit(game_title,(0,0))
    screen_back = screen.blit(back_button,(925,25))
    pygame.display.flip()
    global CurrentMenu
    CurrentMenu = "CommandsMenu"
    print(CurrentMenu + ' loaded')

def Credits():
    screen_blur = screen.blit(blur_wallpaper,(0,-45))
    screen_credits = screen.blit(credits_screen,(0,0))
    screen_title = screen.blit(game_title,(0,0))
    screen_back = screen.blit(back_button,(925,25))
    pygame.display.flip()
    global CurrentMenu
    CurrentMenu = "CreditsMenu"
    print(CurrentMenu + ' loaded')

def WhichOneIsOnHover():
    if play_r.collidepoint(pygame.mouse.get_pos()) and n != 1:
        screen_wallpaper = screen.blit(wallpaper,(0,-45))
        screen_title = screen.blit(game_title,(0,0))
        play_btn = screen.blit(play_2,(0,240))
        options_btn = screen.blit(settings_1,(0,320))
        commandes_btn = screen.blit(commands_1,(0,405))
        credits_btn = screen.blit(credits_1,(0,490))
        pygame.display.flip()

    if settings_r.collidepoint(pygame.mouse.get_pos()) and n != 1:
        screen_wallpaper = screen.blit(wallpaper,(0,-45))
        screen_title = screen.blit(game_title,(0,0))
        play_btn = screen.blit(play_1,(0,240))
        options_btn = screen.blit(settings_2,(0,320))
        commandes_btn = screen.blit(commands_1,(0,405))
        credits_btn = screen.blit(credits_1,(0,490))
        pygame.display.flip()

    if commands_r.collidepoint(pygame.mouse.get_pos()) and n != 1:
        screen_wallpaper = screen.blit(wallpaper,(0,-45))
        screen_title = screen.blit(game_title,(0,0))
        play_btn = screen.blit(play_1,(0,240))
        options_btn = screen.blit(settings_1,(0,320))
        commandes_btn = screen.blit(commands_2,(0,405))
        credits_btn = screen.blit(credits_1,(0,490))
        pygame.display.flip()

    if credits_r.collidepoint(pygame.mouse.get_pos()) and n != 1:
        screen_wallpaper = screen.blit(wallpaper,(0,-45))
        screen_title = screen.blit(game_title,(0,0))
        play_btn = screen.blit(play_1,(0,240))
        options_btn = screen.blit(settings_1,(0,320))
        commandes_btn = screen.blit(commands_1,(0,405))
        credits_btn = screen.blit(credits_2,(0,490))
        pygame.display.flip()

def PairImpair(x):
    if x%2 == 0:
        return True
    if x%2 !=0:
        return False

def LoadingGame():
    for nb in range(3):
        screen_load_1 = screen.blit(loading_1,(0,0))
        pygame.display.flip()
        time.sleep(0.4)
        screen_load_2 = screen.blit(loading_2,(0,0))
        pygame.display.flip()
        time.sleep(0.4)
        screen_load_3 = screen.blit(loading_3,(0,0))
        pygame.display.flip()
        time.sleep(0.4)

def LuffyPos():
    if luffy_pos == luffy_d:
        screen_luffy = screen.blit(luffy_d_blur,(xluffy,yluffy))

xleft_upforest1 = 56
xleft_upforest2 = 184
yleft_upforest = 229

xleft_house1 = 248
xleft_house2 = 568
yleft_house = 189

xstairs = 504
ystairs = 109

xright_house1 = 632
xright_house2 = 824
yright_house = 269

xleft_stairs = 504
yleft_stairs = 149

xleft_downforest1 = 56
xleft_downforest2 = 120
yleft_downforest = 349

xmiddle_downforest1 = 184
xmiddle_downforest2 = 760
ymiddle_downforest = 469

xright_downforest = 824
yright_downforest = 349

yleft_forestborder1_1 = 229
yleft_forestborder1_2 = 349
xleft_forestborder1 = 56

yleft_forestborder2_1 = 389
yleft_forestborder2_2 = 469
xleft_forestborder2 = 184

yleft_forestborder3_1 = 109
yleft_forestborder3_2 = 189
xleft_forestborder3 = 248

yright_righthouse1 = 189
yright_righthouse2 = 229
xright_righthouse = 568

yright_forest1 = 389
yright_forest2 = 469
xright_forest = 760

yright_map1 = 269
yright_map2 = 349
xright_map = 824

def left_upforest():
    if xluffy >= xleft_upforest1 and xluffy <= xleft_upforest2 and yluffy > yleft_upforest:
        return True
    else:
        return False

def left_house():
    if xluffy >= xleft_house1 and xluffy <= xleft_house2 and yluffy > yleft_house:
        return True
    else:
        return False

def stairs():
    if xluffy == xstairs and yluffy > ystairs:
        return True
    else:
        return False

def right_house():
    if xluffy >= xright_house1 and xluffy <= xright_house2 and yluffy > yright_house:
        return True
    else:
        return False

def left_downforest():
    if xluffy >= xleft_downforest1 and xluffy <= xleft_downforest2 and yluffy < yleft_downforest:
        return True
    else:
        return False

def middle_downforest():
    if xluffy >= xmiddle_downforest1 and xluffy <= xmiddle_downforest2 and yluffy < ymiddle_downforest:
        return True
    else:
        return False

def right_downforest():
    if xluffy == xright_downforest and yluffy < yright_downforest:
        return True
    else:
        return False

def left_stairs():
    if xluffy == xleft_stairs and yluffy == yleft_stairs:
        return False

def left_forestborder1():
    if yluffy >= yleft_forestborder1_1 and yluffy <= yleft_forestborder1_2 and xluffy > xleft_forestborder1:
        return True
    else:
        return False

def left_forestborder2():
    if yluffy >= yleft_forestborder2_1 and yluffy <= yleft_forestborder2_2 and xluffy > xleft_forestborder2:
        return True
    else:
        return False

def left_forestborder3():
    if yluffy >= yleft_forestborder3_1 and yluffy <= yleft_forestborder3_2 and xluffy > xleft_forestborder3:
        return True
    else:
        return False

def right_righthouse():
    if yluffy >= yright_righthouse1 and yluffy <= yright_righthouse2 and xluffy < xright_righthouse:
        return True
    else:
        return False

def right_forest():
    if yluffy >= yright_forest1 and yluffy <= yright_forest2 and xluffy < xright_forest:
        return True
    else:
        return False

def right_map():
    if yluffy >= yright_map1 and yluffy <= yright_map2 and xluffy < xright_map:
        return True
    else:
        return False

def inHouse1():
    global lc
    if lc == 1:
        potion_display = screen.blit(potion,(542,163))

def inHouse2():
    global lc
    if lc == 2:
        mcdo_display = screen.blit(mcdo,(371,140))

def inUphouse():
    global lc
    if lc == 3:
        shield_display = screen.blit(shield,(400,207))

def outdoor():
    global lc
    if lc == 0:
        sword_display = screen.blit(sword,(254,128))

mcdo_list = [mcdo2, mcdo3, mcdo4]
potion_list = [potion2, potion3, potion4]
shield_list = [shield2, shield3, shield4]
sword_list = [sword2, sword3, sword4]

def mcdo_found():
    for i in range(3):
        screen_wallpaper = screen.blit(game_intro,(0,0))
        screen.blit(mcdo_list[i],(371,140))
        screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
        pygame.display.update()
        time.sleep(0.2)
    screen_wallpaper = screen.blit(game_intro,(0,0))
    screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
    pygame.display.update()

def potion_found():
    for i in range(3):
        screen_wallpaper = screen.blit(game_intro,(0,0))
        screen.blit(potion_list[i],(542,163))
        screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
        pygame.display.update()
        time.sleep(0.2)
    screen_wallpaper = screen.blit(game_intro,(0,0))
    screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
    pygame.display.update()

def shield_found():
    for i in range(3):
        screen_wallpaper = screen.blit(game_intro,(0,0))
        screen.blit(shield_list[i],(400,207))
        screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
        pygame.display.update()
        time.sleep(0.2)
    screen_wallpaper = screen.blit(game_intro,(0,0))
    screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
    pygame.display.update()

def sword_found():
    for i in range(3):
        screen_wallpaper = screen.blit(game_intro,(0,0))
        screen.blit(sword_list[i],(254,128))
        screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
        pygame.display.update()
        time.sleep(0.2)
    screen_wallpaper = screen.blit(game_intro,(0,0))
    screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
    pygame.display.update()

#Lancement de la boucle Menu

menu_run = True
n = 0
x = 0
o = 0

if n == 0:
    LoadingMenu()

while menu_run:
    for event in pygame.event.get():
        if event.type == QUIT:
            menu_run = False
            pygame.quit()
            sys.exit()

        if back_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and n == 1:
            LoadingMenu()
            n = 0

        if play_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and n != 1:
            menu_run = False
            n = 2

        if settings_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and n != 1:
            if PairImpair(x) == False:
                Settings_on()
            if PairImpair(x) == True:
                Settings_off()
            n = 1
            o = 1

        if commands_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and n != 1:
            Commands()
            n = 1

        if credits_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and n != 1:
            Credits()
            n = 1

        if son_r.collidepoint(pygame.mouse.get_pos()) and event.type == MOUSEBUTTONDOWN and o == 1:
            if PairImpair(x) == True:
                Settings_on()
            if PairImpair(x) == False:
                Settings_off()
            x += 1
            print(x)

        if n == 0:
            o = 0

    WhichOneIsOnHover()
    pygame.display.flip()

#Chargement du jeu

LoadingGame()

#Lancemant de la boucle du jeu "In Game"

ingame = True
i = 0
s1 = 0
lc = 0
itm_mcdo = 0
itm_potion = 0
itm_shield = 0
itm_sword = 0
xluffy = 824
yluffy = 349
luffy_pos = luffy_d

Play_intro()

while ingame:
    for event in pygame.event.get():
        if event.type == QUIT:
            ingame = False
            pygame.quit()
            sys.exit()

        if xluffy == 56 and yluffy == 309 or yluffy == 349 and event.type == KEYDOWN and event.key == K_LEFT:
            if itm_mcdo and itm_potion and itm_shield and itm_sword == 1:
                LoadingGame()
                ingame = False
                pygame.quit()
                sys.exit()

        if xluffy == 371 and yluffy == 173 and lc == 2 and itm_mcdo == 0:
            mcdo_found()
            itm_mcdo = 1
            mcdo = nothing

        if xluffy == 540 and yluffy == 174 and lc == 1 and itm_potion == 0:
            potion_found()
            itm_potion = 1
            potion = nothing

        if xluffy == 400 and yluffy == 230 and lc == 3 and itm_shield == 0:
            shield_found()
            itm_shield = 1
            shield = nothing

        if xluffy == 248 and yluffy == 149 and lc == 0 and itm_sword == 0:
            sword_found()
            itm_sword = 1
            sword = nothing

        if xluffy == 696 and yluffy == 269 and event.type == KEYDOWN and event.key == K_UP:
            s1 += 1
            lc = 2

            xleft_upforest1 = 243
            xleft_upforest2 = 691
            yleft_upforest = 173

            xleft_house1 = 243
            xleft_house2 = 691
            yleft_house = 173

            xstairs = 0
            ystairs = 0

            xright_house1 = 243
            xright_house2 = 691
            yright_house = 173

            xleft_downforest1 = 243
            xleft_downforest2 = 243
            yleft_downforest = 333

            xmiddle_downforest1 = 307
            xmiddle_downforest2 = 627
            ymiddle_downforest = 413

            xright_downforest = 691
            yright_downforest = 333

            xleft_stairs = 0
            yleft_stairs = 0

            yleft_forestborder1_1 = 373
            yleft_forestborder1_2 = 413
            xleft_forestborder1 = 307

            yleft_forestborder2_1 = 173
            yleft_forestborder2_2 = 333
            xleft_forestborder2 = 243

            yleft_forestborder3_1 = 173
            yleft_forestborder3_2 = 333
            xleft_forestborder3 = 243

            yright_righthouse1 = 373
            yright_righthouse2 = 413
            xright_righthouse = 627

            yright_forest1 = 173
            yright_forest2 = 333
            xright_forest = 691

            yright_map1 = 173
            yright_map2 = 333
            xright_map = 691

            if s1 == 1:
                game_intro = second_house
                screen_wallpaper = screen.blit(game_intro,(0,0))
                mcdo_display = screen.blit(mcdo,(371,140))
                xluffy = 435
                yluffy = 413
                screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
                pygame.display.flip()

        if xluffy == 435 and yluffy == 413 and event.type == KEYDOWN and event.key == K_DOWN and s1 == 1:
            s1 = 0
            lc = 0

            xleft_upforest1 = 56
            xleft_upforest2 = 184
            yleft_upforest = 229

            xleft_house1 = 248
            xleft_house2 = 568
            yleft_house = 189

            xstairs = 504
            ystairs = 109

            xright_house1 = 632
            xright_house2 = 824
            yright_house = 269

            xleft_downforest1 = 56
            xleft_downforest2 = 120
            yleft_downforest = 349

            xmiddle_downforest1 = 184
            xmiddle_downforest2 = 760
            ymiddle_downforest = 469

            xright_downforest = 824
            yright_downforest = 349

            xleft_stairs = 504
            yleft_stairs = 149

            yleft_forestborder1_1 = 229
            yleft_forestborder1_2 = 349
            xleft_forestborder1 = 56

            yleft_forestborder2_1 = 389
            yleft_forestborder2_2 = 469
            xleft_forestborder2 = 184

            yleft_forestborder3_1 = 109
            yleft_forestborder3_2 = 189
            xleft_forestborder3 = 248

            yright_righthouse1 = 189
            yright_righthouse2 = 229
            xright_righthouse = 568

            yright_forest1 = 389
            yright_forest2 = 469
            xright_forest = 760

            yright_map1 = 269
            yright_map2 = 349
            xright_map = 824

            game_intro = pygame.image.load("pictures/game_intro.png").convert()
            screen_wallpaper = screen.blit(game_intro,(0,0))
            sword_display = screen.blit(sword,(254,128))
            xluffy = 696
            yluffy = 269
            screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
            pygame.display.flip()

        if xluffy == 376 and yluffy == 189 and event.type == KEYDOWN and event.key == K_UP:
            s1 += 1
            lc = 1

            xleft_upforest1 = 284
            xleft_upforest2 = 668
            yleft_upforest = 174

            xleft_house1 = 284
            xleft_house2 = 668
            yleft_house = 174

            xstairs = 0
            ystairs = 0

            xright_house1 = 284
            xright_house2 = 668
            yright_house = 174

            xleft_downforest1 = 284
            xleft_downforest2 = 284
            yleft_downforest = 334

            xmiddle_downforest1 = 348
            xmiddle_downforest2 = 668
            ymiddle_downforest = 414

            xright_downforest = 0
            yright_downforest = 0

            xleft_stairs = 0
            yleft_stairs = 0

            yleft_forestborder1_1 = 374
            yleft_forestborder1_2 = 414
            xleft_forestborder1 = 348

            yleft_forestborder2_1 = 174
            yleft_forestborder2_2 = 334
            xleft_forestborder2 = 284

            yleft_forestborder3_1 = 174
            yleft_forestborder3_2 = 334
            xleft_forestborder3 = 284

            yright_righthouse1 = 174
            yright_righthouse2 = 414
            xright_righthouse = 668

            yright_forest1 = 174
            yright_forest2 = 414
            xright_forest = 668

            yright_map1 = 174
            yright_map2 = 414
            xright_map = 668

            if s1 == 1:
                game_intro = first_house
                screen_wallpaper = screen.blit(game_intro,(0,0))
                potion_display = screen.blit(potion,(542,163))
                xluffy = 540
                yluffy = 414
                screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
                pygame.display.flip()

        if xluffy == 540 and yluffy == 414 and event.type == KEYDOWN and event.key == K_DOWN and s1 == 1:
            s1 = 0
            lc = 0

            xleft_upforest1 = 56
            xleft_upforest2 = 184
            yleft_upforest = 229

            xleft_house1 = 248
            xleft_house2 = 568
            yleft_house = 189

            xstairs = 504
            ystairs = 109

            xright_house1 = 632
            xright_house2 = 824
            yright_house = 269

            xleft_downforest1 = 56
            xleft_downforest2 = 120
            yleft_downforest = 349

            xmiddle_downforest1 = 184
            xmiddle_downforest2 = 760
            ymiddle_downforest = 469

            xright_downforest = 824
            yright_downforest = 349

            xleft_stairs = 504
            yleft_stairs = 149

            yleft_forestborder1_1 = 229
            yleft_forestborder1_2 = 349
            xleft_forestborder1 = 56

            yleft_forestborder2_1 = 389
            yleft_forestborder2_2 = 469
            xleft_forestborder2 = 184

            yleft_forestborder3_1 = 109
            yleft_forestborder3_2 = 189
            xleft_forestborder3 = 248

            yright_righthouse1 = 189
            yright_righthouse2 = 229
            xright_righthouse = 568

            yright_forest1 = 389
            yright_forest2 = 469
            xright_forest = 760

            yright_map1 = 269
            yright_map2 = 349
            xright_map = 824

            game_intro = pygame.image.load("pictures/game_intro.png").convert()
            screen_wallpaper = screen.blit(game_intro,(0,0))
            sword_display = screen.blit(sword,(254,128))
            xluffy = 376
            yluffy = 189
            screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
            pygame.display.flip()

        if xluffy == 504 and yluffy == 109 and event.type == KEYDOWN and event.key == K_LEFT:
            s1 += 1
            lc = 3

            xleft_upforest1 = 272
            xleft_upforest2 = 592
            yleft_upforest = 230

            xleft_house1 = 272
            xleft_house2 = 592
            yleft_house = 230

            xstairs = 656
            ystairs = 270

            xright_house1 = 272
            xright_house2 = 592
            yright_house = 230

            xleft_stairs = 0
            yleft_stairs = 0

            xleft_downforest1 = 272
            xleft_downforest2 = 592
            yleft_downforest = 430

            xmiddle_downforest1 = 272
            xmiddle_downforest2 = 592
            ymiddle_downforest = 430

            xright_downforest = 656
            yright_downforest = 310

            yleft_forestborder1_1 = 230
            yleft_forestborder1_2 = 440
            xleft_forestborder1 = 272

            yleft_forestborder2_1 = 230
            yleft_forestborder2_2 = 440
            xleft_forestborder2 = 272

            yleft_forestborder3_1 = 230
            yleft_forestborder3_2 = 440
            xleft_forestborder3 = 272

            yright_righthouse1 = 230
            yright_righthouse2 = 230
            xright_righthouse = 592

            yright_forest1 = 350
            yright_forest2 = 430
            xright_forest = 592

            yright_map1 = 270
            yright_map2 = 310
            xright_map = 656

            if s1 == 1:
                game_intro = up_house
                screen_wallpaper = screen.blit(game_intro,(0,0))
                shield_display = screen.blit(shield,(400,207))
                xluffy = 592
                yluffy = 230
                screen_luffy = screen.blit(luffy_l,(xluffy,yluffy))
                pygame.display.flip()

        if xluffy == 592 and yluffy == 230 and event.type == KEYDOWN and event.key == K_RIGHT and s1 == 1:
            s1 = 0
            lc = 0

            xleft_upforest1 = 56
            xleft_upforest2 = 184
            yleft_upforest = 229

            xleft_house1 = 248
            xleft_house2 = 568
            yleft_house = 189

            xstairs = 504
            ystairs = 109

            xright_house1 = 632
            xright_house2 = 824
            yright_house = 269

            xleft_downforest1 = 56
            xleft_downforest2 = 120
            yleft_downforest = 349

            xmiddle_downforest1 = 184
            xmiddle_downforest2 = 760
            ymiddle_downforest = 469

            xright_downforest = 824
            yright_downforest = 349

            xleft_stairs = 504
            yleft_stairs =149

            yleft_forestborder1_1 = 229
            yleft_forestborder1_2 = 349
            xleft_forestborder1 = 56

            yleft_forestborder2_1 = 389
            yleft_forestborder2_2 = 469
            xleft_forestborder2 = 184

            yleft_forestborder3_1 = 109
            yleft_forestborder3_2 = 189
            xleft_forestborder3 = 248

            yright_righthouse1 = 189
            yright_righthouse2 = 229
            xright_righthouse = 568

            yright_forest1 = 389
            yright_forest2 = 469
            xright_forest = 760

            yright_map1 = 269
            yright_map2 = 349
            xright_map = 824

            game_intro = pygame.image.load("pictures/game_intro.png").convert()
            screen_wallpaper = screen.blit(game_intro,(0,0))
            sword_display = screen.blit(sword,(254,128))
            xluffy = 504
            yluffy = 109
            screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
            pygame.display.flip()

        if event.type == KEYDOWN and event.key == K_RIGHT:
            if right_righthouse() or right_forest() or right_map():
                luffy_pos = luffy_r
                xluffy += 8
                screen_wallpaper = screen.blit(game_intro,(0,0))
                inHouse1()
                inHouse2()
                inUphouse()
                outdoor()
                screen_luffy = screen.blit(luffy_r1,(xluffy,yluffy))
                pygame.display.flip()
                time.sleep(0.075)

                if event.type == KEYDOWN and event.key == K_RIGHT:
                    xluffy += 8
                    screen_wallpaper = screen.blit(game_intro,(0,0))
                    inHouse1()
                    inHouse2()
                    inUphouse()
                    outdoor()
                    screen_luffy = screen.blit(luffy_r2,(xluffy,yluffy))
                    pygame.display.flip()
                    time.sleep(0.075)

                    if event.type == KEYDOWN and event.key == K_RIGHT:
                        xluffy += 8
                        screen_wallpaper = screen.blit(game_intro,(0,0))
                        inHouse1()
                        inHouse2()
                        inUphouse()
                        outdoor()
                        screen_luffy = screen.blit(luffy_r1,(xluffy,yluffy))
                        pygame.display.flip()
                        time.sleep(0.075)

                        if event.type == KEYDOWN and event.key == K_RIGHT:
                            xluffy += 8
                            screen_wallpaper = screen.blit(game_intro,(0,0))
                            inHouse1()
                            inHouse2()
                            inUphouse()
                            outdoor()
                            screen_luffy = screen.blit(luffy_r,(xluffy,yluffy))
                            pygame.display.flip()
                            time.sleep(0.075)

                            if event.type == KEYDOWN and event.key == K_RIGHT:
                                xluffy += 8
                                screen_wallpaper = screen.blit(game_intro,(0,0))
                                inHouse1()
                                inHouse2()
                                inUphouse()
                                outdoor()
                                screen_luffy = screen.blit(luffy_r3,(xluffy,yluffy))
                                pygame.display.flip()
                                time.sleep(0.075)

                                if event.type == KEYDOWN and event.key == K_RIGHT:
                                    xluffy += 8
                                    screen_wallpaper = screen.blit(game_intro,(0,0))
                                    inHouse1()
                                    inHouse2()
                                    inUphouse()
                                    outdoor()
                                    screen_luffy = screen.blit(luffy_r4,(xluffy,yluffy))
                                    pygame.display.flip()
                                    time.sleep(0.075)

                                    if event.type == KEYDOWN and event.key == K_RIGHT:
                                        xluffy += 8
                                        screen_wallpaper = screen.blit(game_intro,(0,0))
                                        inHouse1()
                                        inHouse2()
                                        inUphouse()
                                        outdoor()
                                        screen_luffy = screen.blit(luffy_r3,(xluffy,yluffy))
                                        pygame.display.flip()
                                        time.sleep(0.075)

                                        if event.type == KEYDOWN and event.key == K_RIGHT:
                                            xluffy += 8
                                            screen_wallpaper = screen.blit(game_intro,(0,0))
                                            inHouse1()
                                            inHouse2()
                                            inUphouse()
                                            outdoor()
                                            screen_luffy = screen.blit(luffy_r,(xluffy,yluffy))
                                            pygame.display.flip()
                                            time.sleep(0.075)

        print(xluffy)
        print(yluffy)

        if event.type == KEYDOWN and event.key == K_LEFT:
            if left_forestborder1() or left_forestborder2() or left_forestborder3() or left_stairs():
                luffy_pos = luffy_l
                xluffy -= 8
                screen_wallpaper = screen.blit(game_intro,(0,0))
                inHouse1()
                inHouse2()
                inUphouse()
                outdoor()
                screen_luffy = screen.blit(luffy_l1,(xluffy,yluffy))
                pygame.display.flip()
                time.sleep(0.075)

                if event.type == KEYDOWN and event.key == K_LEFT:
                    xluffy -= 8
                    screen_wallpaper = screen.blit(game_intro,(0,0))
                    inHouse1()
                    inHouse2()
                    inUphouse()
                    outdoor()
                    screen_luffy = screen.blit(luffy_l2,(xluffy,yluffy))
                    pygame.display.flip()
                    time.sleep(0.075)

                    if event.type == KEYDOWN and event.key == K_LEFT:
                        xluffy -= 8
                        screen_wallpaper = screen.blit(game_intro,(0,0))
                        inHouse1()
                        inHouse2()
                        inUphouse()
                        outdoor()
                        screen_luffy = screen.blit(luffy_l1,(xluffy,yluffy))
                        pygame.display.flip()
                        time.sleep(0.075)

                        if event.type == KEYDOWN and event.key == K_LEFT:
                            xluffy -= 8
                            screen_wallpaper = screen.blit(game_intro,(0,0))
                            inHouse1()
                            inHouse2()
                            inUphouse()
                            outdoor()
                            screen_luffy = screen.blit(luffy_l,(xluffy,yluffy))
                            pygame.display.flip()
                            time.sleep(0.075)

                            if event.type == KEYDOWN and event.key == K_LEFT:
                                xluffy -= 8
                                screen_wallpaper = screen.blit(game_intro,(0,0))
                                inHouse1()
                                inHouse2()
                                inUphouse()
                                outdoor()
                                screen_luffy = screen.blit(luffy_l3,(xluffy,yluffy))
                                pygame.display.flip()
                                time.sleep(0.075)

                                if event.type == KEYDOWN and event.key == K_LEFT:
                                    xluffy -= 8
                                    screen_wallpaper = screen.blit(game_intro,(0,0))
                                    inHouse1()
                                    inHouse2()
                                    inUphouse()
                                    outdoor()
                                    screen_luffy = screen.blit(luffy_l4,(xluffy,yluffy))
                                    pygame.display.flip()
                                    time.sleep(0.075)

                                    if event.type == KEYDOWN and event.key == K_LEFT:
                                        xluffy -= 8
                                        screen_wallpaper = screen.blit(game_intro,(0,0))
                                        inHouse1()
                                        inHouse2()
                                        inUphouse()
                                        outdoor()
                                        screen_luffy = screen.blit(luffy_l3,(xluffy,yluffy))
                                        pygame.display.flip()
                                        time.sleep(0.075)

                                        if event.type == KEYDOWN and event.key == K_LEFT:
                                            xluffy -= 8
                                            screen_wallpaper = screen.blit(game_intro,(0,0))
                                            inHouse1()
                                            inHouse2()
                                            inUphouse()
                                            outdoor()
                                            screen_luffy = screen.blit(luffy_l,(xluffy,yluffy))
                                            pygame.display.flip()
                                            time.sleep(0.075)

        print(xluffy)
        print(yluffy)

        if event.type == KEYDOWN and event.key == K_UP:
            if left_upforest() or left_house() or right_house() or stairs():
                luffy_pos = luffy_u
                yluffy -= 5
                screen_wallpaper = screen.blit(game_intro,(0,0))
                inHouse1()
                inHouse2()
                inUphouse()
                outdoor()
                screen_luffy = screen.blit(luffy_u1,(xluffy,yluffy))
                pygame.display.flip()
                time.sleep(0.075)

                if event.type == KEYDOWN and event.key == K_UP:
                    yluffy -= 5
                    screen_wallpaper = screen.blit(game_intro,(0,0))
                    inHouse1()
                    inHouse2()
                    inUphouse()
                    outdoor()
                    screen_luffy = screen.blit(luffy_u2,(xluffy,yluffy))
                    pygame.display.flip()
                    time.sleep(0.075)

                    if event.type == KEYDOWN and event.key == K_UP:
                        yluffy -= 5
                        screen_wallpaper = screen.blit(game_intro,(0,0))
                        inHouse1()
                        inHouse2()
                        inUphouse()
                        outdoor()
                        screen_luffy = screen.blit(luffy_u1,(xluffy,yluffy))
                        pygame.display.flip()
                        time.sleep(0.075)

                        if event.type == KEYDOWN and event.key == K_UP:
                            yluffy -= 5
                            screen_wallpaper = screen.blit(game_intro,(0,0))
                            inHouse1()
                            inHouse2()
                            inUphouse()
                            outdoor()
                            screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
                            pygame.display.flip()
                            time.sleep(0.075)

                            if event.type == KEYDOWN and event.key == K_UP:
                                yluffy -= 5
                                screen_wallpaper = screen.blit(game_intro,(0,0))
                                inHouse1()
                                inHouse2()
                                inUphouse()
                                outdoor()
                                screen_luffy = screen.blit(luffy_u3,(xluffy,yluffy))
                                pygame.display.flip()
                                time.sleep(0.075)

                                if event.type == KEYDOWN and event.key == K_UP:
                                    yluffy -= 5
                                    screen_wallpaper = screen.blit(game_intro,(0,0))
                                    inHouse1()
                                    inHouse2()
                                    inUphouse()
                                    outdoor()
                                    screen_luffy = screen.blit(luffy_u4,(xluffy,yluffy))
                                    pygame.display.flip()
                                    time.sleep(0.075)

                                    if event.type == KEYDOWN and event.key == K_UP:
                                        yluffy -= 5
                                        screen_wallpaper = screen.blit(game_intro,(0,0))
                                        inHouse1()
                                        inHouse2()
                                        inUphouse()
                                        outdoor()
                                        screen_luffy = screen.blit(luffy_u3,(xluffy,yluffy))
                                        pygame.display.flip()
                                        time.sleep(0.075)

                                        if event.type == KEYDOWN and event.key == K_UP:
                                            yluffy -= 5
                                            screen_wallpaper = screen.blit(game_intro,(0,0))
                                            inHouse1()
                                            inHouse2()
                                            inUphouse()
                                            outdoor()
                                            screen_luffy = screen.blit(luffy_u,(xluffy,yluffy))
                                            pygame.display.flip()
                                            time.sleep(0.075)

        print(xluffy)
        print(yluffy)

        if event.type == KEYDOWN and event.key == K_DOWN:
            if left_downforest() or middle_downforest() or right_downforest():
                luffy_pos = luffy_d
                yluffy += 5
                screen_wallpaper = screen.blit(game_intro,(0,0))
                inHouse1()
                inHouse2()
                inUphouse()
                outdoor()
                screen_luffy = screen.blit(luffy_d1,(xluffy,yluffy))
                pygame.display.flip()
                time.sleep(0.075)

                if event.type == KEYDOWN and event.key == K_DOWN:
                    yluffy += 5
                    screen_wallpaper = screen.blit(game_intro,(0,0))
                    inHouse1()
                    inHouse2()
                    inUphouse()
                    outdoor()
                    screen_luffy = screen.blit(luffy_d2,(xluffy,yluffy))
                    pygame.display.flip()
                    time.sleep(0.075)

                    if event.type == KEYDOWN and event.key == K_DOWN:
                        yluffy += 5
                        screen_wallpaper = screen.blit(game_intro,(0,0))
                        inHouse1()
                        inHouse2()
                        inUphouse()
                        outdoor()
                        screen_luffy = screen.blit(luffy_d1,(xluffy,yluffy))
                        pygame.display.flip()
                        time.sleep(0.075)

                        if event.type == KEYDOWN and event.key == K_DOWN:
                            yluffy += 5
                            screen_wallpaper = screen.blit(game_intro,(0,0))
                            inHouse1()
                            inHouse2()
                            inUphouse()
                            outdoor()
                            screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
                            pygame.display.flip()
                            time.sleep(0.075)

                            if event.type == KEYDOWN and event.key == K_DOWN:
                                yluffy += 5
                                screen_wallpaper = screen.blit(game_intro,(0,0))
                                inHouse1()
                                inHouse2()
                                inUphouse()
                                outdoor()
                                screen_luffy = screen.blit(luffy_d3,(xluffy,yluffy))
                                pygame.display.flip()
                                time.sleep(0.075)

                                if event.type == KEYDOWN and event.key == K_DOWN:
                                    yluffy += 5
                                    screen_wallpaper = screen.blit(game_intro,(0,0))
                                    inHouse1()
                                    inHouse2()
                                    inUphouse()
                                    outdoor()
                                    screen_luffy = screen.blit(luffy_d4,(xluffy,yluffy))
                                    pygame.display.flip()
                                    time.sleep(0.075)

                                    if event.type == KEYDOWN and event.key == K_DOWN:
                                        yluffy += 5
                                        screen_wallpaper = screen.blit(game_intro,(0,0))
                                        inHouse1()
                                        inHouse2()
                                        inUphouse()
                                        outdoor()
                                        screen_luffy = screen.blit(luffy_d3,(xluffy,yluffy))
                                        pygame.display.flip()
                                        time.sleep(0.075)

                                        if event.type == KEYDOWN and event.key == K_DOWN:
                                            yluffy += 5
                                            screen_wallpaper = screen.blit(game_intro,(0,0))
                                            inHouse1()
                                            inHouse2()
                                            inUphouse()
                                            outdoor()
                                            screen_luffy = screen.blit(luffy_d,(xluffy,yluffy))
                                            pygame.display.flip()
                                            time.sleep(0.075)

            print(xluffy)
            print(yluffy)


import multiprocessing  as mp
import numpy as np
import pandas as pd
import requests
import time
from bs4 import BeautifulSoup

# Get all listings for Berlin
wohnung_url = []

# Time tracking
start_time = time.time()

for page in range(200):
    PAGE_URL \
        = (f'https://www.immobilienscout24.de/Suche/S-T/P-{page}/Wohnung-Miete/Berlin/Berlin')

    res = requests.get(PAGE_URL)
    html_page = res.content
    soup = BeautifulSoup(html_page, 'html.parser')

    expose_keys = []

    href_links = soup.findAll('a')
    for link in href_links:
        expose_keys.append(link['href'])

    expose_keys = pd.Series(expose_keys)
    expose_keys = expose_keys[expose_keys.str.startswith("/expose/")]
    expose_keys = expose_keys.unique()
    expose_keys = [w.replace('/expose/', '') for w in expose_keys]

    for key in expose_keys:
        wohnung_url.append((page, f'https://www.immobilienscout24.de/expose/{key}'))

elapsed_time = time.time() - start_time
fetching_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print(f'Fetching listings time: {fetching_time}')
print(f'Number of listings: {len(wohnung_url)}')

# DF with listings
df_url_listings = pd.DataFrame(wohnung_url)
df_url_listings.columns = ['page', 'url']

# Get content of the listings
def get_listings_data(listing):
    try:
        res = requests.get(listing)
        html_page = res.content
        soup = BeautifulSoup(html_page, 'html.parser')
    except:
        print('Listing Non-extistent')

    try:
        listing_address = soup.find(class_='block font-nowrap print-hide')
        listing_address = listing_address.contents
    except:
        listing_address = 'N/A'

    try:
        listing_zip_district = soup.find(class_='zip-region-and-country')
        listing_zip_district = listing_zip_district.contents
    except:
        listing_zip_district = 'N/A'

    try:
        listing_kalte = soup.find(class_='is24qa-kaltmiete is24-value font-semibold')
        listing_kalte = listing_kalte.contents
    except:
        listing_kalte = 'N/A'

    try:
        listing_zimmer = soup.find(class_='is24qa-zi is24-value font-semibold')
        listing_zimmer = listing_zimmer.contents
    except:
        listing_zimmer = 'N/A'

    try:
        listing_flaeche = soup.find(class_='is24qa-flaeche is24-value font-semibold')
        listing_flaeche = listing_flaeche.contents
    except:
        listing_flaeche = 'N/A'

    try:
        listing_bezugsfrei = soup.find(class_='is24qa-bezugsfrei-ab grid-item three-fifths')
        listing_bezugsfrei = listing_bezugsfrei.contents
    except:
        listing_bezugsfrei = 'N/A'

    try:
        listing_etage = soup.find(class_='is24qa-etage grid-item three-fifths')
        listing_etage = listing_etage.contents
    except:
        listing_etage = 'N/A'

    try:
        listing_zimmer_ii = soup.find(class_='is24qa-zimmer grid-item three-fifths')
        listing_zimmer_ii = listing_zimmer_ii.contents
    except:
        listing_zimmer_ii = 'N/A'

    try:
        listing_kalte_ii = soup.find(class_='is24qa-kaltmiete grid-item three-fifths')
        listing_kalte_ii = listing_kalte_ii.contents
    except:
        listing_kalte_ii = 'N/A'

    try:
        listing_nebenkosten = soup.find(class_='is24qa-nebenkosten grid-item three-fifths')
        listing_nebenkosten = listing_nebenkosten.contents
    except:
        listing_nebenkosten = 'N/A'

    try:
        listing_heizkosten = soup.find(class_='is24qa-heizkosten grid-item three-fifths')
        listing_heizkosten = listing_heizkosten.contents
    except:
        listing_heizkosten = 'N/A'

    try:
        listing_gesamtmiete = soup.find(class_='is24qa-gesamtmiete grid-item three-fifths font-bold')
        listing_gesamtmiete = listing_gesamtmiete.contents
    except:
        listing_gesamtmiete = 'N/A'

    try:
        listing_genossenschaftsanteile = soup.find(class_='is24qa-kaution-o-genossenschaftsanteile')
        listing_genossenschaftsanteile = listing_genossenschaftsanteile.contents
    except:
        listing_genossenschaftsanteile = 'N/A'

    try:
        listing_baujahr = soup.find(class_='is24qa-baujahr grid-item three-fifths')
        listing_baujahr = listing_baujahr.contents
    except:
        listing_baujahr = 'N/A'

    try:
        listing_sanierung = soup.find(class_='is24qa-modernisierung-sanierung grid-item three-fifths')
        listing_sanierung = listing_sanierung.contents
    except:
        listing_sanierung = 'N/A'

    try:
        listing_objektzustand = soup.find(class_='is24qa-objektzustand grid-item three-fifths')
        listing_objektzustand = listing_objektzustand.contents
    except:
        listing_objektzustand = 'N/A'

    try:
        listing_ausstattung = soup.find(class_='is24qa-qualitaet-der-ausstattung grid-item three-fifths')
        listing_ausstattung = listing_ausstattung.contents
    except:
        listing_ausstattung = 'N/A'

    try:
        listing_heizungsart = soup.find(class_='is24qa-heizungsart grid-item three-fifths')
        listing_heizungsart = listing_heizungsart.contents
    except:
        listing_heizungsart = 'N/A'

    try:
        listing_energietraeger = soup.find(class_='is24qa-wesentliche-energietraeger grid-item three-fifths')
        listing_energietraeger = listing_energietraeger.contents
    except:
        listing_energietraeger = 'N/A'

    try:
        listing_energieausweis = soup.find(class_='is24qa-energieausweis grid-item three-fifths')
        listing_energieausweis = listing_energieausweis.contents
    except:
        listing_energieausweis = 'N/A'

    try:
        listing_energieausweistyp = soup.find(class_='is24qa-energieausweistyp grid-item three-fifths')
        listing_energieausweistyp = listing_energieausweistyp.contents
    except:
        listing_energieausweistyp = 'N/A'

    try:
        listing_endenergieverbrauch = soup.find(class_='is24qa-endenergieverbrauch grid-item three-fifths')
        listing_endenergieverbrauch = listing_endenergieverbrauch.contents
    except:
        listing_endenergieverbrauch = 'N/A'

    return np.array((
        listing_zip_district[0],
        listing_address[0],
        listing_kalte[0],
        listing_zimmer[0],
        listing_flaeche[0],
        listing_bezugsfrei[0],
        listing_etage[0],
        listing_zimmer_ii[0],
        listing_kalte_ii[0],
        listing_nebenkosten[0],
        listing_heizkosten[0],
        listing_gesamtmiete[0],
        listing_genossenschaftsanteile[0],
        listing_baujahr[0],
        listing_sanierung[0],
        listing_objektzustand[0],
        listing_ausstattung[0],
        listing_heizungsart[0],
        listing_energietraeger[0],
        listing_energieausweis[0],
        listing_energieausweistyp[0],
        listing_endenergieverbrauch[0],
        listing,
    ))

# DF object that will store the data
df_wohnung_data = pd.DataFrame()
wohnung_listings = df_url_listings['url'].unique()

# Main function with Multiprocessing to fetch all data
def main(df):
    pool = mp.Pool(mp.cpu_count())
    result = pool.map(get_listings_data, wohnung_listings)
    df = df.append(pd.DataFrame(result))
    return df

# Time tracking
start_time = time.time()

# Call the main wrapper with multiprocessing
df_wohnung_data = main(df_wohnung_data)

elapsed_time = time.time() - start_time
fetching_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print(f'Fetching Time: {fetching_time}')

# Column and data adjustments
df_wohnung_data.columns = [
    'zip_district',
    'address',
    'kalte',
    'zimmer',
    'flaeche',
    'bezugsfrei',
    'etage',
    'zimmer_ii',
    'kalte_ii',
    'nebenkosten',
    'heizkosten',
    'gesamtmiete',
    'genossenschaftsanteile',
    'baujahr',
    'sanierung',
    'objektzustand',
    'ausstattung',
    'heizungsart',
    'energietraeger',
    'energieausweis',
    'energieausweistyp',
    'endenergieverbrauch',
    'listing_url',
]

df__district_ = df_wohnung_data['zip_district'].str.split(',', expand=True)
df__district_ = df__district_[[0, 1]]
df_wohnung_data[['zipcode_city', 'District']] = df__district_
df__zipcode = df_wohnung_data['zipcode_city'].str.split(' ', expand=True)
df__zipcode = df__zipcode[[0, 1]]
df_wohnung_data[['zipcode', 'city']] = df__zipcode
df_wohnung_data['kalte'] = df_wohnung_data['kalte'].str.replace('€','')
df_wohnung_data['kalte'] = df_wohnung_data['kalte'].str.replace(' ','')
df_wohnung_data['kalte'] = df_wohnung_data['kalte'].str.replace('.','')
df_wohnung_data['kalte'] = df_wohnung_data['kalte'].str.replace(',','.')
df_wohnung_data['kalte_ii'] = df_wohnung_data['kalte_ii'].str.replace('€','')
df_wohnung_data['kalte_ii'] = df_wohnung_data['kalte_ii'].str.replace(' ','')
df_wohnung_data['kalte_ii'] = df_wohnung_data['kalte_ii'].str.replace('.','')
df_wohnung_data['kalte_ii'] = df_wohnung_data['kalte_ii'].str.replace(',','.')
df_wohnung_data['gesamtmiete'] = df_wohnung_data['gesamtmiete'].str.replace('€','')
df_wohnung_data['gesamtmiete'] = df_wohnung_data['gesamtmiete'].str.replace(' ','')
df_wohnung_data['flaeche'] = df_wohnung_data['flaeche'].str.replace('m²','')
df_wohnung_data['flaeche'] = df_wohnung_data['flaeche'].str.replace(' ','')

# Save to disk in excel and csv
df_wohnung_data.to_excel("immo24_urutu.xlsx", index=False)
df_wohnung_data.to_csv("immo24_urutu.csv", index=False)


from __future__ import absolute_import
from functools import partial
import json
import datetime

import six
from colander import (SchemaNode, Mapping, Sequence, Length, String,
                      null, drop, Invalid, Boolean)
from pyramid.security import Everyone

from daybed.backends.exceptions import ModelNotFound, PolicyNotFound
from . import registry, TypeFieldNode

class DefinitionValidator(SchemaNode):
    def __init__(self):
        super(DefinitionValidator, self).__init__(Mapping())
        self.add(SchemaNode(String(), name='title'))
        self.add(SchemaNode(String(), name='description'))
        self.add(SchemaNode(Sequence(), SchemaNode(TypeFieldNode()),
                            name='fields', validator=Length(min=1)))

class RolesValidator(SchemaNode):
    def __init__(self):
        super(RolesValidator, self).__init__(Mapping(unknown='preserve'))
        self.add(SchemaNode(Sequence(), SchemaNode(String()),
                            name='admins', validator=Length(min=1)))

        # XXX Control that the values of the sequence are valid users.
        # (we need to merge master to fix this. see #86)

class PolicyValidator(SchemaNode):
    def __init__(self):
        super(PolicyValidator, self).__init__(Mapping(unknown='preserve'))

    def _crudSchema(self, domain):
        crudSchema = SchemaNode(Mapping(unknown='raise'),
                                name=domain, missing=drop)
        crudSchema.add(SchemaNode(Boolean(), name='create', missing=drop))
        crudSchema.add(SchemaNode(Boolean(), name='read', missing=drop))
        crudSchema.add(SchemaNode(Boolean(), name='update', missing=drop))
        crudSchema.add(SchemaNode(Boolean(), name='delete', missing=drop))
        return crudSchema

    def deserialize(self, cstruct=null):
        if cstruct:
            self.children = []
            self.add(SchemaNode(String(), name='title', missing=drop))
            self.add(SchemaNode(String(), name='description', missing=drop))
            roles = [key for key in six.iterkeys(cstruct)
                     if key not in ('title', 'description')]
            for key in roles:
                permSchema = SchemaNode(Mapping(unknown='raise'), name=key)
                for domain in ('definition', 'records', 'users', 'policy'):
                    permSchema.add(self._crudSchema(domain))
                self.add(permSchema)
        return super(PolicyValidator, self).deserialize(cstruct)

class RecordValidator(SchemaNode):
    def __init__(self, definition):
        super(RecordValidator, self).__init__(Mapping())
        for field in definition['fields']:
            fieldtype = field.pop('type')
            self.add(registry.validation(fieldtype, **field))

def validate_against_schema(request, schema, data):
    try:
        data_pure = schema.deserialize(data)
        data_clean = post_serialize(data_pure)
        # Attach data_clean to request: see usage in views.
        request.data_clean = data_clean
    except Invalid as e:
        for error in e.children:
            # here we transform the errors we got from colander into cornice
            # errors
            for field, error in error.asdict().items():
                request.errors.add('body', field, error)

def post_serialize(data):
    """Returns the most agnostic version of specified data.
    (remove colander notions, datetimes in ISO, ...)
    """
    clean = dict()
    for k, v in data.items():
        if isinstance(v, (datetime.date, datetime.datetime)):
            clean[k] = v.isoformat()
        elif v is null:
            pass
        else:
            clean[k] = v
    return clean

def validator(request, schema):
    """Validates the request according to the given schema"""
    try:
        body = request.body.decode('utf-8')
        dictbody = json.loads(body) if body else {}
        validate_against_schema(request, schema, dictbody)
    except ValueError as e:
        request.errors.add('body', 'body', six.text_type(e))

#  Validates a request body according model definition schema.
definition_validator = partial(validator, schema=DefinitionValidator())

def record_validator(request):
    """Validates a request body according to its model definition.
    """
    model_id = request.matchdict['model_id']

    try:
        definition = request.db.get_model_definition(model_id)
        schema = RecordValidator(definition)
        validator(request, schema)
    except ModelNotFound:
        request.errors.add('path', 'modelname',
                           'Unknown model %s' % model_id)
        request.errors.status = 404

policy_validator = partial(validator, schema=PolicyValidator())

def model_validator(request):
    """Verify that the model is okay (that we have the right fields) and
    eventually populates it if there is a need to.
    """
    try:
        body = json.loads(request.body.decode('utf-8'))
    except ValueError:
        request.errors.add('body', 'json value error', "body malformed")
        return

    # Check the definition is valid.
    definition = body.get('definition')
    if not definition:
        request.errors.add('body', 'definition', 'definition is required')
    else:
        validate_against_schema(request, DefinitionValidator(), definition)
    request.validated['definition'] = definition

    # Check that the records are valid according to the definition.
    records = body.get('records')
    request.validated['records'] = []
    if records:
        definition_validator = RecordValidator(definition)
        for record in records:
            validate_against_schema(request, definition_validator, record)
            request.validated['records'].append(record)

    # Check that roles are valid.
    if request.user:
        default_roles = {'admins': [request.user['name']]}
    else:
        default_roles = {'admins': [Everyone]}
    roles = body.get('roles', default_roles)
    validate_against_schema(request, RolesValidator(), roles)

    request.validated['roles'] = roles
    policy_id = body.get('policy_id', request.registry.default_policy)

    # Check that the policy exists in our db.
    try:
        request.db.get_policy(policy_id)
    except PolicyNotFound:
        request.errors.add('body', 'policy_id',
                           "policy '%s' doesn't exist" % policy_id)
    request.validated['policy_id'] = policy_id


# -*- coding: utf-8 -*-

import os
import random
import numpy as np
import torch
import torch.utils.data as data
import pymesh
from tqdm import tqdm
from utils import shapenet_labels
import logging
from time import time
import open3d as o3d
from deco import pc_info

def scale_linear_bycolumn(rawdata, high=1.0, low=0.0):
    mins = np.min(rawdata, axis=0)
    maxs = np.max(rawdata, axis=0)
    rng = maxs - mins
    return high - (high-low)*(maxs-rawdata)/(rng+np.finfo(np.float32).eps)

class ClsDataset(data.Dataset):
        '''Object classification on ModelNet'''
        def __init__(self, root, npoints=1024, train=True):
                self.root = root
                self.npoints = npoints
                self.catfile = os.path.join(self.root, 'modelnet_cat2num.txt')
                self.cat = {}

                with open(self.catfile, 'r') as f:
                        for line in f.readlines():
                                lns = line.strip().split()
                                self.cat[lns[0]] = lns[1]
                self.num_classes = len(self.cat)
                self.datapath = []
                FLAG = 'train' if train else 'test'
                for item in os.listdir(self.root):
                        if os.path.isdir(os.path.join(self.root, item)):
                                for f in os.listdir(os.path.join(self.root, item, FLAG)):
                                    if f.endswith('.off'):
                                        self.datapath.append((os.path.join(self.root, item, FLAG, f), int(self.cat[item])))

        def __getitem__(self, idx):
                fn = self.datapath[idx]
                points = pymesh.load_mesh(fn[0]).vertices
                label = fn[1]
                replace = True if points.shape[0]<self.npoints else False
                choice = np.random.choice(points.shape[0], self.npoints, replace=replace)
                points = points[choice, :]
                points = scale_linear_bycolumn(points)
                points = torch.from_numpy(points.astype(np.float32))
                label = torch.from_numpy(np.array([label]).astype(np.int64))
                return points, label

        def __len__(self):
                return len(self.datapath)

class PartDataset(data.Dataset):
        def __init__(self, root, npoints=2048, class_choice=None, train=True):
                '''Part segmentation on ShapeNet'''
                self.root = root
                self.npoints = npoints
                self.catfile = os.path.join(self.root, 'synsetoffset2category.txt')
                self.cat = {}

                with open(self.catfile, 'r') as f:
                        for line in f.readlines():
                                lns = line.strip().split()
                                self.cat[lns[0]] = lns[1]

                if not class_choice is None:
                    self.cat = {k:v for k, v in self.cat.items() if k in class_choice}
                    self.num_classes = shapenet_labels[class_choice[0]]
                else:
                    self.num_classes = 50

                self.meta = {}
                for item in self.cat:
                        self.meta[item] = []
                        point_dir = os.path.join(self.root, self.cat[item], 'points')
                        seg_dir = os.path.join(self.root, self.cat[item], 'points_label')
                fns = sorted(os.listdir(point_dir))
                if train:
                        fns = fns[:int(0.9*len(fns))]
                else:
                        fns = fns[int(0.9*len(fns)):]

                for fn in fns:
                        token = (os.path.splitext(os.path.basename(fn))[0])
                        self.meta[item].append((os.path.join(point_dir, token + '.pts'), os.path.join(seg_dir, token + '.seg')))

                self.datapath = []
                for item in self.cat:
                        for fn in self.meta[item]:
                                self.datapath.append((item, fn[0], fn[1]))

                self.classes = dict(zip(sorted(self.cat), range(len(self.cat))))

        def __getitem__(self, idx):
                fn = self.datapath[idx]
                points = np.loadtxt(fn[1]).astype(np.float32)
                seg = np.loadtxt(fn[2]).astype(np.int64)
                replace = True if points.shape[0]<self.npoints else False
                choice = np.random.choice(len(seg), self.npoints, replace=replace)
                # resample
                points = points[choice, :]
                seg = seg[choice]
                points = torch.from_numpy(points)
                seg = torch.from_numpy(seg)
                return points, seg

        def __len__(self):
                return len(self.datapath)

class S3dDataset(data.Dataset):
        '''Semantic segmentation on S3DIS'''
        def __init__(self, root, npoints=4096, train=True, gen_labels=False):
                self.root = root
                self.npoints = npoints
                self.catfile = os.path.join(self.root, 's3d_cat2num.txt')
                self.cat = {}
                with open(self.catfile, 'r') as f:
                    for line in f.readlines():
                        lns = line.strip().split()
                        self.cat[lns[0]] = lns[1]
                self.num_classes = len(self.cat)
                self.datapath, self.labelspath = [], []
                FLAG = 'train' if train else 'test'
                path = os.path.join(self.root, FLAG)
                for area in os.listdir(path):
                    area_path = os.path.join(path, area)
                    for scene in os.listdir(area_path):
                        if os.path.isdir(os.path.join(area_path, scene)):
                            scene_path = os.path.join(area_path, scene)
                            for scene_component in os.listdir(os.path.join(scene_path, 'Annotations')):
                                if 'labels' not in scene_component and 'preds' not in scene_component:
                                    self.datapath.append(os.path.join(scene_path, 'Annotations', scene_component))

                if gen_labels: # do this only once
                    pbar = tqdm(total=len(self.datapath))
                    ds_i = 0
                    for path in self.datapath:
                        l = path.split('\\')
                        labels_path = os.path.join(l[-5], l[-4], l[-3], l[-2])
                        component_name = l[-1].split('.')[0]
                        class_name = l[-1].split('_')[0]
                        if class_name == '.DS':
                            ds_i += 1
                            continue
                        file_path = os.path.join(labels_path, component_name + '_labels.txt')
                        if os.path.exists(file_path):
                            continue
                        with open(path, 'r') as f:
                            with open(file_path, 'w') as g:
                                for line in f.readlines():
                                    g.write(str(self.cat[class_name]) + '\n')
                        pbar.update()

        def __getitem__(self, idx):
            try:
                fn = self.datapath[idx]
                points = np.loadtxt(fn)[:, :3].astype(np.float32)
                ln = os.path.splitext(fn)[0] + '_labels.txt'
                seg = np.loadtxt(ln).astype(np.int64)
                replace = True if points.shape[0]<self.npoints else False
                choice = np.random.choice(points.shape[0], self.npoints, replace=replace)
                points = points[choice, :]
                points = scale_linear_bycolumn(points)
                seg = seg[choice]
                points = torch.from_numpy(points)
                seg = torch.from_numpy(seg)
                return points, seg

            except Exception as e:
                    print(fn)
                    print(e)
                    raise e

        def __len__(self):
            return len(self.datapath)

class RoomsDataset_mk1:
        '''
                Semantic segmentation on S3DIS

                take sample from a room only ones (simple)
        '''

        def __init__(self, path, npoints=4096, feature_num=3):
                self.path = path
                self.files_paths = [os.path.join(self.path, fn) for fn in os.listdir(path)]
                self.npoints = npoints
                self.feature_num = feature_num

        def __len__(self):
                return len(self.files)

        def __getitem__(self, idx):
                fp = self.files_paths[idx]
                points = np.loadtxt(fp)[:, :3]
                seg = points[:, -1].astype(np.int64)
                points = points[:, :feature_num].astype(np.float32)
                replace = True if points.shape[0]<self.npoints else False
                choice = np.random.choice(points.shape[0], self.npoints, replace=replace)
                points = points[choice, :]
                points = scale_linear_bycolumn(points)
                seg = seg[choice]
                points = torch.from_numpy(points)
                seg = torch.from_numpy(seg)
                return points, seg

class RoomsDataset_mk2:
        '''
                Semantic segmentation on S3DIS

                types of room slicing:
                1) x_size*y_size*z_size
                2) x_mesh_num*y_mesh_num*z_mesh_num

                we have to return a len. we don't want to load all Gbs of data to calculate size
                let me set some size and when we are out of it we cycle (idx % real_size) until
                we met that size.
        '''

        def __init__(self, path, npoints=4096, size=1000, feature_num=3, slicing_sizes=(1.0, 1.0, 1.0), slicing_mesh=None):
                self.path = path
                self.files_paths = [os.path.join(self.path, fn) for fn in os.listdir(self.path)]
                # random.shuffle(self.files_paths)

                self.npoints = npoints
                self.size = size
                # self.feature_num = feature_num
                self.box_gen = None
                self.slicing_sizes = slicing_sizes
                self.slicing_mesh = slicing_mesh
                if slicing_sizes and slicing_mesh:
                        raise Exception("Set only one type of room slicing, please")

        def __len__(self):
                return self.size

        def make_box_gen(self, points):
                """
                Split points in to subarrays to list

                input: points - np.array of shape (point_number, xyz[rgb]+label)
                """
                boxes = []
                if self.slicing_sizes:
                        # there are 8 points to start sized slicing... we will go from the mins to max
                        # also we can start from random point of view
                        mins = np.min(points[:, :3], axis=0)
                        maxs = np.max(points[:, :3], axis=0)
                        ranges = maxs-mins
                        ns=(ranges/np.array(self.slicing_sizes)).astype(np.int64)
                        ns = np.clip(ns, 0, None)
                        for ix in range(ns[0]):
                                for iy in range(ns[1]):
                                        for iz in range(ns[2]):
                                                # print(ix, iy, iz)
                                                # print(self.slicing_sizes)
                                                # print(ns)
                                                # print(ranges)
                                                x_indices = (mins[0]+self.slicing_sizes[0]*ix<points[:, 0]) * (mins[0]+self.slicing_sizes[0]*(ix+1)>points[:, 0])
                                                y_indices = (mins[1]+self.slicing_sizes[1]*iy<points[:, 1]) * (mins[1]+self.slicing_sizes[1]*(iy+1)>points[:, 1])
                                                z_indices = (mins[2]+self.slicing_sizes[2]*iz<points[:, 2]) * (mins[2]+self.slicing_sizes[2]*(iz+1)>points[:, 2])
                                                # print(np.sum(x_indices), np.sum(y_indices), np.sum(z_indices))
                                                # print(np.sum(x_indices*y_indices))
                                                # print(np.sum(x_indices*z_indices))
                                                # print(np.sum(y_indices*z_indices))
                                                # print(np.sum(x_indices*y_indices*z_indices))
                                                indices = x_indices*y_indices*z_indices
                                                # print('indshp', indices.shape)
                                                if np.sum(indices) == 0:
                                                        continue
                                                yield points[indices, :]

        def new_box_gen(self):
                if len(self.files_paths) == 0:
                        self.files_paths =[os.path.join(self.path, fn) for fn in os.listdir(self.path)]
                        # random.shuffle(self.files_paths)
                fp = self.files_paths.pop(0)
                print("Loading file: {}".format(fp))
                points = np.loadtxt(fp)
                self.box_gen = self.make_box_gen(points)

        def reset(self):
                self.files_paths = []
                self.new_box_gen()

        # @pc_info(cmap='tab10', viz=True)
        def __getitem__(self, idx):
                if self.box_gen is None:
                        self.new_box_gen()

                try:
                        box = next(self.box_gen)
                except StopIteration:
                        self.new_box_gen()
                        box = next(self.box_gen)

                points = box
                if self.npoints is not None:
                        replace = True if box.shape[0]<self.npoints else False
                        choice = np.random.choice(box.shape[0], self.npoints, replace=replace)
                        points = box[choice, :]

                xyz = points[:, :6].astype(np.float32)
                seg = points[:, -1].astype(np.int64)
                xyz = scale_linear_bycolumn(xyz)
                return xyz, seg

class S3dDatasetNeiSphe(data.Dataset):
    def __init__(self, root, npoints=4096, train=True, gen_labels=False):
        self.root = root
        self.npoints = npoints
        self.catfile = os.path.join(self.root, 's3d_cat2num.txt')
        self.cat = {}
        with open(self.catfile, 'r') as f:
            for line in f.readlines():
                lns = line.strip().split()
                self.cat[lns[0]] = lns[1]
        self.num_classes = len(self.cat)
        self.datapath, self.labelspath = [], []
        FLAG = 'train' if train else 'test'
        path = os.path.join(self.root, FLAG)
        for area in os.listdir(path):
            area_path = os.path.join(path, area)
            for scene in os.listdir(area_path):
                if os.path.isdir(os.path.join(area_path, scene)):
                    scene_path = os.path.join(area_path, scene)
                    for scene_component in os.listdir(
                            os.path.join(scene_path, 'Annotations')):
                        if not scene_component.endswith(
                                '_labels.txt') and not scene_component.startswith(
                                '.'):
                            self.datapath.append(
                                os.path.join(scene_path, 'Annotations',
                                             scene_component))

        if gen_labels:  # do this only once
            for path in tqdm(self.datapath):
                dir_name, base_name = os.path.split(path)
                root, ext = os.path.splitext(base_name)
                class_name = root.split('_')[0]
                file_path = os.path.join(dir_name, root + '_labels.txt')
                with open(os.path.join(file_path), 'w') as f:
                    f.write(str(self.cat[class_name]))

#     @pc_normalize(norm_type='spherical', center=False, verbose=False)
#     @ps_to_spherical(verbose=False)
    # @pc_info(cmap='tab10', viz=True)
#     @pc_noise(sigma=0.01, seed=42)
#     @pc_rotate(max_x=0, max_y=0, max_z=180, seed=42)
    def __getitem__(self, idx):
        n = 0
        while n != self.npoints:
            fn = self.datapath[idx]
            points = np.loadtxt(fn)[:, :3].astype(np.float32)
            ln = os.path.splitext(fn)[0] + '_labels.txt'
            seg = np.loadtxt(ln).astype(np.int64)
            seg = np.full(points.shape[0], seg)
            pcd = o3d.geometry.PointCloud()
            pcd.points = o3d.utility.Vector3dVector(points)
            pcd_tree = o3d.geometry.KDTreeFlann(pcd)
            idx = np.random.randint(0, points.shape[0])
            n, idxs, ds = pcd_tree.search_knn_vector_3d(points[idx], self.npoints)
            idx = np.random.randint(0, len(self.datapath))  # FIXME workaround
        points = points[idxs]
        seg = seg[idxs]
        points = torch.from_numpy(points)
        seg = torch.from_numpy(seg)
        return points, seg

    def __len__(self):
        return len(self.datapath)

if __name__ == '__main__':

        # c = ClsDataset(root='modelnet40_manually_aligned')
        # ps, seg = c[100]
        # print(ps.type(), ps.size(), l.type(), l.size(), l)

        # d = PartDataset(root='shapenetcore_partanno_segmentation_benchmark_v0')
        # ps, seg = d[10]
        # print(ps.type(), ps.size(), seg.type(), seg.size())

        s = S3dDataset(root='.', train=True, gen_labels=True)
        s = S3dDataset(root='.', train=False, gen_labels=True)
        s = S3dDatasetNeiSphe(root='Stanford3dDataset_v1.2', train=True, gen_labels=True)
        ps, seg = s[100]
        print(ps.type(), ps.size(), seg.type(), seg.size())
        for _ in tqdm(s):
            try:
                pass
            except Exception as e:
                print(e)


import math
import os.path as op
import operator
from scipy.stats import poisson
import random
import time

from array import array
import ROOT
from ROOT import TFile, TH1D, TCanvas, gStyle, gROOT, TF1, TColor, TMinuit, gPad, TPad
from ROOT import TGaxis, TBox, THStack, TLegend, TGraphErrors, TVectorD, TGraph
from ROOT.Math import GaussIntegrator, WrappedTF1, IParamFunction
from ROOT.Math import IParametricFunctionOneDim

gROOT.SetBatch(True)

class BackgroundModel:
    def __init__(self, p1, p2, p3, p4):
        self.p1 = p1
        self.p2 = p2
        self.p3 = p3
        self.p4 = p4

    def model_at(self, x):
        scale = x/14
        a1 = self.p1 * math.pow(1-scale, self.p2)
        a2 = self.p3 + (self.p4 * math.log(scale))
        a3 = math.pow(scale, a2)
        return a1*a3

    def random_at(self, x):
        return poisson.ppf(random.random(), self.model_at(x))

class Fits:
    def __init__(self):

        self.p_n = [0,]*100
        self.e_n = [0,]*100
        self.stored_parameters = [0,]*100

        self.num_bins = 0
        self.xmins = []
        self.xmaxes = []

        self.data = []
        self.errors = []
        self.data_fits = []

        self.model_scale_values = []
        self.final = False

        self.exclude_regions = ((0, 0),)

        self.col1 = 1
        self.col2 = TColor.GetColor(27, 158, 119)
        self.col3 = TColor.GetColor(217, 95, 2)
        self.col4 = TColor.GetColor(117, 112, 179)

    def run_mass_fit(self, peak_scale_initial):
        self.gMinuit = TMinuit(30)
        self.gMinuit.SetPrintLevel(-1)
        self.gMinuit.SetFCN(self.Fitfcn_max_likelihood)

        arglist = array("d", [0,]*10)
        ierflg = ROOT.Long(0)
        arglist[0] = ROOT.Double(1)

        # peak_scale_initial = ROOT.Double(peak_scale_initial)

        tmp = array("d", [0,])
        self.gMinuit.mnexcm("SET NOWarnings", tmp, 0, ierflg);

        self.gMinuit.mnexcm("SET ERR", arglist, 1, ierflg)
        self.gMinuit.mnparm(0, "p1", 5e-6, 1e-7, 0, 0, ierflg)
        self.gMinuit.mnparm(1, "p2", 10, 10, 0, 0, ierflg)
        self.gMinuit.mnparm(2, "p3", -5.3, 1, 0, 0, ierflg)
        self.gMinuit.mnparm(3, "p4", -4e-2, 1e-2, 0, 0, ierflg)
        self.gMinuit.mnparm(4, "p5", peak_scale_initial, peak_scale_initial/50, 0, 0, ierflg)

        arglist[0] = ROOT.Double(0)
        arglist[1] = ROOT.Double(0)

        self.exclude_regions = ((2.2, 3.3),)
        self.gMinuit.FixParameter(2)
        self.gMinuit.FixParameter(3)
        self.gMinuit.FixParameter(4)

        self.gMinuit.mnexcm("simplex", arglist, 2, ierflg)
        self.gMinuit.mnexcm("MIGRAD", arglist, 2, ierflg)

        self.gMinuit.Release(2)
        self.gMinuit.mnexcm("simplex", arglist, 2, ierflg)
        self.gMinuit.mnexcm("MIGRAD", arglist, 2, ierflg)

        self.gMinuit.Release(3)
        self.gMinuit.mnexcm("simplex", arglist, 2, ierflg)
        self.gMinuit.mnexcm("MIGRAD", arglist, 2, ierflg)

        #self.exclude_regions = ((0, 2), (3.3, 100),)
        self.exclude_regions = ()
        self.gMinuit.FixParameter(0)
        self.gMinuit.FixParameter(1)
        self.gMinuit.FixParameter(2)
        self.gMinuit.FixParameter(3)
        self.gMinuit.Release(4)
        self.gMinuit.mnexcm("simplex", arglist, 2, ierflg)
        self.gMinuit.mnexcm("MIGRAD", arglist, 2, ierflg)

        #self.final=True
        self.gMinuit.Release(0)
        self.gMinuit.Release(1)
        self.gMinuit.Release(2)
        self.gMinuit.Release(3)
        self.gMinuit.mnexcm("simplex", arglist, 2, ierflg)
        self.gMinuit.mnexcm("MIGRAD", arglist, 2, ierflg)

    def Fitfcn(self, npar, gin, fcnVal, par, iflag):
        chisq = 0
        mf = ROOT.MyMassSpectrum()
        mf.SetParameters(par)
        ig = GaussIntegrator()
        ig.SetFunction(mf)
        ig.SetRelTolerance(0.00001)

        for i in range(0, self.num_bins):
            val = ig.Integral(self.xmins[i], self.xmaxes[i])/(self.xmaxes[i]-self.xmins[i])
            chiValue = 0
            if self.errors[i]:
                chiValue = (self.data[i]-val)/math.sqrt(val)
            chiValue *= chiValue
            chisq += chiValue
            self.data_fits[i] = val

        fcnVal[0] = chisq

    def Fitfcn_max_likelihood(self, npar, gin, fcnVal, par, iflag):
        likelihood = 0
        mf = ROOT.MyMassSpectrum()
        mf.SetParameters(par)
        ig = GaussIntegrator()
        ig.SetFunction(mf)
        ig.SetRelTolerance(0.00001)

        model_total = 0

        events_total = 0

        for i in range(0, self.num_bins):
            for lower, higher in self.exclude_regions:
                if lower < self.xmins[i] < higher:
                    continue

            model_val = ig.Integral(self.xmins[i], self.xmaxes[i]) / (self.xmaxes[i]-self.xmins[i])
            if self.final:
                print(self.xmins[i], model_val, self.model_scale_values[i]*par[4])
            model_val += self.model_scale_values[i]*par[4]

            self.data_fits[i] = model_val

            if model_val <= 0 or (self.data[i] == 0 and self.xmins[i] < 2):
                continue

            likelihood += self.data[i] * math.log(model_val)

            model_total += model_val
            events_total += self.data[i]

        fcnVal[0] = 2*(model_total-likelihood)

def fit_mass():
    # pre-fitted background parameters to generate random distributions from
    background = BackgroundModel(1.70475e1, 8.43990, -4.63149, -2.43023e-3)

    gROOT.LoadMacro("IABStyle.cpp+g")
    ROOT.IABstyles.global_style()
    TGaxis.SetMaxDigits(3)

    root_file = TFile.Open("data15_13TeV_background.root")
    hist = root_file.Get("mjj_data15_13TeV_00276262_physics_Main_total_final")

    root_file_blackmax = TFile.Open("dataLikeHistograms.QStar3000.root")
    nominal = root_file_blackmax.GetDirectory("Nominal")
    hist_model = nominal.Get("mjj_Scaled_QStar3000_30fb")
    hist_model.Smooth(1)
    hist_model.Scale(1/hist_model.Integral())
    hist_model.Scale(40000)
    # Model data started one bin earlier than actual data
    # so there was a lone low value right at the start
    # of the data+model which was dragging the fit down.
    # This removes that.
    #for b in range(1, hist.GetNbinsX()+1):
    #   if hist.GetBinContent(b) == 0:
    #       hist_model.SetBinContent(b, 0)
    total_peak = 0
    for b in range(1, hist.GetNbinsX()+1):
        if hist.GetBinContent(b) > 0 or hist.GetBinLowEdge(b)/1000 > 2:
            x = hist.GetBinCenter(b)/1000
            bg = background.random_at(x)
            peak = 0
            if hist_model.GetBinContent(b) > 0:
                peak = poisson.ppf(random.random(), hist_model.GetBinContent(b))
            total_peak += peak
            hist.SetBinContent(b, bg+peak)
    print("Total peak:, ", total_peak)
    fits = Fits()
    fits.model_scale_values = [hist_model.GetBinContent(b) for b in range(1, hist_model.GetNbinsX()+1)]

    #hist_model.Scale(2)
    #hist.Add(hist_model)

    nbins = hist.GetNbinsX()
    xwidth = [(hist.GetBinLowEdge(b+1)/1000-hist.GetBinLowEdge(b)/1000)/2 for b in range(1, nbins+1)]
    xmiddle = [hist.GetBinCenter(b)/1000 for b in range(1, nbins+1)]

    fits.xmins = [hist.GetBinLowEdge(b)/1000 for b in range(1, nbins+1)]
    fits.xmaxes = [hist.GetBinLowEdge(b+1)/1000 for b in range(1, nbins+1)]
    fits.data = [hist.GetBinContent(b) for b in range(1, nbins+1)]
    fits.data_fits = [0,]*nbins

    fits.errors = [math.sqrt(x) for x in fits.data]
    fits.num_bins = nbins

    fits.run_mass_fit(1.0)

    test_canvas = TCanvas("TestCanvas", "Ds Fit", 0, 0, 800, 575)

    gStyle.SetPadBorderMode(0)
    gStyle.SetFrameBorderMode(0)

    test_canvas.Divide(1, 2, 0, 0)
    upper_pad = test_canvas.GetPad(1)#TPad("upper_pad", "upper_pad", 0.005, 0.7525, 0.995, 0.995)
    lower_pad = test_canvas.GetPad(2)#TPad("lower_pad", "lower_pad", 0.005, 0.005,  0.995, 0.7475)
    low, high = 0.05, 0.95
    upper_pad.SetPad(low, 0.4, high, high)
    lower_pad.SetPad(low, low, high, 0.4)

    test_canvas.cd(1)

    ROOT.IABstyles.canvas_style(test_canvas, 0.25, 0.05, 0.02, 0.15, 0, 0)

    h_Mjj = TH1D("h_Mjj", "Mass Spectrum", 100, 0.2, 12)
    h_Mjj.GetYaxis().SetTitle("num. events")
    h_Mjj.GetXaxis().SetTitle("M [Tev/c^{-2}]")

    ROOT.IABstyles.h1_style(h_Mjj, ROOT.IABstyles.lWidth/2, ROOT.IABstyles.Scolor, 1, 0, 0, -1111.0, -1111.0, 508, 508, 8, ROOT.IABstyles.Scolor, 0.1, 0)

    h_Mjj.GetYaxis().SetRangeUser(0.1, 1e6)
    h_Mjj.GetXaxis().SetRangeUser(1, 10)
    h_Mjj.GetXaxis().SetTitleOffset(1)
    h_Mjj.GetYaxis().SetTitleOffset(1.1)

    upper_pad.SetLogy(1)
    upper_pad.SetLogx(1)
    lower_pad.SetLogx(1)

    gr = TGraphErrors(fits.num_bins, array("d", xmiddle), array("d", fits.data), array("d", xwidth), array("d", fits.errors))
    ROOT.IABstyles.h1_style(gr, ROOT.IABstyles.lWidth/2, ROOT.IABstyles.Scolor, 1, 0, 0, -1111, -1111, 505, 505, 8, ROOT.IABstyles.Scolor, 0.1, 0)

    grFit = TGraph(fits.num_bins, array("d", xmiddle), array("d", fits.data_fits))
    ROOT.IABstyles.h1_style(grFit, ROOT.IABstyles.lWidth/2, 632, 1, 0, 0, -1111, -1111, 505, 505, 8, 632, 0.1, 0)

    h_Mjj.Draw("axis")
    gr.Draw("P")
    grFit.Draw("c")
    #upper_pad.Draw()

    def gaus(x, par):
        arg = 0
        if par[2] != 0:
            arg = (x[0] - par[1]) / par[2]
        return par[0] * math.exp(-0.5*arg*arg)

    gauss_f = TF1("gauss_f", gaus, 1000, 2900, 100)
    gauss_f.SetParameters(1000, 2900, 100)
    gauss_f.SetLineColor(3)
    gauss_f.Draw("same")

    test_canvas.Update()

    gPad.SetBottomMargin(1e-5)

    test_canvas.cd(2)
    gPad.SetTopMargin(1e-5)

    """h2 = TH1D("h2", "Significance", fits.num_bins, 0.2, 12)
    h2.GetYaxis().SetTitle("Significance")
    for bin_num, (data, theory) in enumerate(zip(fits.data, fits.data_fits), start=1):
        print("bin", bin_num, data, theory, (data-theory)/theory)
        h2.SetBinContent(bin_num, (data-theory)/theory)
    h2.GetXaxis().SetRangeUser(1, 10)

    h2.Draw()"""
    h2 = TH1D("h2", "", 100, 0.2, 12)
    h2.GetXaxis().SetRangeUser(1, 10)
    h2.GetYaxis().SetRangeUser(-10, 10)
    h2.SetStats(False) # don't show stats box
    h2.Draw("axis")
    sig_values = [(data-theory)/theory if data!= 0 else -100 for data, theory in zip(fits.data, fits.data_fits)]
    sig = TGraph(fits.num_bins, array("d", xmiddle), array("d", sig_values))
    #ROOT.IABstyles.h1_style(sig, ROOT.IABstyles.lWidth/2, 632, 1, 0, 0, -1111, -1111, 505, 505, 8, 632, 0.1, 0)
    ROOT.IABstyles.h1_style(gr, ROOT.IABstyles.lWidth/2, ROOT.IABstyles.Scolor, 1, 0, 0, -1111, -1111, 505, 505, 8, ROOT.IABstyles.Scolor, 0.1, 0)
    sig.SetMarkerStyle(22) # triangle
    sig.SetMarkerColor(2)  # red
    sig.SetMarkerSize(0.8)
    sig.Draw("P")
    #lower_pad.Draw()

    # test_canvas.SaveAs("output_qstar.pdf")
    test_canvas.SaveAs("output_qstar.png")

def fit_peak(num_injected_events):
    # pre-fitted background parameters to generate random distributions from
    background = BackgroundModel(1.70475e1, 8.43990, -4.63149, -2.43023e-3)

    ROOT.IABstyles.global_style()
    TGaxis.SetMaxDigits(3)

    root_file = TFile.Open("data15_13TeV_background.root")
    hist = root_file.Get("mjj_data15_13TeV_00276262_physics_Main_total_final")

    root_file_qstar = TFile.Open("dataLikeHistograms.QStar3000.root")
    nominal = root_file_qstar.GetDirectory("Nominal")
    hist_model = nominal.Get("mjj_Scaled_QStar3000_30fb")
    hist_model.Smooth(1)
    hist_model.Scale(1/hist_model.Integral())
    hist_model.Scale(num_injected_events)
    # Model data started one bin earlier than actual data
    # so there was a lone low value right at the start
    # of the data+model which was dragging the fit down.
    # This removes that.
    #for b in range(1, hist.GetNbinsX()+1):
    #   if hist.GetBinContent(b) == 0:
    #       hist_model.SetBinContent(b, 0)
    total_peak = 0
    for b in range(1, hist.GetNbinsX()+1):
        if hist.GetBinContent(b) > 0 or hist.GetBinLowEdge(b)/1000 > 2:
            x = hist.GetBinCenter(b)/1000
            bg = background.random_at(x)
            peak = 0
            if hist_model.GetBinContent(b) > 0:
                peak = poisson.ppf(random.random(), hist_model.GetBinContent(b))
            total_peak += peak
            hist.SetBinContent(b, bg+peak)

    fits = Fits()
    fits.model_scale_values = [hist_model.GetBinContent(b) for b in range(1, hist_model.GetNbinsX()+1)]

    nbins = hist.GetNbinsX()
    xwidth = [(hist.GetBinLowEdge(b+1)/1000-hist.GetBinLowEdge(b)/1000)/2 for b in range(1, nbins+1)]
    xmiddle = [hist.GetBinCenter(b)/1000 for b in range(1, nbins+1)]

    fits.xmins = [hist.GetBinLowEdge(b)/1000 for b in range(1, nbins+1)]
    fits.xmaxes = [hist.GetBinLowEdge(b+1)/1000 for b in range(1, nbins+1)]
    fits.data = [hist.GetBinContent(b) for b in range(1, nbins+1)]
    fits.data_fits = [0,]*nbins

    fits.errors = [math.sqrt(x) for x in fits.data]
    fits.num_bins = nbins

    fits.run_mass_fit(1.0)

    par4 = ROOT.Double(0)
    par4_error = ROOT.Double(0)

    fits.gMinuit.GetParameter(4, par4, par4_error)

    hist_model.Scale(par4)
    return hist_model.Integral()

def fit_many():
    start_time = time.time()
    gROOT.LoadMacro("FitFunction.cpp+g")
    gROOT.LoadMacro("IABStyle.cpp+g")
    with open("results.txt", "w") as out_file:
        for i in range(0, 200):
            if i%10 == 0:
                print("Starting trial {0}".format(i))
            n = fit_peak(40000)
            out_file.write("{0}\n".format(n))

    print("Took {0:.2f} seconds in total".format(time.time()-start_time))

fit_many()


"""Test configuration and mocks for the google integration."""
from unittest.mock import patch

import pytest

TEST_CALENDAR = {
    "id": "qwertyuiopasdfghjklzxcvbnm@import.calendar.google.com",
    "etag": '"3584134138943410"',
    "timeZone": "UTC",
    "accessRole": "reader",
    "foregroundColor": "#000000",
    "selected": True,
    "kind": "calendar#calendarListEntry",
    "backgroundColor": "#16a765",
    "description": "Test Calendar",
    "summary": "We are, we are, a... Test Calendar",
    "colorId": "8",
    "defaultReminders": [],
    "track": True,
}

@pytest.fixture
def test_calendar():
    """Return a test calendar."""
    return TEST_CALENDAR

@pytest.fixture
def mock_next_event():
    """Mock the google calendar data."""
    patch_google_cal = patch(
        "homeassistant.components.google.calendar.GoogleCalendarData"
    )
    with patch_google_cal as google_cal_data:
        yield google_cal_data


from django.test import TestCase
from django.urls import reverse,resolve
from .views import home

class HomeTests(TestCase):
	def test_home_view_status_code(self):
		url=reverse('home')
		print(str(url))
		response=self.client.get(url)
		self.assertEquals(response.status_code,200)

	def test_home_url_resolve_home_view(self):
		view=resolve('/')
		print(str(view))
		self.assertEquals(view.func, home)


# -*- coding: utf-8 -*-
#adapted from https://github.com/sujitpal/dl-models-for-qa/blob/master/src/es-load-flashcards.py

from __future__ import division, print_function
import elasticsearch
import nltk
import os

DATA_DIR = "../data"
EXPL_FILE = "expl-tablestore.csv"
EXPL_INDEX = "expl-idx"

es = elasticsearch.Elasticsearch(hosts=[{
    "host": "localhost",
    "port": "9200"
}])

if es.indices.exists(EXPL_INDEX):
    print("deleting index: %s" % (EXPL_INDEX))
    resp = es.indices.delete(index=EXPL_INDEX)
    print(resp)

body = {
    "settings": {
        "number_of_shards": 5,
        "number_of_replicas": 0
    }
}
print("creating index: %s" % (EXPL_INDEX))
resp = es.indices.create(index=EXPL_INDEX, body=body)
print(resp)

fexpl = open(os.path.join(DATA_DIR, EXPL_FILE), "r", encoding="utf8")
lno = 1
for line in fexpl:
    if lno % 1000 == 0:
        print("# explanation sentences read: %d" % (lno))
    line = line.strip()
    fcid, sent = line.split("\t")
    expl = " ".join(nltk.word_tokenize(sent))
    doc = { "story": expl }
    resp = es.index(index=EXPL_INDEX, doc_type="explanations", id=lno, body=doc)
#    print(resp["created"])
    lno += 1
print("# explanation sentences read and indexed: %d" % (lno))
fexpl.close()
es.indices.refresh(index=EXPL_INDEX)

query = """ { "query": { "match_all": {} } }"""
resp = es.search(index=EXPL_INDEX, doc_type="explanations", body=query)
print("# of records in index: %d" % (resp["hits"]["total"]))


from hashlib import sha256
import hmac
from django.conf import settings
from models import WebauthUser
from django.contrib.auth.models import User

WEBAUTH_VERSION = "WA_3"

class WebauthVersionNotSupported(Exception):
    """
    Error occurs only when the server providing login services had a version that was not equal to ours.
    """
    pass

def WebauthLogin(request, mac, webauth_version, username_base64, username, name_base64, name):
    """
    Completes the login process for a Webauth user; checks the MAC to ensure the user should be able to login,
    then sets the appropriate parameter so that the middleware logs in the user.

    The session will not be active until the middleware runs again (i.e., the next page load.)
    """
    if webauth_version != WEBAUTH_VERSION:
        raise WebauthVersionNotSupported

    expected_mac = hmac.new(settings.WEBAUTH_SHARED_SECRET,
                            username_base64 + '|' + name_base64 + '|' + webauth_version,
                            sha256).hexdigest()

    # Security note: to avoid timing attacks, you should use crypto-specific functions like
    # hmac.compare_digest to check equality. However, if you're using older versions of Python,
    # this isn't available, so you'll probably just use == to check equality. If you're using this
    # code and worried about timing attacks, you're probably doing it wrong.
    if hmac.compare_digest(mac, expected_mac):
        # create Django user for the WebAuth'd person if they don't exist
        # session will not be active till next pageload
        if WebauthUser.objects.filter(username__exact=username).count() == 0:
            WebauthCreate(username, name)

        request.session['wa_username'] = username
        return True
    else:
        return False

def WebauthCreate(username, full_name):
    """
    Creates a WebauthUser from the provided username + full name. If the given username already exists, it converts the user
    into a WebauthUser.
    """
    if WebauthUser.objects.filter(username__exact=username).count() > 0:
        return

    authuser_obj = User.objects.filter(username__exact=username)
    if not authuser_obj.exists():
        user = WebauthUser()
        user.new_webauth(username, full_name)
    else:
        user = authuser_obj.get()
        user.__class__ = WebauthUser
        user.new_webauth(username, full_name)

def WebauthLogout(request):
    """
    Logs out a WebauthUser session.
    """
    del request.session['wa_username']