from __future__ import print_function

import imp
import sys
import os

import const_config
imp.reload(const_config)
from const_config import *

import const_hadoop
imp.reload(const_hadoop)
from const_hadoop import *

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

def get_log_file(config_file,suffix="",output_log_folder="e"):
    """
    Get logging files.

    Parameters
    ----------
     config_file : str
         path to CorrelX configuration file.
     suffix : str
         suffix (with timestamp) to be added to log filename.
     output_log_folder : str
         suffix to be added to log file path.

    Returns
    -------
     file_log : handler to file
         handler to log file.
     temp_log : str
         path to temporary (buffer) file for system calls.
    """

    config = configparser.ConfigParser()
    config.optionxform = str
    config.read(config_file)

    temp_log=config.get(C_CONF_FILES,C_CONF_FILES_TMP_LOG)
    temp_log+=suffix
    file_log_input=config.get(C_CONF_GEN,C_CONF_GEN_LOG_FILE)
    if file_log_input=="sys.stdout":
        file_log = sys.stdout
    else:
        os.system("mkdir -p "+output_log_folder)
        file_log_input=output_log_folder+"/log_"+suffix+"_ext"
        file_log = open(file_log_input,'w')

    return([file_log,temp_log])

def get_nodes_file(config_file):
    """
    Get name of file with list of nodes (from config file).

    Parameters
    ----------
     config_file : str
         path to CorrelX configuration file.

    Returns
    -------
     file_read_nodes : str
         path to hosts file.
    """
    config = configparser.ConfigParser()
    config.optionxform = str
    config.read(config_file)
    file_read_nodes=config.get(C_CONF_GEN,C_CONF_GEN_HOSTS)
    return(file_read_nodes)

def get_config_mod_for_this_master(config_file,config_suffix,master_node,script_arg_zero):
    """
    Overwrite all instances of "localhost" in configuration file with master node name
     into new configuration file (original configuration file is used as a template),
     "~" with home folder for the current user, "localuser" with the current user,
     and "localpath" with the path of the script_arg_zero (mapred_cx.py).

    Parameters
    ----------
     config_file : str
         path to CorrelX configuration file.
     config_suffix : str
         suffix to be added to resulting configuration file.
     master_node : str
         master node name.
     script_arg_zero : str
         path given for the main script (mapred_cx.py).

    Returns
    -------
     new_config_file : str
         configuration file plus suffix.

    Notes
    -----
    |
    | **TO DO:**
    |
    |  Move new configuration file into folder with logs for this job.
    """
    new_config_file = config_file + config_suffix
    abs_local_path = os.path.dirname(os.path.abspath(script_arg_zero))
    abs_local_path = os.path.abspath(abs_local_path+C_CONF_RELATIVE_PATH_LOCALPATH)

    os.system("sed -e 's/"+C_CONF_RES_LOCALHOST+"/" + master_node + "/' " + config_file +" > " + new_config_file)
    os.system("sed -i \"s@~@$HOME@g\" "  + new_config_file)
    os.system("sed -i \"s@"+C_CONF_RES_LOCALUSER+"@$USER@g\" "  + new_config_file)
    os.system("sed -i \"s@"+C_CONF_RES_LOCALPATH+"@" + abs_local_path + "@g\" "  + new_config_file)
    return(new_config_file)

def get_configuration(file_log,config_file,timestamp_str,v=0):
    """
    Read parameters from configuration file "configh.conf".

    Parameters
    ----------
     file_log : handler to file
         handler to log file.
     config_file : str
         path to CorrelX configuration file.
     timestamp_str : str
         suffix to be added to temporary data folder (hwere media will be split).
     v : int
         verbose if 1.

    Returns
    -------
     MAPPER : str
         Python mapper (.py).
     REDUCER : str
         Python reducer (.py).
     DEPENDENCIES : str
         Comma separated list of Python files required for mapper and reducer (1.py,2.py,etc).
     PACKETS_PER_HDFS_BLOCK : int
         Number of VDIF frames per file split.
     CHECKSUM_SIZE : int
         Number of bytes for checksum.
     SRC_DIR : str
         Folder with Python sources for mapper, reducer and dependencies.
     APP_DIR : str
         Folder to place mapper, reducer and dependencies in all nodes (in master-associated folder).
     CONF_DIR : str
         Base working folder for configuration files (to be updated later for this master).
     TEMPLATES_CONF_DIR : str
         Folder with templates for core-site.xml,yarn-site.xml,mapred-site.xml,hdfs-site.xml.
     TEMPLATES_ENV_DIR : str
         Folder with templates for hadoop-env.sh, etc.
     HADOOP_DIR : str
         Path to Hadoop home folder.
     HADOOP_CONF_DIR : str
         Path to Hadoop configuration folder (to be updated later for this master).
     NODES : str
         File to write list of nodes to host the cluster (one node per line).
     MAPPERSH : str
         File to write bash script for mapper (call to python script with all arguments).
     REDUCERSH : str
         File to write bash script for reducer (call to python script with all arguments).
     JOBSH : str
         File to write bash script for job request for Hadoop (call to python script with all arguments).
     PYTHON_X : str
         Path to Python executable.
     USERNAME_MACHINES : str
         Username for ssh into the cluster machines.
     MAX_SLAVES : int
         Maximum number of worker nodes (-1 no maximum).
     SLAVES : str
         Filename for Hadoop slaves file.
     MASTERS : str            Filename for Hadoop masters file.
     MASTER_IS_SLAVE : bool
         Boolean, if 1 master is also launching a nodemanager (doing mapreduce).
     HADOOP_TEMP_DIR : str
         Folder for Hadoop temporary folders.
     DATA_DIR : str
         Path with media input files.
     DATA_DIR_TMP : str
         Path to folder to place splits of input file before moving them to the distributed filesystem.
     HDFS_DATA_DIR : str
         Path in the HDFS distributed filesystem to move input splits.
     HADOOP_START_DELAY : str
         Number of seconds to wait after every interaction with Hadoop during the cluster initialization.
     HADOOP_STOP_DELAY : str
         Number of seconds to wait after every interaction with Hadoop during the cluster termination.
     PREFIX_OUTPUT : str
         Prefix for output file.
     HADOOP_TEXT_DELIMITER : str
         Text delimiter for input splits (lib_mapredcorr.run_mapreduce_sh).
     OUTPUT_DIR : str
         Folder in local filesystem to place output file.
     OUTPUT_SYM : str
         Folder within experiment configuration folders to place symbolic link to output file.
     RUN_PIPELINE : bool
         Boolean, if 1 will run in pipeline mode.
     RUN_HADOOP : bool
         Boolean, if 1 will run Hadoop.
     MAX_CPU_VCORES : int
         Maximum number of virtual CPU cores.
     HDFS_REPLICATION : int
         Number of copies of each input split in HDFS.
     OVER_SLURM : bool
         Boolean, 1 to run in a cluster where the local filesystem is NFS (or synchronized among all nodes).
     HDFS_COPY_DELAY : int
         Number of seconds to wait after every interaction with Hadoop during file distribution to HDFS.
     FFT_AT_MAPPER : bool
         Boolean, if 0 FFT is done at reducer (default).
     INI_FOLDER : str
         Folder with experiment .ini files.
     INI_STATIONS : str
         Stations ini file name.
     INI_SOURCES : str
         Sources ini file name.
     INI_DELAY_MODEL : str
         Delay model ini file name.
     INI_DELAYS : str
         Delay polynomials ini file name.
     INI_MEDIA : str
         Media ini file name.
     INI_CORRELATION : str
         Correlation ini file name.
     INTERNAL_LOG_MAPPER
         [remove] currently default 0.
     INTERNAL_LOG_REDUCER
         [remove] currenlty default 0.
     ADJUST_MAPPERS : float
         Force number of mappers computed automatically to be multiplied by this number.
     ADJUST_REDUCERS : float
         Force number of reducers computed automatically to be multiplied by this number.
     FFTS_PER_CHUNK
        [Remove] Number of DFT windows per mapper output, -1 by default (whole frame)
     TEXT_MODE : bool
        True by default.
     USE_NOHASH_PARTITIONER : bool
        True to use NoHash partitioner.
     USE_LUSTRE_PLUGIN : bool
        True to use Lustre plugin for Hadoop.
     LUSTRE_USER_DIR : str
        Absolute path for the Lustre working path (used in mapreduce job).
     LUSTRE_PREFIX : str
        Path in Lustre to preceed HDFS_DATA_DIR if using Lustre.
     ONE_BASELINE_PER_TASK : int
        0 by default (if 1, old implementation allowed scaling with one baseline per task in the reducers).
     MIN_MAPPER_CHUNK
        [Remove] Chunk constraints for mapper.
     MAX_MAPPER_CHUNK
        [Remove] Chunk constraints for mapper.
     TASK_SCALING_STATIONS: int
        0 by default (if 1, old implementation allowed linear scaling per task in the reducers).
     SORT_OUTPUT : bool
        If 1 will sort lines in output file.
     BM_AVOID_COPY : bool
        If 1 will not split and copy input files if this has already been done previously (for benchmarking).
     BM_DELETE_OUTPUT : bool
        If 1 will not retrieve output file from distributed filesystem (for benchmarking).
     TIMEOUT_STOP : int
        Number of seconds to  wait before terminating nodes during cluster stop routine.
     SINGLE_PRECISION : bool
        If 1 computations will be done in single precision.
     PROFILE_MAP: int
        | if 1 will generate call graphs with timing information for mapper (requires Python Call Graph package),
        | if 2 will use cProfile.
     PROFILE_RED : int
        | if 1 will generate call graphs with timing information for reducer (requires Python Call Graph package),
        | if 2 will use cProfile.

    Notes
    -----
    |
    | **Configuration:**
    |
    |  All constants taken from const_config.py and const_hadoop.py.
    |
    |
    | **TO DO:**
    |
    |  OVER_SLURM: explain better, and check assumptions.
    |  Remove INTERNAL_LOG_MAPPER and INTERNAL_LOG_REDUCER.
    |  Remove FFTS_PER_CHUNK,MIN_MAPPER_CHUNK and MAX_MAPPER_CHUNK.
    |  Check that SINGLE_PRECISION is followed in mapper and reducer.
    """

    #config_file="configh.conf"
    config = configparser.ConfigParser()
    config.optionxform = str
    config.read(config_file)

    if v==1:
        print("\nReading configuration file...",file=file_log)

    # General
    RUN_PIPELINE =           config.getboolean( C_CONF_GEN, C_CONF_GEN_RUN_PIPE)
    SORT_OUTPUT =            config.getboolean( C_CONF_GEN, C_CONF_GEN_SORT_OUTPUT)
    RUN_HADOOP =             config.getboolean( C_CONF_GEN, C_CONF_GEN_RUN_HADOOP)
    OVER_SLURM =             config.getboolean( C_CONF_GEN, C_CONF_GEN_OVER_SLURM)
    USE_NOHASH_PARTITIONER = config.getboolean(C_CONF_GEN, C_CONF_GEN_USE_NOHASH)
    USE_LUSTRE_PLUGIN =      config.getboolean( C_CONF_GEN, C_CONF_GEN_USE_LUSTRE)
    LUSTRE_USER_DIR =        config.get(        C_CONF_GEN, C_CONF_GEN_LUSTRE_USER_FOLDER)
    LUSTRE_PREFIX =          config.get(        C_CONF_GEN, C_CONF_GEN_LUSTRE_PREFIX)
    INTERNAL_LOG_MAPPER =    0                                                                # TO DO: remove
    INTERNAL_LOG_REDUCER =   0                                                                # TO DO: remove

    # Benchmarking
    BM_AVOID_COPY =     config.getboolean( C_CONF_BENCHMARKING, C_CONF_BENCHMARKING_AVOID_COPY)
    BM_DELETE_OUTPUT =  config.getboolean( C_CONF_BENCHMARKING, C_CONF_BENCHMARKING_DELETE_OUTPUT)

    # Profiling
    PROFILE_MAP =       config.getboolean( C_CONF_PROFILING, C_CONF_PROFILING_MAP)
    PROFILE_RED =       config.getboolean( C_CONF_PROFILING, C_CONF_PROFILING_RED)
    use_pycallgraph =   config.getboolean( C_CONF_PROFILING, C_CONF_PROFILING_PYCALLGRAPH)
    if not(use_pycallgraph):
        PROFILE_MAP = 2*int(PROFILE_MAP)
        PROFILE_RED = 2*int(PROFILE_RED)

    # Files
    MAPPER =             config.get(       C_CONF_FILES, C_CONF_FILES_MAP)
    REDUCER =            config.get(       C_CONF_FILES, C_CONF_FILES_RED)
    DEPENDENCIES =       config.get(       C_CONF_FILES, C_CONF_FILES_DEP).split(',')
    MAPPERSH =           config.get(       C_CONF_FILES, C_CONF_FILES_MAP_BASH)
    REDUCERSH =          config.get(       C_CONF_FILES, C_CONF_FILES_RED_BASH)
    JOBSH =              config.get(       C_CONF_FILES, C_CONF_FILES_JOB_BASH)
    PYTHON_X =           config.get(       C_CONF_FILES, C_CONF_FILES_PYTHON)
    NODES =              config.get(       C_CONF_FILES, C_CONF_FILES_NODES)
    SRC_DIR =            config.get(       C_CONF_FILES, C_CONF_FILES_SRC_DIR)
    if SRC_DIR[-1]!="/":
        SRC_DIR+=("/")
    APP_DIR =            config.get(       C_CONF_FILES, C_CONF_FILES_APP_DIR)
    if APP_DIR[-1]!="/":
        APP_DIR+=("/")
    CONF_DIR =           config.get(       C_CONF_FILES, C_CONF_FILES_CONF_DIR)
    if CONF_DIR[-1]!="/":
        CONF_DIR+=("/")
    TEMPLATES_CONF_DIR = config.get(       C_CONF_FILES, C_CONF_FILES_CONF_TEMPLATES)
    if TEMPLATES_CONF_DIR[-1]!="/":
        TEMPLATES_CONF_DIR+=("/")
    TEMPLATES_ENV_DIR = config.get(       C_CONF_FILES, C_CONF_FILES_ENV_TEMPLATES)
    if TEMPLATES_ENV_DIR[-1]!="/":
        TEMPLATES_ENV_DIR+=("/")
    HADOOP_DIR =         config.get(       C_CONF_FILES, C_CONF_FILES_HADOOP_DIR)
    if HADOOP_DIR[-1]!="/":
        HADOOP_DIR+=("/")
    HADOOP_CONF_DIR = HADOOP_DIR+HADOOP_REL_ETC                                   # (!) This is overriten later at get_conf_out_dirs()
    HADOOP_TEMP_DIR =    config.get(       C_CONF_FILES, C_CONF_FILES_TMP_DIR)
    if HADOOP_TEMP_DIR[-1]!="/":
        HADOOP_TEMP_DIR+=("/")
    DATA_DIR_TMP =       config.get(       C_CONF_FILES, C_CONF_FILES_TMP_DATA_DIR)
    if DATA_DIR_TMP[-1]!="/":
        DATA_DIR_TMP+=("/")
    OUTPUT_DIR =         config.get(       C_CONF_FILES, C_CONF_FILES_OUT_DIR)
    if OUTPUT_DIR[-1]!="/":
        OUTPUT_DIR+=("/")
    USERNAME_MACHINES =  config.get(       C_CONF_FILES, C_CONF_FILES_USERNAME_MACHINES)
    PREFIX_OUTPUT =      config.get(       C_CONF_FILES, C_CONF_FILES_PREFIX_OUTPUT)

    # HDFS
    HDFS_DATA_DIR =          config.get(    C_CONF_HDFS, C_CONF_HDFS_INPUT_DATA_DIR)
    HDFS_DATA_DIR_SUFFIX =   config.get(    C_CONF_HDFS, C_CONF_HDFS_DIR_SUFFIX)
    HDFS_DATA_DIR+=HDFS_DATA_DIR_SUFFIX
    if HDFS_DATA_DIR[-1]!="/":
        HDFS_DATA_DIR+=("/")
    PACKETS_PER_HDFS_BLOCK = config.getint( C_CONF_HDFS, C_CONF_HDFS_PACKETS_PER_BLOCK)
    CHECKSUM_SIZE =          config.getint( C_CONF_HDFS, C_CONF_HDFS_CHECKSUM_SIZE) # Has to divide the block size (which is a multiple of 32)
    # PACKET SIZE is generated with "generate_vdif", next cell.

    # Slaves
    SLAVES =                 config.get(    C_CONF_HSLAVES, C_CONF_H_ALL_CONFIG_FILE)
    MAX_SLAVES =             config.getint( C_CONF_HSLAVES, C_CONF_HSLAVES_MAX_SLAVES)

    # Masters
    MASTERS =           config.get(         C_CONF_HMASTERS, C_CONF_H_ALL_CONFIG_FILE)
    MASTER_IS_SLAVE =   config.getboolean(  C_CONF_HMASTERS, C_CONF_HMASTERS_MASTER_IS_SLAVE)

    # Hadoop-other
    HADOOP_START_DELAY =     config.getint(     C_CONF_OTHER, C_CONF_OTHER_START_DELAY)
    HADOOP_STOP_DELAY =      config.getint(     C_CONF_OTHER, C_CONF_OTHER_STOP_DELAY)
    HDFS_COPY_DELAY =        config.getint(     C_CONF_OTHER, C_CONF_OTHER_COPY_DELAY)
    TEXT_MODE =              config.getboolean( C_CONF_OTHER, C_CONF_OTHER_TEXT_MODE)
    HADOOP_TEXT_DELIMITER =  config.get(        C_CONF_OTHER, C_CONF_OTHER_TEXT_DELIMITER)
    MAX_CPU_VCORES =         config.getint(     C_CONF_OTHER, C_CONF_OTHER_MAX_CPU_VCORES)
    FFT_AT_MAPPER =          config.getboolean( C_CONF_OTHER, C_CONF_OTHER_FFT_MAP)
    ADJUST_MAPPERS =         config.getfloat(   C_CONF_OTHER, C_CONF_OTHER_ADJ_MAP)
    ADJUST_REDUCERS =        config.getfloat(   C_CONF_OTHER, C_CONF_OTHER_ADJ_RED)
    ONE_BASELINE_PER_TASK =  config.getboolean( C_CONF_OTHER, C_CONF_OTHER_ONE_BASELINE)
    TASK_SCALING_STATIONS =  config.getboolean( C_CONF_OTHER, C_CONF_OTHER_SCALING_STATIONS)
    TIMEOUT_STOP =           config.getint(     C_CONF_OTHER, C_CONF_OTHER_TIMEOUT_STOP_NODES)
    SINGLE_PRECISION =       config.getboolean( C_CONF_OTHER, C_CONF_OTHER_SINGLE_PRECISION)
    FFTS_PER_CHUNK =         -1                                                                # TO DO: remove
    MIN_MAPPER_CHUNK =       -1                                                                # TO DO: remove
    MAX_MAPPER_CHUNK =       -1                                                                # TO DO: remove

    # Experiment
    INI_FOLDER =                         config.get(   C_CONF_EXP, C_CONF_EXP_FOLDER)
    INI_STATIONS = INI_FOLDER + "/" +    config.get(   C_CONF_EXP, C_CONF_EXP_STATIONS)
    INI_SOURCES = INI_FOLDER + "/" +     config.get(   C_CONF_EXP, C_CONF_EXP_SOURCES)
    INI_DELAYS = INI_FOLDER + "/" +      config.get(   C_CONF_EXP, C_CONF_EXP_DELAYS)
    INI_DELAY_MODEL = INI_FOLDER + "/" + config.get(   C_CONF_EXP, C_CONF_EXP_DELAY_MODEL)
    INI_MEDIA = INI_FOLDER + "/" +       config.get(   C_CONF_EXP, C_CONF_EXP_MEDIA)
    INI_CORRELATION = INI_FOLDER + "/" + config.get(   C_CONF_EXP, C_CONF_EXP_CORRELATION)
    DATA_DIR = INI_FOLDER + "/" +        config.get(   C_CONF_EXP, C_CONF_EXP_MEDIA_SUB) + "/"
    OUT_DIR_PREFIX =                     config.get(   C_CONF_EXP, C_CONF_EXP_MEDIA_SUB_PREFIX)

    #TO DO: vcores value is forced from MAX_CPU_VCORES... maybe it has to be taken out from the configuration file section... fix this.

    HDFS_REPLICATION =                   config.getint( C_CONF_H_HDFS, C_H_HDFS_REPLICATION)

    ##AUTO_STATIONS = config.getint("Correlation",'Corr. same station')
    ##AUTO_THREADS = config.getint("Correlation",'Corr. same thread')

    OUTPUT_SYM=INI_FOLDER+"/"+OUT_DIR_PREFIX+"_"+timestamp_str+"/"

    if v==1:
        #if SAMPLES_PER_PACKET<FFT_SIZE:
        #    print(" (!) The packet has fewer samples than the size of the FFT",file=file_log)

        print(" FFT at mapper:\t\t\t" + str(int(FFT_AT_MAPPER)),file=file_log)
        print(" Hadoop delays: [initialization = " + str(HADOOP_START_DELAY) + " s], [termination = " + str(HADOOP_STOP_DELAY) + " s], [HDFS = " + str(HDFS_COPY_DELAY) + " s]",file=file_log)

        if MAX_SLAVES>0:
            print(" Iterating: NODES (max=" +str(MAX_SLAVES) + ")",file=file_log)
        else:
            print(" Single run max. number of available nodes",file=file_log)

        if OVER_SLURM:
            print(" ---> SLURM - Configured for running on SLURM",file=file_log)
        else:
            print(" ---> VBox - Configured for running on VBox",file=file_log)

        print(" Temp dir overriden (default: /tmp)",file=file_log)

    return([MAPPER, REDUCER, DEPENDENCIES, PACKETS_PER_HDFS_BLOCK,CHECKSUM_SIZE,\
            SRC_DIR,APP_DIR, CONF_DIR, TEMPLATES_CONF_DIR, TEMPLATES_ENV_DIR, HADOOP_DIR,HADOOP_CONF_DIR,NODES,
            MAPPERSH,REDUCERSH,JOBSH,PYTHON_X,\
            USERNAME_MACHINES,MAX_SLAVES,SLAVES,MASTERS,MASTER_IS_SLAVE,HADOOP_TEMP_DIR,DATA_DIR,DATA_DIR_TMP,HDFS_DATA_DIR,HADOOP_START_DELAY,HADOOP_STOP_DELAY,\
            PREFIX_OUTPUT,HADOOP_TEXT_DELIMITER,OUTPUT_DIR,OUTPUT_SYM,RUN_PIPELINE,RUN_HADOOP,MAX_CPU_VCORES,\
            HDFS_REPLICATION,OVER_SLURM,HDFS_COPY_DELAY,\
            FFT_AT_MAPPER,INI_FOLDER,\
            INI_STATIONS, INI_SOURCES, INI_DELAY_MODEL, INI_DELAYS, INI_MEDIA, INI_CORRELATION,\
            INTERNAL_LOG_MAPPER,INTERNAL_LOG_REDUCER,ADJUST_MAPPERS,ADJUST_REDUCERS,FFTS_PER_CHUNK,TEXT_MODE,\
            USE_NOHASH_PARTITIONER,USE_LUSTRE_PLUGIN,LUSTRE_USER_DIR,LUSTRE_PREFIX,ONE_BASELINE_PER_TASK,\
            MIN_MAPPER_CHUNK,MAX_MAPPER_CHUNK,TASK_SCALING_STATIONS,SORT_OUTPUT,BM_AVOID_COPY,\
            BM_DELETE_OUTPUT,TIMEOUT_STOP,SINGLE_PRECISION,PROFILE_MAP,PROFILE_RED])

def update_config_param(source_file,pairs,v=0,file_log=sys.stdout):
    """
    Updates a list of pairs [parameter,value] in a configuration file. Should be valid to any .ini file, but this is
     used to override the configuration on the CorrelX configuration file.

    Parameters
    ----------
     source_file : str
         configuration file (.ini).
     pairs : list
         list of [parameter,value].
     v : int
         verbose if 1.
     file_log : file handler
         handler for log file.

    Returns
    -------
     N/A

    Notes
    -----
    |
    | **TO DO:**
    |
    |  Currently parameters that are not found are not added, this should be reported.
    """
    if v==1:
        print("\nOverriding configuration file!",file=file_log)
    destination_file = source_file

    isamec = ""
    copy_again=1
    destination_file+=".tmp"
    isamec = " mv " + destination_file + " " + source_file

    command = ""
    iteration=0

    for pair in pairs:
        iteration+=1
        if iteration==1:
            #command+="sed -e 's/" + pair[0] + ".*/" + pair[0] + "\\t\\t" + pair[1] + "/g' " + source_file
            command+="sed -e 's;" + pair[0] + ".*;" + pair[0] + "\\t\\t" + pair[1] + ";g' " + source_file
        if iteration>0:
            #command+="|sed -e 's/" + pair[0] + ".*/" + pair[0] + "\\t\\t" + pair[1] + "/g' "
            command+="|sed -e 's;" + pair[0] + ".*;" + pair[0] + "\\t\\t" + pair[1] + ";g' "

    command+= "> " + destination_file

    os.system(command)
    os.system(isamec)

    if v==1:
        for pair in pairs: #[1:]:
            print("  " + pair[0].ljust(49) + " " + pair[1],file=file_log)

    # Now check if any parameter was not found, and thus was not updated
    # Checks lowercase!
    with open(source_file, 'r') as f:
        lines = f.read()
        for pair in pairs:
            count=1
            if pair[0].lower() in lines.lower():
                count=0
            if count==1:
                if v==1:
                    print("  (!) Parameter not found: " + pair[0].ljust(49),file=file_log)

def override_configuration_parameters(forced_configuration_string,config_file,v=1,file_log=sys.stdout):
    """
    This function takes as input the string with the parameters to the main script and overrides the corresponding parameters
    in the configuration files. This is to simplify batch testing.

    Parameters
    ----------
     forced_configuration_string : str
         Comma separated list of parameter0=value0,parameter1=value1,...
     config_file : str
         Path to CorrelX configuration file.
     v : int
         Verbose if 1.
     file_log : file handler
         Handler for log file.

    Output:
    -------
     N/A

    Notes
    -----
    |
    | **Assumptions:**
    |
    |  Assuming that C_H_MAPRED_RED_OPTS is higher than C_H_MAPRED_MAP_OPTS, so the first value is
    |    taken for C_H_MAPRED_CHILD_OPTS.
    |
    |
    | **Notes:**
    |
    |  For new parameters in configh.conf:
    |  (1) Add constants for CLI in const_config.py.
    |  (2) Check/add constants for hadoop configuration files in const_hadoop.py (if applicable).
    |  (3) Add parameter reading in get_configuration().
    |  (4) Add option in if-structure below.
    """
    # Override configuration parameters
    # e.g.: -f stations=3,slowstart=0.4,ppb=1
    read_configuration_pairs=[i.split('=') for i in forced_configuration_string.split(',')]
    forced_configuration_pairs=[]
    unr_parameters=[]
    for pair in read_configuration_pairs:
        if len(pair)<2:
            unr_parameters+=[pair]
        else:
            [parameter,value] = pair

            if parameter==C_ARG_PPB:                                                             # Num. VDIF frames per block
                forced_configuration_pairs+=[[C_CONF_HDFS_PACKETS_PER_BLOCK+":",value]]

            elif parameter==C_ARG_SLOWSTART:                                                     # Mapreduce slowstart
                forced_configuration_pairs+=[[C_H_MAPRED_SLOWSTART+":",value]]

            elif parameter==C_ARG_REPLICATION:                                                   # HDFS replication
                forced_configuration_pairs+=[[C_H_HDFS_REPLICATION+":",value]]

            elif parameter==C_ARG_FFTM:                                                          # FFT at mapper
                forced_configuration_pairs+=[[C_CONF_OTHER_FFT_MAP+":",value]]

            elif parameter==C_ARG_FFTR:                                                          # FFT at reducer
                forced_configuration_pairs+=[[C_CONF_OTHER_FFT_MAP+":",str(int(not(bool(value))))]]

            elif parameter==C_ARG_ADJM:                                                          # Factor to multiply number of mappers
                forced_configuration_pairs+=[[C_CONF_OTHER_ADJ_MAP+":",value]]

            elif parameter==C_ARG_ADJR:                                                          # Factor to multiply number of reducers
                forced_configuration_pairs+=[[C_CONF_OTHER_ADJ_RED+":",value]]

            elif parameter==C_ARG_VCORES:                                                        # Number of cores per node
                forced_configuration_pairs+=[[C_H_YARN_MAX_VCORES+":",value]]
                forced_configuration_pairs+=[[C_H_YARN_RES_VCORES+":",value]]
                forced_configuration_pairs+=[[C_CONF_OTHER_MAX_CPU_VCORES+":",value]]

            elif parameter==C_ARG_VCORESPERMAP:                                                  # Number of cores per map task
                forced_configuration_pairs+=[[C_H_MAPRED_VCORES_MAP+":",value]]

            elif parameter==C_ARG_VCORESPERRED:                                                  # Number of cores per reduce task
                forced_configuration_pairs+=[[C_H_MAPRED_VCORES_RED+":",value]]

            elif parameter==C_ARG_MAPSPERNODE:                                                   # Max. maps per node
                forced_configuration_pairs+=[[C_H_MAPRED_MAP_MAX+":",value]]

            elif parameter==C_ARG_REDUCESPERNODE:                                                # Max. reduces per node
                forced_configuration_pairs+=[[C_H_MAPRED_RED_MAX+":",value]]

            elif parameter==C_ARG_NODEMEM:                                                       # Memory per node (maps+reduces)
                forced_configuration_pairs+=[[C_H_YARN_MAX_MEM_MB+":",value]]
                forced_configuration_pairs+=[[C_H_YARN_RES_MEM_MB+":",value]]

            elif parameter==C_ARG_CONTAINERMEM_MAP:                                              # Memory per container (mapper)
                forced_configuration_pairs+=[[C_H_YARN_MAP_MB+":",value]]

            elif parameter==C_ARG_CONTAINERMEM_RED:                                              # Memory per container (reducer)
                forced_configuration_pairs+=[[C_H_YARN_RED_MB+":",value]]

            elif parameter==C_ARG_CONTAINERMEM_AM:                                               # Memory per container (app manager)
                forced_configuration_pairs+=[[C_H_YARN_AM_RES_MB+":",value]]

            elif parameter==C_ARG_CONTAINERHEAP_MAP:                                             # Memory heap per container (mapper)
                #forced_configuration_pairs+=[[C_H_MAPRED_MAP_OPTS+":",    "-Xms"+value+"m -Xmx"+value+"m"]] # init
                forced_configuration_pairs+=[[C_H_MAPRED_MAP_OPTS+":", "-Xmx"+value+"m"]]                    # no init

            elif parameter==C_ARG_CONTAINERHEAP_RED:                                             # Memory heap per container (mapper)
                #forced_configuration_pairs+=[[C_H_MAPRED_RED_OPTS+":",    "-Xms"+value+"m -Xmx"+value+"m"]] # init
                #forced_configuration_pairs+=[[C_H_MAPRED_CHILD_OPTS+":", "-Xms"+value+"m -Xmx"+value+"m"]] # init
                forced_configuration_pairs+=[[C_H_MAPRED_RED_OPTS+":", "-Xmx"+value+"m"]]                    # no init
                forced_configuration_pairs+=[[C_H_MAPRED_CHILD_OPTS+":", "-Xmx"+value+"m"]]                 # no init

            elif parameter==C_ARG_CONTAINERHEAP_AM:                                             # Memory heap per container (mapper)
                #forced_configuration_pairs+=[[C_H_MAPRED_AM_OPTS+":", "-Xms"+value+"m -Xmx"+value+"m"]] # init
                forced_configuration_pairs+=[[C_H_MAPRED_AM_OPTS+":", "-Xmx"+value+"m"]]                    # no init

                forced_configuration_pairs+=[[C_H_MAPRED_MAP_OPTS+":", "-Xmx"+value+"m"]]                   # no init
                forced_configuration_pairs+=[[C_H_MAPRED_RED_OPTS+":", "-Xmx"+value+"m"]]                   # no init

            elif parameter==C_ARG_SORTMEM:                                                       # Sort memory (after map)
                forced_configuration_pairs+=[[C_H_MAPRED_SORT_MB+":",value]]

            elif parameter==C_ARG_BLOCKSIZE:                                                     # HDFS block size
                forced_configuration_pairs+=[[C_H_HDFS_BLOCKSIZE+":",value]]

            elif parameter==C_ARG_MASTERWORKS:                                                   # Master node is also slave
                forced_configuration_pairs+=[[C_CONF_HMASTERS_MASTER_IS_SLAVE+":",value]]

            elif parameter==C_ARG_TASKSPERJVM:                                                   # Tasks before restaring JVM
                forced_configuration_pairs+=[[C_H_MAPRED_JVM_NUMTASKS+":",value]]

            elif parameter==C_ARG_AVOIDCOPY:                                                     # Do not re-copy input files
                forced_configuration_pairs+=[[C_CONF_BENCHMARKING_AVOID_COPY+":",value]]

            elif parameter==C_ARG_DELETEOUTPUT:                                                  # Delete output file (for benchmarking)
                forced_configuration_pairs+=[[C_CONF_BENCHMARKING_DELETE_OUTPUT+":",value]]

            elif parameter==C_ARG_SCALEST:                                                       # Linear scaling stations
                forced_configuration_pairs+=[[C_CONF_OTHER_SCALING_STATIONS+":",value]]

            elif parameter==C_ARG_MEDIASUFFIX:                                                   # Media suffix folder
                forced_configuration_pairs+=[[C_CONF_HDFS_DIR_SUFFIX+":",value]]

            elif parameter==C_ARG_SINGLEPRECISION:                                               # Single precision in computations
                forced_configuration_pairs+=[[C_CONF_OTHER_SINGLE_PRECISION+":",value]]

            #elif parameter==C_ARG_DATA:                                                          # Data directory
            #    list_value=value.split("/")
            #    value='\/'.join(map(str, list_value))
            #    forced_configuration_pairs+=[["Data directory:",value]]

            elif parameter==C_ARG_EXPER:                                                          # Experiment folder
                forced_configuration_pairs+=[[C_CONF_EXP_FOLDER+":",value]]

            elif parameter==C_ARG_OUT:                                                            # Output folder
                forced_configuration_pairs+=[[C_CONF_FILES_OUT_DIR+":",value]]

            elif parameter==C_ARG_APP:                                                            # Application folder
                forced_configuration_pairs+=[[C_CONF_FILES_APP_DIR+":",value]]

            elif parameter==C_ARG_SERIAL:                                                         # Run pipeline mode
                forced_configuration_pairs+=[[C_CONF_GEN_RUN_PIPE+":",value]]

            elif parameter==C_ARG_PARALLEL:                                                       # Run Hadoop
                forced_configuration_pairs+=[[C_CONF_GEN_RUN_HADOOP+":",value]]

            elif parameter==C_ARG_NM_LOC_PORT:                                                    # Nodemanager ports
                forced_configuration_pairs+=[[C_H_YARN_NM_LOCALIZER_ADDRESS+":",HOSTNAME_HADOOP+":"+value]]
            elif parameter==C_ARG_NM_WEB_PORT:
                forced_configuration_pairs+=[[C_H_YARN_NM_WEBAPP_ADDRESS+":",HOSTNAME_HADOOP+":"+value]]
            elif parameter==C_ARG_SHUFFLE_PORT:
                forced_configuration_pairs+=[[C_H_MAPRED_SHUFFLE_PORT+":",value]]

            else:                                                                                # Unknown parameter, simply report it
                unr_parameters+=[parameter]

    update_config_param(source_file=config_file,pairs=forced_configuration_pairs,v=v,file_log=file_log)

    if v==1:
        if unr_parameters!=[]:
            print("\n Unrecognized parameters:",file=file_log)
            for i in unr_parameters:
                print("  "+''.join(map(str,i)),file=file_log)

def get_list_configuration_files(config_file):
    """
    Get list of Hadoop configuration files.

    Parameters
    ----------
     config_file : str
         Path to CorrelX configuration file.

    Returns
    -------
     list_configurations : list
         List of sections from configuration file associated to lists in "pairs_config" below.
     pairs_config : list
         List of pairs [[param0,value0],[param1,value1]...]...,[[...]...,[...]]] to update
                                 Hadoop config files later.
    """
    config = configparser.ConfigParser()
    config.optionxform = str
    config.read(config_file)

    list_configurations=[i for i in config.sections() if (C_CONF_PREH in i)and(C_CONF_SUF_SLAVES not in i)and(C_CONF_SUF_OTHER not in i)]

    pairs_config = [[list(j) for j in config.items(i)] for i in list_configurations  ]

    return([list_configurations,pairs_config])

def is_this_node_master(master,temp_log,v=0,file_log=sys.stdout):
    """
    Devised in case the script was run in parallel at many nodes.
    Currenlty simply used to enforce that only one node is running as master.

    Parameters
    ----------
     master : str
         master node name.
     temp_log : str
         path to temporary file for system calls (buffer).
     v : int
         verbose if 1.
     file_log : file handler
         handler for log file.

    Returns
    -------
     this_is_master : int
         1 if current node is the master, 0 otherwise.
     my_name : str
         current node name.
     my_ip : str
         current node IP address.

    Notes
    -----
    |
    | **TO DO:**
    |
    |  Simplify this.
    """
    os.system("hostname > " + temp_log)
    os.system("hostname -I >> " + temp_log)
    this_is_master = 0
    local_info=[]
    thiss=" not "
    with open(temp_log, 'r') as f_tmp:
        for line in f_tmp:
            local_info += line.strip().split(",")
            if master in line:
                this_is_master = 1
                thiss=" "

    my_name = local_info[0]
    my_ip = local_info[1]
    if v==1:
        print("\nChecking if this node is master:",file=file_log)
        print(" Node " + my_name + " (" + my_ip + ") is" + thiss + "master",file=file_log)

    return([this_is_master,my_name,my_ip])

def overwrite_nodes_file(nodes_list,nodes_file,v=0,file_log=sys.stdout):
    """
    Overwrite nodes file (in case less nodes are requested than are available).

    Parameters
    ----------
     nodes_list : list of str
         names of the nodes in the allocation.
     nodes_file : str
         path to nodes file.
     v : int
         verbose if 1.
     file_log : file handler
         handler for log file.
    """
    if v==1:
        print("\nCreating nodes file:",file=file_log)
        print(" " + nodes_file,file=file_log)

    with open(nodes_file, 'w') as f:
        for i in nodes_list:
            print(i,file=f)
            if v==1:
                print("  " + i,file=file_log)
    if v==1:
        print(" Nodes file overwritten!",file=file_log)

def create_directories(directories,v=0,file_log=sys.stdout):
    """
    Create directories from a list of str with their paths.
    """
    if v==1:
        print("\nCreating directories:",file=file_log)
    for i in directories:
        if not os.path.isdir(i):
            os.system("mkdir -p " + i)
        if v==1:
            print(" "+i,file=file_log)

def get_conf_out_dirs(master_name,hadoop_dir,app_dir,conf_dir,suffix_conf,output_dir,suffix_out,v=1,file_log=sys.stdout):
    """
    Get paths for configuration and output folders.
    App and Conf directories are modified with master's name.

    Parameters
    ----------
     master_name : str
         master node hostname.
     hadoop_dir : str
         hadoop base folder.
     app_dir : str
         base app folder.
     conf_dir : str
         base configuration folder.
     suffix_conf : str
         suffix for configuration folder.
     output_dir : str
         base output folder.
     suffix_out : str
         suffix for output folder (only the string after the last "/" is taken).
     v : int
         verbose if 1.
     file_log : file handler
         handler for log file.

    Returns
    -------
     app_dir : str
         path to app folder (modified for this master).
     conf_dir : str
         path to configuration folder (for modified config file and bash scripts).
     hadoop_conf_dir : str
         path to hadoop configuration folder for master node.
     hadoop_default_conf_dir : str
         path to hadoop default configuration folder (to be used at slaves nodes).
     output_dir : str
         path in local filesystem for output file.

    Notes
    -----
     Having a different folder associated to the master node allows to run multiple deployments in the same cluster if
      the local filesystem is NFS.
    """

    if "/" in suffix_out:
        suffix_out=suffix_out.split('/')[-1]

    conf_dir+=master_name + "/"                                        # Create conf dir for each node (still shared...)
    app_dir+=master_name + "/"                                         # Create app dir for each node (still shared...)
    hadoop_conf_dir = conf_dir + "etc_hadoop_" + suffix_conf +"/"      # Hadoop conf dir for this host
    output_dir = output_dir + suffix_out +"/"                          # Create more folders for different experiments

    create_directories(directories=[conf_dir,hadoop_conf_dir,output_dir],v=v,file_log=file_log)

    # This is for slaves, for distributing files
    hadoop_default_conf_dir = hadoop_dir+HADOOP_REL_ETC

    return([app_dir,conf_dir,hadoop_conf_dir,hadoop_default_conf_dir,output_dir])

def reduce_list_nodes(num_slaves,nodes_list,v=1,file_log=sys.stdout):
    """
    Reduce list of nodes given a maximum number of nodes.

    Parameters
    ----------
     num_slaves : int
         maximum number of slaves (-1 for no maximum).
     nodes_list : list of str
         names of nodes.

    Output:
    -------
     num_slaves:  number of nodes in updated list.
     nodes_list:  updated nodes_list.
    """

    if num_slaves>0 and num_slaves<len(nodes_list):
        nodes_list=nodes_list[:num_slaves]
        if v==1:
            print("\nForcing number of nodes: ".ljust(24) + str(num_slaves),file=file_log)
    else:
        num_slaves=len(nodes_list)

    return([num_slaves,nodes_list])


from django.db import models

# Create your models here.
class Piece(models.Model):
    PAINTING = 'PA'
    ABSTRACT = 'AB'
    DRAWINGS = 'DR'
    INSTALLATIONS = 'IN'
    SCULPTURES = 'SC'
    PHOTOGRAPHY = 'PH'
    KIND = [
        (PAINTING, 'Painting'),
        (SCULPTURES, 'Sculptures'),
        (DRAWINGS, 'Drawings'),
        (ABSTRACT, 'Abstract'),
        (INSTALLATIONS, 'Installations'),
        (PHOTOGRAPHY, 'Photography'),
    ]
    image = models.ImageField(upload_to = 'images')
    title = models.CharField(max_length=120)
    description = models.TextField()
    price = models.TextField()
    artist = models.CharField(max_length=100)
    kind = models.CharField(
        max_length=2,
        choices=KIND,
        default=PAINTING,
    )


#!/usr/bin/env python3
import aws_cdk as cdk

from aws_config_to_s3.aws_config_to_s3_stack import AwsConfigToS3Stack

app = cdk.App()
AwsConfigToS3Stack(app,
                   'AwsConfigToS3Stack',
                   description='This stack creates a Lambda function to query AWS Config advanced query and store the results in an S3 bucket'
                   )

app.synth()


from tkinter import *

from simulation.simulation import Simulation
from simulation.object import Object, ObjectDimension
from simulation.tkinter_window import Window

Object.CANVAS_WIDTH = 1200
Object.CANVAS_HEIGHT = 720

root = Tk()
root.geometry(str(Object.CANVAS_WIDTH) + "x" + str(Object.CANVAS_HEIGHT))

center_x, center_y = Object.CANVAS_WIDTH / 2, Object.CANVAS_HEIGHT / 2

center_part = ObjectDimension(center_x, center_y, 50, 50)

motor_radius = 15
margin_delta = 30

motor_left = ObjectDimension(center_x - margin_delta - motor_radius / 2, center_y - motor_radius, motor_radius * 2,
                             motor_radius * 2)
motor_right = ObjectDimension(center_x + margin_delta - motor_radius, center_y - motor_radius, motor_radius * 2,
                              motor_radius * 2)

corner_radius = 15

corner_left = ObjectDimension(margin_delta, margin_delta, corner_radius * 2, corner_radius * 2)
corner_right = ObjectDimension(Object.CANVAS_WIDTH - margin_delta - corner_radius * 2, margin_delta, corner_radius * 2,
                               corner_radius * 2)

simulation = Simulation(motor_left, motor_right, corner_left, corner_right)
app = Window(root, simulation)

root.after(16, app.frame)
root.mainloop()


from typing import List
import collections, math, string

class Solution:
    def ladderLength(self, beginWord: str, endWord: str, wordList: List[str]) -> int:
        def isAdjacent(word1, word2):
            count = 0
            for char1, char2 in zip(word1, word2):
                if char1 != char2:
                    count += 1
            return count == 1

        def genNeighbor(word):
            word = list(word)
            neighbors = []
            for i in range(len(word)):
                originChar = word[i]
                for char in string.ascii_lowercase:
                    if char == originChar:
                        continue
                    word[i] = char
                    neighbors.append("".join(word))
                word[i] = originChar
            return neighbors

        wordSet = set(wordList)
        if endWord not in wordSet:
            return 0

        # wordSet.add(beginWord)

        queue = collections.deque([(beginWord, 1)])
        while queue:
            word, dist = queue.popleft()
            for neighbor in genNeighbor(word):
                if neighbor == endWord:
                    return dist + 1
                if neighbor in wordSet:
                    queue.append((neighbor, dist + 1))
                    wordSet.remove(neighbor)
        return 0

soln = Solution()
print(soln.ladderLength("hit", "cog", ["hot","dot","dog","lot","log","cog"]))


from host import FileNode
from host.models.Cloud import Cloud
from common.BaseCommand import BaseCommand
from common_util import ResultAndData, Error, Success
from argparse import Namespace

__author__ = 'zadjii'

class DebugMirrorsCommand(BaseCommand):
    def add_parser(self, subparsers):
        dbg_mirrors = subparsers.add_parser('dbg-mirrors', description='Debug print all the mirrors on this device')
        return dbg_mirrors

    def do_command_with_args(self, instance, args):
        # type: (Instance, Namespace) -> ResultAndData
        db = instance.get_db()
        mirrors = db.session.query(Cloud).all()
        for mirror in mirrors:
            print(
                '[{:3}]\t{}/{}, {}, {}, {}\n\t{}'
                .format(
                    mirror.id
                    , mirror.username
                    , mirror.name
                    , mirror.created_on
                    , mirror.last_update
                    , mirror.completed_mirroring
                    , [child.name for child in mirror.children.all()]
                )
            )
        return Success()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

class Solution:
    """
    给你两个二进制字符串，返回它们的和（用二进制表示）。

    输入为 非空 字符串且只包含数字 1 和 0。
    """

    def addBinary(self, a: str, b: str) -> str:
        if not a or not b:
            return ""
        index_a = len(a) - 1
        index_b = len(b) - 1

        ans = ""

        temp = 0

        while index_a >= 0 and index_b >= 0:
            num_a = int(a[index_a])
            num_b = int(b[index_b])
            sum_ = num_a + num_b + temp
            temp = sum_ // 2
            sum_ = sum_ % 2
            ans = str(sum_) + ans
            index_a -= 1
            index_b -= 1

        if index_b >= 0:
            index_a = index_b
            a = b

        while index_a >= 0:
            num_a = int(a[index_a])
            sum_ = num_a + temp
            temp = sum_ // 2
            sum_ = sum_ % 2
            ans = str(sum_) + ans
            index_a -= 1

        if temp > 0:
            ans = str(temp) + ans
        return ans


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import nltk;

# In[2]:

from textblob import TextBlob
import pandas as pd

# In[10]:

df=pd.read_csv('train_file.csv', encoding = "ISO-8859-1")

# In[11]:

train = df

# In[12]:

# Removing punctuations
data=train.iloc[:,1:4]
data.replace("[^a-zA-Z]"," ",regex=True, inplace=True)

# Renaming column names for ease of access
list1= [i for i in range(3)]
new_Index=[str(i) for i in list1]
data.columns= new_Index
data.head(5)

# In[13]:

# Convertng headlines to lower case
for index in new_Index:
    data[index]=data[index].str.lower()
data.head(1)

# In[14]:

title = []
for row in range(0,len(data.index)):
    title.append(''.join(str(x) for x in data.iloc[row,0]))

headline = []
for row in range(0,len(data.index)):
    headline.append(''.join(str(x) for x in data.iloc[row,1]))

df=pd.read_csv('test_file.csv', encoding = "ISO-8859-1")

# In[41]:

test = df

# In[42]:

# Removing punctuations
data=test.iloc[:,1:4]
data.replace("[^a-zA-Z]"," ",regex=True, inplace=True)

# Renaming column names for ease of access
list1= [i for i in range(3)]
new_Index=[str(i) for i in list1]
data.columns= new_Index
data.head(5)

# In[43]:

# Convertng headlines to lower case
for index in new_Index:
    data[index]=data[index].str.lower()
data.head(1)

# In[44]:

title = []
for row in range(0,len(data.index)):
    title.append(''.join(str(x) for x in data.iloc[row,0]))

# In[48]:
headline = []
for row in range(0,len(data.index)):
    headline.append(''.join(str(x) for x in data.iloc[row,1]))

headsent = []
for i in range(0,len(data.index)):
    analysis = TextBlob(headline[i])
    headsent.insert(i,analysis.sentiment.polarity)

titlesent = []
for i in range(0,len(data.index)):
    abc = TextBlob(title[i])
    titlesent.insert(i,abc.sentiment.polarity)

df['SentimentTitle'] = titlesent

# In[68]:

df['SentimentHeadline'] = headsent

df.head(7)

# In[72]:

df.to_csv(r'C:\Octave\Machine Learning\dataset\test.csv', index = False)

# In[ ]:


import matplotlib.pyplot as plt
from datetime import date, timedelta
import telebot
from telebot import types
import os
from ast import literal_eval
import requests
from bs4 import BeautifulSoup
from flask import Flask, request

TOKEN = '1$$$$$$$$$$$$$$$$$$$$$$$$$$$'
bot = telebot.TeleBot(TOKEN)
server = Flask(__name__)
# bot.set_webhook()

today = date.today().strftime('%d.%m.%Y')
day7 = date.today() - timedelta(days=6)
day6 = date.today() - timedelta(days=5)
day5 = date.today() - timedelta(days=4)
day4 = date.today() - timedelta(days=3)
day3 = date.today() - timedelta(days=2)
day2 = date.today() - timedelta(days=1)

r = requests.get('http://cgo-sreznevskyi.kyiv.ua/index.php?fn=k_meteo&f=kyiv')

html = BeautifulSoup(r.text, 'lxml')

parsing_str = html.select('tr')[6].text
table_list = parsing_str.split('\xa0')
table_list2 = table_list[:8]

day_values = ('Дата: ' + table_list2[0] + '\n' + 'Температура повітря, oC: ' + table_list2[1] + '\n' + 'Вологість повітря, % : ' + table_list2[2] + '\n' + 'Напрям вітру: ' + table_list2[3] +
              '\n' + 'Максимальна швидкість вітру, м/с: ' + table_list2[4] + '\n' + 'Атмосферний тиск, гПа: ' + table_list2[5] + '\n' + 'Атмосферний тиск, мм рт.ст.: ' + table_list2[6])

for line in open("./db_weather.txt", "r", encoding="cp1251"):
    new = literal_eval(line)

for row in new:
    for elem in row:

        if elem == day7.strftime('%d.%m.%Y'):
            data1 = row[0]
            temp1 = float(row[1])
            pressure1 = int(row[6])
            humidity1 = int(row[2])
            wind1 = row[3]
        if elem == day6.strftime('%d.%m.%Y'):

            data2 = row[0]
            temp2 = float(row[1])
            pressure2 = int(row[6])
            humidity2 = int(row[2])
            wind2 = row[3]
        if elem == day5.strftime('%d.%m.%Y'):
            data3 = row[0]
            temp3 = float(row[1])
            pressure3 = int(row[6])
            humidity3 = int(row[2])
            wind3 = row[3]
        if elem == day4.strftime('%d.%m.%Y'):
            data4 = row[0]
            temp4 = float(row[1])
            pressure4 = int(row[6])
            humidity4 = int(row[2])
            wind4 = row[3]
        if elem == day3.strftime('%d.%m.%Y'):
            data5 = row[0]
            temp5 = float(row[1])
            pressure5 = int(row[6])
            humidity5 = int(row[2])
            wind5 = row[3]
        if elem == day2.strftime('%d.%m.%Y'):
            data6 = row[0]
            temp6 = float(row[1])
            pressure6 = int(row[6])
            humidity6 = int(row[2])
            wind6 = row[3]
        if elem == today:
            data7 = row[0]
            temp7 = float(row[1])
            pressure7 = int(row[6])
            humidity7 = int(row[2])
            wind7 = row[3]

def press():

    fig, ax = plt.subplots()
    x = [data1, data2, data3, data4, data5, data6, data7]
    y = [pressure1, pressure2, pressure3,
         pressure4, pressure5, pressure6, pressure7]
    plt.plot(x, y)
    plt.scatter(x, y)
    ax.set_facecolor('seashell')
    fig.set_facecolor('floralwhite')
    fig.set_figwidth(12)
    fig.set_figheight(6)
    plt.title(r'Давление за прошедшую неделю')
    plt.ylabel(r'мм. рт.ст.')
    plt.xlabel(r'Дата')
    plt.grid(True)
    plt.savefig('pressure_figure.png')

press()

def humid():
    fig, ax = plt.subplots()
    x = [data1, data2, data3, data4, data5, data6, data7]
    y = [humidity1, humidity2, humidity3,
         humidity4, humidity5, humidity6, humidity7]
    plt.plot(x, y)
    plt.scatter(x, y)
    ax.set_facecolor('seashell')
    fig.set_facecolor('floralwhite')
    fig.set_figwidth(12)
    fig.set_figheight(6)
    plt.title(r'Влажность за прошедшую неделю')
    plt.ylabel(r'%')
    plt.xlabel(r'Дата')
    plt.grid(True)

    plt.savefig('humidity.png')

humid()

def temperature():
    fig, ax = plt.subplots()
    x = [data1, data2, data3, data4, data5, data6, data7]
    y = [temp1, temp2, temp3,
         temp4, temp5, temp6, temp7]
    plt.plot(x, y)
    plt.scatter(x, y)
    ax.set_facecolor('seashell')
    fig.set_facecolor('floralwhite')
    fig.set_figwidth(12)
    fig.set_figheight(6)
    plt.title(r'Температура за прошедшую неделю')
    plt.ylabel(r'oC')
    plt.xlabel(r'Дата')
    plt.grid(True)

    plt.savefig('Tempereture.png')

temperature()

@bot.message_handler(commands=["start"])
def start(m, res=False):
    # Добавляем две кнопки
    markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
    item1 = types.KeyboardButton("Давление")
    item2 = types.KeyboardButton("Ветер")
    item3 = types.KeyboardButton("Влажность")
    item4 = types.KeyboardButton("Температура")
    item5 = types.KeyboardButton("Выбрать дату")
    markup.add(item1)
    markup.add(item2)
    markup.add(item3)
    markup.add(item4)
    markup.add(item5)
    bot.send_message(
        m.chat.id, day_values,  reply_markup=markup)

@bot.message_handler(content_types=["text"])
def handle_text(message):
    # Если юзер прислал 1,
    if message.text.strip() == 'Давление':
        press()
        bot.send_photo(message.chat.id, open('./pressure_figure.png', 'rb'))
        plt.close()
        os.remove('./pressure_figure.png')
    # Если юзер прислал 2,
    elif message.text.strip() == 'Ветер':
        bot.send_message(
            message.chat.id, (f'{data1}: {wind1} \n{data2}: {wind2}\n{data3}: {wind3}\n{data4}: {wind4}\n{data5}: {wind5}\n{data6}: {wind6}\n{data7}: {wind7}\n'))
    # Если юзер прислал 3,
    elif message.text.strip() == 'Влажность':
        humid()
        bot.send_photo(message.chat.id, open('./humidity.png', 'rb'))
        plt.close()
        os.remove('./humidity.png')
    elif message.text.strip() == 'Температура':
        temperature()
        bot.send_photo(message.chat.id, open('./Tempereture.png', 'rb'))
        plt.close()
        os.remove('./Tempereture.png')
    # Отсылаем юзеру сообщение
    elif message.text.strip() == 'Выбрать дату':
        bot.send_message(
            message.chat.id, 'Архив данных ведется с 01.12.2021г!\nВы можете выбрать любую дату в формате день.месяц.год! Например: 20.03.2022!\nВыбранная дата будет последним днем недельного отрезка.\nВведите дату:')
        bot.register_next_step_handler(message, handle_text2)

@bot.message_handler(content_types=["text"])
def handle_text2(m):
    try:

        for row in new:
            for elem in row:
                if m.text.strip() == elem:
                    slice_ = new[(new.index(row)):(new.index(row))+7]

                    pressure_slice = [int(slice_[6][6]), int(slice_[5][6]), int(slice_[4][6]), int(
                        slice_[3][6]), int(slice_[2][6]), int(slice_[1][6]), int(slice_[0][6])]

                    humidity_slice = [int(slice_[6][2]), int(slice_[5][2]), int(slice_[4][2]), int(
                        slice_[3][2]), int(slice_[2][2]), int(slice_[1][2]), int(slice_[0][2])]

                    wind_slice = str(
                        f'{slice_[6][0]}: {slice_[6][3]}\n{slice_[5][0]}: {slice_[5][3]}\n{slice_[4][0]}: {slice_[4][3]}\n{slice_[3][0]}: {slice_[3][3]}\n{slice_[2][0]}: {slice_[2][3]}\n{slice_[1][0]}: {slice_[1][3]}\n{slice_[0][0]}: {slice_[0][3]} \n')

        fig, ax = plt.subplots()
        x = [slice_[6][0], slice_[5][0], slice_[4][0], slice_[
            3][0], slice_[2][0], slice_[1][0], slice_[0][0]]
        y = pressure_slice
        plt.plot(x, y)
        plt.scatter(x, y)
        ax.set_facecolor('seashell')
        fig.set_facecolor('floralwhite')
        fig.set_figwidth(12)
        fig.set_figheight(6)
        plt.title(r'Давление')
        plt.ylabel(r'мм. рт.ст.')
        plt.xlabel(r'Дата')
        plt.grid(True)
        plt.savefig('pressure_figure.png')
        bot.send_photo(m.chat.id, open(
            './pressure_figure.png', 'rb'))
        plt.close()
        os.remove('./pressure_figure.png')

        bot.send_message(
            m.chat.id, wind_slice)

        fig, ax = plt.subplots()
        x = [slice_[6][0], slice_[5][0], slice_[4][0], slice_[
            3][0], slice_[2][0], slice_[1][0], slice_[0][0]]
        y = humidity_slice
        plt.plot(x, y)
        plt.scatter(x, y)
        ax.set_facecolor('seashell')
        fig.set_facecolor('floralwhite')
        fig.set_figwidth(12)
        fig.set_figheight(6)
        plt.title(r'Влажность')
        plt.ylabel(r'%')
        plt.xlabel(r'Дата')
        plt.grid(True)
        plt.savefig('humidity.png')
        bot.send_photo(m.chat.id, open('./humidity.png', 'rb'))
        plt.close()
        os.remove('./humidity.png')
    except Exception:
        bot.send_message(
            m.chat.id, 'Такой даты нет. нажмите кнопку "Выбор даты" и повторите ввод даты')

@server.route('/' + TOKEN, methods=['POST'])
def getMessage():
    json_string = request.get_data().decode('utf-8')
    update = telebot.types.Update.de_json(json_string)
    bot.process_new_updates([update])
    return "!", 200

@server.route("/")
def webhook():
    bot.remove_webhook()
    bot.set_webhook(url='https://test1test120.herokuapp.com/' + TOKEN)
    return "!", 200

if __name__ == "__main__":
    server.run(host="0.0.0.0", port=int(os.environ.get('PORT', 5000)))
    # Запускаем бота
bot.polling(none_stop=True, interval=0)