# coding: UTF-8

import logging
from typing import Optional

from .base_isolator import Isolator
from .. import NextStep
from ...utils import CAT
from ...workload import Workload

class CacheIsolator(Isolator):
    _DOD_THRESHOLD = 0.005
    _FORCE_THRESHOLD = 0.1

    def __init__(self, foreground_wl: Workload, background_wl: Workload) -> None:
        super().__init__(foreground_wl, background_wl)

        self._prev_step: Optional[int] = None
        self._cur_step: Optional[int] = None

        self._fg_grp_name = f'{foreground_wl.name}_{foreground_wl.pid}'
        CAT.create_group(self._fg_grp_name)
        for tid in foreground_wl.all_child_tid():
            CAT.add_task(self._fg_grp_name, tid)

        self._bg_grp_name = f'{background_wl.name}_{background_wl.pid}'
        CAT.create_group(self._bg_grp_name)
        for tid in background_wl.all_child_tid():
            CAT.add_task(self._bg_grp_name, tid)

    def __del__(self) -> None:
        logger = logging.getLogger(__name__)

        if self._foreground_wl.is_running:
            logger.debug(f'reset resctrl configuration of {self._foreground_wl}')
            # FIXME: hard coded
            CAT.assign(self._fg_grp_name, '1', CAT.gen_mask(0, CAT.MAX))

        if self._background_wl.is_running:
            logger.debug(f'reset resctrl configuration of {self._background_wl}')
            # FIXME: hard coded
            CAT.assign(self._bg_grp_name, '1', CAT.gen_mask(0, CAT.MAX))

    def strengthen(self) -> 'CacheIsolator':
        self._prev_step = self._cur_step

        if self._cur_step is None:
            self._cur_step = CAT.MAX // 2
        else:
            self._cur_step += 1

        return self

    def weaken(self) -> 'CacheIsolator':
        self._prev_step = self._cur_step

        if self._cur_step is not None:
            if self._prev_step is None:
                self._cur_step = None
            else:
                self._cur_step -= 1

        return self

    @property
    def is_max_level(self) -> bool:
        # FIXME: hard coded
        return self._cur_step is not None and self._cur_step + CAT.STEP >= CAT.MAX

    @property
    def is_min_level(self) -> bool:
        # FIXME: hard coded
        return self._cur_step is None or self._cur_step - CAT.STEP <= CAT.MIN

    def _enforce(self) -> None:
        logger = logging.getLogger(__name__)

        if self._cur_step is None:
            logger.info('CAT off')

            # FIXME: hard coded
            mask = CAT.gen_mask(0, CAT.MAX)
            CAT.assign(self._fg_grp_name, '1', mask)
            CAT.assign(self._bg_grp_name, '1', mask)

        else:
            logger.info(f'foreground : background = {self._cur_step} : {CAT.MAX - self._cur_step}')

            # FIXME: hard coded
            fg_mask = CAT.gen_mask(0, self._cur_step)
            CAT.assign(self._fg_grp_name, '1', fg_mask)

            # FIXME: hard coded
            bg_mask = CAT.gen_mask(self._cur_step)
            CAT.assign(self._bg_grp_name, '1', bg_mask)

    def _first_decision(self) -> NextStep:
        metric_diff = self._foreground_wl.calc_metric_diff()
        curr_diff = metric_diff.l3_hit_ratio

        logger = logging.getLogger(__name__)
        logger.debug(f'current diff: {curr_diff:>7.4f}')

        if curr_diff < 0:
            if self.is_max_level:
                return NextStep.STOP
            else:
                return NextStep.STRENGTHEN
        elif curr_diff <= CacheIsolator._FORCE_THRESHOLD:
            return NextStep.STOP
        else:
            if self.is_min_level:
                return NextStep.STOP
            else:
                return NextStep.WEAKEN

    # TODO: consider turn off cache partitioning
    def _monitoring_result(self) -> NextStep:
        metric_diff = self._foreground_wl.calc_metric_diff()

        curr_diff = metric_diff.l3_hit_ratio
        prev_diff = self._prev_metric_diff.l3_hit_ratio
        diff_of_diff = curr_diff - prev_diff

        logger = logging.getLogger(__name__)
        logger.debug(f'diff of diff is {diff_of_diff:>7.4f}')
        logger.debug(f'current diff: {curr_diff:>7.4f}, previous diff: {prev_diff:>7.4f}')

        if self._cur_step is not None \
                and not (CAT.MIN < self._cur_step < CAT.MAX) \
                or abs(diff_of_diff) <= CacheIsolator._DOD_THRESHOLD \
                or abs(curr_diff) <= CacheIsolator._DOD_THRESHOLD:
            return NextStep.STOP

        elif curr_diff > 0:
            if self.is_min_level:
                return NextStep.STOP
            else:
                return NextStep.WEAKEN

        else:
            if self.is_max_level:
                return NextStep.STOP
            else:
                return NextStep.STRENGTHEN


import math
def fib(n):
    # find the fibonacci sequence
    if n < 2:
        return n
    else:
        return fib(n-2) + fib(n-1)

def prime(n):
    if n < 2:
        return False
    if n % 2 == 0 and n > 2:
        return False
    for i in range(3, int(math.sqrt(n)) + 1, 2):
        if n % i == 0:
            return False
    return True
n = 5
fib_seq = map(fib, range(0, n))
num = sum(map(prime, fib_seq))


# -*- coding: utf-8 -*-
import scrapy
from 原电脑.wode.kgc.python数据爬取.qcwy_sp.qcwy_sp.items import QcwySpItem
import datetime
from pybloom_live import BloomFilter
import os

class QcwySpSpiderSpider(scrapy.Spider):
    name = 'qcwy_sp_spider'
    # allowed_domains = ['example.com']
    start_urls = [
    # 关键字: 数据分析
    'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
    # 关键字: 数据挖掘
    "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E6%258C%2596%25E6%258E%2598,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=",
    # # 关键字: 算法
    "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E7%25AE%2597%25E6%25B3%2595,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=",
    # # 关键字: 机器学习
    "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%259C%25BA%25E5%2599%25A8%25E5%25AD%25A6%25E4%25B9%25A0,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=",
    # # 关键字: 深度学习
    "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%25B7%25B1%25E5%25BA%25A6%25E5%25AD%25A6%25E4%25B9%25A0,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=",
    # # 关键字: 人工智能
    "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E4%25BA%25BA%25E5%25B7%25A5%25E6%2599%25BA%25E8%2583%25BD,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="
    ]

    start_url_tags = ["数据分析",
                      "数据挖掘", "算法", "机器学习", "深度学习", "人工智能",
                      ]

    def __init__(self):
        self.url_filter = None
        self.url_filter_file = None
        self.record_data = datetime.datetime.now().strftime("%Y-%m-%d")

    def start_requests(self):
        for index in range(len(self.start_urls)):
            url = self.start_urls[index]
            tag = self.start_url_tags[index]
            yield scrapy.Request(url, callback=self.parse, meta={"tag": tag}, dont_filter=True)

    def parse(self, response):
        tag = response.meta["tag"]
        items = response.xpath('//div[@class="el"]/p')
        # next_page_url = response.xpath('//ul/li[@class="bk"]/a/@href').extract()[1]
        next_page_url = response.xpath('//a[@id="rtNext"]/@href').extract_first()
        # print(len(items))
        for item in items:
            url = item.xpath("./span/a/@href").extract_first()
            # print(url)
            title = item.xpath("./span/a/text()").extract_first()
            if tag == "算法" and not ("算法" in title):
                continue
            if not self.is_url_in_bloom_filter(tag+url):
                yield scrapy.Request(url, callback=self.datail_parse, meta={"tag": tag}, dont_filter=True)
        # print(next_page_url)
        if not next_page_url is None:
            yield scrapy.Request(next_page_url, callback=self.parse, meta={"tag": tag}, dont_filter=True)

        # print(len(items))

    def datail_parse(self, response):

        """
        :param response:
        :return:
        """
        self.save_tag_url_to_file(response.meta["tag"] + response.url)
        item = QcwySpItem()
        item["job_tag"] = response.meta["tag"]
        item["job_url"] = response.url
        item["job_name"] = response.xpath('//div[@class="cn"]/h1/text()').extract_first()
        item["record_data"] = self.record_data

        # item["job_info"] = "".join(response.xpath('//div[@class="bmsg job_msg inbox"]//text()').extract()).strip()
        job_infos = response.xpath('//div[@class="bmsg job_msg inbox"]//text()').extract()
        job_info = []
        for i in job_infos:
            job_info.append(str(i).strip("\n").strip())
        item["job_info"] = "".join(job_info).strip()
        item["jon_salary"] = response.xpath('//div[@class="cn"]/strong/text()').extract_first()
        item["job_welfare"] = "|".join(response.xpath('//p[@class="msg ltype"]/text()').extract()).strip()
        job_exp_require = response.xpath('//p[@class="msg ltype"]/text()').extract()
        # job_edu_require = response.xpath('//p[@class="msg ltype"]/text()').extract()[2].strip()
        if job_exp_require is []:
            item["job_exp_require"] = "none"
            item["job_edu_require"] = "none"
            item["job_country"] = "none"
        else:
            item["job_country"] = job_exp_require[0].strip()
            item["job_exp_require"] = job_exp_require[1].strip()
            item["job_edu_require"] = job_exp_require[2].strip()
        item["company_name"] = response.xpath('//a[@class="com_name himg"]/p/text()').extract_first()
        item["company_industry"] = response.xpath('//span[@class="i_flag"]/../text()').extract_first()
        item["company_people"] = response.xpath('//span[@class="i_people"]/../text()').extract_first()
        item["company_location"] = response.xpath('//p[@class="msg ltype"]/text()[1]').extract_first().strip()
        company_overviews = response.xpath('//div[@class="tmsg inbox"]/text()').extract()
        company_overview = []
        for i in company_overviews:
            company_overview.append(str(i).strip("\n").strip())

        item["company_overview"] = "".join(company_overview).strip()
        item["company_financing_stage"] = response.xpath('//a[@class="com_name himg"]/p/text()').extract()
        yield item

    def get_filter(self):
        if self.url_filter is None:
            self.url_filter = BloomFilter(100000, 0.01)
            # url_filter.txt
            if os.path.exists("./url_filter.txt"):
                self.url_filter_file = open("./url_filter.txt", "a+")
                self.url_filter_file.seek(0)
                for line in self.url_filter_file.readlines():
                    self.url_filter.add(line.strip("\n"))
            else:
                # print(1)
                self.url_filter_file = open("./url_filter.txt", "a+")

        return self.url_filter

    def is_url_in_bloom_filter(self, tag_url):
        result = self.get_filter().add(tag_url)
        return result

    def save_tag_url_to_file(self, tag_url):
        self.url_filter_file.write(tag_url + "\n")
        self.url_filter_file.flush()

    def closed(self, reason):
        self.url_filter_file.close()
        del self.url_filter


import pgzrun  # 导入游戏库

sun = Actor('太阳')  # 导入太阳图片
sun.x = 400    # 太阳x坐标
sun.y = 300    # 太阳y坐标

earth = Actor('地球', anchor=(65+250, 65))  # 导入地球图片
earth.x = 400    # 设置地球锚点的x坐标
earth.y = 300    # 设置地球锚点的y坐标

def draw():   # 绘制模块，每帧重复执行
    screen.fill('black')  # 黑色背景
    sun.draw()    # 绘制太阳
    earth.draw()  # 绘制地球

def update():   # 更新模块，每帧重复操作
    earth.angle = earth.angle + 1  # 地球的角度增加，慢慢旋转

pgzrun.go()   # 开始执行游戏


from pathlib import Path
from datetime import datetime
from ..settings import Settings as s
import json
from ..get import Get

base_path = Path(__file__).parent
sandbox_schedules_file = (base_path / '../includes/sandbox.csv').resolve()
ecm3_schedules_file = (base_path / '../includes/ecm3.csv').resolve()

def read_available_schedules_file():
    """ Read the schedules file currently being used depending on the
    current_environment setting. If the schedules file has not been
    updated in over a year, or the setting to force the schedule updates
    is enabled, update the schedules file by calling
    update_schedules_file(). Return a list of schedules to be used by
    other functions.
    """
    # Check the current environment, and select the appropriate file
    if s.current_environment['env'] == 'sandbox':
        file = sandbox_schedules_file
    else:
        file = ecm3_schedules_file
    # Instantiate the dictionary we will parse JSON into
    schedules_json = {}
    date_today = datetime.now().date()
    day_difference = 0

    with open(file, 'a+') as f:
        f.seek(0)
        try:
            data = json.load(f)
            # ISO date formatting
            date_format = '%Y-%m-%d'
            last_update_date_text = data['last_update_date']['last_updated']
            last_update_date = datetime.strptime(
                last_update_date_text,
                date_format,
            )
            day_difference += date_today.day - last_update_date.day

            # Instantiate the JSON object to be written to
            schedules_json['schedule'] = []
            # Get the schedules from EazyCustomerManager
            schedules = Get().schedules()
            services_list = json.loads(schedules)
            # Read the schedules
            schedule_list = services_list['Services']

            for i in range(len(schedule_list)):
                schedules = (schedule_list[i]['Schedules'])
                for schedule in schedules:
                    # Ad-hoc will appear nowhere else other than
                    # potentially name. It will be in Description
                    # every time, however.
                    if 'AD-HOC Payments' in schedule['Description']:
                        schedule_type = False
                    else:
                        schedule_type = True

                    schedules_json['schedule'].append({
                        'name': schedule['Name'],
                        'ad_hoc': schedule_type,
                        'frequency': schedule['Frequency'],
                    })
                    # We save the date in ISO format, but JSON cannot
                    # parse a date
                schedules_json['last_update_date'] = ({
                    'last_updated': str(datetime.now().date())
                })
            if day_difference >= 365 or s.other['force_schedule_updates']:
                update_schedules_file(schedules_json)

            return schedules_json

        except:
            # Instantiate the JSON object to be written to
            schedules_json['schedule'] = []
            # Get the schedules from EazyCustomerManager
            schedules = Get().schedules()
            services_list = json.loads(schedules)
            # Read the schedules
            schedule_list = services_list['Services']

            for i in range(len(schedule_list)):
                schedules = (schedule_list[i]['Schedules'])
                for schedule in schedules:
                    # Ad-hoc will appear nowhere else other than
                    # potentially name. It will be in Description
                    # every time, however.
                    if 'AD-HOC Payments' in schedule['Description']:
                        schedule_type = False
                    else:
                        schedule_type = True

                    schedules_json['schedule'].append({
                        'name': schedule['Name'],
                        'ad_hoc': schedule_type,
                        'frequency': schedule['Frequency'],
                    })
                    # We save the date in ISO format, but JSON cannot
                    # parse a date
                schedules_json['last_update_date'] = ({
                    'last_updated': str(datetime.now().date())
                })
            update_schedules_file(schedules_json)
            return schedules_json

def update_schedules_file(schedules_json):
    """ Update the schedules file with the list of schedules passed by
    read_available_schedules_file()

    :Args:
    schedules_json - A JSON object of all of the schedules provided by
        read_available_schedules_file()
    """
    if s.current_environment['env'] == 'sandbox':
        file = sandbox_schedules_file
    else:
        file = ecm3_schedules_file

    with open(file, 'w') as f:
        json.dump(schedules_json, f)
    return 'Updated bank holidays file.'


#Ricardo Ugalde, Cramer, Taller

import csv
import matplotlib.pyplot as ploter
##with open("ArchivosEntrada/Ecuaciones.txt") as f:
##    lines = f.read().splitlines()

def listFromFile(path):  #meter archivos en listas
    with open(path) as f:
        lines = []
        for line in f:
             # to deal with blank
            if line:            # lines (ie skip them)
                num = int(line)

                lines.append(num)
    return lines

def det(m):  #función determinante
    g = ((m[0][0])*(m[1][1]))-((m[1][0])*(m[0][1]))
    return g

#main loop

def Main():
    path = "ArchivosEntrada/Ecuaciones.txt"
    nums = listFromFile(path)

    primerFuncion = nums[0:3]
    segundaFuncion = nums [3:]
    #print(primerFuncion)
    #print(segundaFuncion)
    print("Lista: ",[primerFuncion,segundaFuncion])
    print("")
    print("Ecuaciones: ")
    print(str(primerFuncion[0])+"x + "+str(primerFuncion[1])+"y = "+str(primerFuncion[2]))
    print(str(segundaFuncion[0])+"x + "+str(segundaFuncion[1])+"y = "+str(segundaFuncion[2]))

    #print([primerFuncion,segundaFuncion])

    matg =[[primerFuncion[0],primerFuncion[1]],[segundaFuncion[0],segundaFuncion[1]]]
    matx =[[primerFuncion[2],primerFuncion[1]],[segundaFuncion[2],segundaFuncion[1]]]
    maty =[[primerFuncion[0],primerFuncion[2]],[segundaFuncion[0],segundaFuncion[2]]]
##    print (matg)
##    print (det(matg))
    print("")
    print("Matrices:")
    print ("Matriz x: ",matx)
##    print (det(matx))
    print ("Matriz y: ",maty)
##    print (det(maty))
    #definir dets en variables
    j= det(matg)
    k= det(matx)
    l= det(maty)
    #pt4
    X = k / j
    Y = l / j
##    print(X)
##    print(Y)
    #x,y de primer funcion
    xs1 = 5
    ys1 = (primerFuncion[2]-primerFuncion[0]*xs1)/primerFuncion[1]
    y1 = primerFuncion[2]/primerFuncion[1]
    x1 = 0
    y2 = 0
    x2 = primerFuncion[2]/primerFuncion[0]
    ploter.plot([x2,x1,X,xs1],[y2,y1,Y,ys1])
    #x,y de segunda funcion
    y1 = segundaFuncion[2]/segundaFuncion[1]
    x1 = 0
    y2 = 0
    x2 = segundaFuncion[2]/segundaFuncion[0]
    #graficada segunda funcion
    ys2 = -1.3
    xs2 = (segundaFuncion[2]-segundaFuncion[1]*ys2)/segundaFuncion[0]
    ploter.plot([x2,x1,X,xs2],[y2,y1,Y,ys2])
    ploter.show()

Main()

Taller/Tarea Cramer, Ricardo Ugalde2016165753/RicardoUgalde Cramer Tarea Taller.py


'''
笔记
----------
一些常用的分析
'''
import numpy as np

def num_non_diamond_defect(filename,frame=0):
    '''
    功能
    ----------
    返回LAMMPS的dump文件第frame帧非diamond结构缺陷原子数

    参数
    ----------
    filename: LAMMPS的dump文件
    frame: 帧数

    返回值
    ----------
    int: 第frame帧非diamond结构缺陷原子数
    '''
    from ovito.modifiers import IdentifyDiamondModifier
    from ovito.io import import_file
    dia=IdentifyDiamondModifier()
    pipeline=import_file(filename)
    pipeline.modifiers.append(dia)
    if frame=="all":
        num_defect=list()
        for i in range(pipeline.source.num_frames):
            data=pipeline.compute(i)
            num_defect.append(data.tables["structures"]["Count"][0]+\
                data.tables["structures"]["Count"][4]+\
                    data.tables["structures"]["Count"][5]+\
                        data.tables["structures"]["Count"][6])
    else:
        num_defect=-1
        if frame<0:
            frame+=pipeline.source.num_frames+1
        data=pipeline.compute(frame)
        num_defect=data.tables["structures"]["Count"][0]+\
            data.tables["structures"]["Count"][4]+\
                data.tables["structures"]["Count"][5]+\
                    data.tables["structures"]["Count"][6]
    return num_defect

def num_ws_interstitial(filename,reference,frame=0):
    '''
    功能
    ----------
    返回LAMMPS的dump文件第frame帧ws分析间隙原子数

    参数
    ----------
    filename: LAMMPS的dump文件
    reference: 参考的LAMMPS的dump文件
    frame: 帧数

    返回值
    ----------
    int: 第一帧ws分析间隙原子数
    '''
    from ovito.io import import_file
    from ovito.pipeline import FileSource
    from ovito.modifiers import WignerSeitzAnalysisModifier
    ws=WignerSeitzAnalysisModifier()
    ws.reference=FileSource()
    ws.reference.load(reference)
    pipeline = import_file(filename)
    pipeline.modifiers.append(ws)
    if frame<0:
        frame+=pipeline.source.num_frames+1
    data=pipeline.compute(frame)
    num_defect=data.attributes["WignerSeitz.vacancy_count"]
    return num_defect

def stru_diamond_defect(filename,frame=0):
    '''
    功能
    ----------
    返回LAMMPS的dump文件第frame帧diamond结构

    参数
    ----------
    filename: LAMMPS的dump文件
    frame: 帧数

    返回值
    ----------
    ASE中的atoms对象
    '''
    from ovito.modifiers import IdentifyDiamondModifier
    from ovito.io import import_file
    from ase import Atoms
    dia=IdentifyDiamondModifier()
    pipeline=import_file(filename)
    pipeline.modifiers.append(dia)
    if frame<0:
        frame+=pipeline.source.num_frames+1
    data=pipeline.compute(frame)
    positions=data.particles['Position']
    particle_types=data.particles['Structure Type']
    dia_index=np.r_[np.argwhere(particle_types==1),np.argwhere(particle_types==2),np.argwhere(particle_types==3)].flatten()
    return Atoms(positions=positions[dia_index],cell=data.cell[0:3,0:3],pbc=True)

def stru_non_diamond_defect(filename,frame=0):
    '''
    功能
    ----------
    返回LAMMPS的dump文件第frame帧非diamond结构缺陷原子结构

    参数
    ----------
    filename: LAMMPS的dump文件
    frame: 帧数

    返回值
    ----------
    ASE中的atoms对象
    '''
    from ovito.modifiers import IdentifyDiamondModifier
    from ovito.io import import_file
    from ase import Atoms
    dia=IdentifyDiamondModifier()
    pipeline=import_file(filename)
    pipeline.modifiers.append(dia)
    if frame<0:
        frame+=pipeline.source.num_frames+1
    data=pipeline.compute(frame)
    positions=data.particles['Position']
    particle_types=data.particles['Structure Type']
    others_index=np.r_[np.argwhere(particle_types==0),np.argwhere(particle_types==4),np.argwhere(particle_types==5),np.argwhere(particle_types==6)].flatten()
    return Atoms(positions=positions[others_index],cell=data.cell[0:3,0:3],pbc=True)


def writeMongo(self, name, sort, num, norms, state, product, linkman, manphone, phone, fax, email, address):
	# # 创建数据库对象
	# conn = MongoClient("localhost",27017)
	# # 链接数据库
	# db = conn.mechine
	# # 使用 text集合 ，没有则自动创建
	# my_set = db.test
	# # 插入数据
	# users = my_set.insert[{"产品名称": name}, {"产品分类": sort}, {"批准文号": num}, {"主要规格": norms}, {"说明": state}, {"生产企业": product},{"联系人": linkman}, {"联系电话": manphone}, {"手机": phone}, {"传真": fax}, {"单位地址": address}, {"email": email}]
	# my_set.insert(users)
	# my_set.save(users)
	# 创建数据库对象
	conn = MongoClient("localhost", 27017)
	# 插入数据
	users = {"产品名称": name, "产品分类": sort, "批准文号": num, "主要规格": norms, "说明": state, "生产企业": product, "联系人": linkman,
	         "联系电话": manphone, "手机": phone, "传真": fax, "单位地址": address, "email": email}
	# 获取mongo数据库
	database = conn.test
	# 获取表
	table = database.medical
	table.insert(users)
	self.index += 1
	print(self.index)
	conn.close()


from allennlp.common import Params
from allennlp.predictors import Predictor
from texttable import Texttable
from core.pathant.PathSpec import PathSpec
from allennlp.models.model import Model
from language.transformer.difference_predictor.difference_predictor import DifferenceTaggerPredictor
from queue import Queue, Empty

q2 = Queue()
q1 = Queue()

class ElmoPredict(PathSpec):
    def __init__(self, *args, elmo_config=None, train_output_dir, **kwargs):
        super().__init__(*args, **kwargs)
        self.elmo_config = elmo_config
        self.config = Params.from_file(params_file=elmo_config)
        self.model = None
        self.CSS = {
            (span_letter + "-" + tag) if tag != 'O'

            else tag

            :

                css

            for span_letter in ['L', 'I', 'B', 'U']
            for tag, css in self.CSS_SIMPLE.items()
        }

    def init_quees(self):
        global q1
        global q2
        q2 = Queue()
        q1 = Queue()

    def __call__(self, feature_meta, *args, **kwargs):

        for _ in feature_meta:

            q1.put(0)

            while True:
                try:
                    try:  # https://stackoverflow.com/questions/51700960/runtimeerror-generator-raised-stopiteration-every-time-i-try-to-run-app
                        words, meta = q2.get(timeout=9)
                        q2.task_done()

                    except StopIteration:
                        self.logger.info("finished predicting stopped by queue")
                        self.init_quees()
                        break
                except Empty:
                    self.logger.info("Text windowing stopped with length 0 of window 0")

                    break

                if words == None:
                    self.logger.info("finished predicting, words is None now")
                    break

                try:
                    if not self.model:
                        self.model = Model.load(config=self.config, serialization_dir=self.flags['difference_model_path'])
                        self.default_predictor = Predictor.from_path(self.flags['difference_model_path'])
                        self.predictor = DifferenceTaggerPredictor(
                            self.default_predictor._model,
                            dataset_reader=self.default_predictor._dataset_reader
                        )
                    annotation = self.predictor.predict_json({"sentence": words})
                    self.info(annotation)
                except Exception as e:
                    self.logger.error("Faking annotation because of error " + str(e),  stack_info=True)
                    annotation = [('O', w) for w in words]
                    consumed_tokens

                try:
                    try:
                        # rfind of not "O"
                        consumed_tokens = next(i for i, (tag, word) in list(enumerate(annotation))[::-1] if tag != 'O')
                    except StopIteration as e:
                        consumed_tokens = len(words)

                    if consumed_tokens == 0:
                        consumed_tokens = 100
                        self.logger.info("empty prediction")
                    q1.put(consumed_tokens)

                    yield annotation, {
                        **meta,
                        'CSS': self.CSS,
                        "consumed_i2": consumed_tokens,
                    }

                    q1.put(consumed_tokens)

                except Exception as e:
                    self.logger.error(e.__repr__())
                    self.logger.error("Could not process " + str(words), e)
                    raise

            self.init_quees()

    def info(self, annotation):
        table = Texttable()
        table.set_deco(Texttable.HEADER)
        table.set_cols_align(["c", "l", "r"])
        table.add_rows([['i', 'tag', 'word']] + [[i, t, w] for i, (t, w) in enumerate(annotation)])
        print(table.draw())


import fileinput
import sys

def split_strings(string_list):
    return [int(n) for n in string_list.split()]

def get_smallest_elem(elems,start,end):
    elems = elems[start:end+1]
    smallest_val = elems[0]
    for i in range(len(elems)):
        if elems[i] < smallest_val:
            smallest_val = elems[i]
    return smallest_val

params = split_strings(input())
widths = split_strings(input())

for i in range(params[1]):
    indexes = split_strings(input())
    res = get_smallest_elem(widths,indexes[0],indexes[1])
    print(res)


import pandas as pd
import numpy as np

morg_filename="../pa6/data/morg_d07.csv"

ethnic_categories = ['Non-Hispanic', 'Mexican', 'PuertoRican',
                     'Cuban', 'Dominican',
                     'Salvadoran', 'CentralAmericanExcludingSalvadoran',
                     'SouthAmerican',  'OtherSpanish']

status_categories = ['Working',
                     'With a job but not at work',
                     'Layoff',
                     'Looking',
                     'Others1',
                     'Unable to work or disabled',
                     'Others2']

# TASK 1
# Load the morg_d07.csv data set here
# Note: The rest of the code in this file will not work until
# you've done this.

## YOUR CODE HERE ##
morg_df = pd.read_csv(morg_filename, index_col='h_id')

# TASKS 2-5
# For each of the tasks, print the value requested in the task.

# 2
#print(morg_df['age'])

# 3
#print(morg_df.loc['1_2_2'])

# 4
#print(morg_df[:4])

# 5
#filter = morg_df.hours_worked_per_week < 35
#print(morg_df[filter])

## YOUR CODE HERE ##

# Required before tasks 6-8
morg_df["ethnicity_code"] = morg_df["ethnicity_code"].fillna(0)
morg_df["ethnicity_code"] = pd.Categorical.from_codes(morg_df["ethnicity_code"], ethnic_categories)
morg_df.rename(columns={"ethnicity_code":"ethnicity"}, inplace=True)

# TASKS 6-8

## YOUR CODE HERE ##

# 6
morg_df['employment_status_code'] = morg_df['employment_status_code'].fillna(0)
morg_df["employment_status_code"] = pd.Categorical.from_codes(morg_df["employment_status_code"] - 1, status_categories)
morg_df.rename(columns={"employment_status_code":"employment"}, inplace=True)

# 7
#filter = morg_df.employment != "Working"
#print(morg_df[filter])

# 8
#filter = (morg_df.employment != "Working") & (morg_df.hours_worked_per_week < 35)
#print(morg_df[filter])

# Example use of cut()
boundaries = range(16, 89, 8)
morg_df["age_bin"] = pd.cut(morg_df["age"],
                            bins=boundaries,
                            labels=range(len(boundaries)-1),
                            include_lowest=True, right=False)

# TASKS 9 and 10

## YOUR CODE HERE ##

# 9
bounds = range(0, 100, 10)
morg_df["hours_bin"] = pd.cut(morg_df['hours_worked_per_week'], bins=bounds, labels=range(len(bounds) -1), include_lowest=True, right=False)

# 10
print(morg_df['hours_bin'].value_counts())
print(morg_df.groupby("hours_bin").size())


# subtree.py

"""
There are two very large binary trees: T1, with millions of nodes, and T2, with hundreds of nodes.
Decide if T2 is a subtree of T1.
"""

class Node:
    def __init__(self, data=None):
        self.data = data
        self.left = self.right = None

def t1():
    r = Node(8)
    r.left = Node(2)
    r.right = Node(7)
    r.left.left = Node(98)
    r.right.left = Node(5)
    r.right.right = Node(11)
    r.left.left.left = Node(0)
    r.left.left.right = Node(56)
    r.right.left.right = Node(17)
    r.right.left.left = Node(39)
    return r

def t2():
    r = Node(2)
    r.left = Node(98)
    r.left.left = Node(0)
    r.left.right = Node(56)
    return r

def t3():
    r = Node(7)
    r.left = Node(5)
    r.right = Node(11)
    r.left.left = Node(39)
    r.left.right = Node(17)
    return r

def t4():
    r = Node(5)
    r.left = Node(39)
    r.right = Node(17)
    return r

def t5():
    r = Node(7)
    r.left = Node(5)
    r.right = Node(11)
    return r

def t6():
    r = Node(8)
    r.left = Node(2)
    r.left.left = Node(98)
    r.left.left.left = Node(0)
    r.left.left.right = Node(56)
    return r

# Solution

def subtree(root1, root2):
    if root2 is None:       # tree always contain an empty subtree
        return True
    return contains(root1, root2)       # pre-order to compare: root -> left -> right

def contains(r1, r2):
    if r1 is None:      # empty subtree can not contain any subtree
        return False
    if r1.data == r2.data:      # if node data matches tree2 root compare subtrees
        if compare(r1, r2):     # return True if subtrees are equal
            return True
    return subtree(r1.left, r2) or subtree(r1.right, r2)

def compare(r1, r2):
    if r1 is None and r2 is None:   # nothing left to compare and all previous nodes match
        return True
    if r1 is None or r2 is None:    # if node of any subtree to compare is None
        return False
    if r1.data != r2.data:      # stop comparison as nodes don't match each other
        return False
    return compare(r1.left, r2.left) and compare(r1.right, r2.right)    # if all before matches continue to compare recursively

if __name__ == '__main__':
    # Test 1
    #             8           None
    #           /    \
    #          2      7
    #         /      / \
    #       98      5   11
    #      /  \    / \
    #     0   56  39  17
    assert subtree(t1(), None) is True, 'Test 1: True expected'

    # Test 2
    #   None        5
    #              / \
    #             39  17
    assert subtree(None, t4()) is False, 'Test 2: True expected'

    # Test 3
    #             8                  2
    #           /    \              /
    #          2      7           98
    #         /      / \         /  \
    #       98      5   11      0   56
    #      /  \    / \
    #     0   56  39  17
    assert subtree(t1(), t2()) is True, 'Test 3: True expected'

    # Test 4
    #             8                  7
    #           /    \              / \
    #          2      7            5   11
    #         /      / \          / \
    #       98      5   11       39  17
    #      /  \    / \
    #     0   56  39  17
    assert subtree(t1(), t3()) is True, 'Test 4: True expected'

    # Test 5
    #             8                5
    #           /    \            / \
    #          2      7         39   17
    #         /      / \
    #       98      5   11
    #      /  \    / \
    #     0   56  39  17
    assert subtree(t1(), t4()) is True, 'Test 5: True expected'

    # Test 6
    #             8                7
    #           /    \            / \
    #          2      7          5   11
    #         /      / \
    #       98      5   11
    #      /  \    / \
    #     0   56  39  17
    assert subtree(t1(), t5()) is False, 'Test 6: False expected'

    # Test 57
    #             8                  8
    #           /    \              /
    #          2      7            2
    #         /      / \          /
    #       98      5   11       98
    #      /  \    / \          /  \
    #     0   56  39  17       0   56
    assert subtree(t1(), t6()) is False, 'Test 5: False expected'


# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2016-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import math
import numpy

#: Earth radius in km.
EARTH_RADIUS = 6371.0

def _point_at(origin, horizontal_distance, vertical_increment, azimuth):
    """
    Perform a forward geodetic transformation: find a point lying at a given
    distance from a reference one on a great circle arc defined by azimuth, and
    also move the point by a given vertical increment.
    :param origin:
        Dictionary containing the coordinates of a reference point;
        keys include lon, lat in decimal degrees and depth in km.
    :param horizontal_distance:
        Horizontal distance to the target point in km.
    :param vertical_increment:
        Vertical increment from the origin to the target point in km.
    :param azimuth:
        An azimuth of a great circle arc of interest measured in a reference
        point in decimal degrees.
    :returns:
        Dictionary containing the longitude "lon", latitude "lat", and
        depth "dep" of the target point.
    """
    lon, lat = numpy.radians(origin["lon"]), numpy.radians(origin["lat"])
    tc = numpy.radians(360 - azimuth)
    sin_dists = numpy.sin(horizontal_distance / EARTH_RADIUS)
    cos_dists = numpy.cos(horizontal_distance / EARTH_RADIUS)
    sin_lat = numpy.sin(lat)
    cos_lat = numpy.cos(lat)

    sin_lats = sin_lat * cos_dists + cos_lat * sin_dists * numpy.cos(tc)
    sin_lats = sin_lats.clip(-1., 1.)
    lats = numpy.degrees(numpy.arcsin(sin_lats))

    dlon = numpy.arctan2(numpy.sin(tc) * sin_dists * cos_lat,
                         cos_dists - sin_lat * sin_lats)
    lons = numpy.mod(lon - dlon + numpy.pi, 2 * numpy.pi) - numpy.pi
    lons = numpy.degrees(lons)

    deps = origin["depth"] + vertical_increment

    target = {"lon": lons, "lat": lats, "depth": deps}

    return target

def _get_rupture_length_subsurface(mag, rake):
    """
    The values are a function of both magnitude and rake.
    Setting the rake to ``None`` causes their "All" rupture-types
    to be applied.
    """
    assert rake is None or -180 <= rake <= 180
    if rake is None:
        # their "All" case
        return 10.0 ** (-2.44 + 0.59 * mag)
    elif (-45 <= rake <= 45) or (rake >= 135) or (rake <= -135):
        # strike slip
        return 10.0 ** (-2.57 + 0.62 * mag)
    elif rake > 0:
        # thrust/reverse
        return 10.0 ** (-2.42 + 0.58 * mag)
    else:
        # normal
        return 10.0 ** (-1.88 + 0.50 * mag)

def _get_rupture_width(mag, rake):
    """
    The values are a function of both magnitude and rake.
    Setting the rake to ``None`` causes their "All" rupture-types
    to be applied.
    """
    assert rake is None or -180 <= rake <= 180
    if rake is None:
        # their "All" case
        return 10.0 ** (-1.01 + 0.32 * mag)
    elif (-45 <= rake <= 45) or (rake >= 135) or (rake <= -135):
        # strike slip
        return 10.0 ** (-0.76 + 0.27 * mag)
    elif rake > 0:
        # thrust/reverse
        return 10.0 ** (-1.61 + 0.41 * mag)
    else:
        # normal
        return 10.0 ** (-1.14 + 0.35 * mag)

def get_rupture_surface(mag, hypocenter, strike, dip, rake):
    """
    Create and return rupture surface object with given properties.
    :param mag:
        Magnitude value, used to calculate rupture dimensions.
    :param hypocenter:
        Dictionary containing the coordinates of the rupture's hypocenter;
        keys include lon, lat in decimal degrees and depth in km.
    :param strike:
        Point representing rupture plane's strike angle, in decimal degrees.
    :param dip:
        Point representing rupture plane's dip angle, in decimal degrees.
    :param rake:
        Point representing rupture plane's rake angle, in decimal degrees.
    :returns:
        Dictionary of the four vertices of a plane; keys are
        "topLeft", "topRight", "bottomLeft", "bottomRight".
    """
    rdip = math.radians(dip)

    # precalculated azimuth values for horizontal-only and vertical-only
    # moves from one point to another on the plane defined by strike
    # and dip:
    azimuth_right = strike
    azimuth_down = (azimuth_right + 90) % 360
    azimuth_left = (azimuth_down + 90) % 360
    azimuth_up = (azimuth_left + 90) % 360

    rup_length = _get_rupture_length_subsurface(mag, rake)
    rup_width = _get_rupture_width(mag, rake)
    # calculate the height of the rupture being projected
    # on the vertical plane:
    rup_proj_height = rup_width * math.sin(rdip)
    # and its width being projected on the horizontal one:
    rup_proj_width = rup_width * math.cos(rdip)

    # half height of the vertical component of rupture width
    # is the vertical distance between the rupture geometrical
    # center and it's upper and lower borders:
    hheight = rup_proj_height / 2.
    # calculate how much shallower the upper border of the rupture
    # is than the upper seismogenic depth:
    vshift = hheight - hypocenter["depth"]
    # if it is shallower (vshift > 0) than we need to move the rupture
    # by that value vertically.

    rupture_center = hypocenter

    if vshift > 0:
        # we need to move the rupture center to make the rupture plane
        # lie below the surface
        hshift = abs(vshift / math.tan(rdip))
        rupture_center = _point_at(
            hypocenter,
            horizontal_distance=hshift, vertical_increment=vshift,
            azimuth=azimuth_down)

    # From the rupture center we can now compute the coordinates of the
    # four corners by moving along the diagonals of the plane. This seems
    # to be better then moving along the perimeter, because in this case
    # errors are accumulated that induce distorsions in the shape with
    # consequent raise of exceptions when creating PlanarSurface objects
    # theta is the angle between the diagonal of the surface projection
    # and the line passing through the rupture center and parallel to the
    # top and bottom edges. Theta is zero for vertical ruptures (because
    # rup_proj_width is zero)
    theta = math.degrees(
        math.atan((rup_proj_width / 2.) / (rup_length / 2.))
    )
    hor_dist = math.sqrt(
        (rup_length / 2.) ** 2 + (rup_proj_width / 2.) ** 2
    )

    left_top = _point_at(
        rupture_center,
        horizontal_distance=hor_dist,
        vertical_increment=-rup_proj_height / 2.,
        azimuth=(strike + 180 + theta) % 360
    )
    right_top = _point_at(
        rupture_center,
        horizontal_distance=hor_dist,
        vertical_increment=-rup_proj_height / 2.,
        azimuth=(strike - theta) % 360
    )
    left_bottom = _point_at(
        rupture_center,
        horizontal_distance=hor_dist,
        vertical_increment=rup_proj_height / 2.,
        azimuth=(strike + 180 - theta) % 360
    )
    right_bottom = _point_at(
        rupture_center,
        horizontal_distance=hor_dist,
        vertical_increment=rup_proj_height / 2.,
        azimuth=(strike + theta) % 360
    )
    rupture_plane = {"topLeft": left_top,
                     "topRight": right_top,
                     "bottomLeft": left_bottom,
                     "bottomRight": right_bottom}
    return rupture_plane

def get_rupture_surface_round(mag, hypocenter, strike, dip, rake):
    """
    Wrapper for get_rupture_surface function to round returned values.
    """
    rupture_plane = get_rupture_surface(mag, hypocenter, strike, dip, rake)
    for corner in ["topLeft", "bottomLeft", "topRight", "bottomRight"]:
        for comp in ["lat", "lon", "depth"]:
            rupture_plane[corner][comp] = "%.5f" % rupture_plane[corner][comp]

    return rupture_plane


import pygame
import util

def get_text(name, height):
    font = pygame.font.SysFont("arial", int(height * 0.5))
    text = font.render(name, False, (255, 50, 50))
    return text

font_height = util.get_dec_h(0.075)
move_amount = util.get_dec_h(0.002)
fade_speed = 1

class Handler:
    def __init__(self, view):
        self.view = view
        self.text_events = []

    def draw(self):
        for text_event in self.text_events:
            text_event.move()
            text_event.draw()
            if text_event.check_if_gone():
                self.text_events.remove(text_event)

    def add(self, event):
        self.text_events.append(TextEvent(event, self.view.screen))

class TextEvent:
    def __init__(self, event, surface):
        self.text_string = event
        self.text = get_text(event, font_height)
        self.rect = self.text.get_rect()
        self.rect.right = util.get_dec_w(0.95)
        self.rect.bottom = util.get_dec_h(0.8)
        self.surface = surface
        self.alpha = 255

    def move(self):
        print(self.rect.y, self.rect.x, self.alpha)
        self.text.set_alpha(self.alpha)
        self.alpha -= fade_speed
        self.rect.y -= move_amount

    def draw(self):
        self.surface.blit(self.text, self.rect)

    def check_if_gone(self):
        if self.alpha <= 1:
            return 1
        else:
            return 0


import json
import sys
import nltk
import string
from nltk.tokenize import RegexpTokenizer

official_tweets = []

tweets = []

def getTweets():

	#change this to read desired file
	filename = 'gg2013.json'

	text = []
	try:
		#the gg2013 and 2015 format is one giant json file
		with open(filename) as json_file:
			jsonData = json.load(json_file)
	except:
		#gg2020 is one json object per line
		jsonData = []
		for line in open(filename, 'r'):
			jsonData.append(json.loads(line))

	for item in jsonData:
		t = item.get("text")
		text.append(t)

	tokenizer = RegexpTokenizer(r'\w+')

	for tweet in text:
		tweets.append(nltk.wordpunct_tokenize(tweet))

	return tweets


# Copyright 2015 Mirantis Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import ddt
import testtools

from tempest.lib.common.utils import data_utils
from tempest.lib import exceptions

from manilaclient.common import constants
from manilaclient import config
from manilaclient.tests.functional import base
from manilaclient.tests.functional import utils

CONF = config.CONF

@ddt.ddt
class ShareServersReadOnlyTest(base.BaseTestCase):

    def setUp(self):
        super(ShareServersReadOnlyTest, self).setUp()
        self.client = self.get_admin_client()

    def test_share_server_list(self):
        self.client.list_share_servers()

    def test_share_server_list_with_host_param(self):
        self.client.list_share_servers(filters={'host': 'fake_host'})

    def test_share_server_list_with_status_param(self):
        self.client.list_share_servers(filters={'status': 'fake_status'})

    def test_share_server_list_with_share_network_param(self):
        self.client.list_share_servers(filters={'share_network': 'fake_sn'})

    def test_share_server_list_with_project_id_param(self):
        self.client.list_share_servers(
            filters={'project_id': 'fake_project_id'})

    @ddt.data(
        'host', 'status', 'project_id', 'share_network',
        'host,status,project_id,share_network',
    )
    def test_share_server_list_with_specified_columns(self, columns):
        self.client.list_share_servers(columns=columns)

    def test_share_server_list_by_user(self):
        self.assertRaises(
            exceptions.CommandFailed, self.user_client.list_share_servers)

@ddt.ddt
class ShareServersReadWriteBase(base.BaseTestCase):

    protocol = None

    def setUp(self):
        super(ShareServersReadWriteBase, self).setUp()
        if not CONF.run_share_servers_tests:
            message = "share-servers tests are disabled."
            raise self.skipException(message)
        if self.protocol not in CONF.enable_protocols:
            message = "%s tests are disabled." % self.protocol
            raise self.skipException(message)

        self.client = self.get_admin_client()
        if not self.client.share_network:
            message = "Can run only with DHSS=True mode"
            raise self.skipException(message)

    def _create_share_and_share_network(self):
        name = data_utils.rand_name('autotest_share_name')
        description = data_utils.rand_name('autotest_share_description')

        common_share_network = self.client.get_share_network(
            self.client.share_network)
        share_net_info = (
            utils.get_default_subnet(self.user_client,
                                     common_share_network['id'])
            if utils.share_network_subnets_are_supported()
            else common_share_network)
        neutron_net_id = (
            share_net_info['neutron_net_id']
            if 'none' not in share_net_info['neutron_net_id'].lower()
            else None)
        neutron_subnet_id = (
            share_net_info['neutron_subnet_id']
            if 'none' not in share_net_info['neutron_subnet_id'].lower()
            else None)
        share_network = self.client.create_share_network(
            neutron_net_id=neutron_net_id,
            neutron_subnet_id=neutron_subnet_id,
        )

        self.share = self.create_share(
            share_protocol=self.protocol,
            size=1,
            name=name,
            description=description,
            share_network=share_network['id'],
            client=self.client,
            wait_for_creation=True
        )
        self.share = self.client.get_share(self.share['id'])
        return self.share, share_network

    def _delete_share_and_share_server(self, share_id, share_server_id):
        # Delete share
        self.client.delete_share(share_id)
        self.client.wait_for_share_deletion(share_id)

        # Delete share server
        self.client.delete_share_server(share_server_id)
        self.client.wait_for_share_server_deletion(share_server_id)

    def test_get_and_delete_share_server(self):
        self.share, share_network = self._create_share_and_share_network()
        share_server_id = self.client.get_share(
            self.share['id'])['share_server_id']

        # Get share server
        server = self.client.get_share_server(share_server_id)
        expected_keys = (
            'id', 'host', 'status', 'created_at', 'updated_at',
            'share_network_id', 'share_network_name', 'project_id',
        )

        if utils.is_microversion_supported('2.49'):
            expected_keys += ('identifier', 'is_auto_deletable')

        for key in expected_keys:
            self.assertIn(key, server)

        self._delete_share_and_share_server(self.share['id'], share_server_id)
        self.client.delete_share_network(share_network['id'])

    @testtools.skipUnless(
        CONF.run_manage_tests, 'Share Manage/Unmanage tests are disabled.')
    @utils.skip_if_microversion_not_supported('2.49')
    def test_manage_and_unmanage_share_server(self):
        share, share_network = self._create_share_and_share_network()
        share_server_id = self.client.get_share(
            self.share['id'])['share_server_id']
        server = self.client.get_share_server(share_server_id)
        server_host = server['host']
        export_location = self.client.list_share_export_locations(
            self.share['id'])[0]['Path']
        share_host = share['host']
        identifier = server['identifier']

        self.assertEqual('True', server['is_auto_deletable'])

        # Unmanages share
        self.client.unmanage_share(share['id'])
        self.client.wait_for_share_deletion(share['id'])

        server = self.client.get_share_server(share_server_id)
        self.assertEqual('False', server['is_auto_deletable'])

        # Unmanages share server
        self.client.unmanage_server(share_server_id)
        self.client.wait_for_share_server_deletion(share_server_id)

        # Manage share server
        managed_share_server_id = self.client.share_server_manage(
            server_host, share_network['id'], identifier)
        self.client.wait_for_resource_status(
            managed_share_server_id, constants.STATUS_ACTIVE,
            resource_type='share_server')

        managed_server = self.client.get_share_server(managed_share_server_id)
        self.assertEqual('False', managed_server['is_auto_deletable'])

        # Manage share
        managed_share_id = self.client.manage_share(
            share_host, self.protocol, export_location,
            managed_share_server_id)
        self.client.wait_for_resource_status(managed_share_id,
                                             constants.STATUS_AVAILABLE)

        self._delete_share_and_share_server(managed_share_id,
                                            managed_share_server_id)
        self.client.delete_share_network(share_network['id'])

class ShareServersReadWriteNFSTest(ShareServersReadWriteBase):
    protocol = 'nfs'

class ShareServersReadWriteCIFSTest(ShareServersReadWriteBase):
    protocol = 'cifs'

def load_tests(loader, tests, _):
    result = []
    for test_case in tests:
        if type(test_case._tests[0]) is ShareServersReadWriteBase:
            continue
        result.append(test_case)
    return loader.suiteClass(result)


# -*- coding: utf-8 -*-

# Laser
_laser_type_list = [

		# 2019
		#('LASER CO2 FRACCIONAL',	'LASER CO2 FRACCIONAL'),

		# 2018
		# Consultation
		('consultation',			'consultation'),

		# Laser
		('laser_co2','Laser Co2'),
		('laser_excilite','Laser Excilite'),
		('laser_ipl','Laser Ipl'),
		('laser_ndyag','Laser Ndyag'),
		('laser_quick','QuickLaser'),

		# Medical
		('medical',					'Tratamiento médico'),
		('criosurgery', 			'Criocirugía'),
		('botulinum_toxin', 		'Toxina botulínica'),
		('hyaluronic_acid', 		'Acido hialurónico'),
		('hyaluronic_acid_repair', 	'Acido hialurónico - Repair'),
		('intravenous_vitamin', 	'Vitamina intravenosa'),
		('lepismatic', 				'Lepismático'),
		('mesotherapy_nctf', 		'Mesoterapia NCTF'),
		('plasma', 					'Plasma'),
		('sclerotherapy', 			'Escleroterapia'),
		('infiltration_keloid', 	'Infiltracion Queloide'),
		('infiltration_scar', 		'Infiltracion Cicatriz'),
		# 12 Jul 2019
		('infiltration', 	'Infiltracion'),

		# Cosmetology
		('cosmetology','Cosmiatría'),

		# Cos
		('diamond_tip',									'diamond_tip'),
		('triactive_carboxytherapy_reductionchamber',	'triactive_carboxytherapy_reductionchamber'),
		('carboxytherapy',								'carboxytherapy'),
		('triactive_carboxytherapy',					'triactive_carboxytherapy'),

		('none',		'none'),
]

_time_list = [
				('15 min','15 min'),
				('30 min','30 min'),
				('60 min','60 min'),
				('none',''),
]

#------------------------------------------------ Vars ----------------------
_h_subfamily = {
					'product' : 		'Producto',
					'consultation' : 	'Consulta',
					'cosmetology' : 	'Cosmeatria',

					'laser_co2' : 		'Laser Co2',
					'laser_excilite' : 	'Laser Exc',
					'laser_ipl' : 		'Laser Ipl',
					'laser_ndyag' : 	'Laser Ndyag',
					'laser_quick' : 	'Quick Laser',

					'criosurgery' : 			'Criocirugia',
					'intravenous_vitamin' : 	'Vitamina Intravenosa',
					'botulinum_toxin' : 		'Toxina Botulinica',
					'hyaluronic_acid' : 		'Acido Hialuronico',
					'mesotherapy_nctf' :		'Mesoterapia NCTF',

					False: False,
}

# Zone
_level_list = [
		('1',	'1'),
		('2',	'2'),
		('3',	'3'),
		('4',	'4'),
		('5',	'5'),
]

# Zone
_zone_list = [
		# Infiltrations
		('any',		'any'),

		# Quick
		('face_all_neck',	'Todo Rostro Cuello'),
		('face_all_hands',	'Todo Rostro Manos'),
		('neck_hands',		'Cuello Manos'),

		# Other
		('areolas',		'areolas'),
		('armpits',		'armpits'),
		('arms',		'arms'),
		('back',		'back'),
		('belly',		'belly'),
		('body_local',	'Localizado Cuerpo'),
		('breast',		'breast'),
		('cheekbones',	'Pómulos'),
		('face',		'face'),
		('face_all',	'face_all'),
		('face_hands',	'face_hands'),
		('face_local',	'face_local'),
		('face_neck',	'face_neck'),
		('face_neck_hands',	'face_neck_hands'),
		('feet',		'feet'),
		('gluteus',		'gluteus'),
		('hands',		'Manos'),
		('head',		'head'),
		('legs',		'legs'),
		('neck',		'Cuello'),
		('package',		'package'),
		('shoulders',	'shoulders'),
		('vagina',		'vagina'),

		('1_zone',		'1_zone'),
		('1_hypodermic','1_hypodermic'),
		('1_hypodermic_repair','1_hypodermic_repair'),
		('x',		'x'),

		('na',		'na'),

		# Cosmetology
		('body_all',		'body_all'),
		('body',		'body'),
		('face_doublechin_neck',		'face_doublechin_neck'),

		('none',		'none'),
		('',		''),
]

_pathology_list = [

		# 28 Jun 2019
		('varices',			'varices'),

		# Standard
		('varicose',			'varicose'),

		# Med
		('keloid',		'keloid'),

		# Quick
		#('tatoo',			'Tatuaje'),
		('tatoo',			'tatoo'),
		#('tatoo_1',		'tatoo_1'),
		#('tatoo_2',		'tatoo_2'),
		#('tatoo_3',		'tatoo_3'),
		#('tatoo_4',		'tatoo_4'),

		#('rejuvenation',	'Rejuvenecimiento'),
		('rejuvenation',	'rejuvenation'),
		#('rejuvenation_1',	'rejuvenation_1'),
		#('rejuvenation_2',	'rejuvenation_2'),
		#('rejuvenation_3',	'rejuvenation_3'),
		#('rejuvenation_4',	'rejuvenation_4'),

		# Other
		#('wart',		'Verruga'),
		('wart',		'wart'),
		('wart_1',		'wart_1'),
		('wart_2',		'wart_2'),
		('wart_3',		'wart_3'),
		('wart_4',		'wart_4'),
		('wart_5',		'wart_5'),

		#('keratosis',	'Queratosis'),
		('keratosis',	'keratosis'),
		('keratosis_1',	'keratosis_1'),
		('keratosis_2',	'keratosis_2'),
		('keratosis_3',	'keratosis_3'),
		('keratosis_4',	'keratosis_4'),
		('keratosis_5',	'keratosis_5'),

		#('stains',		'Manchas'),
		('stains',		'stains'),
		('stains_1',	'stains_1'),
		('stains_2',	'stains_2'),
		('stains_3',	'stains_3'),
		('stains_4',	'stains_4'),
		('stains_5',	'stains_5'),

		#('cyst',		'Quiste'),
		('cyst',		'cyst'),
		('cyst_1',		'cyst_1'),
		('cyst_2',		'cyst_2'),
		('cyst_3',		'cyst_3'),
		('cyst_4',		'cyst_4'),
		('cyst_5',		'cyst_5'),

		#('acne_sequels',	'Acné y secuelas'),
		('acne_sequels',	'acne_sequels'),
		('acne_sequels_1',	'acne_sequels_1'),
		('acne_sequels_2',	'acne_sequels_2'),
		('acne_sequels_3',	'acne_sequels_3'),
		('acne_sequels_4',	'acne_sequels_4'),
		('acne_sequels_5',	'acne_sequels_5'),

		#('mole',		'Lunar'),
		('mole',		'mole'),
		('mole_1',		'mole_1'),
		('mole_2',		'mole_2'),
		('mole_3',		'mole_3'),
		('mole_4',		'mole_4'),
		('mole_5',		'mole_5'),

		#('scar',		'Cicatriz'),
		('scar',		'scar'),
		('scar_1',		'scar_1'),
		('scar_2',		'scar_2'),
		('scar_3',		'scar_3'),
		('scar_4',		'scar_4'),
		('scar_5',		'scar_5'),

		('rejuvenation_face',	'rejuvenation_face'),
		('rejuvenation_face_repair',	'rejuvenation_face_repair'),

		('rejuvenation_face_1',	'rejuvenation_face_1'),
		('rejuvenation_face_2',	'rejuvenation_face_2'),
		('rejuvenation_face_3',	'rejuvenation_face_3'),
		('rejuvenation_face_4',	'rejuvenation_face_4'),
		('rejuvenation_face_5',	'rejuvenation_face_5'),

		('rejuvenation_neck',	'rejuvenation_neck'),
		('rejuvenation_neck_1',	'rejuvenation_neck_1'),
		('rejuvenation_neck_2',	'rejuvenation_neck_2'),
		('rejuvenation_neck_3',	'rejuvenation_neck_3'),
		('rejuvenation_neck_4',	'rejuvenation_neck_4'),
		('rejuvenation_neck_5',	'rejuvenation_neck_5'),

		('rejuvenation_hands',	'rejuvenation_hands'),
		('rejuvenation_hands_1',	'rejuvenation_hands_1'),
		('rejuvenation_hands_2',	'rejuvenation_hands_2'),
		('rejuvenation_hands_3',	'rejuvenation_hands_3'),
		('rejuvenation_hands_4',	'rejuvenation_hands_4'),
		('rejuvenation_hands_5',	'rejuvenation_hands_5'),

		('rejuvenation_face_neck',	'rejuvenation_face_neck'),

		('acne',			'acne'),

		('acne_active',			'acne_active'),
		('alopecia',			'alopecia'),

		('depilation',			'depilation'),

		('emangiomas',		'emangiomas'),

		('polyp',		'polyp'),
		('ruby_point',	'ruby_point'),

		('monalisa_touch',	'monalisa_touch'),

		('psoriasis',			'psoriasis'),

		('rejuvenation_face_hands',			'rejuvenation_face_hands'),
		('rejuvenation_face_neck_hands',	'rejuvenation_face_neck_hands'),
		('rosacea',			'rosacea'),
		('ruby_points',		'ruby_points'),

		('telangiectasia',	'telangiectasia'),

		('vitiligo',			'vitiligo'),

		('x',		'x'),
		('rejuvenation_capilar',		'rejuvenation_capilar'),
		('na',		'na'),

		# Cosmetology
		('deep_face_cleansing',		'deep_face_cleansing'),

		('reaffirmation',		'reaffirmation'),

		('reduction_weight_measures',		'reduction_weight_measures'),

		('',		''),
		('',		''),
		('',		''),

		('none',		'none'),

		# Legacy
		('body_local',		'body_local'),
		('face_local',		'face_local'),

	]

_sessions_list = [
		#('one',			'one'),
		#('six',			'six'),

		('1',			'1'),
		('10',			'10'),

		('2',			'2'),
		('3',			'3'),

		# Cosmetology
		('6',			'6'),
		('12',			'12'),

		('none',		'none'),
		]

_family_list = [

		# 6 Feb 2019
		('credit_note',		'credit_note'),

		# 12 Jul 2018
		('other',			'other'),

		# Service
		('consultation',	'consultation'),
		('consultation_gyn',	'consultation_gyn'),
		('laser',			'laser'),
		('medical',			'medical'),
		('cosmetology',		'cosmetology'),

		# Product
		#('topical',		'Tópico'),
		#('kit',			'Kit'),
		#('card',			'Tarjeta'),
		('topical',			'topical'),
		('kit',				'kit'),
		('card',			'card'),

		('none',		'none'),
]

_treatment_list = [

		# 19 Jul 2018
		('infiltration',			'infiltration'),

		# 12 Jul 2018
		('other',		'other'),

		# 29 May 2018
		('infiltration_scar',		'infiltration_scar'),
		('infiltration_keloid',		'infiltration_keloid'),

		# 28 May 2018
		('product',		'product'),
		('vip_card',		'vip_card'),

		('consultation',		'consultation'),

		('laser_quick',			'laser_quick'),
		('laser_co2',			'laser_co2'),
		('laser_excilite',		'laser_excilite'),
		('laser_ipl',			'laser_ipl'),
		('laser_ndyag',			'laser_ndyag'),
		('laser_m22',			'laser_m22'),

		('criosurgery',			'criosurgery'),
		('hyaluronic_acid',		'hyaluronic_acid'),
		('sclerotherapy',		'sclerotherapy'),
		('lepismatic',			'lepismatic'),
		('plasma',				'plasma'),
		('botulinum_toxin',		'botulinum_toxin'),
		('intravenous_vitamin',	'intravenous_vitamin'),

		# 2017
		('mesotherapy_nctf',		'mesotherapy_nctf'),
		('hyaluronic_acid_repair',		'hyaluronic_acid_repair'),

		# Cosmetology
		('triactive_carboxytherapy_reductionchamber',			'triactive_carboxytherapy_reductionchamber'),
		('carboxytherapy',			'carboxytherapy'),
		('diamond_tip',			'diamond_tip'),
		('triactive_carboxytherapy',			'triactive_carboxytherapy'),
		('',			''),

		#('none',			'none'),
		('none',		'none'),
]

#------------------------------------------------ Getters ----------------------
def get_family_list():
	"""
	Get Family list
	"""
	return _family_list

def get_treatment_list():
	"""
	Get Treatment list
	"""
	return _treatment_list

def get_zone_list():
	"""
	Get Zone list
	"""
	return _zone_list

def get_pathology_list():
	"""
	Get Pathology list
	"""
	return _pathology_list

def get_level_list():
	"""
	Get Level list
	"""
	return _level_list

def get_laser_type_list():
	"""
	Get laser type list
	"""
	return _laser_type_list

def get_time_list():
	"""
	Get time list
	"""
	return _time_list


import pytest

from aiogram.utils.deep_linking import decode_payload, encode_payload, filter_payload
from aiogram.utils.deep_linking import get_start_link, get_startgroup_link
from tests.types import dataset

# enable asyncio mode
pytestmark = pytest.mark.asyncio

PAYLOADS = [
    'foo',
    'AAbbCCddEEff1122334455',
    'aaBBccDDeeFF5544332211',
    -12345678901234567890,
    12345678901234567890,
]

WRONG_PAYLOADS = [
    '@BotFather',
    'spaces spaces spaces',
    1234567890123456789.0,
]

@pytest.fixture(params=PAYLOADS, name='payload')
def payload_fixture(request):
    return request.param

@pytest.fixture(params=WRONG_PAYLOADS, name='wrong_payload')
def wrong_payload_fixture(request):
    return request.param

@pytest.fixture(autouse=True)
def get_bot_user_fixture(monkeypatch):
    """ Monkey patching of bot.me calling. """
    from aiogram.utils import deep_linking

    async def get_bot_user_mock():
        from aiogram.types import User
        return User(**dataset.USER)

    monkeypatch.setattr(deep_linking, '_get_bot_user', get_bot_user_mock)

class TestDeepLinking:
    async def test_get_start_link(self, payload):
        link = await get_start_link(payload)
        assert link == f'https://t.me/{dataset.USER["username"]}?start={payload}'

    async def test_wrong_symbols(self, wrong_payload):
        with pytest.raises(ValueError):
            await get_start_link(wrong_payload)

    async def test_get_startgroup_link(self, payload):
        link = await get_startgroup_link(payload)
        assert link == f'https://t.me/{dataset.USER["username"]}?startgroup={payload}'

    async def test_filter_encode_and_decode(self, payload):
        _payload = filter_payload(payload)
        encoded = encode_payload(_payload)
        decoded = decode_payload(encoded)
        assert decoded == str(payload)

    async def test_get_start_link_with_encoding(self, payload):
        # define link
        link = await get_start_link(payload, encode=True)

        # define reference link
        payload = filter_payload(payload)
        encoded_payload = encode_payload(payload)

        assert link == f'https://t.me/{dataset.USER["username"]}?start={encoded_payload}'


#!/usr/bin/python

import sys
import os
import shutil
import json
import pickle
import warnings
import datetime
import numpy as np
import pandas as pd
from tqdm import tqdm
import joblib

from sklearn.isotonic import IsotonicRegression

import xgboost as xgb
import catboost as cb
from sklearn import ensemble
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    import lightgbm as lgb

def load_data(path_data):
    # Load preprocessed data

    df = pd.read_csv(path_data, header=[0,1], index_col=[0,1], parse_dates=True)

    return df

class Trial():
    def __init__(self, params_json):
        self.params_json = params_json

        # Mandatory input variables
        self.trial_name = params_json['trial_name']
        self.trial_comment = params_json['trial_comment']
        self.path_result = params_json['path_result']
        self.path_preprocessed_data = params_json['path_preprocessed_data']
        self.splits = params_json['splits']
        self.sites = params_json['sites']
        self.features = params_json['features']
        self.target = params_json['target']
        self.model_params = params_json['model_params']
        self.regression_params = params_json['regression_params']
        self.save_options = params_json['save_options']

        if 'parallel_processing' in params_json:
            self.parallel_processing = params_json['parallel_processing']
        else:
            self.parallel_processing = {'backend': 'threading',
                                        'n_workers': 1}

        if 'quantile' in self.regression_params['type']:
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            if len(alpha_q) == 0:
                raise ValueError('Number of quantiles needs to be larger than zero.')

        # Optional input variables
        if 'variables_lags' in params_json:
            self.variables_lags = params_json['variables_lags']
        else:
            self.variables_lags = None
        if 'diff_target_with_physical' in params_json:
            self.diff_target_with_physical = params_json['diff_target_with_physical']
        else:
            self.diff_target_with_physical = False
        if 'target_smoothing_window' in params_json:
            self.target_smoothing_window = params_json['target_smoothing_window']
        else:
            self.target_smoothing_window = 1
        if 'train_only_zenith_angle_below' in params_json:
            self.train_only_zenith_angle_below = params_json['train_only_zenith_angle_below']
        else:
            self.train_only_zenith_angle_below = False
        if 'weight_params' in params_json:
            self.weight_params = params_json['weight_params']
        else:
            self.weight_params = False
        # Checks

        # runtime
        self.parallel_backend = params_json.get("parallel_backend", "threading")

    def generate_dataset(self, df, split, site):

        def add_lags(df, variables_lags):
            # Lagged features
            vspec = pd.DataFrame([(k, lag) for k, v in variables_lags.items() for lag in v],
                                 columns=["Variable", "Lag"])\
                              .set_index("Variable")\
                              .sort_values("Lag")
            for lag, variables in vspec.groupby("Lag").groups.items():
                shifted = df.loc[:, sorted(variables)].groupby('ref_datetime').shift(lag)
                shifted.columns = ['%s_lag%s' % (variable, lag) for variable in sorted(variables)]
                df = pd.concat([df, shifted], axis=1)
            return df

        # Make target into list if not already
        if self.diff_target_with_physical and not ('Physical_Forecast' in self.features):
            df_X = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], self.features+['Physical_Forecast']]
        else:
            df_X = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], self.features]

        df_y = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], [self.target]]

        # Add lagged variables
        if self.variables_lags is not None:
            df_X = add_lags(df_X, self.variables_lags)

        # Remove samples where either all features are nan or target is nan
        is_nan = df_X.isna().all(axis=1) | df_y.isna().all(axis=1)
        df_model = pd.concat([df_X, df_y], axis=1)[~is_nan]

        # Keep all timestamps for which zenith <= prescribed value (day timestamps)
        if self.train_only_zenith_angle_below:
            idx_day = df_model[df_model['zenith'] <= self.train_only_zenith_angle_below].index
            df_model = df_model.loc[idx_day, :]

        # Create target and feature DataFrames
        if self.diff_target_with_physical:
            df_model[self.target] = df_model[self.target]-df_model['Physical_Forecast']

        # Use mean window to smooth target
        df_model[self.target] = df_model[self.target].rolling(self.target_smoothing_window, win_type='boxcar', center=True, min_periods=0).mean()

        # Apply sample weighting
        if self.weight_params:
            weight_end = self.weight_params['weight_end']
            weight_shape = self.weight_params['weight_shape']
            valid_times = df_model.index.get_level_values('valid_datetime')
            days = np.array((valid_times[-1]-valid_times).total_seconds()/(60*60*24))
            weight = (1-weight_end)*np.exp(-days/weight_shape)+weight_end
        else:
            weight = None

        return df_X, df_y, df_model, weight

    def generate_dataset_split_site(self, df, split_set='train'):
        # Generate train and valid splits

        print('Generating dataset...')
        dfs_X_split, dfs_y_split, dfs_model_split, weight_split = [], [], [], []
        with tqdm(total=len(self.splits[split_set])*len(self.sites)) as pbar:
            for split in self.splits[split_set]:
                dfs_X_site, dfs_y_site, dfs_model_site, weight_site = [], [], [], []
                for site in self.sites:

                    df_X, df_y, df_model, weight = self.generate_dataset(df, split, site)

                    dfs_X_site.append(df_X)
                    dfs_y_site.append(df_y)
                    dfs_model_site.append(df_model)
                    weight_site.append(weight)

                    pbar.update(1)

                dfs_X_split.append(dfs_X_site)
                dfs_y_split.append(dfs_y_site)
                dfs_model_split.append(dfs_model_site)
                weight_split.append(weight_site)

        return dfs_X_split, dfs_y_split, dfs_model_split, weight_split

    def build_model_dataset(self, df_model_train, model, df_model_valid=None, weight=None):
        # Build up dataset adapted to models
        train_set, valid_sets = {}, {}
        if model == 'lightgbm':
            train_set = lgb.Dataset(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight, params={'verbose': -1}, free_raw_data=False)
            if df_model_valid is not None:
                valid_set = lgb.Dataset(df_model_valid[self.features], label=df_model_valid[[self.target]], params={'verbose': -1}, free_raw_data=False)
                valid_sets = [train_set, valid_set]
            else:
                vaild_sets['lightgbm'] = [train_set_lgb]
        elif model == 'xgboost':
            train_set = xgb.DMatrix(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight)
            if df_model_valid is not None:
                valid_set = xgb.DMatrix(df_model_valid[self.features], label=df_model_valid[[self.target]])
                valid_sets = [(train_set, 'train'), (valid_set, 'valid')]
            else:
                valid_sets = [(train_set, 'train')]
        elif model == 'catboost':
            train_set = cb.Pool(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight)
            if df_model_valid is not None:
                valid_set = cb.Pool(df_model_valid[self.features], label=df_model_valid[[self.target]])
                valid_sets = [valid_set]
        elif model == 'skboost' in self.model_params:
            train_set = [df_model_train[self.features], df_model_train[self.target], weight]

        return train_set, valid_sets

    def train_on_objective(self, train_set, valid_sets, model, objective='mean', alpha=None):

        if model == 'lightgbm':
            with warnings.catch_warnings():
                if self.model_params['lightgbm']['verbose'] == -1:
                    warnings.simplefilter("ignore")
                if objective == 'mean':
                    objective_lgb = 'mean_squared_error'
                    eval_key_name = 'l2'
                elif objective == 'quantile':
                    objective_lgb = 'quantile'
                    eval_key_name = 'quantile'
                    self.model_params['lightgbm']['alpha'] = alpha
                else:
                    raise ValueError("'objective' for lightgbm must be either 'mean' or 'quantile'")
                self.model_params['lightgbm']['objective'] = objective_lgb

                evals_result = {}
                gbm = lgb.train(self.model_params['lightgbm'],
                                train_set,
                                valid_sets=valid_sets,
                                valid_names=None,
                                evals_result=evals_result,
                                verbose_eval=False,
                                callbacks=None)
                evals_result = {key: value[eval_key_name] for key, value in evals_result.items()}

        elif model == 'xgboost':
            if objective == 'mean':
                self.model_params['xgboost']['objective'] = 'reg:squarederror'
            else:
                raise ValueError("'objective' for xgboost must be 'mean'.")
            evals_result = {}
            gbm = xgb.train(self.model_params['xgboost'],
                            train_set,
                            self.model_params['xgboost']['num_round'],
                            evals=valid_sets,
                            evals_result=evals_result,
                            verbose_eval=False)
            evals_result = None #TODO Add evals for xgboost

        elif model=='catboost':
            if objective == 'mean':
                objective_cb = 'Lq:q=2'
            elif objective == 'quantile':
                objective_cb = 'Quantile:alpha={0:g}'.format(alpha)
            else:
                raise ValueError("'objective' must be one of ['mean', 'quantile']")
            self.model_params['catboost']['objective'] = objective_cb

            gbm = cb.train(pool=train_set,
                           params=self.model_params['catboost'],
                           eval_set=valid_sets,
                           verbose=False)
            evals_result = {key: value[objective_cb] for key, value in gbm.evals_result_.items()}

        elif model=='skboost':
            if objective == 'mean':
                self.model_params['skboost']['loss'] = 'ls'
                self.model_params['skboost']['criterion'] = 'friedman_mse'
            elif objective == 'quantile':
                self.model_params['skboost']['loss'] = 'quantile'
                self.model_params['skboost']['alpha'] = alpha
                self.model_params['skboost']['criterion'] = 'mae' #TODO Check how `criterion` affects quantile loss.
            else:
                raise ValueError("'objective' must be one of ['mean', 'quantile']")

            gbm = ensemble.GradientBoostingRegressor(**self.model_params['skboost'])
            gbm.fit(train_set[0], train_set[1], sample_weight=train_set[2])
            evals_result = None #TODO Add evals for skboost

        else:
            raise ValueError("'objective' for skboost must be either 'mean' or 'quantile'")

        return gbm, evals_result

    def train(self, train_set, valid_sets, model):

        gbm_q, evals_result_q = {}, {}
        if 'mean' in self.regression_params['type']:
            # Train model for mean
            gbm, evals_result = self.train_on_objective(train_set, valid_sets, model, objective='mean')

            gbm_q['mean'] = gbm
            evals_result_q['mean'] = evals_result #TODO change this similarly to quantile case.

        if 'quantile' in self.regression_params['type']:
            # Train models for different quantiles
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])

            with joblib.parallel_backend(self.parallel_processing['backend']):
                results = joblib.Parallel(n_jobs=self.parallel_processing['n_workers'])(
                                joblib.delayed(self.train_on_objective)(
                                    train_set, valid_sets,
                                    model, objective='quantile', alpha=alpha)
                                    for alpha in alpha_q)
            for (gbm, evals_result), alpha in zip(results, alpha_q):
                gbm_q['quantile{0:.2f}'.format(alpha)] = gbm
                evals_result_q['quantile{0:.2f}'.format(alpha)] = evals_result

        if not (('mean' in self.regression_params['type']) or ('quantile' in self.regression_params['type'])):
            raise ValueError('Value of regression parameter "objective" not recognized.')

        return gbm_q, evals_result_q

    def train_model_split_site(self, dfs_model_train_split, dfs_model_valid_split=None, weight_train_split=None):

        print('Training...')
        gbm_model, evals_result_model = {}, {}
        with tqdm(total=len(self.model_params.keys())*len(dfs_model_train_split)*len(dfs_model_train_split[0])) as pbar:
            for model in self.model_params.keys():
                gbm_split, evals_result_split = [], []
                for idx_split, dfs_model_train_site in enumerate(dfs_model_train_split):

                    gbm_site, evals_result_site = [], []
                    for idx_site, df_model_train in enumerate(dfs_model_train_site):

                        if dfs_model_valid_split is not None:
                            df_model_valid = dfs_model_valid_split[idx_split][idx_site]
                        else:
                            df_model_valid = None

                        if weight_train_split is not None:
                            weight = weight_train_split[idx_split][idx_site]
                        else:
                            weight = None

                        train_set, valid_sets = self.build_model_dataset(df_model_train, model, df_model_valid=df_model_valid, weight=weight)
                        gbm_q, evals_result_q = self.train(train_set, valid_sets, model) #TODO Make it possible to train starting from an existing model. E.g. LightGBM has a `input_model` option.

                        #TODO Add support for categorical_features.
                        gbm_site.append(gbm_q)
                        evals_result_site.append(evals_result_q)

                        pbar.update(1)

                    gbm_split.append(gbm_site)
                    evals_result_split.append(evals_result_site)

                gbm_model[model] = gbm_split
                evals_result_model[model] = evals_result_split

        return gbm_model, evals_result_model

    def predict(self, df_X, gbm_q, model):
        # Use trained models to predict
        #TODO Use SHAP to estimate contribution of different features. https://github.com/slundberg/shap

        def post_process(y_pred):

            if self.diff_target_with_physical:
                y_pred = y_pred+df_X['Physical_Forecast'].values

            if not self.regression_params['target_min_max'] == [None, None]:
                target_min_max = self.regression_params['target_min_max']

                if target_min_max[1] == 'clearsky':
                    idx_clearsky = y_pred > df_X['Clearsky_Forecast'].values
                    y_pred[idx_clearsky] = df_X['Clearsky_Forecast'].values[idx_clearsky]

                    if not target_min_max[0] == None:
                        y_pred = y_pred.clip(min=target_min_max[0], max=None)

                else:
                    y_pred = y_pred.clip(min=target_min_max[0], max=target_min_max[1])

            return y_pred

        # Make DataFrame to store the predictions in
        idx_q_start = 0
        columns = []
        if 'mean' in self.regression_params['type']:
            idx_q_start += 1
            columns.append('mean')

        if 'quantile' in self.regression_params['type']:
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            columns.extend(['quantile{0}'.format(int(round(100*alpha))) for alpha in alpha_q])

        df_index = pd.DataFrame(index=df_X.index, columns=columns)

        # Keep all timestamps for which zenith <= prescribed value (day timestamps)
        if self.train_only_zenith_angle_below:
            idx_day = df_X['zenith'] <= self.train_only_zenith_angle_below
            idx_night = df_X['zenith'] > self.train_only_zenith_angle_below
            df_X = df_X[idx_day]

        df_y_pred_qs = {}

        y_pred_q = []
        for q in gbm_q.keys():
            if model == 'lightgbm':
                y_pred = gbm_q[q].predict(df_X[self.features])
            elif model == 'xgboost':
                if self.regression_params['type'][0] == 'mean':
                    y_pred = gbm_q[q].predict(xgb.DMatrix(df_X[self.features]))
            elif model == 'catboost':
                y_pred = gbm_q[q].predict(df_X[self.features])
            elif model == 'skboost':
                y_pred = gbm_q[q].predict(df_X[self.features])
            else:
                raise ValueError()

            y_pred = post_process(y_pred)
            y_pred_q.append(y_pred)

        # Convert list to numpy 2D-array
        y_pred_q = np.stack(y_pred_q, axis=-1)

        if 'quantile_postprocess' in self.regression_params.keys():
            if self.regression_params['quantile_postprocess'] == 'none':
                pass
            elif self.regression_params['quantile_postprocess'] == 'sorting':
                # Lazy post-sorting of quantiles
                y_pred_q = np.sort(y_pred_q, axis=-1)
            elif self.regression_params['quantile_postprocess'] == 'isotonic_regression':
                # Isotonic regression
                regressor = IsotonicRegression()
                y_pred_q = np.stack([regressor.fit_transform(alpha_q, y_pred_q[sample,:]) for sample in range(idx_q_start, y_pred_q.shape[0])])

        # Create prediction output dataframe
        df_y_pred_q = df_index
        if self.train_only_zenith_angle_below:
            df_y_pred_q[idx_day] = y_pred_q
            df_y_pred_q[idx_night] = 0
        else:
            df_y_pred_q.values[:] = y_pred_q

        df_y_pred_q = df_y_pred_q.astype('float64')

        return df_y_pred_q

    def predict_model_split_site(self, dfs_X_split, gbm_model):
        # Use trained models to predict for their corresponding split

        dfs_y_pred_model = {}
        print('Predicting...')
        with tqdm(total=len(self.model_params.keys())*len(dfs_X_split[0])*len(dfs_X_split)) as pbar:
            for model in self.model_params.keys():
                dfs_y_pred_split = []
                gbm_split = gbm_model[model]
                for dfs_X_site, gbm_site in zip(dfs_X_split, gbm_split):
                    dfs_y_pred_site = []
                    for dfs_X, gbm_q, in zip(dfs_X_site, gbm_site):
                        df_y_pred_q = self.predict(dfs_X, gbm_q, model)
                        dfs_y_pred_site.append(df_y_pred_q)

                        pbar.update(1)

                    dfs_y_pred_split.append(dfs_y_pred_site)

                dfs_y_pred_model[model] = dfs_y_pred_split

        return dfs_y_pred_model

    def calculate_loss(self, dfs_y_true_split, dfs_y_pred_model):

        print('Calculating loss...')
        if 'mean' in self.regression_params['type']:

            dfs_loss_model = {}
            for model in self.model_params.keys():
                dfs_loss_split = []
                dfs_y_pred_split = dfs_y_pred_model[model]
                for dfs_y_true_site, dfs_y_pred_site in zip(dfs_y_true_split, dfs_y_pred_split):
                    dfs_loss_site = []
                    for df_y_true, df_y_pred in zip(dfs_y_true_site, dfs_y_pred_site):
                        y_true = df_y_true[[self.target]].values
                        y_pred = df_y_pred.values

                        loss = (y_pred-y_true)**2

                        df_loss = pd.DataFrame(data=loss, index=df_y_pred.index, columns=df_y_pred.columns)

                        dfs_loss_site.append(df_loss)

                    dfs_loss_split.append(dfs_loss_site)

                dfs_loss_model[model] = dfs_loss_split

        if 'quantile' in self.regression_params['type']:
            # Evaluation using pinball loss function

            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            a = alpha_q.reshape(1,-1)

            dfs_loss_model = {}
            for model in self.model_params.keys():

                dfs_loss_split = []
                dfs_y_pred_split = dfs_y_pred_model[model]
                for dfs_y_true_site, dfs_y_pred_site in zip(dfs_y_true_split, dfs_y_pred_split):
                    dfs_loss_site = []
                    for df_y_true, df_y_pred in zip(dfs_y_true_site, dfs_y_pred_site):
                        y_true = df_y_true[[self.target]].values
                        y_pred = df_y_pred.values

                        # Pinball loss with nan if true label is nan
                        with np.errstate(invalid='ignore'):
                            loss = np.where(np.isnan(y_true),
                                            np.nan,
                                            np.where(y_true < y_pred,
                                                    (1-a)*(y_pred-y_true),
                                                    a*(y_true-y_pred)))

                            df_loss = pd.DataFrame(data=loss, index=df_y_pred.index, columns=df_y_pred.columns)

                        dfs_loss_site.append(df_loss)

                    dfs_loss_split.append(dfs_loss_site)

                dfs_loss_model[model] = dfs_loss_split

        return dfs_loss_model

    def calculate_score(self, dfs_loss_model):

        flatten = lambda l: [item for sublist in l for item in sublist]
        score_model = {}
        for model in self.model_params.keys():
            score_model[model] = pd.concat(flatten(dfs_loss_model[model])).mean().mean()

        return score_model

    def save_result(self, params_json, result_data, result_prediction, result_model, result_evals, result_loss):

        print('Saving results...')
        trial_path = self.path_result+self.trial_name
        if os.path.exists(trial_path):
            shutil.rmtree(trial_path)
        os.makedirs(trial_path)

        file_name_json = '/params_'+self.trial_name+'.json'
        with open(trial_path+file_name_json, 'w') as file:
            json.dump(params_json, file, indent=4)

        if self.save_options['data'] == True:
            for key in result_data.keys():
                os.makedirs(trial_path+'/'+key)
                for split in range(len(result_data[key])):
                    file_name = key+'_split_{0}.csv'.format(split)
                    df = pd.concat(result_data[key][split], axis=1, keys=self.sites)
                    df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['prediction'] == True:
            for key in result_prediction.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_prediction[key][model])):
                            file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                            df = pd.concat(result_prediction[key][model][split], axis=1, keys=self.sites)
                            df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['model'] == True:
            for key in result_model.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_model[key][model])):
                        for site in range(len(result_model[key][model][0])):
                            for q in result_model[key][model][0][0].keys():
                                if model in ['lightgbm', 'xgboost', 'catboost']:
                                    file_name = key+'_'+model+'_q_'+q+'_split_{0}_site_{1}.txt'.format(split, site)
                                    result_model[key][model][split][site][q].save_model(trial_path+'/'+key+'/'+file_name)
                                if model == 'skboost':
                                    file_name = key+'_'+model+'_q_'+q+'_split_{0}_site_{1}.pkl'.format(split, site)
                                    with open(trial_path+'/'+key+'/'+file_name, 'wb') as f:
                                        pickle.dump(result_model[key][model][split][site][q], f)
        if self.save_options['evals'] == True:
            for key in result_evals.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_evals[key][model])):
                        file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                        data = result_evals[key][model][split]
                        data = {(level1_key, level2_key, level3_key): pd.Series(values)
                                for level1_key, level2_dict in zip(self.sites,data)
                                for level2_key, level3_dict in level2_dict.items()
                                for level3_key, values in level3_dict.items()}
                        df = pd.DataFrame(data)
                        df.index.name = 'trees'
                        df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['loss'] == True:
            for key in result_loss.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_loss[key][model])):
                        file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                        df_loss = pd.concat(result_loss[key][model][split], axis=1, keys=self.sites)
                        df_loss.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['overall_score'] == True:
            score_train_model = self.calculate_score(result_loss['dfs_loss_train'])
            score_valid_model = self.calculate_score(result_loss['dfs_loss_valid'])
            file_name = self.path_result+'/trial-scores.txt'

            for model in score_train_model.keys():
                if not os.path.exists(file_name):
                    with open(file_name, 'w') as file:
                        file.write('Name: {0}; Comment: {1}; Model: {2}; Train score {3}; valid score {4};\n'.format(self.trial_name, self.trial_comment, model, score_train_model[model], score_valid_model[model]))
                else:
                    with open(file_name, 'a') as file:
                        file.write('Name: {0}; Comment: {1}; Model: {2}; Train score {3}; valid score {4};\n'.format(self.trial_name, self.trial_comment, model, score_train_model[model], score_valid_model[model]))
        else:
            score_train_model = None
            score_valid_model = None
        print('Results saved to: '+trial_path)

        return score_train_model, score_valid_model

    def run(self, df):

        print('Running trial pipeline for trial: {0}...'.format(self.trial_name))
        dfs_X_train_split, dfs_y_train_split, dfs_model_train_split, weight_train_split = self.generate_dataset_split_site(df, split_set='train')
        dfs_X_valid_split, dfs_y_valid_split, dfs_model_valid_split, _ = self.generate_dataset_split_site(df, split_set='valid')

        gbm_model, evals_result_model = self.train_model_split_site(dfs_model_train_split, dfs_model_valid_split=dfs_model_valid_split, weight_train_split=weight_train_split)

        dfs_y_pred_train_model = self.predict_model_split_site(dfs_X_train_split, gbm_model)
        dfs_y_pred_valid_model = self.predict_model_split_site(dfs_X_valid_split, gbm_model)

        dfs_loss_train_model = self.calculate_loss(dfs_y_train_split, dfs_y_pred_train_model)
        dfs_loss_valid_model = self.calculate_loss(dfs_y_valid_split, dfs_y_pred_valid_model)

        result_data = {'dfs_X_train': dfs_X_train_split,
                    'dfs_X_valid': dfs_X_valid_split,
                    'dfs_y_train': dfs_y_train_split,
                    'dfs_y_valid': dfs_y_valid_split}
        result_model = {'gbm_model': gbm_model}
        result_evals = {'evals_result': evals_result_model}
        result_prediction = {'dfs_y_pred_train': dfs_y_pred_train_model,
                                'dfs_y_pred_valid': dfs_y_pred_valid_model}
        result_loss = {'dfs_loss_train': dfs_loss_train_model,
                    'dfs_loss_valid': dfs_loss_valid_model}

        score_train_model, score_valid_model = self.save_result(self.params_json, result_data, result_prediction, result_model, result_evals, result_loss)

        return score_train_model, score_valid_model

if __name__ == '__main__':
    params_path = sys.argv[1]
    with open(params_path, 'r', encoding='utf-8') as file:
        params_json = json.loads(file.read())

    df = load_data(params_json['path_preprocessed_data']+params_json['filename_preprocessed_data'])
    trial = Trial(params_json)
    trial.run(df)


import gym
import numpy as np
import tensorflow as tf

__all__ = [ "make_env", "SkipWrapper" ]

def make_env(id, wrappers):
  env = gym.make(id)
  for wrapper in wrappers:
    env = wrapper(env)
  return env

def SkipWrapper(repeat_count):
  class SkipWrapper(gym.Wrapper):
    """
        Generic common frame skipping wrapper.
        Will perform action for `x` additional steps and return all the
        observations. After resetting returns same observation in a collection
        of length (`repeat_count` + 1) with no actions taken.
    """
    def __init__(self, env):
      super(SkipWrapper, self).__init__(env)
      self.repeat_count = repeat_count
      self.stepcount = 0

    def _step(self, action):
      done = False
      total_reward = 0
      current_step = 0
      observations = []
      while current_step < (self.repeat_count + 1):
        self.stepcount += 1
        obs, reward, done, info = self.env.step(action)
        total_reward += reward
        observations.append(obs)
        current_step += 1
      if "skip.stepcount" in info:
        raise gym.error.Error("Key 'skip.stepcount' already in info. Make"\
              "sure you are not stacking the SkipWrapper wrappers.")
      info["skip.stepcount"] = self.stepcount
      return np.asarray(observations), total_reward, done, info

    def _reset(self):
      self.stepcount = 0
      return np.asarray([self.env.reset()] * (self.repeat_count + 1))

  return SkipWrapper


import os
import numpy as np
from PIL import Image

size=(200,200,3)
# photos=np.ones(200,200,3)
rootDir = 'Image_folder'

for dirName, subdirList, fileList in os.walk(rootDir):
    print('Found directory: %s' % dirName)
    for fname in fileList:
        print('-------------------\t%s' % fname)
        img=Image.open(f"{rootDir}/{fname}")
        arr=np.array(img)
        print(arr)
        img=Image.fromarray(arr)
        img.thumbnail(size)
        print(img)
        img.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Sep 24 16:26:02 2014

@author: hang
"""

import is_prime
s = 0
for i in range(1000):
    if is_prime.is_prime(i) == True:
        s = s + i
print(s)