import fcntl
import json
import logging
import multiprocessing
import os
import signal
import subprocess
import sys
import tempfile
import zipfile
from typing import Any, Dict, List, Optional, Tuple, Union
from urllib.parse import urldefrag

from celery.exceptions import SoftTimeLimitExceeded  # type: ignore

import toil.server.wes.amazon_wes_utils as amazon_wes_utils
from toil.common import Toil
from toil.jobStores.utils import generate_locator
from toil.server.celery_app import celery
from toil.server.utils import (WorkflowStateMachine,
                               connect_to_workflow_state_store,
                               download_file_from_internet,
                               download_file_from_s3,
                               get_file_class,
                               get_iso_time,
                               link_file)

logger = logging.getLogger(__name__)

# How many seconds should we give a Toil workflow to gracefully shut down
# before we kill it?
# Ought to be long enough to let it clean up its job store, but shorter than
# our patience for CANCELING WES workflows to time out to CANCELED.
WAIT_FOR_DEATH_TIMEOUT = 20

class ToilWorkflowRunner:
    """
    A class to represent a workflow runner to run the requested workflow.

    Responsible for parsing the user request into a shell command, executing
    that command, and collecting the outputs of the resulting workflow run.
    """

    def __init__(self, base_scratch_dir: str, state_store_url: str, workflow_id: str, request: Dict[str, Any], engine_options: List[str]):
        """
        Make a new ToilWorkflowRunner to actually run a workflow leader based
        on a WES request.

        :param base_scratch_dir: Base work directory. Workflow scratch directory
               will be in here under the workflow ID.
        :param state_store_url: URL to the state store through which we will
               communicate about workflow state with the WES server.
        :param workflow_id: ID of the workflow run.
        :param request: WES request information.
        :param engine_options: Extra options to pass to Toil.
        """

        # Find the scratch directory inside the base scratch directory
        self.scratch_dir = os.path.join(base_scratch_dir, workflow_id)

        # Connect to the workflow state store
        self.store = connect_to_workflow_state_store(state_store_url, workflow_id)
        # And use a state machine over that to look at workflow state
        self.state_machine = WorkflowStateMachine(self.store)

        self.request = request
        self.engine_options = engine_options

        self.wf_type: str = request["workflow_type"].lower().strip()
        self.version: str = request["workflow_type_version"]

        self.exec_dir = os.path.join(self.scratch_dir, "execution")
        self.out_dir = os.path.join(self.scratch_dir, "outputs")

        # Compose the right kind of job store to use it the user doesn't specify one.
        default_type = os.getenv('TOIL_WES_JOB_STORE_TYPE', 'file')
        self.default_job_store = generate_locator(default_type, local_suggestion=os.path.join(self.scratch_dir, "toil_job_store"))

        self.job_store = self.default_job_store

    def write_scratch_file(self, filename: str, contents: str) -> None:
        """
        Write a file to the scratch directory.
        """
        with open(os.path.join(self.scratch_dir, filename), "w") as f:
            f.write(contents)

    def get_state(self) -> str:
        return self.state_machine.get_current_state()

    def write_workflow(self, src_url: str) -> str:
        """
        Fetch the workflow file from its source and write it to a destination
        file.
        """
        logger.info(f"Processing workflow_url: '{src_url}'...")
        dest = os.path.join(self.exec_dir, os.path.basename(src_url))

        if src_url.startswith("file://"):
            logger.info("Linking workflow from filesystem.")
            link_file(src=src_url[7:], dest=dest)
        elif src_url.startswith(("http://", "https://")):
            logger.info("Downloading workflow_url from the Internet.")
            download_file_from_internet(src=src_url, dest=dest, content_type="text/")
        elif src_url.startswith("s3://"):
            logger.info("Downloading workflow_url from Amazon S3.")
            download_file_from_s3(src=src_url, dest=dest)
        else:
            logger.info("Using workflow from relative URL.")
            dest = os.path.join(self.exec_dir, src_url)

        # Make sure that the destination file actually exists
        if not os.path.isfile(urldefrag(dest)[0]):
            raise RuntimeError(f"Cannot resolve workflow file from: '{src_url}'.")

        return dest

    def sort_options(self, workflow_engine_parameters: Optional[Dict[str, Optional[str]]] = None) -> List[str]:
        """
        Sort the command line arguments in the order that can be recognized by
        the workflow execution engine.

        :param workflow_engine_parameters: User-specified parameters for this
        particular workflow. Keys are command-line options, and values are
        option arguments, or None for options that are flags.
        """
        options = []

        # First, we pass the default engine parameters
        options.extend(self.engine_options)

        if workflow_engine_parameters:
            # Then, we pass the user options specific for this workflow run.
            # This should override the default
            for key, value in workflow_engine_parameters.items():
                if value is None:  # flags
                    options.append(key)
                else:
                    options.append(f"{key}={value}")

        # We want to clean always by default, unless a particular job store or
        # a clean option was passed.
        clean = None

        # Parse options and drop options we may need to override.
        for option in options:
            if option.startswith("--jobStore="):
                self.job_store = option[11:]
                options.remove(option)
            if option.startswith(("--outdir=", "-o=")):
                # We need to generate this one ourselves.
                options.remove(option)
            if option.startswith("--clean="):
                clean = option[8:]

        cloud = False
        job_store_type, _ = Toil.parseLocator(self.job_store)
        if job_store_type in ("aws", "google", "azure"):
            cloud = True

        if self.job_store == self.default_job_store and clean is None:
            # User didn't specify a clean option, and we're on a default,
            # randomly generated job store, so we should clean it up even if we
            # crash.
            options.append("--clean=always")

        if self.wf_type in ("cwl", "wdl"):
            if not cloud:
                options.append("--outdir=" + self.out_dir)
            options.append("--jobStore=" + self.job_store)
        else:
            # TODO: find a way to communicate the out_dir to the Toil workflow.

            # append the positional jobStore argument at the end for Toil workflows
            options.append(self.job_store)

        return options

    def initialize_run(self) -> List[str]:
        """
        Write workflow and input files and construct a list of shell commands
        to be executed. Return that list of shell commands that should be
        executed in order to complete this workflow run.
        """
        # Obtain CWL-style workflow parameters from the request. Default to an
        # empty dict if not found, because we want to tolerate omitting this.
        workflow_params = self.request.get("workflow_params", {})

        # And any workflow engine parameters the user specified.
        workflow_engine_parameters = self.request.get("workflow_engine_parameters", {})

        # Obtain main workflow file to a path (no-scheme file URL).
        workflow_url = self.write_workflow(src_url=self.request["workflow_url"])

        if os.path.basename(workflow_url) == "workflow.zip" and zipfile.is_zipfile(workflow_url):
            # We've been sent a zip file. We should interpret this as an Amazon Genomics CLI-style zip file.
            # Extract everything next to the zip and find and open relvant files.
            logger.info("Extracting and parsing Amazon-style workflow bundle...")
            zip_info = amazon_wes_utils.parse_workflow_zip_file(workflow_url, self.wf_type)

            # Now parse Amazon's internal format into our own.

            # Find the real workflow source for the entrypoint file
            if 'workflowSource' in zip_info['files']:
                # They sent a file, which has been opened, so grab its path
                workflow_url = zip_info['files']['workflowSource'].name
                logger.info("Workflow source file: '%s'", workflow_url)
            elif 'workflowUrl' in zip_info['data']:
                # They are pointing to another URL.
                # TODO: What does Amazon expect this to mean? Are we supposed to recurse?
                # For now just forward it.
                workflow_url = zip_info['data']['workflowUrl']
                logger.info("Workflow reference URL: '%s'", workflow_url)
            else:
                # The parser is supposed to throw so we can't get here
                raise RuntimeError("Parser could not find workflow source or URL in zip")

            if 'workflowInputFiles' in zip_info['files'] and len(zip_info['files']['workflowInputFiles']) > 0:
                # The bundle contains a list of input files.
                # We interpret these as JSON, and layer them on top of each
                # other, and then apply workflow_params from the request.
                logger.info("Workflow came with %d bundled inputs files; coalescing final parameters",
                            len(zip_info['files']['workflowInputFiles']))

                coalesced_parameters = {}
                for binary_file in zip_info['files']['workflowInputFiles']:
                    try:
                        # Read each input file as a JSON
                        loaded_parameters = json.load(binary_file)
                    except json.JSONDecodeError as e:
                        raise RuntimeError(f"Could not parse inputs JSON {os.path.basename(binary_file.name)}: {e}")
                    # And merge them together in order
                    coalesced_parameters.update(loaded_parameters)
                # Then apply and replace the parameters that came with the request
                coalesced_parameters.update(workflow_params)
                workflow_params = coalesced_parameters

            if 'workflowOptions' in zip_info['files']:
                # The bundle contains an options JSON. We interpret these as
                # defaults for workflow_engine_parameters.
                logger.info(f"Workflow came with bundled options JSON")
                try:
                    # Read as a JSON
                    loaded_options = json.load(zip_info['files']['workflowOptions'])
                except json.JSONDecodeError as e:
                    raise RuntimeError(f"Could not parse options JSON: {e}")
                # Apply and replace the engine parameters that came with the
                # request.
                loaded_options.update(workflow_engine_parameters)
                workflow_engine_parameters = loaded_options

        # Write out CWL-style input file
        input_json = os.path.join(self.exec_dir, "wes_inputs.json")
        with open(input_json, "w") as f:
            json.dump(workflow_params, f)

        # create output directory
        if not os.path.exists(self.out_dir):
            os.makedirs(self.out_dir)

        # sort commands
        options = self.sort_options(workflow_engine_parameters)

        # construct and return the command to run
        if self.wf_type == "cwl":
            command_args = (
                ["toil-cwl-runner"] + options + [workflow_url, input_json]
            )
        elif self.wf_type == "wdl":
            command_args = (
                ["toil-wdl-runner"] + options + [workflow_url, input_json]
            )
        elif self.wf_type == "py":
            command_args = ["python", workflow_url] + options
        else:
            # This shouldn't happen if this was executed from the Toil WES server.
            raise RuntimeError(f"Unsupported workflow type: {self.wf_type}")

        return command_args

    def call_cmd(self, cmd: Union[List[str], str], cwd: str) -> "subprocess.Popen[bytes]":
        """
        Calls a command with Popen. Writes stdout, stderr, and the command to
        separate files.
        """
        stdout_f = os.path.join(self.scratch_dir, "stdout")
        stderr_f = os.path.join(self.scratch_dir, "stderr")

        with open(stdout_f, "w") as stdout, open(stderr_f, "w") as stderr:
            logger.info(f"Calling: '{' '.join(cmd)}'")
            process = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, close_fds=True, cwd=cwd)

        return process

    def run(self) -> None:
        """
        Construct a command to run a the requested workflow with the options,
        run it, and deposit the outputs in the output directory.
        """

        # the task has been picked up by the runner and is currently preparing to run
        self.state_machine.send_initialize()
        commands = self.initialize_run()

        # store the job store location
        self.store.set("job_store", self.job_store)

        # Check if we are supposed to cancel
        state = self.get_state()
        if state in ("CANCELING", "CANCELED"):
            logger.info("Workflow canceled.")
            return

        # Otherwise start to run
        self.state_machine.send_run()
        process = self.call_cmd(cmd=commands, cwd=self.exec_dir)

        self.store.set("start_time", get_iso_time())
        self.store.set("cmd", " ".join(commands))

        try:
            exit_code = process.wait()
        except (KeyboardInterrupt, SystemExit, SoftTimeLimitExceeded) as e:
            logger.warning("Workflow interrupted: %s", type(e))
            process.terminate()

            try:
                process.wait(timeout=WAIT_FOR_DEATH_TIMEOUT)
            except subprocess.TimeoutExpired:
                # We need to actually stop now.
                process.kill()

            logger.info("Child process terminated by interruption.")
            exit_code = 130

        self.store.set("end_time", get_iso_time())
        self.store.set("exit_code", str(exit_code))

        logger.info('Toil child finished with code %s', exit_code)

        if exit_code == 0:
            self.state_machine.send_complete()
        # non-zero exit code indicates failure
        elif exit_code == 130:
            self.state_machine.send_canceled()
        else:
            self.state_machine.send_executor_error()

    def write_output_files(self) -> None:
        """
        Fetch all the files that this workflow generated and output information
        about them to `outputs.json`.
        """
        output_obj = {}
        job_store_type, _ = Toil.parseLocator(self.job_store)

        # For CWL workflows, the stdout should be a JSON object containing the outputs
        if self.wf_type == "cwl":
            try:
                with open(os.path.join(self.scratch_dir, "stdout")) as f:
                    output_obj = json.load(f)
            except Exception as e:
                logger.warning("Failed to read outputs object from stdout:", exc_info=e)

        elif job_store_type == "file":
            for file in os.listdir(self.out_dir):
                location = os.path.join(self.out_dir, file)
                output_obj[file] = {
                    "location": location,
                    "size": os.stat(location).st_size,
                    "class": get_file_class(location),
                }

        # TODO: fetch files from other job stores

        self.write_scratch_file("outputs.json", json.dumps(output_obj))

def run_wes_task(base_scratch_dir: str, state_store_url: str, workflow_id: str, request: Dict[str, Any], engine_options: List[str]) -> str:
    """
    Run a requested workflow.

    :param base_scratch_dir: Directory where the workflow's scratch dir will live, under the workflow's ID.

    :param state_store_url: URL/path at which the server and Celery task communicate about workflow state.

    :param workflow_id: ID of the workflow run.

    :returns: the state of the workflow run.
    """

    logger.info("Starting WES workflow")

    runner = ToilWorkflowRunner(base_scratch_dir, state_store_url, workflow_id,
                                request=request, engine_options=engine_options)

    try:
        runner.run()

        state = runner.get_state()
        logger.info(f"Workflow run completed with state: '{state}'.")

        if state == "COMPLETE":
            logger.info(f"Fetching output files.")
            runner.write_output_files()
    except (KeyboardInterrupt, SystemExit, SoftTimeLimitExceeded):
        # We canceled the workflow run
        logger.info('Canceling the workflow')
        runner.state_machine.send_canceled()
    except Exception:
        # The workflow run broke. We still count as the executor here.
        logger.exception('Running Toil produced an exception.')
        runner.state_machine.send_executor_error()
        raise

    return runner.get_state()

# Wrap the task function as a Celery task
run_wes = celery.task(name="run_wes")(run_wes_task)

def cancel_run(task_id: str) -> None:
    """
    Send a SIGTERM signal to the process that is running task_id.
    """
    celery.control.terminate(task_id, signal='SIGUSR1')

class TaskRunner:
    """
    Abstraction over the Celery API. Runs our run_wes task and allows canceling it.

    We can swap this out in the server to allow testing without Celery.
    """

    @staticmethod
    def run(args: Tuple[str, str, str, Dict[str, Any], List[str]], task_id: str) -> None:
        """
        Run the given task args with the given ID on Celery.
        """
        run_wes.apply_async(args=args,
                            task_id=task_id,
                            ignore_result=True)

    @staticmethod
    def cancel(task_id: str) -> None:
        """
        Cancel the task with the given ID on Celery.
        """
        cancel_run(task_id)

    @staticmethod
    def is_ok(task_id: str) -> bool:
        """
        Make sure that the task running system is working for the given task.
        If the task system has detected an internal failure, return False.
        """
        # Nothing to do for Celery
        return True

# If Celery can't be set up, we can just use this fake version instead.

class MultiprocessingTaskRunner(TaskRunner):
    """
    Version of TaskRunner that just runs tasks with Multiprocessing.

    Can't use threading because there's no way to send a cancel signal or
    exception to a Python thread, if loops in the task (i.e.
    ToilWorkflowRunner) don't poll for it.
    """

    _id_to_process: Dict[str, multiprocessing.Process] = {}
    _id_to_log: Dict[str, str] = {}

    @staticmethod
    def set_up_and_run_task(output_path: str, args: Tuple[str, str, str, Dict[str, Any], List[str]]) -> None:
        """
        Set up logging for the process into the given file and then call
        run_wes_task with the given arguments.

        If the process finishes successfully, it will clean up the log, but if
        the process crashes, the caller must clean up the log.
        """

        # Multiprocessing and the server manage to hide actual task output from
        # the tests. Logging messages will appear in pytest's "live" log but
        # not in the captured log. And replacing sys.stdout and sys.stderr
        # don't seem to work to collect tracebacks. So we make sure to
        # configure logging in the multiprocessing process and log to a file
        # that we were told about, so the server can come get the log if we
        # unexpectedly die.

        output_file = open(output_path, 'w')
        output_file.write('Initializing task log\n')
        output_file.flush()

        # Take over logging.
        root_logger = logging.getLogger()
        root_logger.setLevel(logging.DEBUG)
        handler = logging.StreamHandler(output_file)
        root_logger.addHandler(handler)

        def handle_sigterm(_: Any, __: Any) -> None:
            """
            Multiprocessing will send SIGTERM to stop us; translate that to
            something the run_wes_task shutdown code understands to avoid
            always waiting for the cancel timeout.

            Caller still needs to handle a sigterm exit code, in case we get
            canceled before the handler is set!
            """
            sys.exit()

        signal.signal(signal.SIGTERM, handle_sigterm)

        try:
            logger.info('Running task')
            output_file.flush()
            run_wes_task(*args)
        except Exception:
            logger.exception('Exception in task!')
            raise
        else:
            # If the task does not crash, clean up the log
            os.unlink(output_path)
        finally:
            logger.debug('Finishing task log')
            output_file.flush()
            output_file.close()

    @classmethod
    def run(cls, args: Tuple[str, str, str, Dict[str, Any], List[str]], task_id: str) -> None:
        """
        Run the given task args with the given ID.
        """

        # We need to send the child process's output somewhere or it will get lost during testing
        fd, path = tempfile.mkstemp()
        os.close(fd)
        # Store the log filename before the process, like is_ok() expects.
        cls._id_to_log[task_id] = path

        logger.info("Starting task %s in a process that should log to %s", task_id, path)

        cls._id_to_process[task_id] = multiprocessing.Process(target=cls.set_up_and_run_task, args=(path, args))
        cls._id_to_process[task_id].start()

    @classmethod
    def cancel(cls, task_id: str) -> None:
        """
        Cancel the task with the given ID.
        """
        if task_id in cls._id_to_process:
            logger.info("Stopping process for task %s", task_id)
            cls._id_to_process[task_id].terminate()
        else:
            logger.error("Tried to kill nonexistent task %s", task_id)

    @classmethod
    def is_ok(cls, task_id: str) -> bool:
        """
        Make sure that the task running system is working for the given task.
        If the task system has detected an internal failure, return False.
        """

        process = cls._id_to_process.get(task_id)
        if process is None:
            # Never heard of this task, so it's not broken.
            return True

        # If the process exited normally, or with an error code consistent with
        # being canceled by cancel(), then it is OK.
        ACCEPTABLE_EXIT_CODES = [0, -signal.SIGTERM]

        if process.exitcode is not None and process.exitcode not in ACCEPTABLE_EXIT_CODES:
            # Something went wring in the task and it couldn't handle it.
            logger.error("Process for running %s failed with code %s", task_id, process.exitcode)
            try:
                for line in open(cls._id_to_log[task_id]):
                    # Dump the task log
                    logger.error("Task said: %s", line.rstrip())
                # Remove the task log because it is no longer needed
                os.unlink(cls._id_to_log[task_id])
            except FileNotFoundError:
                # Log is already gone
                pass
            return False

        return True


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 15 21:48:18 2018

@author: wuwangchuxin
"""
import numpy as np
import pandas as pd
import statsmodels.api as sm
import math
#import matplotlib.pyplot as plt
import os
os.chdir('D:/multiple_factors_models/')
#import date_process_class as dpc
from scipy import stats as ss
import matplotlib.pyplot as plt

# 添加路径，方便导入自定义模块，比如 import date_process_class as dpc
import sys
sys.path.append('D:/multiple_factors_models')

from pylab import mpl
mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['axes.unicode_minus'] = False

add_winddata = 'C:/Users/wuwangchuxin/Desktop/TF_SummerIntern/MF_data/wind/'
add_ready = 'C:/Users/wuwangchuxin/Desktop/TF_SummerIntern/MF_data/prepared_data/del_cixin/'
add_pic = 'C:/Users/wuwangchuxin/Desktop/TF_SummerIntern/20190326pic/'

# load data
##################################################################################
## 估值指标
#pe = np.load(add_ready+'windfactors_pe.npy')
ep = np.load(add_ready+'windfactors_ep.npy')
#pb = np.load(add_ready+'windfactors_pb.npy')
bp = np.load(add_ready+'windfactors_bp.npy')
#ps = np.load(add_ready+'windfactors_ps.npy')
sp = np.load(add_ready+'windfactors_sp.npy')
pcf_ocf = np.load(add_ready+'windfactors_pcf_ocf.npy') #市现率
dividendyield = np.load(add_ready+'windfactors_dividendyield.npy') #股息率
evtoebitda = np.load(add_ready+'windfactors_evtoebitda.npy')  #企业倍数

#盈利能力
roe = np.load(add_ready+'windfactors_roe.npy') #净资产收益率（TTM）
roa = np.load(add_ready+'windfactors_roaebit.npy') #总资产收益率（TTM）
roic = np.load(add_ready+'windfactors_roicebit.npy') #投入资本回报率ROIC(TTM)
gpmgr = np.load(add_ready+'windfactors_gpmgr.npy') #增长率_毛利率(TTM)
grossprofitmargin = np.load(add_ready+'windfactors_grossprofitmargin.npy') #销售毛利率(TTM)
orgr = np.load(add_ready+'windfactors_orgr.npy') #增长率_营业收入(TTM)
cfogr = np.load(add_ready+'windfactors_cfogr.npy') #增长率_经营活动产生的现金流量净额(TTM)
#经营效率
invturn = np.load(add_ready+'windfactors_invturn.npy') #存货周转率(TTM)
arturn = np.load(add_ready+'windfactors_arturn.npy') #应收账款周转率(TTM)
taturn = np.load(add_ready+'windfactors_taturn.npy') #总资产周转率(TTM)
#盈余质量
profittomv = np.load(add_ready+'windfactors_profittomv.npy') #收益市值比(TTM)
#投融资决策
tagr = np.load(add_ready+'windfactors_tagr.npy') #增长率-总资产
#无形资产
stmnote_RDexptosales = np.load(add_ready+'windfactors_stmnote_RDexptosales.npy') #研发支出总额占营业收入比例
#杠杆因子
debttoasset = np.load(add_ready+'windfactors_debttoasset.npy') #资产负债率
current = np.load(add_ready+'windfactors_current.npy') #流动比率

#利用市场信号修正策略
#市场参与者行为
netprofit_fy1_6m = np.load(add_ready+'windfactors_netprofit_fy1_6m.npy') #一致预测净利润（FY1）的变化率_6M
#市场价格信号
revs60 = np.load(add_ready+'windfactors_revs60.npy') #过去3个月的价格动量
#市场情绪信号
rstr12 = np.load(add_ready+'windfactors_rstr12.npy') #12月相对强势

#其它指标
#float_mv = np.load(add_ready+'wind_float_mv.npy') #流动市值
#
#return_month = np.load(add_ready+'wind_return_month.npy') #月收益率
#
#stockcode = np.load(add_ready+'stockscode.npy').reshape(-1,1) #股票代码
#trade_date = np.load(add_ready+'month_end_tdate.npy').reshape(1,-1) #月末交易日
#
#industry_sw1 = np.load(add_ready+'industry_sw1.npy') #申万一级行业哑变量
#industry_sw1_name = np.load(add_ready+'industry_sw1_name.npy').reshape(1,-1) #申万一级行业分类名称
##industry = pd.read_excel(add_winddata+'industry_sw1_class.xlsx')  #原始数据
#
#start_date = np.load(add_ready+'stock_tdate_start.npy').reshape(-1,1) #个股有效期起始时间
#end_date = np.load(add_ready+'stock_tdate_end.npy').reshape(-1,1) #个股有效期终止时间
#
#weights_000300 = np.load(add_ready+'weights_000300.npy') #沪深300指数个股权重
#weights_000300_stocklist = np.load(add_ready+'weights_000300_stocklist.npy')
#
#hs300_sw_1class_weight = np.load(add_ready+'hs300_sw_1class_weight') #沪深300指数申万一级行业分类权重
#hs300_sw_1class_weight_industrynames = np.save(add_ready+'hs300_sw_1class_weight_industrynames')
##################################################################################

class Clean_Data:
    #数据格式：ndarray，3225*115,3225只个股，115个月
    def __init__(self,arr):
        self.arr=arr
        self.trade_date = np.load(add_ready+'matcol_month_end_tdate.npy').reshape(1,-1) #月末交易日
        self.start_date = np.load(add_ready+'stock_tdate_start.npy').reshape(-1,1) #个股有效期起始时间
        self.end_date = np.load(add_ready+'stock_tdate_end.npy').reshape(-1,1) #个股有效期终止时间

    def Median_deextremum(self,n=5):
        '''中位数去极值法'''
        med = np.nanmedian(self.arr,axis=0)
        mad = np.nanmedian(np.abs(self.arr - med),axis=0)
        res = np.where(self.arr>(med+n*mad),(med+n*mad),
                       np.where(self.arr<(med-n*mad),(med-n*mad),self.arr))
        return res

    def Ordinal_values(self):
        # 求数据集的排序值
        # argsort 不能忽略nan值
        res = self.arr.copy()
        not_nan_num = np.sum(~np.isnan(self.arr),axis=0) #非空值数量
        #将nan值赋值为大数字然后排序
        mid = np.argsort(np.where(np.isnan(self.arr),99999,self.arr),axis=0)
        #末尾插入一列递增数列，作为名次标记
        mid2 = np.insert(mid,mid.shape[1],values=np.arange(1,mid.shape[0]+1),axis=1)
        # 逐列进行计算
        for n in np.arange(self.arr.shape[1]):
            mid_tmp = mid2[:,[n,self.arr.shape[1]]] #当前列和标记列，即所在列的位置和排名
            #第一列排序，其位置归位，标记列为排名。得到原来所在位置的元素在所在列的排名
            mid_tmp2 = mid_tmp[mid_tmp[:,0].argsort(axis=0)][:,1]
            #恢复为空值的列
            mid_tmp3 = np.where(mid_tmp2>=not_nan_num[n]+1,np.nan,mid_tmp2)
            res[:,n] = mid_tmp3
        return res

    def Z_score(self):
        '''z_score标准化'''
        return (self.arr - np.nanmean(self.arr))/np.nanstd(self.arr)

    def Fill_na(self):
        '''将本来应该有但是却为nan值的位置填充为0'''
        mid = np.where((self.trade_date>=self.start_date) & (self.trade_date<=self.end_date),
                       self.arr,99999)
        res = np.where(np.isnan(mid),0,self.arr)
        return res

    @staticmethod
    def Round_df(df_factor):
        return df_factor.applymap(lambda x:'%.4f'%x)

class Single_factors_test_regression:
    '''单因子测试之回归法'''
    def __init__(self,factor_arr):
        def Data_preposs(f_arr):
            '''完整的数据预处理流程'''
            f_a1 = Clean_Data(f_arr).Median_deextremum()
            f_a2 = Clean_Data(f_a1).Z_score()
            f_a3 = Clean_Data(f_a2).Fill_na()
            f_a4 = np.around(f_a3,decimals=4)
            return f_a4
        self.factor_arr=Data_preposs(factor_arr) #3171*115
        self.industry_sw1 = np.load(add_ready+'industry_sw1.npy') #3171*28
        self.float_mv = np.load(add_ready+'wind_float_mv.npy') #3171*115
        r_month = np.load(add_ready+'wind_return_month.npy')
        self.return_month = Data_preposs(r_month) #月收益率 #3171*115
        self.trade_date = np.load(add_ready+'matcol_month_end_tdate.npy').reshape(1,-1) #月末交易日 115

    @staticmethod
    def OLS_regression(x,y):
        # 普通最小二乘法回归
        X = sm.add_constant(x)
        regr = sm.OLS(y, X).fit() #regr.resid,残差；regr.params，beta值;regr.tvalues,T值
        return regr

    @staticmethod
    def WLS_regression(x,y,w):
        #加权最小二乘法回归
        #w=data.iloc[:,2].tolist()
        #w=np.array([i**0.5 for i in w])
        X = sm.add_constant(x)
        regr = sm.WLS(y,X,weights=w).fit()
        #results.tvalues T值 regr.resid,残差；regr.params，beta值;results.t_test([1,0])
        return regr

    def single_factor_regress(self):
        ''' 单因子回归，包含OLS和WLS回归，回归自变量加入申万一级行业哑变量，WLS以流通市值的平方根作为权值'''
        res_regr=pd.DataFrame(index=np.arange(self.factor_arr.shape[1]-1),
                              columns=['trade_month','Beta_OLS','Tvalue_OLS',
                                       'Beta_WLS','Tvalue_WLS','IC','RANKIC'])
        for n in np.arange(self.factor_arr.shape[1]-1):
            #剔除空值
            nona_index = (~np.isnan(self.factor_arr[:,n])) & (~np.isnan(self.return_month[:,n+1]))
            factor_arr_nona = self.factor_arr[nona_index,n][:,np.newaxis]
            return_month_nona = self.return_month[nona_index,n+1][:,np.newaxis]
            industry_sw1_nona = self.industry_sw1[nona_index,:]
            X = np.hstack((industry_sw1_nona,factor_arr_nona))
            #OLS
            res_ols = self.OLS_regression(X,return_month_nona)
            #WLS
            w=self.float_mv[:,n][nona_index]
            w = np.where(np.isnan(w),np.nanmean(w),w) #用均值填充nan值
            w=np.array([i**0.5 for i in w])
            res_wls = self.WLS_regression(X,return_month_nona,w)
            # IC,先用因子值对行业和市值回归取残差，然后再和次月收益求IC
            float_mv_nona = self.float_mv[nona_index,n][:,np.newaxis]
            float_mv_nona=np.where(np.isnan(float_mv_nona),np.nanmean(float_mv_nona),float_mv_nona)
            X_IC = np.hstack((industry_sw1_nona,float_mv_nona))
            res_ols_IC = self.OLS_regression(X_IC,factor_arr_nona)
            mid_ic = np.corrcoef(res_ols_IC.resid,return_month_nona[:,0])[0,1]
            # RANKIC，同IC值，先取残差再求值
            rank_ic = pd.DataFrame([res_ols_IC.resid,
                                    return_month_nona[:,0]]).T.corr('spearman').iloc[0,1]
            #结果
            res_regr.iloc[n,:] = [self.trade_date[0,n],res_ols.params[-1],res_ols.tvalues[-1],
                                  res_wls.params[-1],res_wls.tvalues[-1],mid_ic,rank_ic]
        return res_regr

    @staticmethod
    def T_analysis(regress_df):
        # T abs mean
        T_res = pd.DataFrame(index=['OLS','WLS'],
                              columns=['t_abs_mean','port_greater_2','beta_mean',
                                       'beta_t','t_mean_div_std'])
        L1 = ['Tvalue_OLS','Tvalue_WLS']
        for col1 in L1:
            t_abs_mean = np.mean(regress_df[col1].apply(lambda x:abs(x)))
            port_greater_2 = np.sum(regress_df[col1].apply(lambda x:abs(x))>2)/len(regress_df[col1])
#            t_mean_div_std = t_abs_mean/np.std(regress_df[col1].apply(lambda x:abs(x)))
            t_mean_div_std = abs(np.mean(regress_df[col1]))/np.std(regress_df[col1])
            T_res.loc[col1[-3:],'t_abs_mean'] = t_abs_mean
            T_res.loc[col1[-3:],'port_greater_2'] = port_greater_2
            T_res.loc[col1[-3:],'t_mean_div_std'] = t_mean_div_std

        L2 = ['Beta_OLS','Beta_WLS']
        for col2 in L2:
            beta_mean = np.mean(regress_df[col2])
            beta_t = ss.ttest_1samp(regress_df[col2], popmean = 0)[0]
            T_res.loc[col2[-3:],'beta_mean'] = beta_mean
            T_res.loc[col2[-3:],'beta_t'] = beta_t
        return T_res

    @staticmethod
    def IC_analysis(regress_df):
        IC_res=pd.DataFrame(index=['fac'],columns=['IC_mean','IC_std','IC_IR','IC_positive_port',
                            'RANKIC_mean','RANKIC_std','RANKIC_IR','RANKIC_positive_port'])
        IC_mean = regress_df['IC'].mean()
        IC_std = regress_df['IC'].std()
        IC_IR = IC_mean/IC_std
        IC_positive_port = np.sum(regress_df['IC']>0)/len(regress_df['IC'])

        RANKIC_mean = regress_df['RANKIC'].mean()
        RANKIC_std = regress_df['RANKIC'].std()
        RANKIC_IR = RANKIC_mean/RANKIC_std
        RANKIC_positive_port = np.sum(regress_df['RANKIC']>0)/len(regress_df['RANKIC'])

        IC_res.loc['fac',:] = IC_mean,IC_std,IC_IR,IC_positive_port,RANKIC_mean,RANKIC_std,RANKIC_IR,RANKIC_positive_port
        return IC_res

    @staticmethod
    def Style_rotation(regress_df):
        '''风格因子显著性正负和方向转换统计'''
        SR_res = pd.DataFrame(index=['OLS','WLS'],
                              columns=['pos_cor_sig_port','neg_cor_sig_port',
                                       'syn_sig_port','exch_sig_port'])
        L3 = ['Tvalue_OLS','Tvalue_WLS']
        for col3 in L3:
            pos_port = sum(regress_df[col3]>2)/len(regress_df[col3])
            neg_port = sum(regress_df[col3]<-2)/len(regress_df[col3])
            SR_res.loc[col3[-3:],'pos_cor_sig_port'] = pos_port
            SR_res.loc[col3[-3:],'neg_cor_sig_port'] = neg_port

            mid_Tvalues = regress_df[np.logical_or(regress_df[col3]>2,regress_df[col3]<-2)]
            mid_Tvalues.reset_index(drop=True,inplace=True)
            syn_sig_num=0
            exch_sig_num=0
            for mid_index in range(1,mid_Tvalues.shape[0]):
                # 同向
                if mid_Tvalues.loc[mid_index,col3]*mid_Tvalues.loc[mid_index-1,col3]>0:
                    syn_sig_num+=1
                else:
                    exch_sig_num+=1
            SR_res.loc[col3[-3:],'syn_sig_port'] = syn_sig_num/(len(regress_df[col3])-1)
            SR_res.loc[col3[-3:],'exch_sig_port'] = exch_sig_num/(len(regress_df[col3])-1)
        return SR_res

class Single_factors_test_group:
    '''单因子测试之分组法
       数据格式：ndarray，3225*115,3225只个股，115个月 self=Single_factors_test_group(ep)'''
    capital_initial = 100000000
    def __init__(self,factor_arr):
        self.factor_arr=factor_arr
        self.return_month = np.load(add_ready+'wind_return_month.npy') #月收益率
        self.trade_date = np.load(add_ready+'matcol_month_end_tdate.npy')#.reshape(1,-1) #月末交易日
        self.industry = pd.read_excel(add_winddata+'industry_sw1_class.xlsx')
        self.stockcode = np.load(add_ready+'matind_stockscode.npy')#.reshape(-1,1) #股票代码
        self.hs300_sw_1class_weight = np.load(add_ready+'hs300_sw_1class_weight.npy') #沪深300指数申万一级行业分类权重
        self.industrynames = np.load(add_ready+'hs300_sw_1class_weight_industrynames.npy')
        self.hs300_mkt = pd.read_csv(add_winddata+'market_data_hs300.csv')
        self.zz500_mkt = pd.read_csv(add_winddata+'market_data_zz500.csv')
        self.szzz_mkt = pd.read_csv(add_winddata+'market_data_szzz.csv')

    def group_net_value(self):
        '''行业中性分组，行业所占权重和hs300相同，行业内个股等权重；
           按单因子值排序分五组构建投资组合，因此每个中性分组的权重之和都是20%
           并构建多空组合，添加沪深300和中证500指数作为比较'''
        # hs300指数申万一级行业分类权重
        industry_w_df=pd.DataFrame(self.hs300_sw_1class_weight,index=self.industrynames,
                                  columns= self.trade_date)
        industry_w_df.fillna(0,inplace=True)

        factor_df = pd.DataFrame(self.factor_arr,index=self.stockcode,columns=self.trade_date)
        return_month_df = pd.DataFrame(self.return_month,index=self.stockcode,columns=self.trade_date)
        #资金变化结果df
        capital_df = pd.DataFrame(index = self.trade_date,
                                   columns=['group1','group2','group3','group4','group5'])
        capital_df.iloc[0,:]=self.capital_initial
        for n in range(len(factor_df.columns)-1):
            mid_df=pd.merge(factor_df[[factor_df.columns[n]]],
                        return_month_df[[factor_df.columns[n+1]]],
                        left_index=True,right_index=True)
            mid_df = pd.merge(mid_df,self.industry,left_index=True,right_on='code',how='inner')
            mid_df.rename(columns={factor_df.columns[n]:'factor',
                                   factor_df.columns[n+1]:'return_month'},inplace = True)
            mid_df.dropna(inplace=True)
            grouped =mid_df.groupby(by='industry_1class') #按行业分组
            grouped_df = pd.DataFrame(columns=['factor','return_month','group_NO','weight_stock'])
            #hs300指数申万一级行业中性分组，被分组隔断的个股权重按照比例分配
            for indus_name,value in grouped:
                base_weight_stock = industry_w_df.loc[indus_name][n]/value.shape[0]
                value.sort_values(by='factor',ascending=False,inplace=True) #因子倒序排列
                value.reset_index(drop=True,inplace=True)
                group_amount = value.shape[0]/5 #每组股票数量
                if group_amount>=1:
                    if int(group_amount)==group_amount:
                        for group_n in range(5):
                            mid_indus_group = value.loc[group_n*group_amount:(group_n+1)*group_amount-1,
                                                        ['factor','return_month']]
                            mid_indus_group['group_NO'] = group_n+1
                            mid_indus_group['weight_stock'] = base_weight_stock
                            grouped_df = grouped_df.append(mid_indus_group)
                    else:
                        for group_n in range(5):
                            int_start =  int(group_amount*group_n)
                            int_end = int(group_amount*(group_n+1))
                            dec_start = 1-math.modf(group_amount*group_n)[0]
                            dec_end = math.modf(group_amount*(group_n+1))[0]
                            if group_n==0:
                                mid_indus_group = value.loc[:int_end,['factor','return_month']]
                                mid_indus_group['group_NO'] = 1
                                # 正常的
                                mid_indus_group.loc[:int_end-1,'weight_stock']= base_weight_stock
                                # 最后一个分隔的
                                mid_indus_group.loc[int_end,'weight_stock']= base_weight_stock*dec_end
                                grouped_df = grouped_df.append(mid_indus_group)
                            elif group_n == 4:
                                mid_indus_group = value.loc[int_start:,['factor','return_month']]
                                mid_indus_group['group_NO'] = group_n+1
                                mid_indus_group.loc[int_start+1:,'weight_stock']= base_weight_stock
                                mid_indus_group.loc[int_start,'weight_stock']=base_weight_stock*dec_start
                                grouped_df = grouped_df.append(mid_indus_group)
                            else:
                                mid_indus_group = value.loc[int_start:int_end,['factor','return_month']]
                                mid_indus_group['group_NO'] = group_n+1
                                mid_indus_group['weight_stock'] = [x*base_weight_stock for x in
                                              [dec_start]+[1]*(int_end-int_start-1)+[dec_end]]
                                grouped_df = grouped_df.append(mid_indus_group)
                else:
                    midl = [x*group_amount for x in range(1,6)]
                    for group_n in range(5):
                        int_start =  int(group_amount*group_n)
                        int_end = int(group_amount*(group_n+1))
                        dec_start = 1-math.modf(group_amount*group_n)[0]
                        dec_end = math.modf(group_amount*(group_n+1))[0]
                        if group_n==0:
                            mid_indus_group = pd.DataFrame(value.loc[0,['factor','return_month']]).T
                            mid_indus_group['group_NO'] = 1
                            mid_indus_group.loc[0,'weight_stock']= base_weight_stock*group_amount
                        elif group_n == 4:
                            mid_indus_group = pd.DataFrame(value[['factor','return_month']].iloc[-1,:]).T
                            mid_indus_group['group_NO'] = 5
                            mid_indus_group['weight_stock']= base_weight_stock*group_amount
                        else:
                            if int(midl[group_n])>int(midl[group_n-1]):
                                mid_indus_group = value.loc[int_start:int_end,['factor','return_month']]
                                mid_indus_group['group_NO'] = group_n+1
                                mid_indus_group['weight_stock'] = [x*base_weight_stock for x in
                                                  [dec_start,dec_end]]
                            elif int(midl[group_n])==int(midl[group_n-1]):
                                mid_indus_group = pd.DataFrame(value.loc[int_start,['factor','return_month']]).T
                                mid_indus_group['group_NO'] = group_n+1
                                mid_indus_group['weight_stock'] = base_weight_stock*group_amount
                        grouped_df = grouped_df.append(mid_indus_group)
            grouped_df.reset_index(drop=True,inplace=True)
            #资金等权,即每个分组的资金都是1亿；因为每个分组的权重一样，所以总资金1亿每个分组2千万和每个分组1亿相同
            for group_num in range(5):
                grouped_indus = grouped_df[grouped_df['group_NO']==(group_num+1)]
                capital_df.iloc[n+1,group_num] = sum(capital_df.iloc[n,group_num]*(
                         grouped_indus['weight_stock']/grouped_indus['weight_stock'].sum())
                         *(1+grouped_indus['return_month']/100))
            print (n,'done')
        # 计算多空组合
        if capital_df.iloc[-1,0]>capital_df.iloc[-1,-1]:
            capital_df['long_short'] = capital_df['group1']-capital_df['group5']+self.capital_initial
        else:
            capital_df['long_short'] = capital_df['group5']-capital_df['group1']+self.capital_initial
        netvalue_df = capital_df/self.capital_initial
        # 加上基准
        netvalue_df['hs300']=(self.hs300_mkt['chg']/100+1).cumprod().values/(self.hs300_mkt['chg'][0]/100+1)
#        netvalue_df['zz500']=(self.zz500_mkt['chg']/100+1).cumprod().values/(self.zz500_mkt['chg'][0]/100+1)
        netvalue_df['szzz']=(self.szzz_mkt['chg']/100+1).cumprod().values/(self.szzz_mkt['chg'][0]/100+1)
        return netvalue_df

    @staticmethod
    def backtest_indicates(netvalue_df):
        #年化收益，最大回撤，sharp ratio
        return_monthly = (netvalue_df/netvalue_df.shift(1)-1).fillna(0)
        res = pd.DataFrame(columns=['annual_return','sharp_ratio','max_drawdown','volatility','win_rate'],
                           index = netvalue_df.columns)
        for column_name in netvalue_df.columns:
            mkt_series = netvalue_df[column_name]
            res.loc[column_name,'annual_return']=pow(mkt_series[-1],1/(9+7/12))-1 #年化收益
            res.loc[column_name,'sharp_ratio'] = \
              np.sqrt(12)*return_monthly[column_name].mean()/return_monthly[column_name].std() #sharp
            # 最大回撤
            Drawdown = []
            for nn in range(1,len(mkt_series)):
                max_s = max(mkt_series[:nn])
                if max_s>mkt_series[nn]:
                    DD = (max_s-mkt_series[nn])/max_s
                else:
                    DD = 0
                Drawdown.append(DD*100)
            res.loc[column_name,'max_drawdown']= max(Drawdown)
        res['volatility'] = np.sqrt(12)*return_monthly.std()*100 #波动率
        res['win_rate'] = (netvalue_df.iloc[1:] - netvalue_df.shift(1).iloc[1:]>0) \
                            .sum()/netvalue_df.iloc[1:].shape[0] #胜率
        return res.applymap(lambda x:'%.4f'%x)

    @staticmethod
    def draw(netvalue_df,pic_name):
#         画图
        ax1 = plt.figure(figsize=(16, 9)).add_subplot(1,1,1)
        netvalue_df.plot(ax=ax1,grid=True)
        ax1.set_xlabel('交易日期', fontsize=16) #x轴名称
        ax1.set_ylabel('净值', fontsize=16) #x轴名称
        plt.title("%s分组净值曲线"%pic_name,fontsize=20) #标题
        plt.legend(loc='best')
        plt.savefig(add_pic+pic_name+'.png',dpi=400,bbox_inches='tight')

if __name__=='__main__':

    #因子倒数值
    # 回归法

    # 估值指标
    #ep #市盈率
    #bp #市净率
    #sp #市销率
    #pcf_ocf #市现率
    #dividendyield #股息率
    #evtoebitda #企业倍数
    #盈利能力
    #roe #净资产收益率（TTM）
    #roa #总资产收益率（TTM）
    #roic #投入资本回报率ROIC(TTM)
    #gpmgr #增长率_毛利率(TTM)
    #grossprofitmargin #销售毛利率(TTM)
    #orgr #增长率_营业收入(TTM)
    #cfogr #增长率_经营活动产生的现金流量净额(TTM)
    ##经营效率
    #invturn #存货周转率(TTM)
    #arturn #应收账款周转率(TTM)
    #taturn #总资产周转率(TTM)
    ##盈余质量
    #profittomv #收益市值比(TTM)
    ##投融资决策
    #tagr #增长率-总资产
    ##无形资产
    #stmnote_RDexptosales #研发支出总额占营业收入比例
    ##杠杆因子
    #debttoasset #资产负债率
    #current #流动比率
    ##市场参与者行为
    #netprofit_fy1_6m #一致预测净利润（FY1）的变化率_6M
    #revs60 #过去3个月的价格动量
    #rstr12 #12月相对强势

    factors_origin = ['ep','bp','sp','pcf_ocf','dividendyield','evtoebitda'] #估值因子
    factors_origin = ['roe','roa','roic','gpmgr','grossprofitmargin','orgr','cfogr']#盈利能力
    factors_origin = ['invturn','arturn','taturn'] #经营效率
    factors_origin = ['profittomv','tagr','stmnote_RDexptosales']#盈余质量#投融资决策#无形资产stmnote_RDexptosales无数据
    factors_origin = ['debttoasset','current'] #杠杆因子
    factors_origin = ['netprofit_fy1_6m','revs60','rstr12'] #市场参与者行为

    sf_res_num_T = pd.DataFrame()
    sf_res_num_IC = pd.DataFrame()
    sf_res_num_exch = pd.DataFrame()
    for fac_str in factors_origin:
        fac = eval(fac_str)
        # 数据清洗
        fac_med_de = Clean_Data(fac).Median_deextremum()
        fac_m_zscore = Clean_Data(fac_med_de).Z_score()
        fac_num = Clean_Data(fac_m_zscore).Fill_na()
        # 单因子回归
        Single_factors_test_ins = Single_factors_test_regression(fac_num)
        res_fac_num = Single_factors_test_ins.single_factor_regress()
        # T值分析
        T_fac_num_res = Single_factors_test_ins.T_analysis(res_fac_num)
        T_fac_num_res = Clean_Data.Round_df(T_fac_num_res) #保留四位小数
        T_fac_num_res['factor'] = fac_str #添加因子名称
        # IC值分析
        IC_fac_num_res = Single_factors_test_ins.IC_analysis(res_fac_num)
        IC_fac_num_res = Clean_Data.Round_df(IC_fac_num_res)
        IC_fac_num_res['factor'] = fac_str
        #方向转换统计
        exch_fac_num_res = Single_factors_test_ins.Style_rotation(res_fac_num)
        exch_fac_num_res = Clean_Data.Round_df(exch_fac_num_res) #保留四位小数
        exch_fac_num_res['factor'] = fac_str #添加因子名称
        #结果
        sf_res_num_T = sf_res_num_T.append(T_fac_num_res)
        sf_res_num_IC = sf_res_num_IC.append(IC_fac_num_res)
        sf_res_num_exch = sf_res_num_exch.append(exch_fac_num_res)

    #因子倒数
    #分组法
    sf_res_group = pd.DataFrame()
    for fac_str in factors_origin:
        fac = eval(fac_str)

        sf_group_ins = Single_factors_test_group(fac)
        sf_netvalue_df = sf_group_ins.group_net_value()
        #分组回测结果指标
        sf_group_res = sf_group_ins.backtest_indicates(sf_netvalue_df)
        sf_group_res['factor'] = fac_str
        sf_res_group = sf_res_group.append(sf_group_res)
        #画图
        sf_group_ins.draw(sf_netvalue_df,fac_str)

    sf_res_group = sf_res_group.reset_index()

#    #因子倒数序数值
#    #回归法
#    sf_res_ord_T = pd.DataFrame()
#    sf_res_ord_IC = pd.DataFrame()
#    sf_res_ord_exch = pd.DataFrame()
#    for fac_str in factors_origin:
#        fac = eval(fac_str)
#        # 数据清洗
#        fac_ord = Clean_Data(fac).Ordinal_values()
#        fac_o_zscore = Clean_Data(fac_ord).Z_score()
#        fac_ordinal = Clean_Data(fac_o_zscore).Fill_na()
#        # 单因子回归
#        res_fac_ord = Single_factors_test_regression(fac_ordinal).single_factor_regress()
#        # T值分析
#        T_fac_ord_res = Single_factors_test_ins.T_analysis(res_fac_ord)
#        T_fac_ord_res = Clean_Data.Round_df(T_fac_ord_res) #保留四位小数
#        T_fac_ord_res['factor'] = fac_str #添加因子名称
#        # IC值分析
#        IC_fac_ord_res = Single_factors_test_ins.IC_analysis(res_fac_ord)
#        IC_fac_ord_res = Clean_Data.Round_df(IC_fac_ord_res)
#        IC_fac_ord_res['factor'] = fac_str
#        #方向转换统计
#        exch_fac_ord_res = Single_factors_test_ins.Style_rotation(res_fac_ord)
#        exch_fac_ord_res = Clean_Data.Round_df(exch_fac_ord_res) #保留四位小数
#        exch_fac_ord_res['factor'] = fac_str #添加因子名称
#        #结果
#        sf_res_ord_T = sf_res_ord_T.append(T_fac_ord_res)
#        sf_res_ord_IC = sf_res_ord_IC.append(IC_fac_ord_res)
#        sf_res_ord_exch = sf_res_ord_exch.append(exch_fac_ord_res)


import collections

def left_of_expandable(index, list, placeholder=None):
    new_index = index - 1
    while new_index < 0:
        list.insert(0, placeholder)
        new_index += 1
    return new_index

def left_of_unexpandable(index, list, placeholder=None):
    new_index = index - 1
    if new_index < 0:
        raise ValueError('Index out of left range.')
    return new_index

def right_of_expandable(index, list, placeholder=None):
    new_index = index + 1
    while new_index >= len(list):
        list.append(placeholder)
    return new_index

def right_of_unexpandable(index, list, placeholder=None):
    new_index = index + 1
    if new_index >= len(list):
        raise ValueError('Index out of right range.')
    return new_index

ExpansionStrategy = collections.namedtuple('ExpansionStrategy', 'move_left_op move_right_op')

EXPAND_ALL = ExpansionStrategy(move_left_op=left_of_expandable, move_right_op=right_of_expandable)

EXPAND_NONE = ExpansionStrategy(move_left_op=left_of_unexpandable, move_right_op=right_of_unexpandable)

EXPAND_LEFT_ONLY = ExpansionStrategy(move_left_op=left_of_expandable, move_right_op=right_of_unexpandable)

EXPAND_RIGHT_ONLY = ExpansionStrategy(move_left_op=left_of_unexpandable, move_right_op=right_of_expandable)


from rest_framework.views import APIView
from rest_framework.response import Response
from . import models, serializers
from space_manager.rooms import models as rooms_models
from space_manager.membership import models as membership_models
from space_manager.branches import models as branch_models
from space_manager.users import models as user_models
from rest_framework import status
from datetime import datetime, timedelta
from random import *
from operator import eq

class SeatDetail(APIView):
    def find_seat(self, seat_id):
        try:
            seat = models.Seat.objects.get(id=seat_id)
            return seat
        except models.Seat.DoesNotExist:
            return None

    def get(self, request, seat_id, format=None):
        seat = self.find_seat(seat_id)

        if (seat is None):
            return Response(status=status.HTTP_404_NOT_FOUND)

        serializer = serializers.SeatSerializerForAdmin(seat)

        return Response(data=serializer.data, status=status.HTTP_200_OK)

class SeatLog(APIView):
    def get(self, request, user_id, format=None):
        user = user_models.User.objects.get(id=user_id)

        seat_logs = models.Log.objects.filter(user=user)

        serializer = serializers.LogSerializer(seat_logs, many=True)

        return Response(status=status.HTTP_200_OK, data=serializer.data)

class MySeat(APIView):
    def get(self, request, format=None):
        user = request.user
        now = datetime.today()

        try:
            using_seat = models.Seat.objects.get(
                now_user=user, end_datetime__gte=now, usable=True)
        except models.Seat.DoesNotExist:
            return Response(status=status.HTTP_204_NO_CONTENT)

        serializer = serializers.SeatSerializer(using_seat)

        return Response(status=status.HTTP_200_OK, data=serializer.data)

class NowUsing(APIView):
    # 현재 사용하고 있는 좌석 가져오기

    def get(self, request, format=None):
        user = request.user

        #배정 액션 가져오기
        try:
            allocate_action = models.Action.objects.get(
                en_substance='allocation')
        except models.Action.DoesNotExist:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        #사용자의 가장 최근 좌석 이용 기록 가져오기
        try:
            user_recent_log = models.Log.objects.filter(
                user=user).latest('created_at')

        #좌석 이용기록이 하나도 없으면 처음 자리 잡음으로 그전에 자리 잡은 자리가 없음, True 리턴
        except models.Log.DoesNotExist:
            return Response(status=status.HTTP_204_NO_CONTENT)

        #사용자의 가장 최근 좌석 이용 기록이 배정이라면 해당 로그 리턴 => 변수 배정하고 반납하지 않은채 이용시간이 자나갈수 있음
        if user_recent_log.action == allocate_action:
            serializer = serializers.DetailLogSerializer(user_recent_log)

            if user_recent_log.reg_datetime + timedelta(
                    hours=24) < datetime.now():
                # 배정하고 반납하지 않은채 이용시간이 지나간 경우, 현재 잡은 자리가 없는 걸로
                return Response(status=status.HTTP_204_NO_CONTENT)
            else:
                return Response(
                    data=serializer.data, status=status.HTTP_200_OK)

        else:
            return Response(status=status.HTTP_204_NO_CONTENT)

class Seats(APIView):
    #  해당열람실에 좌석 추가하기
    #  data :seat_number,left,top,rotate
    def post(self, request, room_id, format=None):

        user = request.user

        # 슈퍼 유저인지 검사
        if (user.is_superuser == False):
            return Response(status=status.HTTP_401_UNAUTHORIZED)

        try:
            room = rooms_models.Room.objects.get(id=room_id)
        except rooms_models.Room.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        data = request.data
        data['room'] = room_id
        data['branch'] = room.branch.id

        serializer = serializers.InputSeatSerializer(data=data)
        if serializer.is_valid():

            serializer.save()

            return Response(
                data=serializer.data, status=status.HTTP_201_CREATED)

        else:

            return Response(
                data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def get(self, request, room_id, format=None):
        # 해당 열람실에 해당되는 좌석 가져오기
        # data : seat_number,left,top,rotate,usable,discard
        try:
            seats = models.Seat.objects.filter(room__id=room_id)
        except models.Seat.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        serializer = serializers.ShowSeatSerializer(seats, many=True)

        return Response(data=serializer.data, status=status.HTTP_200_OK)

class Seat(APIView):
    def find_seat(self, seat_id):
        try:
            seat = models.Seat.objects.get(id=seat_id)
            return seat
        except models.Seat.DoesNotExist:
            return None

    def get(self, request, seat_id, format=None):
        # 해당 좌석 정보 가져오기
        # data : seat_number,left,top,rotate,usable,discard,room,branch,
        try:
            seat = models.Seat.objects.get(id=seat_id)
        except mdoels.Seat.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        serializer = serializers.SeatSerializer(seat)

        return Response(data=serializer.data, status=status.HTTP_200_OK)

    def put(self, request, seat_id, format=None):
        # 해당 좌석 수정하기
        # data : seat_number,left,top,rotate,usable,discard,room,branch

        user = request.user
        # 슈퍼 유저인지 검사
        if (user.is_superuser == False):
            return Response(status=status.HTTP_401_UNAUTHORIZED)

        seat = self.find_seat(seat_id)

        if seat is None:
            return Response(status=status.HTTP_404_NOT_FOUND)

        serializer = serializers.SeatSerializer(
            seat, data=request.data, partial=True)

        if serializer.is_valid():
            serializer.save()

            return Response(
                data=serializer.data, status=status.HTTP_202_ACCEPTED)
        else:
            return Response(
                data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)

# class Allocation(APIView):

#     # 멤버쉽 가져오기
#     def get_memberships(self, user):

#         now = datetime.today()
#         try:
#             membership = membership_models.Membership.objects.filter(
#                 end_date__gte=now,  #맴버쉽 만료시각이 현재보다 크고
#                 start_date__lte=now,  # 시작시간이 현재보다 작고
#                 is_usable=True,  #이용가능하고
#                 user=user)

#             return membership

#         except membership_models.Membership.DoesNotExist:
#             return None

#     # 성별에 따른 좌석 이용 가능여부 체크
#     def can_user_seat(self, user, seat):
#         gender = user.gender

#         if gender == 'male':  # 유저가 남자일 경우
#             if seat.for_who.male_available:
#                 return True
#             else:
#                 return False

#         elif gender == 'female':
#             if seat.for_who.female_available:
#                 return True
#             else:
#                 return False

#     #사용 가능한 좌석인지 체크
#     def is_usable_seat(self, seat):
#         usable = seat.usable
#         discard = seat.discard

#         if usable is True and discard is False:
#             return True
#         else:
#             return False

#     # 비어있는 좌석 체크(now_user가 없거나 24시간이 지났거나)
#     def is_empty_seat(self, seat):
#         if (seat.end_datetime is None):
#             return True

#         elif (seat.now_user is not None
#               and seat.end_datetime > datetime.now()):
#             return False
#         else:
#             return True

#     def is_usable_room(self, seat):
#         room = seat.room
#         if room.usable is False:
#             return False
#         else:
#             return True

#     def seat_before_return(self, user):

#         #사용자의 가장 최근 좌석 이용 기록 가져오기
#         try:
#             user_recent_log = models.Log.objects.filter(
#                 user=user).latest('created_at')

#         #좌석 이용기록이 하나도 없으면 처음 자리 잡음으로 그전에 자리 잡은 자리가 없음, True 리턴
#         except models.Log.DoesNotExist:
#             return True

#         #반납 액션 가져오기
#         try:
#             return_action = models.Action.objects.get(en_substance='return')
#         except models.Action.DoesNotExist:
#             return False

#         #사용자의 가장 최근 좌석 이용 기록이 반납이라면 처음 자리 잡는 것으로 확인, True리턴 => 변수 배정하고 반납하지 않은채 이용시간이 자나갈수 있음
#         #
#         if user_recent_log.action == return_action:
#             return True
#         elif user_recent_log.reg_datetime + timedelta(hours=24) < datetime.now(
#         ):  # 배정하고 반납하지 않은채 이용시간이 지나간 경우, 처음 자리 잡는 걸로
#             return True

#         # 프론트 단에서 시간이 지난 좌석일 경우 빈좌석으로 표시

#         #이전에 잡은 자리가 있음, 빈좌석 이미지를 가지고 옴
#         try:
#             seat_return_image = models.SeatImage.objects.get(
#                 action=return_action)
#         except models.SeatImage.DoesNotExist:
#             return False

#         #반납 기록 로그 생성
#         new_log = models.Log.objects.create(
#             action=return_action,
#             user=user,
#             seat=user_recent_log.seat,
#             reg_datetime=datetime.now())

#         result = new_log.save()

#         self.set_seat_state(user_recent_log.seat, user, seat_return_image,
#                             datetime.now())

#         return True

#     def set_seat_state(self, seat, user, seat_image, sel_datetime):

#         seat.now_user = user
#         seat.seat_image = seat_image
#         seat.end_datetime = sel_datetime
#         seat.save()

#     def allocate(self,
#                  user,
#                  seat,
#                  end_datetime=datetime.now() + timedelta(hours=24)):

#         # 배정 액션 얻기
#         try:
#             allocate_action = models.Action.objects.get(
#                 en_substance='allocation')
#         except models.Action.DoesNotExist:
#             return Response(status=status.HTTP_400_BAD_REQUEST)

#         seat_images = models.SeatImage.objects.filter(gender=user.gender)

#         count_images = len(seat_images)
#         rand_limit = count_images - 1
#         random_image_number = randint(0, rand_limit)
#         sel_image = seat_images[random_image_number]

#         # 새 배정 기록 록그 생성
#         new_log = models.Log.objects.create(
#             action=allocate_action,
#             user=user,
#             seat=seat,
#             reg_datetime=datetime.now())

#         new_log.save()

#         # 좌석 상태 배정상태로 변경
#         self.set_seat_state(seat, user, sel_image, end_datetime)

#         return Response(status=status.HTTP_201_CREATED)

#     def post(self, request, seat_id, user_id, format=None):

#         # 기본 좌석 마감시각
#         # 멤버쉽 등록기간이 좌석 마감시각보다 작을 경우 멤버쉽 마감시각으로 변경
#         e_datetime = datetime.now() + timedelta(hours=24)

#         creator = request.user
#         # creator는 본인이 될수도 있고 superuser or staff가 될수도 있음

#         user = user_models.User.objects.get(id=user_id)

#         # 해당 좌석 찾기
#         try:
#             seat = models.Seat.objects.get(
#                 id=seat_id, usable=True, discard=False)
#             # find out by id, usable, discard
#         except models.Seat.DoesNotExist:
#             return Response(status=status.HTTP_400_BAD_REQUEST)

#         # 성별에 따른 좌석 이용 가능여부 체크
#         if self.can_user_seat(user, seat) is False:
#             return Response(status=status.HTTP_403_FORBIDDEN)

#         # 해당 좌석이 이용불가 상태 혹은 폐기 상태가 아닌지 확인
#         if self.is_usable_seat(seat) is False:
#             return Response(status=status.HTTP_403_FORBIDDEN)

#         # 좌석이 빈좌석인지 확인(now_user가 없거나 24시간이 지났거나)
#         if self.is_empty_seat(seat) is False:
#             return Response(status=status.HTTP_403_FORBIDDEN)

#         # 슈퍼유저인지 스탭인지 또는 본인이 맞는지 확인
#         if creator.is_superuser is False or creator.is_staff is False:  #생성자가 슈퍼유저 혹은 스탭이 아니라면
#             if creator != user:  #유저가 생성자 본인이 아니라면
#                 return Response(status=status.HTTP_401_UNAUTHORIZED)
#             else:  #유저와 생성자가 같다면(본인이 직접 배정할때)
#                 #멤버쉽 체크, 멤버쉽 끝나는 기간이 24시간 이내라면 종료시간을 멤버십 끝나는 시간으로
#                 #멤버쉽 등록할 때 잡은 자리가 있으면 24시간 뒤로 변경할 것

#                 #불러오는 멤버쉽 정보는 제일 최근 정보만 가져옴
#                 #예를 들면 5월1일, 6월 1일에 등록한 멤버쉽 있으면 5월달 것을 먼저 불러옴(현재 5월 가정)
#                 memberships = self.get_memberships(user)

#                 print(memberships)

#                 if memberships is None:  #맴버쉽에 등록되어있지 않을 때
#                     return Response(status=status.HTTP_406_NOT_ACCEPTABLE
#                                     )  # 프론트단에서 멤버쉽 등록창으로 리다이렉트
#                 elif len(memberships) == 0:  #현재 사용가능한 맴버쉽이 남아 있지 않을 때
#                     return Response(status=status.HTTP_406_NOT_ACCEPTABLE)

#                 #멤버쉽 만료기간이 24시간 이내로 남지 않았다면(하루등록 이용자도 포함)
#                 if memberships[0].end_date < (
#                         datetime.now() + timedelta(hours=24)):

#                     if len(memberships) > 1:  #다음 멤버쉽이 하나 더 이상 있다면

#                         #현재 맴버쉽 만료시각보다 다음 맴버쉽 시작 시간이 더크다면
#                         if memberships[0].end_date < memberships[1].start_date:
#                             e_datetime = memberships[0].end_date

#                         # 다음 멤버쉽 만료시각이 기본 좌석 만료시각보다 크다면 24시간
#                         # 그렇지 않다면 다음 맴버쉽 만료시각도 체크하고 다음 멤버쉽도

#                         elif memberships[1].end_date < datetime.now(
#                         ) + timedelta(hours=24):
#                             # 다음 멤버쉽 만료시각이 기본 좌석 만료시각 보다 작다면 다음 멤버쉽 만료시각으로
#                             e_datetie = memberships[1].end_date

#                     else:  #다음 멤버쉽이 없다면 현재 맴버쉽의 만료시각으로
#                         e_datetime = memberships[0].end_date

#                 #멤버쉽중 만료시각 가장 최근것으로 불러옴

#         # allocation action 찾기, log에 기록할 때 필요함
#         try:
#             allocate_action = models.Action.objects.get(
#                 en_substance='allocation')
#         except models.Action.DoesNotExist:
#             return Response(status=status.HTTP_400_BAD_REQUEST)

#         #########################################################

#         # 성별을 통한 배정 이미지 찾기
#         if user.gender is None:
#             seat_images = models.SeatImage.objects.exclude(
#                 gender='Not-specified')
#         else:
#             seat_images = models.SeatImage.objects.filter(gender=user.gender)

#         count_images = len(seat_images)
#         rand_limit = count_images - 1
#         random_image_number = randint(0, rand_limit)
#         sel_image = seat_images[random_image_number]  #선택된 이미지

#         # end_time = datetime.now() + timedelta(hours=24)  #오늘 날짜시간 +24시간

#         # 열람실 설정에 이상 없는지 확인
#         if self.is_usable_room(seat) is False:
#             return Response(status=status.HTTP_403_FORBIDDEN)

#         # 배정전 잡은자리 있으면 반납처리
#         if self.seat_before_return(user) is False:
#             return Response(status=status.HTTP_400_BAD_REQUEST)
#         else:
#             return self.allocate(user, seat)  # 배정
class Allocation(APIView):
    def put(self, request, seat_id, user_id, format=None):

        # one_day_to_seconds = 86400
        # membership_end_date = datetime.strptime(request.data['end_date'],
        #                                         '%Y-%m-%d %H:%M:%S')
        # one_day_after = datetime.now() + timedelta(hours=24)
        # delta = membership_end_date - one_day_after
        # if delta.seconds > one_day_to_seconds:  # 하루 좌석 잡을 경우의 종료시각보다 멤버십이 더 길게 있을 경우
        #     target_end_datetime = one_day_after
        # else:
        #     target_end_datetime = membership_end_date
        # print(delta.seconds)

        # 만료시각이 하루 이하이면 만료시각으로 자리 잡도록 함

        end_datetime = datetime.strptime(request.data['end_datetime'],
                                         '%Y-%m-%d %H:%M:%S')

        try:
            target_user = user_models.User.objects.get(id=user_id)
        except user_models.User.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        # 성별을 통한 배정 이미지 찾기
        if target_user.gender is None:
            return Response(status=status.HTTP_400_BAD_REQUEST)
        else:
            seat_images = models.SeatImage.objects.filter(
                gender=target_user.gender)

        count_images = len(seat_images)
        rand_limit = count_images - 1
        random_image_number = randint(0, rand_limit)
        target_seat_img = seat_images[random_image_number]  #선택된 이미지

        # 좌석 찾기
        try:
            target_seat = models.Seat.objects.get(id=seat_id)
        except models.Seat.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        try:
            target_action = models.Action.objects.get(
                en_substance='allocation')
        except models.Action.DoesNotExist:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        target_seat.seat_image = target_seat_img
        target_seat.now_user = target_user
        target_seat.end_datetime = end_datetime

        target_seat.save()
        serializer = serializers.SeatSerializer(target_seat)

        action = target_action
        user = target_user
        seat = target_seat
        reg_datetime = datetime.now()

        new_seat_log = models.Log.objects.create(
            action=action, user=user, seat=seat, reg_datetime=reg_datetime)

        new_seat_log.save()

        return Response(data=serializer.data, status=status.HTTP_202_ACCEPTED)

class ReturnSeat(APIView):
    #좌석 반납
    def put(self, request, seat_id, format=None):
        # creator = request.user
        # 기록할 때 쓸것

        try:
            target_seat = models.Seat.objects.get(id=seat_id)
        except models.Seat.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        target_user = target_seat.now_user

        try:
            target_action = models.Action.objects.get(en_substance='return')
        except models.Action.DoesNotExist:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        try:
            target_seat_img = models.SeatImage.objects.get(
                action=target_action)
        except models.Action.DoesNotExist:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        target_seat.end_datetime = datetime.now()
        target_seat.seat_image = target_seat_img
        target_seat.now_user = None

        target_seat.save()
        serializer = serializers.SeatSerializer(target_seat)

        action = target_action
        user = target_user
        seat = target_seat
        reg_datetime = datetime.now()

        new_seat_log = models.Log.objects.create(
            action=action, user=user, seat=seat, reg_datetime=reg_datetime)

        new_seat_log.save()

        return Response(data=serializer.data, status=status.HTTP_202_ACCEPTED)

# class ReturnSeat(APIView):
#     # 좌석 반납, 슈퍼유저기능 포함
#     def post(self, request, user_id, format=None):

#         creator = request.user

#         try:
#             user = user_models.User.objects.get(id=user_id)
#         except user_models.User.DoesNotExist:
#             return Response(status=status.HTTP_404_NOT_FOUND)

#         if creator != user:
#             if request.user.is_superuser is False:
#                 if request.user.is_staff is False:
#                     return Response(status=status.HTTP_403_FORBIDDEN)

#         # 잡혀져 있는 좌석이 있는지 확인
#         try:
#             user_recent_logs = models.Log.objects.filter(user=user)[:1]
#         except models.Log.DoesNotExist:
#             return Response(status=status.HTTP_404_NOT_FOUND)

#         #가장 최근 로그중 잡혀져있는 로그가 있다면
#         user_recent_log = user_recent_logs[0]

#         allocate_action = models.Action.objects.get(en_substance='allocation')
#         return_action = models.Action.objects.get(en_substance='return')
#         return_seat_image = models.SeatImage.objects.get(action=return_action)

#         #가장 최근 로그가 배정이 아니라면 오류
#         if allocate_action != user_recent_log.action:
#             return Response(status=status.HTTP_406_NOT_ACCEPTABLE)

#         #반납로그생성

#         seat = models.Seat.objects.get(id=user_recent_log.seat.id)

#         seat.seat_image = return_seat_image
#         seat.now_user = None
#         seat.end_datetime = None

#         seat.save()

#         new_return = models.Log.objects.create(
#             action=return_action,
#             user=user,
#             seat=user_recent_log.seat,
#         )

#         new_return.save()

#         return Response(status=status.HTTP_201_CREATED)

#여기서부터 확인!!!!!!!!!!!


from pymongo import MongoClient

# creating connections for communicating with Mongo DB
client = MongoClient('localhost:27017')
db = client.EmployeeData

db.Employees.update_one(
        {"id": '001'},
        {
        "$set": {
            "name":'AAA',
            "age":51,
            "country":'Korea'
        }
        }
    )


import folium

def gen_map():
    """
    マップの基本部分を生成
    Returns:
        m(folium.Map)
    """
    copyright_st = '©' \
                   'Map tiles by Stamen Design,' \
                   ' under CC BY 3.0.' \
                   'Data by OpenStreetMap,' \
                   'under ODbL.'

    m = folium.Map(location=[35.681167, 139.767052],
                   zoom_start=5,
                   attr=copyright_st,
                   tiles='https://stamen-tiles-{s}.a.ssl.fastly.net/toner-lite/{z}/{x}/{y}.png')
    return m

def plot_points(df):
    """
    施設座標のマーカーをプロット
    Args:
        rm: RoadMakerクラス
    """
    m = gen_map()
    for feature in df.itertuples():
        folium.Marker((feature.coordinates[1], feature.coordinates[0]),
                      popup=f"{feature.Index, feature.name}\n{feature.coordinates}").add_to(m)
    m.save("plot_admin.html")

def plot_oneline(rm):
    """
    路線別で区間の曲線と施設座標のマーカーをプロット
    Args:
        rm: RoadMakerクラス
    """
    m = gen_map()

    for feature in rm.df_point.itertuples():
        folium.Marker((feature.coordinates[1], feature.coordinates[0]),
                      popup=f"{feature.Index, feature.name}\n{feature.coordinates}").add_to(m)
    for feature in rm.df_path.itertuples():
        folium.PolyLine(locations=[(x[1], x[0]) for x in feature.path],
                        popup=f"{feature.Index}",
                        color="green").add_to(m)
    m.save("plot_admin.html")


import uvicorn
from typing import Optional

from fastapi import FastAPI
from pydantic import BaseModel
app = FastAPI()
m = []
@app.get("/")
def Root():
    return "Hello World"

@app.get("/api/products") # список товаров
def watch_all_products():
    global m
    if not m:
        return "Товары отсутствуют"
    return m

@app.get("/api/products/{id}") # информация о товаре по id
def watch_current_product(id: int):
    global m
    for i in range(len(m)):
        if m[i]["id"] == id:
            return m[i]
    return "Данный товар отсутствует"

@app.post("/api/products/new") # создание нового товара
def create_new_product(id: int, name: str, description: str, prise:float):
    global m
    mgl = {
        "id": id,
        "name": name,
        "description": description,
        "prise": prise
    }
    m.append(mgl)
    return m

@app.put("/api/products/edit/{id}") # обновление информации о товаре
def edit_product(id: int, name: str, description: str, prise:float):
    global m
    for i in range(len(m)):
        if m[i]["id"] == id:
            m[i]["name"] = name
            m[i]["description"] = description
            m[i]["prise"] = prise
            return m[i]
    return "Данный товар отсутствует"

@app.delete("/api/products/delete/{id}") # удаление товара
def delete_product(id: int):
    global m
    for i in range(len(m)):
        if m[i]["id"] == id:
            del m[i]
            return "ok"
    return "Данный товар отсутствует"

if __name__ == '__main__':
    uvicorn.run(app, host = "127.0.0.1", port=8000)


"""
ELR Scorer
==========

Scorer for MRF based models. Supports FSDM and ELR-based models.

:Author: Faegheh Hasibi
"""

from __future__ import division

import argparse
import json
import math
from collections import defaultdict
from pprint import pprint

from nordlys.core.ml.instance import Instance
from nordlys.core.ml.instances import Instances
from nordlys.core.retrieval.elastic import Elastic
from nordlys.core.retrieval.elastic_cache import ElasticCache
from nordlys.core.retrieval.scorer import ScorerPRMS
from nordlys.core.utils.entity_utils import EntityUtils
from nordlys.core.utils.file_utils import FileUtils
from nordlys.logic.elr.top_fields import TopFields

class ScorerELR(object):
    DEBUG = 0

    def __init__(self, elastic_uri, query_annot, query_len, params, n_fields=10):
        self.elastic_uri = elastic_uri
        self.query_annot = query_annot
        self.query_len = query_len
        self.n_fields = n_fields
        self.lambda_T = params[0]
        self.lambda_E = params[1]

        self.E = self.normalize_el_scores(query_annot)

    @staticmethod
    def normalize_el_scores(scores):
        """Normalize entity linking score, so that sum of all scores equal to 1"""
        normalized_scores = {}
        sum_score = sum(scores.values())
        for item, score in scores.items():
            normalized_scores[item] = score / sum_score
        return normalized_scores

    def get_field_weights(self, uri):
        """
        Gets PRMS mapping probability for a clique type

        :return Dictionary {field: weight, ..}
        """
        print("Computing field weights ...")
        # top_fields = TopFields(self.elastic_uri).get_top_term(uri, self.n_fields)
        top_fields = ["catchall"]
        scorer_prms = ScorerPRMS(self.elastic_uri, None, {'fields': top_fields})
        field_weights = scorer_prms.get_mapping_prob(uri)
        return field_weights

    def get_uri_prob(self, doc_id, field, e, lambd=0.1):
        """
        P(e|d_f) = P(e|d_f)= (1 - lambda) tf(e, d_f)+ lambda df(f, e) / df(f)

        :param doc_id: document id
        :param field: field name
        :param e: entity uri
        :param lambd: smoothing parameter
        :return: P(e|d_f)
        """
        if self.DEBUG:
            print("\t\tf:", field)
        tf = self.elastic_uri.term_freqs(doc_id, field)
        tf_e_d_f = 1 if tf.get(e, 0) > 0 else 0
        df_f_e = self.elastic_uri.doc_freq(e, field)
        df_f = self.elastic_uri.doc_count(field)
        p_e_d_f = ((1 - lambd) * tf_e_d_f) + (lambd * df_f_e / df_f)
        if self.DEBUG:
            print("\t\t\ttf(e,d_f):", tf_e_d_f, "df(f, e):", df_f_e, "df(f):", df_f, "P(e|d_f):", p_e_d_f)
        return p_e_d_f

    def get_p_e_d(self, e, field_weights, doc_id):
        """
        p(e|d) = sum_{f in F} p(e|d_f) p(f|e)

        :param e: entity URI
        :param field_weights: Dictionary {f: p_f_t, ...}
        :param doc_id: entity id
        :return p(e|d)
        """
        if self.DEBUG:
            print("\te:", e)
        p_e_d = 0
        for f, p_f_e in field_weights.items():
            p_e_d_f = self.get_uri_prob(doc_id, f, e)
            p_e_d += p_e_d_f * p_f_e
            if self.DEBUG:
                print("\t\tp(e|d_f):", p_e_d_f, "p(f|e):", p_f_e, "p(e|d_f).p(f|e):", p_e_d_f * p_f_e)
        if self.DEBUG:
            print("\tp(e|d):", p_e_d)
        return p_e_d

    def score_doc(self, doc_id, field_mappings=None):
        """
        P(q|e) = lambda_T sum_{t}P(t|d) + lambda_E sum_{e}P(e|d)

        :param doc_id: document id
        :param term_sim: term-base similarity (e.g. LM, SDM, and FSDM)
        :param: length of query
        :return: p(q|d)
        """
        if self.DEBUG:
            print("Scoring doc ID=" + doc_id)

        p_E_d = 0
        if self.lambda_E != 0:
            for e, score in self.E.items():
                field_weights = field_mappings[e] if field_mappings else self.get_field_weights(e)
                catchall_weight = {"catchall": field_weights.get("catchall", 0)}
                p_e_d = self.get_p_e_d(e, catchall_weight, doc_id)
                if p_e_d != 0:
                    p_E_d += score * math.log(p_e_d)
        return p_E_d

        # p_T_d = term_sim / self.query_len
        # p_q_d = (self.lambda_T * p_T_d) + (self.lambda_E * p_E_d)
        # if self.DEBUG:
        #     print("\t\tp(E|d):", p_E_d)
        #
        # # Adds the current doc to instances
        # # self.gen_ins(doc_id, {"p_T_d": p_T_d, "p_E_d": p_E_d})
        # return p_q_d

def load_run(run_file):
    """Loads a run file to the memory

    :param run_file: A trec run file
    :return {query: {enId: score, ..}, ...}
    """
    run = defaultdict(dict)
    with open(run_file, "r") as f:
        for line in f:
            cols = line.strip().split()
            run[cols[0]][cols[2]] = float(cols[4])
    return run

def load_annot(annot_file, th=0.1):
    """Reads TAGME annotation file and generates a dictionary

    :return {qid: {en_id: score, ..}, ..}
    """
    annots = defaultdict(dict)
    tagme_annots = json.load(open(annot_file, "r"))
    for qid, annot in tagme_annots.items():
        for item in annot["annots"]:
            if item["score"] >= th:
                en_id = item["entity"]
                annots[qid][en_id] = item["score"]
    return annots

def get_mapping_query(query_annots, mappings):
    query_mappings = {}
    for en_id in query_annots:
        query_mappings[en_id] = mappings[en_id]
    return query_mappings

def trec_format(results, query_id, run_id):
    """Outputs results in TREC format"""
    out_str = ""
    rank = 1
    for doc_id, score in sorted(results.items(), key=lambda x: x[1], reverse=True):
        out_str += query_id + "\tQ0\t" + doc_id + "\t" + str(rank) + "\t" + str(score) + "\t" + run_id + "\n"
        rank += 1
    return out_str

def arg_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("config", help="config file", type=str)
    args = parser.parse_args()
    return args

def main(args):
    config = FileUtils.load_config(args.config)
    elastic_term = ElasticCache(config["text_index"])
    lambdas = config.get("lambdas", [0.9, 0.1])

    queries = json.load(open(config["query_file"], "r"))
    mappings = json.load(open(config["mapping_file"], "r"))
    annots = load_annot(config["annot_file"])
    run = load_run(config["run_file"])

    instances = Instances()
    # gets the results
    out_file = open(config["output_file"], "w")
    qid_int = 0
    for qid, query in sorted(queries.items()):
        print("Scoring ", qid, "...")
        results, libsvm_str = {}, ""
        query_len = len(elastic_term.analyze_query(query).split())
        scorer = ScorerELR(ElasticCache(config["uri_index"]), annots[qid], query_len, lambdas)
        for doc_id, p_T_d in sorted(run[qid].items()):
            query_mappings = get_mapping_query(annots[qid], mappings)
            p_E_d = scorer.score_doc(doc_id, query_mappings)
            properties = {'doc_id': doc_id, 'query': query, 'qid': qid, 'qid_int': qid_int}
            features = {'p_T_d': p_T_d, 'p_E_d': p_E_d}
            ins = Instance(qid + "_" + doc_id, features=features, properties=properties)
            instances.add_instance(ins)
            # libsvm_str += ins.to_libsvm(qid_prop="qod_int")
            results[doc_id] = (lambdas[0] * p_T_d) + (lambdas[1] * p_E_d)
        qid_int += 1

        # Write trec format
        out_str = trec_format(results, qid, "elr")
        out_file.write(out_str)

    out_file.close()
    print("Output file:", config["output_file"])
    instances.to_json(config["json_file"])
    print("Output file:", config["json_file"])

if __name__ == "__main__":
    main(arg_parser())


"""
    VintedPyInterface
    By Alix Hamidou 2023
"""
from fake_useragent import UserAgent
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from dataclasses import dataclass

import time

"""
    VintedPyInterface
    By Alix Hamidou 2023
"""

__version__ = "0.1.0"
__author__ = 'Alix Hamidou'

@dataclass
class SearchSetting:
    category: str
    size: str
    brand: str

    def dict(self) -> dict[str, str]:
        return {
            "categorie": self.category,
            "taille": self.size,
            "marque": self.brand
        }

@dataclass
class Article:
    imgLink: str
    price: float
    link: str
    title: str

class VintedPyInterface:
    driver: webdriver.Chrome

    def __init__(self) -> None:
        ua = UserAgent()
        user_agent = ua.random

        options = Options()
        options.add_argument(f"user-agent={user_agent}")
        options.add_argument("--headless")  # Lancer Chrome en mode sans tête

        self.driver = webdriver.Chrome(options=options)
        return

    def Search(self, search: SearchSetting) -> list[Article]:
        url: str = "https://www.vinted.fr/catalog?"
        params = [f"{v}" for v in search.dict().values()]
        url += "search_text=" + "+".join(params) + "&order=newest_first"

        self.driver.get(url)
        time.sleep(5)
        articlesHTML: list[webdriver.remote.webelement.WebElement] = self.driver.find_elements(By.CLASS_NAME, "feed-grid__item")
        articles: list[Article] = []
        for articleHTML in articlesHTML:
            try:
                imgLink: str = articleHTML.find_element(By.CLASS_NAME, "web_ui__ItemBox__image-container").find_element(By.TAG_NAME, "img").get_attribute("src")
                price: float = float(articleHTML.find_elements(By.CLASS_NAME, "web_ui__Cell__cell")[1].text.split("\n")[0].replace("€", "").replace(",", ".").strip())
                link: str = articleHTML.find_element(By.CLASS_NAME, "web_ui__ItemBox__overlay").get_attribute("href")
                title: str = articleHTML.find_element(By.CLASS_NAME, "web_ui__ItemBox__overlay").get_attribute("title")

                articles.append(Article(imgLink, price, link, title))
            except:
                pass
        return articles


def find_feed(board,n,lvl):
    visited = []
    q = []

    for i in range(n):
        if 9 in board[i]:
            q.append([i,board[i].index(9),0])
            board[i][board[i].index(9)] = 0

    while q:
        c = q.pop(0)

        if 0 < board[c[0]][c[1]] and board[c[0]][c[1]] < lvl:
            board[c[0]][c[1]] = 9

            print()
            print("time:",c[2])
            print()

            for qwe in range(n):
                 print(board[qwe])

            return c[2]

        if c not in visited:
            visited.append(c)

            if 0 < c[0] and [c[0]-1,c[1]] not in visited and [c[0]-1,c[1]] not in q and board[c[0]-1][c[1]] <= lvl:
                q.append([c[0]-1,c[1],c[2]+1])

            if 0 < c[1] and [c[0],c[1]-1] not in visited and [c[0],c[1]-1] not in q and board[c[0]][c[1]-1] <= lvl:
                q.append([c[0],c[1]-1,c[2]+1])

            if c[0] < n-1 and [c[0]+1,c[1]] not in visited and [c[0]+1,c[1]] not in q and board[c[0]+1][c[1]] <= lvl:
                q.append([c[0]+1,c[1],c[2]+1])

            if c[1] < n-1 and [c[0],c[1]+1] not in visited and [c[0],c[1]+1] not in q and board[c[0]][c[1]+1] <= lvl:
                q.append([c[0],c[1]+1,c[2]+1])

n = int(input())
board = []
lvl, ate, time = 2, 0, 0
for _ in range(n):
    board.append(list(map(int,input().split())))

while list(filter(lambda x:0 < x < lvl,sum(board,[]))) != []:

    print()
    print("========================================")
    print(lvl,ate)

    time += find_feed(board,n,lvl)
    ate += 1

    if lvl == ate:
        lvl += 1
        ate = 0

print(time)


#!/usr/bin/env python3.7
# -*- coding: utf-8 -*-
from collections import namedtuple
from distutils.version import LooseVersion
from importlib import import_module
from subprocess import PIPE, call
from sys import executable

InstallStatus = namedtuple('InstallStatus', ('status', 'package', 'version'))
InstallStatus.__str__ = lambda self: 'For package {3} installed version {4} is {0}{1}\033[0m'.format(
    '\033[94m' if self.status else '\033[91m', 'ok' if self.status else 'not ok', *self)

def install(package: str, version: str = '') -> InstallStatus:
    required_version = LooseVersion(version or '0')
    installed_version = get_version(package)
    if installed_version is None or installed_version < required_version:
        call([executable, '-m', 'pip', 'install', package, '--upgrade'], stdout=PIPE, stderr=PIPE)
        installed_version = get_version(package)
    return InstallStatus(installed_version >= required_version if installed_version else False,
                         package, str(installed_version))

def get_version(package: str) -> LooseVersion:
    try:
        installed = import_module(package)
    except ImportError:
        pass
    else:
        return LooseVersion(installed.__version__)

if __name__ == '__main__':
    from sys import argv

    for arg in argv[1:]:
        print(install(*arg.split('.', 1)))


import sys
import opai

def main():
    args = sys.argv
    # 女の子のお悩み相談解決ソリューションズ㈱（以下、オナソリュ）の提唱するフサフサをインスタンス名とする
    fusafusa = opai.New(args[1])
    fusafusa.show()

if __name__ == '__main__':
    main()


class ZigZagConversion(object):

  def zigZagConversion(self, string, numberOfRows):
    if numberOfRows == 1:
        return string
    self.rows = [[] for counter in range(numberOfRows)]
    currentRow = 0
    direction = 1
    for _, character in enumerate(string):
        self.rows[currentRow].append(character)
        if currentRow == numberOfRows-1 and direction == 1:
            currentRow -= 1
            direction = -1
        elif currentRow == 0 and direction == -1:
            currentRow += 1
            direction = 1
        elif direction == 1:
            currentRow += 1
        else:
            currentRow -= 1
    return ''.join([item for sublist in self.rows for item in sublist])

from nose.tools import assert_equals, assert_raises

class TestZigZagConversion(object):

  def testZigZagConversion(self):
    zigZagConversion = ZigZagConversion()

    assert_equals(zigZagConversion.zigZagConversion("PAYPALISHIRING", 3), "PAHNAPLSIIGYIR")

    assert_equals(zigZagConversion.zigZagConversion("ABC", 1), "ABC")

    print ("All test cases passed!")

def main():
  testZigZagConversion = TestZigZagConversion()
  testZigZagConversion.testZigZagConversion()

if __name__ == '__main__':
  main()


# The following inputs are for testing
A=[5,1,5,2,5,3,5,4]
# A=[1,2,2,3]

for i in set(A):
	if A.count(i)==len(A)//2:
		ans=i

print("Answer is ---> %d"%ans)

# ------------------------------------------------------------------------

# Final code as in leetcode

# class Solution:
#     def repeatedNTimes(self, A: List[int]) -> int:
#         for i in set(A):
#             if A.count(i)==len(A)//2:
#                 return(i)

Leetcode Python/N-Repeated Element in Size 2N Array.py


# Justin G. Fr. Michael Goetz CSS
# 3/24/2021
# A program that takes in the names of students and their grades and creates an object with that information using the class Student()
import csv

class Student():
  """A class that takes in a student's name and grades and creates an object which can be accessed to find the student's average, highest, lowest, or median marks"""

  def __init__(self, name, grades):
    """Initialization of the Student class by taking in the student name and a list of grades"""
    self._name = name
    self._grades = grades
    # this list is created so that writing to .csv files easier
    self._csv = []
    self._csv.append(name)
    for grade in grades:
      self._csv.append(grade)

  # def __str__(self):
  #   return f"{self._name} {self._grades}"

  def get_avg(self):
    """Gets the average of the student's grades"""
    int_grades = []
    for grade in self._grades:
      int_grades.append(int(grade))
    avg = (sum(int_grades)) / len(self._grades)
    # print(f"{self._name}'s average is approximately {round(avg, 2)}")
    return avg

  def get_highest(self):
    """Gets the student's highest grade"""
    highest = 0
    for grade in self._grades:
      if int(grade) > highest:
        highest = int(grade)
    # print(f"{self._name}'s highest grade is {highest}")
    return highest

  def get_lowest(self):
    """Gets the student's lowest grade"""
    lowest = 100
    for grade in self._grades:
      if int(grade) < lowest:
        lowest = int(grade)
    # print(f"{self._name}'s lowest grade is {lowest}")
    return lowest

  def get_median(self):
    """Gets the student's median grade"""
    mid = (len(self._grades) - 1) // 2
    int_grades = []
    for grade in self._grades:
      int_grades.append(int(grade))
    sorted_list = sorted(int_grades)
    # print(f"{self._name}'s median grade is {sorted(self._grades)[mid]}")
    return sorted_list[mid]

if __name__ == '__main__':
  with open('student_grades.csv', 'r') as in_file:
    text = csv.reader(in_file)

    student_names = []
    student_grades = []
    for line in text:
      student_names.append(line[0])
      line.pop(0)
      student_grades.append(line)

  students = []
  for i in range(len(student_names)):
    students.append(Student(student_names[i], student_grades[i]))

  print(students[6].get_median())


import sys, re
from math import ceil, floor, sqrt, pi, factorial, gcd,sin,cos,tan,asin,acos,atan2,exp,log,log10
from collections import deque, Counter, defaultdict
from itertools import product, accumulate
from functools import reduce,lru_cache
from bisect import bisect
from heapq import heapify, heappop, heappush
sys.setrecursionlimit(5 * 10 ** 5)
# from pypyjit import set_param
# set_param('max_unroll_recursion=-1')
input = lambda: sys.stdin.readline().rstrip()
ii = lambda: int(input())
mi = lambda: map(int, input().split())
li = lambda: list(mi())
inf = 2 ** 63 - 1
tokens = (i for line in iter(input, "") for i in line.split())

def solve(T: int, L: int, X: int, Y: int, Q: int, E: "List[int]"):
    def pos_e(t):
        s = 2/T*t*pi
        z = -cos(s)*(L/2)+L/2
        y = -sin(s)*(L/2)
        return(y,z)

    def ans(y,z):
        return atan2(z,sqrt((Y-y)**2+X**2))/pi*180

    for e in E:
        y,z = pos_e(e)
        s = ans(y,z)
        print(s)

def main():
    T = int(next(tokens))  # type: int
    L = int(next(tokens))  # type: int
    X = int(next(tokens))  # type: int
    Y = int(next(tokens))  # type: int
    Q = int(next(tokens))  # type: int
    E = [int(next(tokens)) for _ in range(Q)]  # type: "List[int]"
    solve(T, L, X, Y, Q, E)
    return

main()


"""
This modules does XXX
"""

import pickle
from io import BufferedWriter
from typing import Any, Union

import pandas as pd
from astropy.io import fits
from numpy import str_

from . import pickle_protocol_version

def open_pickle(filename):
    if filename.split(".")[-1] == "p":
        a = pd.read_pickle(filename)
        return a
    elif filename.split(".")[-1] == "fits":
        data = fits.getdata(filename)
        header = fits.getheader(filename)
        return data, header

def save_pickle(filename: Union[str_, str], output: Any, header: None = None) -> None:
    if filename.split(".")[-1] == "p":
        pickle.dump(output, open(filename, "wb"), protocol=pickle_protocol_version)
    if filename.split(".")[-1] == "fits":  # for futur work
        pass

def pickle_dump(obj: Any, obj_file: BufferedWriter, protocol: None = None) -> None:
    if protocol is None:
        protocol = pickle_protocol_version
    pickle.dump(obj, obj_file, protocol=protocol)


# By: Jordan Dood and Christine Johnson
# 2/6/2022
# This code takes two polynomials as vectors and multiplies them using several
# different algorithms and returns a polynomial as a vector.
#
# This file runs the naive O(n^2) algorithm

import Polynomial

def naive(poly1, poly2):
    """
    :argument poly1: List
    :argument poly2: List
    :returns: List
    """
    # Code to match the lengths, if needed
    if len(poly1) != len(poly2):
        if len(poly1) > len(poly2):
            poly2 = extend(poly2, len(poly1))
        else:
            poly1 = extend(poly1, len(poly2))

    # Create and populate result list with 0
    result = []
    for i in range(len(poly2)*2 - 1):
        result.append(0)

    # Double loop multiplication of terms
    for i in range(len(poly1)):
        term1 = poly1[i]
        for j in range(len(poly2)):
            result[i+j] += poly2[j]*term1
    return result

# Function to extend a polynomial
def extend(poly, number):
    for i in range(number-len(poly)):
        poly.append(0)
    return poly

# Driver for testing
if __name__ == '__main__':
    test1 = [3, -4, 5]  # 5X^2 - 4X + 3
    test2 = [-8, 10]    # 10X -8
    test3 = [0]         # 0x
    print(naive(test1, test2))  # 50X^3 - 80X^2 + 62X -24 --> [-24, 62, -80, 50, 0]
    print(naive(test1, test3))  # [0, 0, 0, 0, 0]


from collections import defaultdict

from baseline.relation import VideoRelation
from baseline.trajectory import Trajectory

def greedy_relational_association(short_term_relations, truncate_per_segment=100):
    # group short-term relations by their staring frames
    pstart_relations = defaultdict(list)
    for r in short_term_relations:
        pstart_relations[r['duration'][0]].append(r)

    video_relation_list = []
    last_modify_rel_list = []
    for pstart in sorted(pstart_relations.keys()):
        last_modify_rel_list.sort(key=lambda r: r.score(), reverse=True)
        sorted_relations = sorted(pstart_relations[pstart], key=lambda r: r['score'], reverse=True)
        sorted_relations = sorted_relations[:truncate_per_segment]

        cur_modify_rel_list = []
        for rel in sorted_relations:
            conf_score = rel['score']
            sub, pred, obj = rel['triplet']
            _, pend = rel['duration']
            straj = Trajectory(pstart, pend, rel['sub_traj'])
            otraj = Trajectory(pstart, pend, rel['obj_traj'])

            for r in last_modify_rel_list:
                if r.triplet() == tuple(rel['triplet']) and r.both_overlap(straj, otraj, iou_thr=0.5):
                    # merge
                    r.extend(straj, otraj, conf_score)
                    last_modify_rel_list.remove(r)
                    cur_modify_rel_list.append(r)
                    break
            else:
                r = VideoRelation(sub, pred, obj, straj, otraj, conf_score)
                video_relation_list.append(r)
                cur_modify_rel_list.append(r)

        last_modify_rel_list = cur_modify_rel_list

    return [r.serialize() for r in video_relation_list]


"""
Crie uma função que remove duplicatas de uma lista
>>> lst_fruits = ['banana', 'banana', 'caqui', 'caramelo']
>>> dedup(lst_fruits)
['banana', 'caqui', 'caramelo']
"""

def dedup(lst):
    """

    :param lst:
    :return: lst without duplicated elements

    Linear and space
    """
    result = []
    set_result = set()

    for e in lst:
        if e not in set_result:
            result.append(e)
            set_result.add(e)

    return result