#!/usr/bin/env python3
import argparse
import sys
import os
from pathlib import Path
import json
import io
import re
import tempfile
import shutil
import copy

re_has_whitespace = re.compile(r"\s+")
re_has_indent = re.compile(r"\s{4}\s+")
re_empty_line = re.compile(r"^\s*$")

def parse_args(args):
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--package", required=False, help="the namespace of the package to process")
    parser.add_argument("-b", "--build", required=True, help="the path of the build folder")
    parser.add_argument("-f", "--force", action="store_true", required=False, help="force generation, as opposed to generating only if more recent")
    return parser.parse_args(args)

def error(msg):
    print("[ERROR] " + msg, file=sys.stderr)

def progress(msg):
    print("[PROGRESS] " + msg)

def get_package_filename(build, package):
    return str(Path(build) / "packages" / (package + ".json"))

def get_source_filename(build, source):
    unmodified = str(Path(build) / "files" / (source + ".yapl"))
    if os.path.exists(unmodified):
        return unmodified
    return str(Path(build) / "files" / "modified_locally" / (source + ".yapl"))

def validate_parsed_args(parsed_args):
    if not os.path.isdir(parsed_args.build):
        error("The build parameter must specify a directory file")
        return 1
    if parsed_args.package is not None:
        if not os.path.isfile(get_package_filename(parsed_args.build, parsed_args.package)):
            error("The parameters must specify a package file")
            return 1
    return 0

def main(parsed_args):
    progress("Starting")
    progress("Preparing output directories")
    modules_path = Path(parsed_args.build) / "modules"
    modules_path.mkdir(parents=True, exist_ok=True)
    if parsed_args.force:
        shutil.rmtree(modules_path, ignore_errors=False, onerror=None)
        modules_path.mkdir(parents=True, exist_ok=True)
    if parsed_args.package:
        process_package(parsed_args.build, parsed_args.package)
    else:
        packages_path = Path(parsed_args.build) / "packages"
        packages = list(packages_path.glob("*.json"))
        n = 1
        for package in packages:
            progress("Processing package {} ({} of {})".format(package, n, len(packages)))
            n = n + 1
            process_package(parsed_args.build, Path(package).stem)
    progress("Finished")
    return 0

def load_package(build, package):
    package_filename = get_package_filename(build, package)
    with io.open(package_filename) as json_file:
        return json.load(json_file)

def load_sourcefile(build, hash):
    source_filename = get_source_filename(build, hash)
    with io.open(source_filename, "r") as yapl_file:
        return yapl_file.read()

def dedent(l):
    if l == "":
        return l
    elif l[0] == "\t":
        return l[1:]
    elif l[:4] == "    ":
        return l[4:]
    elif l.strip() == "":
        return ""
    else:
        assert False, "expected leading whitespace, not '{}'".format(l)
        return l

def process_package(build, package):
    progress("Processing package " + package)
    package_descriptor = load_package(build, package)
    source_files = list(package_descriptor["source_files"].items())
    modules_path = Path(build) / "modules"
    n = 1
    for source_filename, sourcefile_info in source_files:
        progress("Processing sourcefile {} ({} of {})".format(source_filename, n, len(source_files)))
        n = n + 1
        revision = sourcefile_info["revision"]
        if os.path.isdir(modules_path / revision):
            # TODO: this can lead to inconsistency
            continue
        def save_modules(build, package_descriptor, source_filename, sourcefile_info):
            revision = sourcefile_info["revision"]
            source_file = load_sourcefile(build, revision)
            lines = source_file.split("\n")
            extractor = None
            parent_metadata = {
                "package":package_descriptor["identifier"],
                "identifier":package_descriptor["identifier"],
                "source": {
                    "filename": source_filename,
                    "revision": revision,
                },
                "symbols": copy.deepcopy(package_descriptor["symbols"])
            }
            for line in lines:
                if line.startswith("#!"):
                    # ignore hash-bangs
                    pass
                elif extractor is None and re_has_indent.match(line):
                    pass
                elif extractor is not None:
                    if not extractor.process_line(line):
                        extractor.save(build)
                        extractor = None
                elif ModuleExtractor.matches(line):
                    extractor = ModuleExtractor(parent_metadata, line)
        save_modules(build, package_descriptor, source_filename, sourcefile_info)

class Extractor:
    def __init__(self, parent_metadata, category):
        self._parent_metadata = parent_metadata
        self._metadata = {
            "identifier":None, # to be filled by subclasses
            "category":category,
            "source":parent_metadata["source"],
        }
        if "module" in parent_metadata:
            self._metadata["module"] = parent_metadata["module"]
        elif "package" in parent_metadata:
            self._metadata["package"] = parent_metadata["package"]
        self._lines = []

    def process_line(self, line):
        if line.startswith("}"):
            return False
        self._lines.append(dedent(line))
        return True

    def get_path(self, build):
        _filename = self.get_filename(build)
        _path = Path(_filename).parents[0]
        _path.mkdir(parents=True, exist_ok=True)
        return str(_path)

    def get_filename(self, build):
        assert False

    def save(self, build):
        _path = self.get_path(build)
        Path(_path).mkdir(parents=True, exist_ok=True)
        def strip_empty_lines(lines):
            # this is a brute force evil algorithm, fix this one day.
            while lines and re_empty_line.match(lines[0]):
                lines.pop(0)
            while lines and re_empty_line.match(lines[-1]):
                lines.pop(-1)
            lines.append("")
            return lines
        _lines = strip_empty_lines(self._lines)
        _filename = self.get_filename(build)
        with io.open(_filename + ".yapl", "w") as yapl_file:
            yapl_file.write("\n".join(strip_empty_lines(_lines)))
        with io.open(_filename + ".json", 'w') as metadata_file:
            json.dump(self._metadata, metadata_file, sort_keys=True, indent=4)

    @classmethod
    def matches(cls, line):
        m = cls._expression.match(line)
        return m is not None and m and True

class FunctionExtractor(Extractor):

    _expression = re.compile(r"^(private\s+){0,1}\s*("
                             r"((function)|(constructor)|(destructor)|(method)|(generator)|(closure))"
                             r"(\-((function)|(constructor)|(destructor)|(method)|(generator)|(closure)))*"
                             r")\s+(.+)")
    _identifier = re.compile(r"\s*([^\(]+)(.*)")

    def __init__(self, parent_metadata, line):
        super().__init__(parent_metadata, "functions")
        m = FunctionExtractor._expression.match(line)
        _private = m.group(1) is not None and m.group(1).startswith("private")
        remainder = m.group(len(m.groups()))
        m = FunctionExtractor._identifier.match(remainder)
        _name = m.group(1)
        remainder = m.group(len(m.groups()))
        self._metadata["identifier"] = parent_metadata["identifier"] + "." + _name
        self._metadata["function"] = {
            "name":_name,
            "private":_private,
        }

    def process_line(self, line):
        if line.startswith("}"):
            return False
        return True

    def save(self, build):
        return

class ModuleFunctionExtractor(FunctionExtractor):

    def __init__(self, parent, line):
        super().__init__(parent._metadata, line)
        parent.declare_symbol("functions", self._metadata["function"]["name"], self._metadata["identifier"])

class VariableExtractor(Extractor):

    _expression = re.compile(r"^(private\s+){0,1}\s*("
                             r"(\w+)\s*:\s*"
                             r"(\w+)\s*"
                             r"((:{0,1}=)\s*(.*))"
                             r")$"
                             )

    def __init__(self, parent_metadata, line):
        super().__init__(parent_metadata, "variables")
        m = VariableExtractor._expression.match(line)
        _private = m.group(1) is not None and m.group(1).startswith("private")
        _name = m.group(3)
        _type = m.group(4)
        _op = m.group(6)
        _mutable = m.group(6) is not None and _op == ":="
        _remainder = m.group(len(m.groups()))
        # TODO: extends
        self._metadata["identifier"] = parent_metadata["identifier"] + "." + _name
        self._metadata["scope"] = parent_metadata["identifier"]
        self._metadata["variable"] = {
            "type":_type,
            "name":_name,
            "private":_private,
            "mutable":_mutable
        }
        self._lines.append("{}:{} {} {}".format(_name, _type, _op, _remainder))

    def process_line(self, line):
        if re_has_indent.match(line):
            self._lines.append(dedent(line))
            return True
        return False

    def save(self, build):
        return

class ClassExtractor(Extractor):

    _expression = re.compile(r"^(private\s+){0,1}\s*("
                          r"((abstract\s+class)|(class)|(interface)|(structure))"
                          r")\s+(.+)")
    _identifier = re.compile(r"(\w+).*")

    def __init__(self, parent_metadata, line):
        super().__init__(parent_metadata, "classes")
        m = ClassExtractor._expression.match(line)
        _private = m.group(1) is not None and m.group(1).startswith("private")
        remainder = m.group(len(m.groups()))
        _name = ClassExtractor._identifier.match(remainder).group(1)
        self._metadata["identifier"] = parent_metadata["identifier"] + "." + _name
        self._metadata["class"] = {
            "name":_name,
            "private":_private,
        }

    def save(self, build):
        return

class ModuleClassExtractor(ClassExtractor):

    def __init__(self, parent, line):
        super().__init__(parent._metadata, line)
        self._parent = parent
        parent.declare_symbol("classes", self._metadata["class"]["name"], self._metadata["identifier"])

class ModuleVariableExtractor(VariableExtractor):

    def __init__(self, parent, line):
        super().__init__(parent._metadata, line)
        parent.declare_symbol("variables", self._metadata["variable"]["name"], self._metadata["identifier"])

class TypeExtractor(Extractor):

    _expression = re.compile(r"^(private\s+){0,1}\s*("
                             r"(type)|(alias)"
                             r")\s+(.+)$"
                             )

    _identifier = re.compile(r"\s*([^:\{\(\s=]+)(.*)")

    def __init__(self, parent_metadata, line):
        super().__init__(parent_metadata, "types")
        m = TypeExtractor._expression.match(line)
        _private = m.group(1) is not None and m.group(1).startswith("private")
        remainder = m.group(len(m.groups()))
        m = TypeExtractor._identifier.match(remainder)
        _name = m.group(1)
        self._metadata["identifier"] = parent_metadata["identifier"] + "." + _name
        self._metadata["type"] = {
            "name":_name,
            "private":_private
        }

    def process_line(self, line):
        super().process_line(line)
        return False

    def save(self, build):
        return

class ModuleTypeExtractor(TypeExtractor):

    def __init__(self, parent, line):
        super().__init__(parent._metadata, line)
        parent.declare_symbol("types", self._metadata["type"]["name"], self._metadata["identifier"])

class ModuleExtractor(Extractor):

    _expression = re.compile(r'^((module)|(service)|(process)|(restservice))\s+(\w+)\s*\{\s*$')
    _import = re.compile(r'^\s{4}import\s+(.+)')
    _import_from = re.compile(r'^\s{4}import\s+(.*)\sfrom\s+(.+)')

    def __init__(self, parent_metadata, line):
        super().__init__(parent_metadata, "modules")
        m = ModuleExtractor._expression.match(line)
        _name = m.group(len(m.groups()))
        assert _name == str(Path(parent_metadata["source"]["filename"]).stem)
        self._metadata["identifier"] = parent_metadata["identifier"] + "." + _name
        self._metadata["symbols"] = copy.deepcopy(parent_metadata["symbols"])

    def declare_symbol(self, symbol_category, symbol_name, symbol_identifier):
        self._metadata["symbols"][symbol_name] = symbol_identifier

    def get_filename(self, build):
        return str(Path(build) / "modules" / self._metadata["source"]["revision"] / "module")

    def process_line(self, line):
        if ModuleExtractor._import.match(line):
            m = ModuleExtractor._import.match(line)
            import_from = None
            if ModuleExtractor._import_from.match(line):
                x = ModuleExtractor._import_from.match(line)
                without_from = x.group(1)
                import_from = x.group(2)
            else:
                without_from = m.group(1)
            for symbol in without_from.split(","):
                symbol = symbol.strip()
                if re_has_whitespace.match(symbol):
                    symbol = re.split(r"\s+", symbol)[0]
                if import_from is None:
                    symbol = import_from
                    if import_from in self._metadata["symbols"]:
                        import_from = self._metadata["symbols"][import_from]
                    self.declare_symbol("imports", symbol, import_from)
                else:
                    if symbol == "*":
                        if import_from in self._metadata["symbols"]:
                            import_from = self._metadata["symbols"][import_from]
                        self.declare_symbol("imports", "*." + import_from, import_from + ".*")
                    else:
                        if import_from in self._metadata["symbols"]:
                            import_from = self._metadata["symbols"][import_from]
                        import_from = import_from + "." + symbol
                        self.declare_symbol("imports", symbol, import_from)
            return True
        return super().process_line(line)

    def save(self, build):
        extractor = None
        for line in self._lines:
            if extractor is None and re_has_indent.match(line):
                continue
            if extractor is not None:
                if not extractor.process_line(line):
                    extractor.save(build)
                    extractor = None
            elif ModuleVariableExtractor.matches(line):
                extractor = ModuleVariableExtractor(self, line)
            elif ModuleFunctionExtractor.matches(line):
                extractor = ModuleFunctionExtractor(self, line)
            elif ModuleClassExtractor.matches(line):
                extractor = ModuleClassExtractor(self, line)
            elif ModuleTypeExtractor.matches(line):
                extractor = ModuleTypeExtractor(self, line)
        if extractor is not None:
            more = extractor.process_line("")
            assert (not more), "expected the extractor to complete"
            extractor.save(build)
        super().save(build)

if __name__ == "__main__":
    parsed_args = parse_args(sys.argv[1:])
    exit_code = validate_parsed_args(parsed_args)
    if exit_code == 0:
        exit_code = main(parsed_args)
    sys.exit(exit_code)


from typing import Dict, List

from aiplayground.api.bots import Bot
from aiplayground.api.tournaments.models import Participant, Tournament, Match, PlayerQueueEntry, MatchState
from collections import defaultdict

import operator

from aiplayground.exceptions import AlreadyInTournament
from aiplayground.logging import logger
from aiplayground.types import PlayerSID

def add_player(bot: Bot, tournament: Tournament) -> Participant:
    logger.debug("Getting tournament lock: %s", tournament.id)
    with tournament.lock():
        logger.debug("Got lock for tournament: %s", tournament.id)
        participants = tournament.participants
        participant_ids = {participant.bot.id for participant in participants}
        if bot.id in participant_ids:
            raise AlreadyInTournament
        index = max(x.index for x in participants) + 1 if participants else 1
        participant = Participant.create(index=index, bot=bot, tournament=tournament)
        for opponent in participants:
            if opponent.disqualified:
                continue
            Match.create(
                index=100000 * index + opponent.index,
                tournament=tournament,
                players=[participant, opponent],
                state=MatchState.pending,
            )
        return participant

def pick_match(tournament: Tournament) -> Match:
    with tournament.lock():
        queued_players = PlayerQueueEntry.list(tournament_id=tournament.id)
        participants_by_id = {participant.id: participant for participant in tournament.participants}
        participant_sids: Dict[Participant, List[PlayerSID]] = defaultdict(default_factory=list)
        for player in queued_players:
            participant = participants_by_id[player.participant_id]
            participant_sids[participant].append(player.sid)
        online_participants = set(participant_sids)
        matches = [
            match
            for match in tournament.matches
            if match.state == MatchState.pending and not set(match.players) - online_participants
        ]
        sorted_matches = sorted(matches, key=operator.attrgetter("index"))
        match = sorted_matches[0]
        match.state = MatchState.running
        match.save()
        return match


import streamlit as st
import time
import re
import chardet
import pandas as pd
import numpy as np
import geopandas as gpd
import matplotlib.pyplot as plt
from matplotlib.patches import ConnectionPatch
from functools import wraps
from shapely.geometry import Point

def main():
    if 'run' not in st.session_state:
        st.session_state.run = 0

    if 'layer_selector' not in st.session_state:
        st.session_state.layer_selector = 0

    st.title('区域划分工具')

    region_dict = read_layer_and_check('qgis')
    selected_name, submit = layer_selector(region_dict)
    input_mode = input_mode_selector()

    if st.session_state.layer_selector:
        if input_mode == '文本输入':
            run_manual_input(region_dict, selected_name)
        else:
            run_file_input(region_dict, selected_name)

    st.write('执行次数为：', st.session_state.run)

@st.cache
def read_layer_and_check(geofolder):
    try:
        dictionary = dict(pd.read_csv(f'.//{geofolder}//图层信息.csv', encoding='gb18030').loc[:, ['字段名称', '图层名称']].values)
        key_list = dictionary.keys()
        file_extension = 'shp' if geofolder == 'mapinfo' else 'gpkg'
        for index, name in enumerate(key_list):
            gdf = gpd.read_file(f'.//{geofolder}//{dictionary[name]}.{file_extension}', encoding='utf-8')
            if name not in list(gdf):
                st.error(f'图层字段<{name}>不在图层<{dictionary[name]}.{file_extension}>中')
            else:
                dictionary[name] = [dictionary[name]]
                dictionary.setdefault(name, []).append(gdf)
        return dictionary
    except IOError:
        st.error(f'找不到图层信息')

def layer_selector(region_dictionary):
    st.header('1、图层展示')
    with st.form(key='selector'):
        # st.subheader('图层信息选择')
        region_name = st.multiselect(
            "请选择图层",
            region_dictionary.keys(),
            default=['区县', '三方区域', '规划区域'],
        )
        submit = st.form_submit_button(label='确认', on_click=layer_selector_counter)
    figure = layer_ploting(region_dictionary, region_name, 3)
    if region_name:
        name_list = '、'.join(region_name)
        st.write(f'选择的图层为：{name_list}')
        st.pyplot(figure)
    return region_name, submit

@st.cache(suppress_st_warning=True, allow_output_mutation=True)
def layer_ploting(region_dictionary, region_name, fig_cols):
    plt.rcParams['font.size'] = 5
    num_fig = len(region_name)
    if num_fig > 0:
        nrows = (num_fig - 1) // fig_cols + 1
        fig, ax = plt.subplots(nrows, fig_cols, figsize=(3 * fig_cols, 3 * nrows))
        for i, field_name in enumerate(region_name):
            geo_df = region_dictionary[field_name][1]
            if nrows == 1:
                ax_i = ax[i]
            else:
                ax_rows, ax_cols = i // fig_cols, i % fig_cols
                ax_i = ax[ax_rows][ax_cols]
            ax_i.set_xlim(119.1, 120.3)
            ax_i.set_ylim(31.1, 32.1)
            geo_df.plot(ax=ax_i, column=field_name, cmap='Spectral')
        # 去掉坐标轴
        mod_num = num_fig % fig_cols
        if mod_num != 0:
            if nrows == 1:
                for n in range(mod_num, fig_cols):
                    ax[n].axis('off')
            else:
                for n in range(mod_num, fig_cols):
                    ax[nrows - 1][n].axis('off')
    else:
        fig, ax = plt.subplots()
        ax.axis('off')
    # st.write("Cache miss: layer_ploting")
    return fig

def input_mode_selector():
    st.header('2、数据选择')
    st.sidebar.header('输入模式选择')
    return st.sidebar.radio(
        '请选择输入方式',
        ('文件导入', '文本输入'),
        help='首次执行请先在图层选择处点击确认。'
    )

def run_manual_input(region_dictionary, region_name):
    st.write('数据选择模式：文本输入')
    input_text = st.sidebar.text_input(
        '输入经纬度',
        value='例如：119.934	31.8528 119.939	31.84',
        help='输入经纬度数据，可直接复制粘贴excel表格中的经度、纬度2列数据'
    )
    df_source = text_to_df(input_text)
    st.write('数据源：')
    st.table(df_source)

    if not st.sidebar.button('执行区域划分'):
        st.stop()
    else:
        st.sidebar.header('输出结果')
        result = region_division(df_source, region_dictionary, region_name)

        st.header('3、输出表格')
        st.table(result)

        st.header('4、地图展示')
        st.map(result.rename(columns={'经度': 'lon', '纬度': 'lat'}))

        st.sidebar.header('数据下载')
        name_list = '、'.join(region_name)
        st.sidebar.download_button(
            label='下载结果',
            data=ouput(result),
            file_name=f'区域划分结果-{name_list}.csv',
            mime='text/csv',
        )

def text_to_df(text):
    search_result = re.findall(r'(?P<lon>1[12][0-9].\d+)[\s,，]*(?P<lat>3[12].\d+)', text)
    if search_result:
        point = {}
        for lon_lat in search_result:
            point.setdefault('经度', []).append(float(lon_lat[0]))
            point.setdefault('纬度', []).append(float(lon_lat[1]))
        return pd.DataFrame(data=point)
    else:
        st.error('输入格式错误')

def run_file_input(region_dictionary, region_name):
    st.write('数据选择模式：文件导入')
    file_obj = st.sidebar.file_uploader(
        '上传一个表格',
        type=['csv', 'xlsx', 'xls'],
        help='上传文件格式为csv、xlsx、xls，需包含表头为经度、纬度的2列数据',
    )

    if file_obj:
        # 清理数据、执行区域划分
        df_source = read_df(file_obj)
        if df_source is None:
            st.stop()
        st.sidebar.header('输出结果')
        result = region_division(df_source, region_dictionary, region_name)
        # 显示数据源
        render_rows = 10 if df_source.shape[0] >= 10 else df_source.shape[0] // 5 * 5
        rows = st.sidebar.slider(
            '选择数据源显示行数',
            0, 50, render_rows, 5
        )
        st.write(f'数据源（前{rows}行）：')
        st.dataframe(df_source.head(rows))
        # 结果采样
        st.header('3、输出表格')
        sample_rows = st.sidebar.slider(
            '选择结果采样行数',
            0, 50, render_rows, 5
        )
        st.write(f'随机采样{sample_rows}行：')
        df_sample = result.sample(sample_rows)
        st.dataframe(df_sample)
        # 结果可视化
        st.header('4、统计图表')
        summary, rail_data = reslut_summary(result, region_name)
        fig_list = summary_ploting(summary, rail_data)
        for figure in fig_list:
            st.pyplot(figure)
        # 数据下载
        st.sidebar.header('数据下载')
        name_list = '、'.join(region_name)
        st.sidebar.download_button(
            label='下载明细结果',
            data=ouput(result),
            file_name=f'区域划分结果-{name_list}.csv',
            mime='text/csv',
            help='区域划分的明细数据',
            )
        st.sidebar.download_button(
            label='下载统计结果',
            data=output_summary(summary),
            file_name=f'区域划分统计结果-{name_list}.csv',
            mime='text/csv',
            help='统计每个图层各个区域的数量',
        )

def time_costing(step):
    def func_name(func):
        @wraps(func)
        def core(*args, **kwargs):
            start = time.time()
            res = func(*args, **kwargs)
            region_name = args[2]
            if isinstance(region_name, str):
                region_name = [region_name]
            elif isinstance(region_name, list):
                pass
            st.sidebar.write('、'.join(region_name) + '已划分')
            st.sidebar.write(f'{step}耗时：{float(time.time() - start):.3f}秒')
            return res
        return core
    return func_name

LONLAT_STR_FORMAT = {'经度': 'string', '纬度': 'string'}
LONLAT_FLOAT_FORMAT = {'经度': 'float64', '纬度': 'float64'}

def df_clean(df):
    if {'经度', '纬度'}.issubset(set(list(df))):
        return df.pipe(clean_lotlan).astype(LONLAT_FLOAT_FORMAT)
    else:
        st.error('当前表格格式错误')
        st.sidebar.error('当前表格格式错误')

def clean_lotlan(df_cell):
    for col_name in list(df_cell.loc[:, ['经度', '纬度']]):
        df_cell[col_name] = df_cell.astype({col_name: 'string'})[col_name].str.replace(r'\s', '', regex=True)
    df_cell_split_list = df_cell['经度'].str.contains('/')
    df_cell_split = df_cell[df_cell_split_list]
    if not df_cell_split.empty:
        df_comb = pd.DataFrame([], index=df_cell_split.index)
        for col_name in list(df_cell_split.loc[:, ['经度', '纬度']]):
            df_comb = pd.concat([df_comb, (df_cell_split[col_name].str.split('/', expand=True)
                                           .stack().reset_index(level=1).rename(columns={0: col_name}))], axis=1)
        df_cell = pd.concat([df_cell[~df_cell_split_list],
                             df_cell_split.iloc[:, :3].join(df_comb.drop(['level_1'], axis=1))]).reset_index(drop=True)
    return df_cell

@st.cache(suppress_st_warning=True)
def read_df(file):
    f_ext = file.name.split('.')[1]
    df = None
    if f_ext == 'csv':
        encode = str.lower(chardet.detect(file.readline())["encoding"]).replace('-', '_')
        file.seek(0)
        if encode == 'utf-8':
            df = pd_read(file, f_ext, 'utf-8')
        elif encode == 'gb2312':
            try:
                df = pd_read(file, f_ext, 'gbk')
            except UnicodeDecodeError:
                df = pd_read(file, f_ext, 'gb18030')
        elif encode == 'utf_8_sig':
            df = pd_read(file, f_ext, 'utf_8_sig')
        elif encode == "iso-8859-1":
            df = pd_read(file, f_ext, 'gbk')
        else:
            st.error('文件编码错误')
    elif f_ext in ['xlsx', 'xls']:
        df = pd_read(file, f_ext)
    else:
        st.error('文件格式错误')
    # st.write("Cache miss:read_df")
    return df

def pd_read(file, extension, encode_n=None):
    try:
        if extension == 'csv':
            return pd.read_csv(file, dtype=LONLAT_STR_FORMAT, encoding=encode_n, low_memory=False)
        elif extension in ['xlsx', 'xls']:
            return pd.read_excel(file, dtype=LONLAT_STR_FORMAT)
        else:
            st.error('文件格式错误')
    except ValueError:
        st.error('文件读取错误')

@st.cache(suppress_st_warning=True, allow_output_mutation=True)
@time_costing('区域划分')
def region_division(df, region_dictionary, region_name):
    lanlot_cols = ['经度', '纬度']
    df = df_clean(df)
    if isinstance(region_name, str):
        region_name = [region_name]
    elif isinstance(region_name, list):
        pass
    else:
        st.error('错误：区域名称错误')
    df_dropdu = df.drop_duplicates(subset=lanlot_cols).reset_index(drop=True)
    my_bar = st.sidebar.progress(0)
    for index, name in enumerate(region_name):
        gdf_region = region_dictionary[name][1]
        gdf_region = gdf_region.to_crs('EPSG:2381') if gdf_region.crs is None else gdf_region.to_crs('EPSG:2381')
        lanlot = gpd.GeoSeries([Point(x, y) for x, y in zip(df_dropdu[lanlot_cols[0]], df_dropdu[lanlot_cols[1]])])
        lanlot_region = gpd.sjoin(lanlot.reset_index().rename(columns={0: 'geometry'})
                                  .set_crs('epsg:4326').to_crs('EPSG:2381'), gdf_region.loc[:, [name, 'geometry']])
        df_dropdu = df_dropdu.join(lanlot_region.set_index('index').loc[:, name])
        my_bar.progress((index + 1) / len(region_name))
    df = df.merge(df_dropdu.loc[:, lanlot_cols + region_name], how='left', on=lanlot_cols)
    # st.write("Cache miss: region_division")
    run_counter()
    return df

def run_counter():
    st.session_state.run += 1

def layer_selector_counter():
    st.session_state.layer_selector += 1

def ouput(df):
    return df.to_csv(index=False).encode('utf-8-sig')

def output_summary(summary):
    df_summary = pd.DataFrame([])
    for key in summary.keys():
        df_summary = pd.concat([df_summary, summary[key]], axis=1)
    return df_summary.to_csv(index=False).encode('utf-8-sig')

@st.cache(suppress_st_warning=True)
def reslut_summary(df, region_name):
    for name in region_name:
        if name == '规划区域':
            df['规划区域'] = df['规划区域'].fillna('农村')
        elif name == '网格区域':
            df['网格区域'] = df['网格区域'].fillna('网格外')
        elif name == '高铁周边':
            df['高铁周边'] = df['高铁周边'].fillna('铁路外')
        else:
            df[name] = df[name].fillna('其他')

    county_order = ['天宁', '钟楼', '武进', '新北', '经开', '金坛', '溧阳', '其他']
    third_party_order = ['华星', '华苏-武进', '华苏-金坛', '华苏-溧阳', '其他']
    planning_region_order = ['主城区', '一般城区', '县城', '乡镇', '农村']
    grid_order = ['网格内', '网格边界200米', '网格外']
    rail_surrounding_order = ['京沪周边500米', '京沪周边1.5公里', '沪宁周边500米', '沪宁周边1.5公里', '宁杭周边500米', '宁杭周边1.5公里', '铁路外']
    tag_order = ['主城区', '县城', '其他']

    name_list = ['区县', '三方区域', '规划区域', '网格区域', '高铁周边', '标签区域']
    order_list = [county_order, third_party_order, planning_region_order, grid_order, rail_surrounding_order, tag_order]

    region_order_dict = dict(zip(name_list, order_list))
    summary = {}
    for name in region_name:
        summary[name] = (df.groupby(name)['ECGI'].count().reset_index(name='数量')
                         .assign(temp=lambda x: x[name].astype('category').cat.set_categories(region_order_dict[name]))
                         .sort_values(by=['temp'], ignore_index=True).drop('temp', axis=1))
    rail_data = summary.pop('高铁周边') if summary.get('高铁周边') is not None else None
    # st.write("Cache miss: reslut_summary")
    return summary, rail_data

@st.cache(suppress_st_warning=True, allow_output_mutation=True)
def summary_ploting(summary, rail_data):
    plt.rcParams['font.family'] = 'sans-serif'
    plt.rcParams['font.sans-serif'] = ['SimHei']
    plt.rcParams['axes.unicode_minus'] = False
    fig_list = []
    region_name = list(summary.keys())
    num_name = len(region_name)
    nrows = (num_name - 1) // 4 + 1
    if num_name > 0:
        fig, ax = plt.subplots(nrows, 2, figsize=(10, 4.8 * nrows))
        # 每2组画环形饼图（剔除高铁）
        for index in range(0, num_name, 2):
            name_1 = region_name[index]
            name_2 = region_name[index + 1] if index < num_name - 1 else None
            if nrows == 1:
                ax_i = ax[index // 2]
            else:
                ax_rows, ax_cols = index // 2 // 2, index // 2 % 2
                ax_i = ax[ax_rows][ax_cols]
            if name_2 is not None:
                size = 0.3
                labels_1, vals_1 = summary[name_1][name_1].to_list(), summary[name_1]['数量'].values
                labels_2, vals_2 = summary[name_2][name_2].to_list(), summary[name_2]['数量'].values
                num_label_1, num_label_2 = len(labels_1), len(labels_2)
                cmap = plt.get_cmap("tab20c")

                if num_label_1 <= num_label_2:
                    outer_labels, outer_vals = labels_1, vals_1
                    inner_labels, inner_vals = labels_2, vals_2
                    outer_colors = cmap(tab20c_color_array(num_label_1, 'outer'))
                    inner_colors = cmap(tab20c_color_array(num_label_2, 'inner'))
                else:
                    outer_labels, outer_vals = labels_2, vals_2
                    inner_labels, inner_vals = labels_1, vals_1
                    outer_colors = cmap(tab20c_color_array(num_label_2, 'outer'))
                    inner_colors = cmap(tab20c_color_array(num_label_1, 'inner'))

                wedges1, texts1, autotexts1 = ax_i.pie(
                    inner_vals, radius=1 - size, labels=inner_labels, colors=inner_colors,
                    autopct=lambda pct: pct_func(pct, inner_vals), pctdistance=0.75, labeldistance=0.3,
                    startangle=90, wedgeprops=dict(width=size, edgecolor='w')
                )
                wedges2, texts2, autotexts2 = ax_i.pie(
                    outer_vals, radius=1, labels=outer_labels, colors=outer_colors,
                    autopct=lambda pct: pct_func(pct, outer_vals), pctdistance=0.85,
                    startangle=90, wedgeprops=dict(width=size, edgecolor='w')
                )
                plt.setp(autotexts1, size=10, weight="bold", color="w")
                plt.setp(autotexts2, size=10, weight="bold", color="w")
                plt.setp(texts1, size=10, color="k")
                plt.setp(texts2, size=10, color="k")
                ax_i.set(aspect="equal")
            else:
                # 单独剩一个画传统饼图
                labels_1, vals_1 = summary[name_1][name_1].to_list(), summary[name_1]['数量'].values
                num_label_1 = len(labels_1)
                cmap = plt.get_cmap("tab20c")
                outer_colors = cmap(tab20c_color_array(num_label_1, 'inner'))
                wedges, texts, autotexts = ax_i.pie(vals_1, radius=1, labels=labels_1, colors=outer_colors,
                                                    autopct=lambda pct: pct_func(pct, vals_1), startangle=90)
                plt.setp(autotexts, size=10, weight="bold", color="w")
                plt.setp(texts, size=10, weight="bold", color="k")
                ax_i.set(aspect="equal")
        plt.axis('off')
        fig_list.append(fig)
        # 画高铁复合饼图
        if rail_data is not None:
            fig = plt.figure(figsize=(10, 4.8))
            ax1 = fig.add_subplot(121)
            ax2 = fig.add_subplot(122)
            fig.subplots_adjust(wspace=0)

            merged_label = ['高铁周边', '铁路外']
            df_rail = rail_data.query('高铁周边 != "铁路外"')
            merged_val = [df_rail['数量'].sum(), rail_data.query('高铁周边 == "铁路外"')['数量'].sum()]
            angle = -180 * merged_val[0] / merged_val[1]
            explode = [0.1, 0]
            cmap = plt.get_cmap("tab20c")
            merged_colors = cmap([4, 0])

            wedges1, texts1, autotexts1 = ax1.pie(merged_val, radius=1, labels=merged_label, colors=merged_colors,
                                                  autopct=lambda pct: pct_func(pct, merged_val),
                                                  startangle=angle, explode=explode)
            plt.setp(autotexts1, size=10, weight="bold", color="w")
            plt.setp(texts1, size=12, color="k")

            detail_label, detail_val = df_rail['高铁周边'].to_list(), df_rail['数量'].values
            num_label = len(detail_label)
            cmap = plt.get_cmap("tab20c")
            detail_colors = cmap(tab20c_color_array(num_label, 'inner'))

            r2 = 0.8
            wedges2, texts2, autotexts2 = ax2.pie(detail_val, radius=r2, labels=detail_label, colors=detail_colors,
                                                  autopct=lambda pct: pct_func(pct, detail_val),
                                                  startangle=90, counterclock=False)
            plt.setp(autotexts2, size=10, weight="bold", color="w")
            plt.setp(texts2, size=10, color="k")

            # 饼图边缘的数据
            theta1 = ax1.patches[0].theta1
            theta2 = ax1.patches[0].theta2
            center = ax1.patches[0].center
            r = ax1.patches[0].r
            width = 0.2
            # 上边缘的连线
            x = r * np.cos(np.pi / 180 * theta2) + center[0]
            y = r * np.sin(np.pi / 180 * theta2) + center[1]
            con_a = ConnectionPatch(xyA=(-width / 2, r2), xyB=(x, y), coordsA='data', coordsB='data', axesA=ax2, axesB=ax1)
            # 下边缘的连线
            x = r * np.cos(np.pi / 180 * theta1) + center[0]
            y = r * np.sin(np.pi / 180 * theta1) + center[1]
            con_b = ConnectionPatch(xyA=(-width / 2, -r2), xyB=(x, y), coordsA='data', coordsB='data', axesA=ax2, axesB=ax1)

            for con in [con_a, con_b]:
                con.set_linewidth(1)  # 连线宽度
                con.set_color = ([0, 0, 0])  # 连线颜色
                ax2.add_artist(con)  # 添加连线

            fig_list.append(fig)
    else:
        pass
    # st.write("Cache miss: summary_ploting")
    return fig_list

def pct_func(pct, allvals):
    absolute = int(round(pct/100.*np.sum(allvals)))
    return "{:d}\n{:.1f}%".format(absolute, pct)

def tab20c_color_array(num_label, outer_or_inner):
    array = np.empty((0, 5))
    if outer_or_inner == 'outer':
        outer_layer_num = (num_label - 1) // 5 + 1
        for i in range(outer_layer_num):
            array = np.append(array, np.arange(5) * 4 + i)
        array = np.sort(array).astype(int)
    elif outer_or_inner == 'inner':
        inner_layer_num = (num_label - 1) // 10 + 1
        for i in range(inner_layer_num):
            if i == 0:
                array = np.append(np.arange(5) * 4 + 1, np.arange(5) * 4 + 2)
            else:
                array = np.append(array, np.arange(5) * 4 + i + 2)
    return np.sort(array)

if __name__ == "__main__":
    main()


import time, os, boto3, json, decimal
from boto3.dynamodb.conditions import  Key
from helpers import send_to_datastream
from helpers import _get_body
from helpers import _get_response
from helpers import DecimalEncoder

try:
    dynamodb = boto3.resource('dynamodb')
    phase_status_table = dynamodb.Table(os.getenv('PHASE_STATUS_TABLE'))
except Exception as e:
    print(e)

def post_phase_status(event, context):
    body = _get_body(event)

    try:
        site = body['site']
        message = body['message']
    except Exception as e:
        return _get_response(400, 'Unable to parse all required arguments. ')

    timestamp = time.time()
    payload = {
        "site": site,
        "timestamp": timestamp,
        "message": message,
    }

    # Send to datastream
    topic = "phase_status"
    send_to_datastream(site, payload, topic)

    # save in database
    # Convert floats into decimals for dynamodb
    payload["ttl"] = timestamp + 86400  # ttl = one day
    dynamodb_entry = json.loads(json.dumps(payload, cls=DecimalEncoder), parse_float=decimal.Decimal)
    table_response = phase_status_table.put_item(Item=dynamodb_entry)

    return _get_response(200, 'Phase status broadcasted to sites successfully.')

def get_phase_status(event, context):
    try:
        site = event['pathParameters']['site']
    except Exception as e:
        return _get_response(400, 'Missing path parameter site')

    max_age_seconds = event.get('queryStringParameters', {}).get('max_age_seconds', 3600)  # default max age is 1 hour

    timestamp_cutoff = int(time.time() - int(max_age_seconds))
    phase_status = phase_status_table.query(
        Limit=3,
        ScanIndexForward=False,  # sort by most recent first
        KeyConditionExpression=Key('site').eq(site) & Key('timestamp').gt(timestamp_cutoff)
    )
    return _get_response(200, phase_status['Items'])

if __name__=="__main__":

    phase_status_table = dynamodb.Table('phase-status-dev')
    payload = json.dumps({
        "site": "tst",
        "message": "a phase message 2",
    })
    #post_phase_status({"body": payload}, {})
    event = {
        "pathParameters": {
            "site": "tst"
        },
        "queryStringParameters": {
            "max_age_seconds": "3600"
        }
    }
    context = {}
    ps = get_phase_status(event, context)
    print(ps['body'])


# -*- coding:utf8 -*-
import tweepy
import os
import sys
import json
import time
import urllib2
import requests

"""
ref. http://kslee7746.tistory.com/entry/python-tweepy-%EC%82%AC%EC%9A%A9%ED%95%9C-%ED%8A%B8%EC%9C%84%ED%84%B0-%ED%81%AC%EB%A1%A4%EB%A7%81crawling

ref. https://proinlab.com/archives/1562

ref. http://kyeoneee.tistory.com/9

"""
reload(sys)
sys.setdefaultencoding('utf-8')

hotTopics = json.loads(urllib2.urlopen("http://polatics.news/get_hot_topics").read())
track_list = [topic["topic"] for topic in hotTopics]

# api 인증 요청
consumer_token = 	""
consumer_secret = ""
auth = tweepy.OAuthHandler(consumer_token, consumer_secret)

# access 토큰 요청
access_token = ""
access_token_secret = ""
auth.set_access_token(access_token, access_token_secret)

f = open("twitter_%s.txt"%(time.strftime("%H-%d-%m-%Y")), "a");
print (time.strftime("%H:%d:%m:%Y"))

# api 생성
api = tweepy.API(auth)

data_len = 0
buf = []
class StreamListener(tweepy.StreamListener):
	def on_data(self, data):
		global data_len
		global track_list
		global buf
		if data_len == 1000:
			json_results = json.dumps(buf)
			post_data= {'twitter': json_results}
			res = requests.post("http://polatics.news/add_twitter", data=post_data)
			buf = []
			data_len = 0
			print("전송 " + res.text)
			return
		json_data = json.loads(data)
		#print ("=======================================================")
		#print ("핫토픽: " + ",".join([ht for ht in track_list if ht in json.loads(data)["text"]]))
		#print (json.loads(data)["text"])
		#print ("유저아이디: " + json.loads(data)["user"]["name"])
		ret = {}
		ret["created_at"] = json_data["created_at"]
		ret["text"] = json_data["text"]
		ret["name"] = json_data["user"]["name"]
		ret["screen_name"] = json_data["user"]["screen_name"]
		ret["topic"] = [ht for ht in track_list if ht in json.loads(data)["text"]]

		if len(ret["topic"]) > 0:
			buf.append(ret)

		f.write(data.encode("utf-8"))
		data_len = data_len + 1

	def on_error(self, status_code):
		if status_code == 420:
			return False

location = "%s,%s,%s" % ("35.95", "128.25", "1000km")  # 대한민국 중심 좌표, 반지름

if __name__ == "__main__":
	strmr = StreamListener()
	strmr = tweepy.Stream(auth=api.auth, listener=strmr)
	strmr.filter(track=track_list)

"""
keyword = "박근혜 OR 문재인"  # 검색어

wfile = open(os.getcwd()+"/twitter.txt", mode='w')

cursor = tweepy.Cursor(api.search,
                       q=keyword,
                       since='2017-10-01', # 2015-01-01 이후에 작성된 트윗들로 가져옴
                       count=100,  # 페이지당 반환할 트위터 수 최대 100
                       geocode=location,
                       include_entities=True)

for i, tweet in enumerate(cursor.items()):
    print("{}: {}".format(i, tweet.text))
    wfile.write(tweet.text + '\n')
wfile.close()
"""


import setuptools
from setuptools import find_namespace_packages

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="JarvisAI",
    version="3.9",
    author="Dipesh",
    author_email="dipeshpal17@gmail.com",
    description="JarvisAI is python library to build your own AI virtual assistant with natural language processing.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/Dipeshpal/Jarvis_AI",
    include_package_data=True,
    packages=find_namespace_packages(include=['JarvisAI.*', 'JarvisAI']),
    install_requires=['numpy', 'gtts==2.2.1', 'playsound==1.2.2',
                      'SpeechRecognition==3.8.1', 'pipwin==0.5.0', 'lxml==4.6.1', 'pyjokes',
                      'beautifulsoup4==4.9.3', 'wikipedia==1.4.0', 'auto_face_recognition', 'transformers==4.3.2',
                      'lazyme==0.0.23', 'librosa==0.8.0', "torch==1.7.1", "requests", "opencv-contrib-python==4.5.2.52",
                      "opencv-python==4.5.2.52", "cvzone==1.1.1", "pyttsx3", "googlesearch-python", "spacy",
                      "mediapipe==0.8.8", "googlesearch-python==1.0.1", "youtube-search-python==1.5.3", "shutup==0.1.3"],

    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
    project_urls={
        'Documentation': 'https://github.com/Dipeshpal/Jarvis_AI',
        'Donate': 'https://www.buymeacoffee.com/dipeshpal',
        'Say Thanks!': 'https://youtube.com/techportofficial',
        'Source': 'https://github.com/Dipeshpal/Jarvis_AI',
    },
)


__author__ = 'Vincent'

from scrapy.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.selector import HtmlXPathSelector
from mediascrap.items import NewsItem
import datetime

from pymongo import MongoClient

class ChokomagSpider(CrawlSpider):
    """
    A spider crawls domains (in accordance with some  rules we will define
    to collect all the pages that we wish to extract our LemondeItems instances
    from. Most of this crawling logic is provided by Scrapy in the CrawlSpider class, so we
    can extend this class when writing our first spider.
    """

    name = "chokomag"
    allowed_domains =["chokomag.com"]
    start_urls =[
       "http://chokomag.com/beaute/cheveux/",
        "http://chokomag.com/tag/make-up/",

        "http://chokomag.com/beaute/box/",
    "http://chokomag.com/beaute/soins/",
    "http://chokomag.com/beaute/parfums/",
    "http://chokomag.com/mode/",
    "http://chokomag.com/tag/concours/",
    "http://chokomag.com/non-classe/"
    ]
    rules = [
        #site which should be saved
        Rule(
            LinkExtractor(
                allow = ['/(\d+|\w+)']),
                'parse_page')]

    def parse_page(self,response):
        hxs = HtmlXPathSelector(response)

        body = ''.join(hxs.select('//div[@class="entry-content clearfix"]/p//text()').extract()).strip()

        item = NewsItem()

        if len(body)> 0 :
            item['body'] = body
            item['url'] = response.url
            item['timeOfScrap'] = datetime.datetime.now()

            return item
        else :
            pass


import cv2
import os
import argparse

def image_folder_to_video(folder_path, output_path):
    # Get the list of image filenames
    filenames = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.lower().endswith('.jpg')]
    filenames.sort()  # Sort the filenames

    # Get the dimensions of the first image
    image = cv2.imread(filenames[0])
    height, width, _ = image.shape

    # Define the codec and create a video writer object
    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    out = cv2.VideoWriter(output_path, fourcc, 30.0, (width, height))

    # Add each image to the video with 0.2 seconds of duration
    for filename in filenames:
        image = cv2.imread(filename)
        for i in range(15):
            out.write(image)

    # Release the video writer object
    out.release()

def main():
    parser = argparse.ArgumentParser(description='Convert a folder of images to a video.')
    parser.add_argument('input_folder', metavar='input_folder', type=str,
                        help='The path to the folder containing the input images.')
    parser.add_argument('output_video', metavar='output_video', type=str,
                        help='The path to the output video file.')
    args = parser.parse_args()
    input_folder = args.input_folder
    output_video = args.output_video
    image_folder_to_video(input_folder, output_video)

if __name__ == "__main__":
    main()


# class s_(object):

import functools
import numbers
import operator

import numpy

import cupy
from cupy._creation import from_data
from cupy._manipulation import join

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`cupy.r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        arrays = []
        scalars = []
        if isinstance(key, str):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, str):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)
                arrays.append(newobj)

            objs.append(newobj)

        final_dtype = numpy.result_type(*arrays, *[key[k] for k in scalars])
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return join.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :obj:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :obj:`r_`.

This implementation is partially borrowed from NumPy's one.

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :obj:`numpy.c_`

Examples
--------
>>> a = cupy.array([[1, 2, 3]], dtype=np.int32)
>>> b = cupy.array([[4, 5, 6]], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__()

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :obj:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)

"""

def indices(dimensions, dtype=int):
    """Returns an array representing the indices of a grid.

    Computes an array where the subarrays contain index values 0,1,...
    varying only along the corresponding axis.

    Args:
        dimensions: The shape of the grid.
        dtype: Data type specifier. It is int by default.

    Returns:
        ndarray:
        The array of grid indices,
        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.

    Examples
    --------
    >>> grid = cupy.indices((2, 3))
    >>> grid.shape
    (2, 2, 3)
    >>> grid[0]        # row indices
    array([[0, 0, 0],
           [1, 1, 1]])
    >>> grid[1]        # column indices
    array([[0, 1, 2],
           [0, 1, 2]])

    .. seealso:: :func:`numpy.indices`

    """
    dimensions = tuple(dimensions)
    N = len(dimensions)
    shape = (1,) * N
    res = cupy.empty((N,) + dimensions, dtype=dtype)
    for i, dim in enumerate(dimensions):
        res[i] = cupy.arange(dim, dtype=dtype).reshape(
            shape[:i] + (dim,) + shape[i + 1:]
        )
    return res

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.ix_`

    """
    # TODO(niboshi): Avoid nonzero which may synchronize the device.
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = from_data.asarray(new)
        if new.ndim != 1:
            raise ValueError('Cross index must be 1 dimensional')
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()  # may synchronize
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

def ravel_multi_index(multi_index, dims, mode='wrap', order='C'):
    """
    Converts a tuple of index arrays into an array of flat indices, applying
    boundary modes to the multi-index.

    Args:
        multi_index (tuple of cupy.ndarray) : A tuple of integer arrays, one
            array for each dimension.
        dims (tuple of ints): The shape of array into which the indices from
            ``multi_index`` apply.
        mode ('raise', 'wrap' or 'clip'), optional: Specifies how out-of-bounds
            indices are handled.  Can specify either one mode or a tuple of
            modes, one mode per index:

            - *'raise'* -- raise an error
            - *'wrap'* -- wrap around (default)
            - *'clip'* -- clip to the range

            In 'clip' mode, a negative index which would normally wrap will
            clip to 0 instead.
        order ('C' or 'F'), optional: Determines whether the multi-index should
            be viewed as indexing in row-major (C-style) or column-major
            (Fortran-style) order.

    Returns:
        raveled_indices (cupy.ndarray): An array of indices into the flattened
        version of an array of dimensions ``dims``.

    .. warning::

        This function may synchronize the device when ``mode == 'raise'``.

    Notes
    -----
    Note that the default `mode` (``'wrap'``) is different than in NumPy. This
    is done to avoid potential device synchronization.

    Examples
    --------
    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (7,6))
    array([22, 41, 37])
    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (7,6),
    ...                        order='F')
    array([31, 41, 13])
    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (4,6),
    ...                        mode='clip')
    array([22, 23, 19])
    >>> cupy.ravel_multi_index(cupy.asarray([[3,6,6],[4,5,1]]), (4,4),
    ...                        mode=('clip', 'wrap'))
    array([12, 13, 13])
    >>> cupy.ravel_multi_index(cupy.asarray((3,1,4,1)), (6,7,8,9))
    array(1621)

    .. seealso:: :func:`numpy.ravel_multi_index`, :func:`unravel_index`
    """

    ndim = len(dims)
    if len(multi_index) != ndim:
        raise ValueError(
            "parameter multi_index must be a sequence of "
            "length {}".format(ndim))

    for d in dims:
        if not isinstance(d, numbers.Integral):
            raise TypeError(
                "{} object cannot be interpreted as an integer".format(
                    type(d)))

    if isinstance(mode, str):
        mode = (mode, ) * ndim

    if functools.reduce(operator.mul, dims) > cupy.iinfo(cupy.int64).max:
        raise ValueError("invalid dims: array size defined by dims is larger "
                         "than the maximum possible size")

    s = 1
    ravel_strides = [1] * ndim

    order = 'C' if order is None else order.upper()
    if order == 'C':
        for i in range(ndim - 2, -1, -1):
            s = s * dims[i + 1]
            ravel_strides[i] = s
    elif order == 'F':
        for i in range(1, ndim):
            s = s * dims[i - 1]
            ravel_strides[i] = s
    else:
        raise ValueError('order not understood')

    multi_index = cupy.broadcast_arrays(*multi_index)
    raveled_indices = cupy.zeros(multi_index[0].shape, dtype=cupy.int64)
    for d, stride, idx, _mode in zip(dims, ravel_strides, multi_index, mode):

        if not isinstance(idx, cupy.ndarray):
            raise TypeError("elements of multi_index must be cupy arrays")
        if not cupy.can_cast(idx, cupy.int64, 'same_kind'):
            raise TypeError(
                'multi_index entries could not be cast from dtype(\'{}\') to '
                'dtype(\'{}\') according to the rule \'same_kind\''.format(
                    idx.dtype, cupy.int64().dtype))
        idx = idx.astype(cupy.int64, copy=False)

        if _mode == "raise":
            if cupy.any(cupy.logical_or(idx >= d, idx < 0)):
                raise ValueError("invalid entry in coordinates array")
        elif _mode == "clip":
            idx = cupy.clip(idx, 0, d - 1)
        elif _mode == 'wrap':
            idx = idx % d
        else:
            raise ValueError('Unrecognized mode: {}'.format(_mode))
        raveled_indices += stride * idx
    return raveled_indices

def unravel_index(indices, dims, order='C'):
    """Converts array of flat indices into a tuple of coordinate arrays.

    Args:
        indices (cupy.ndarray): An integer array whose elements are indices
            into the flattened version of an array of dimensions :obj:`dims`.
        dims (tuple of ints): The shape of the array to use for unraveling
            indices.
        order ('C' or 'F'): Determines whether the indices should be viewed as
            indexing in row-major (C-style) or column-major (Fortran-style)
            order.

    Returns:
        tuple of ndarrays:
        Each array in the tuple has the same shape as the indices array.

    Examples
    --------
    >>> cupy.unravel_index(cupy.array([22, 41, 37]), (7, 6))
    (array([3, 6, 6]), array([4, 5, 1]))
    >>> cupy.unravel_index(cupy.array([31, 41, 13]), (7, 6), order='F')
    (array([3, 6, 6]), array([4, 5, 1]))

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.unravel_index`, :func:`ravel_multi_index`

    """
    order = 'C' if order is None else order.upper()
    if order == 'C':
        dims = reversed(dims)
    elif order == 'F':
        pass
    else:
        raise ValueError('order not understood')

    if not cupy.can_cast(indices, cupy.int64, 'same_kind'):
        raise TypeError(
            'Iterator operand 0 dtype could not be cast '
            'from dtype(\'{}\') to dtype(\'{}\') '
            'according to the rule \'same_kind\''.format(
                indices.dtype, cupy.int64().dtype))

    if (indices < 0).any():  # synchronize!
        raise ValueError('invalid entry in index array')

    unraveled_coords = []
    for dim in dims:
        unraveled_coords.append(indices % dim)
        indices = indices // dim

    if (indices > 0).any():  # synchronize!
        raise ValueError('invalid entry in index array')

    if order == 'C':
        unraveled_coords = reversed(unraveled_coords)
    return tuple(unraveled_coords)

def mask_indices(n, mask_func, k=0):
    """
    Return the indices to access (n, n) arrays, given a masking function.

    Assume `mask_func` is a function that, for a square array a of
    size ``(n, n)`` with a possible offset argument `k`, when called
    as ``mask_func(a, k)`` returns a new array with zeros in certain
    locations (functions like :func:`~cupy.triu` or :func:`~cupy.tril` do
    precisely this). Then this function returns the indices where the non-zero
    values would be located.

    Args:
        n (int): The returned indices will be valid to access arrays
            of shape (n, n).
        mask_func (callable): A function whose call signature is
            similar to that of :func:`~cupy.triu`, :func:`~tril`.  That is,
            ``mask_func(x, k)`` returns a boolean array, shaped like
            `x`.  `k` is an optional argument to the function.
        k (scalar): An optional argument which is passed through to
            `mask_func`. Functions like :func:`~cupy.triu`, :func:`~cupy.tril`
            take a second argument that is interpreted as an offset.

    Returns:
        tuple of arrays: The `n` arrays of indices corresponding to
        the locations where ``mask_func(np.ones((n, n)), k)`` is
        True.

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.mask_indices`
    """
    a = cupy.ones((n, n), dtype=cupy.int8)
    return mask_func(a, k).nonzero()

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

def tril_indices(n, k=0, m=None):
    """Returns the indices of the lower triangular matrix.
    Here, the first group of elements contains row coordinates
    of all indices and the second group of elements
    contains column coordinates.

    Parameters
    ----------
    n : int
        The row dimension of the arrays for which the returned
        indices will be valid.
    k : int, optional
        Diagonal above which to zero elements. `k = 0`
        (the default) is the main diagonal, `k < 0` is
        below it and `k > 0` is above.
    m : int, optional
        The column dimension of the arrays for which the
        returned arrays will be valid. By default, `m = n`.

    Returns
    -------
    y : tuple of ndarrays
        The indices for the triangle. The returned tuple
        contains two arrays, each with the indices along
        one dimension of the array.

    See Also
    --------
    numpy.tril_indices

    """

    tri_ = cupy.tri(n, m, k=k, dtype=bool)

    return tuple(cupy.broadcast_to(inds, tri_.shape)[tri_]
                 for inds in cupy.indices(tri_.shape, dtype=int))

def tril_indices_from(arr, k=0):
    """Returns the indices for the lower-triangle of arr.

    Parameters
    ----------
    arr : cupy.ndarray
          The indices are valid for square arrays
          whose dimensions are the same as arr.
    k : int, optional
        Diagonal offset.

    See Also
    --------
    numpy.tril_indices_from

    """

    if arr.ndim != 2:
        raise ValueError("input array must be 2-d")
    return tril_indices(arr.shape[-2], k=k, m=arr.shape[-1])

def triu_indices(n, k=0, m=None):
    """Returns the indices of the upper triangular matrix.
    Here, the first group of elements contains row coordinates
    of all indices and the second group of elements
    contains column coordinates.

    Parameters
    ----------
    n : int
        The size of the arrays for which the returned indices will
        be valid.
    k : int, optional
        Refers to the diagonal offset. By default, `k = 0` i.e.
        the main dialogal. The positive value of `k`
        denotes the diagonals above the main diagonal, while the negative
        value includes the diagonals below the main diagonal.
    m : int, optional
        The column dimension of the arrays for which the
        returned arrays will be valid. By default, `m = n`.

    Returns
    -------
    y : tuple of ndarrays
        The indices for the triangle. The returned tuple
        contains two arrays, each with the indices along
        one dimension of the array.

    See Also
    --------
    numpy.triu_indices

    """

    tri_ = ~cupy.tri(n, m, k=k - 1, dtype=bool)

    return tuple(cupy.broadcast_to(inds, tri_.shape)[tri_]
                 for inds in cupy.indices(tri_.shape, dtype=int))

def triu_indices_from(arr, k=0):
    """Returns indices for the upper-triangle of arr.

    Parameters
    ----------
    arr : cupy.ndarray
          The indices are valid for square arrays.
    k : int, optional
        Diagonal offset (see 'triu_indices` for details).

    Returns
    -------
    triu_indices_from : tuple of ndarrays
        Indices for the upper-triangle of `arr`.

    See Also
    --------
    numpy.triu_indices_from

    """

    if arr.ndim != 2:
        raise ValueError("input array must be 2-d")
    return triu_indices(arr.shape[-2], k=k, m=arr.shape[-1])


"""
FILES to YAML
"""

import argparse
import json
import json.decoder
import os
from pathlib import Path
import yaml
import yaml.scanner

def walk_thru(startdir: str) -> list:
    p = Path(startdir)
    a = [str(el).replace('\\', '/').replace(startdir, '') for el in p.rglob('*')]
    return a

def read_file_content(filepath: str):
    with open(filepath) as file:
        content = file.read()

    try:
        template = yaml.safe_load(content)
        return template
    except yaml.scanner.ScannerError:
        pass

    try:
        template = json.loads(content)
        return template
    except json.decoder.JSONDecodeError:
        pass

    return content

def dir_tree_dict(startdir: str) -> dict:
    d = {}
    for item in dirs_and_files:
        p = d
        for x in item.split('/'):
            if os.path.isdir(startdir + item):
                p = p.setdefault(x, {})
            else:
                content = read_file_content(startdir + item)
                p = p.setdefault(x, content)
    return d

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('startdir', type=str)
    args = parser.parse_args()
    startdir = args.startdir + '/'

    dirs_and_files = walk_thru(startdir)

    res = dir_tree_dict(startdir)

    res = {startdir.split('/')[-2]: res}

    with open('week5/res.yaml', 'w', newline='') as newfile:
        yaml.dump(res, newfile, default_flow_style=False)


from django.urls import path
from rest_framework.routers import DefaultRouter

from src.dates_api.views import DateViewSet, PopularMonthListView

app_name = "dates_api"

router = DefaultRouter()
router.register("dates", DateViewSet, basename="dates")

urlpatterns = [
    path("popular/", PopularMonthListView.as_view(), name="popular_month"),
]
urlpatterns += router.urls


import pandas as pd
import datetime as dt
from kucoincli.client import Client

def test_lending_liquidity(quote='USDT'):
    """Obtain max point-in-time liquidity for lending markets in USDT terms"""
    client = Client()
    l = client.symbols(marginable=True).baseCurrency
    liq = {}

    for curr in l:
        try:
            df = client.lending_rate(curr)
            stats = client.get_stats(curr + '-' + quote)
            max_borrow = (((stats.buy + stats.sell) / 2) * df['size'].sum())
            liq[curr] = max_borrow
        except:
            pass

    return pd.Series(liq).sort_values(ascending=False)

def test_trading_liquidity(lookback=90, interval='1day'):
    """Calculate mean turnover for marginable currencies in `interval` granularity over `lookback` days"""
    client = Client()
    l = client.symbols(marginable=True).index
    liq = {}

    start = dt.datetime.now() - dt.timedelta(days=lookback)

    for curr in l:
        mean_vol = client.ohlcv(
            tickers=curr,
            interval=interval,
            start=start
        ).turnover.mean()
        liq[curr] = mean_vol

    return pd.Series(liq).sort_values(ascending=False)

def get_csv_data(path):
    """Reads in CSV file exported from SQL db"""
    df = pd.read_csv(path, index_col="time")
    df.index = pd.to_datetime(df.index)
    df = df.astype(float)
    return df.sort_index(ascending=True)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 14 13:57:29 2019
@author: Witold Klimczyk

# ICEM
foil = Airfoil(filein = r'E:\propeller\mh_airofils\mh117/mh117.txt', t = 0.001, chord = 0.2)
foil.runFluent(15,.2,1)#

# XFOIL
foil2 = Airfoil(ftype = 'XFOIL', filein = r'E:\AIRFOIL\airfoils/naca0012.txt', t = 0.001, chord = 0.2)

# x,y

X = pd.read_csv(f'http://airfoiltools.com/airfoil/seligdatfile?airfoil={foilname}-il')
X.to_csv(r'E:\AIRFOIL\temp.csv', header = False, index = False)
X = np.loadtxt(r'E:\AIRFOIL\temp.csv')
foil = Airfoil( 'XFOIL',  r'E:\AIRFOIL\temp.csv')

"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
rc('text', usetex=True)
import subprocess
import os
from subprocess import run, PIPE
import gc
import pandas as pd
from urllib.error import HTTPError

from fluentScheme import generateScheme
from icemScheme import generateICEMScheme

class Airfoil():
    def __init__(self, ftype = 'ICEM', filein = None, x = None, y = None,  T_req = None, camber = None,
                 chord = None, beta = None, z = 0, fileoutICEM = None, t = 0, dx = 0, dy = 0, split = False, origin = 0,
                 camb = False, r_LE = None, verbose = False, workingdir = r'E:\AIRFOIL', xfoildir = None):
        """
        inputs:
            - ftype/name: 'ICEM' / 'XFOIL' / 'XY', specifies type of airofil input data or its name to download from airfoiltools
            - filein: '.txt' file with points coords (can be non-txt)
            - chord: dimensionless chord
            - beta: originally used for propeller pitch, for wing stands for twist
            - z: specifies third coordinae used for 3d wing stacking
            - fileoutICEM: full path and name for ICEM output file format, no extension (only name)
            - TEcut: specifies location of vertical cut
            - t: float: te thickness
            - T_req: maximum thickness to match required absolute thickness
            - origin: float: used to keep particular airfoil poitn in center, e.g. origin = .25 keeps quarter chord in center
            - camb: True/False: if we want to scael camber with thickness
            - workingdir: specify if other than current
            - xfoildir: contains xfoil.exe and uses this directory to save .txt files, if not given assumes it is in folder XFOIL under the same directory as current working folder
        attributes:
            - x: x-coords
            - y: y-coords
            - z: z-coords
        """
        gc.collect()
        self.camber = camber
        self.chord = chord
        self.z = z
        self.filein = filein

        self.workingdir = workingdir if workingdir != None else os.getcwd().strip('\\python')
        print('workingdir {}'.format(self.workingdir))
        self.xfoildir = self.workingdir + '/XFOIL/'
        self.filebuffer = self.xfoildir + '/xfoilairfoil.txt'
        self.filecp = self.xfoildir + '/xfoilairfoilcp.txt'
        self.fileCptex = self.xfoildir + '/xfoilairfoilcptex.txt'
        self.camber_t = self.xfoildir + '/camber_t.txt'
        self.xfoilpath = self.xfoildir + '/xfoil.exe'

        # directories to check before analysis
        self.fileFig = self.workingdir + '/saved_plots/airfoil'
        self.meshin = self.workingdir + '/mesh/fluent.msh'
        self.meshDir = self.workingdir + '/mesh/'
        self.fileoutICEM = self.workingdir+'/mesh/foilICEM' if fileoutICEM is None else fileoutICEM
        self.fluentdir = self.workingdir + '/fluent/'
        self.ftype = ftype
        self.verbose = verbose

        self.camber = None
        self.thickness = None
        self.split = False
        self.t = t

        if ftype == 'ICEM':
            self.readICEM()
        elif ftype == 'XFOIL':
            self.readXFOIL()
            self.saveXFOIL()
        elif ftype == 'XY':
            self.x = x
            self.y = y
        else:
            try:
                X = pd.read_csv(f'http://airfoiltools.com/airfoil/seligdatfile?airfoil={ftype}-il')
                X.to_csv(r'E:\AIRFOIL\temp.csv', header = False, index = False)
                X = np.loadtxt(r'E:\AIRFOIL\temp.csv')
                self.x = X[:,0]
                self.y = X[:,1]
                self.z = self.z
            except HTTPError:
                print('error reading airofil from web')
                return None

        # chord scaling
        if chord is None:
            self.chord = np.max(self.x) - np.min(self.x)
            if self.verbose:
                print('evaluated chord is {:.2f}'.format(self.chord))
        else:
            self.chord = np.max(self.x) - np.min(self.x)
            self.scale_XFOIL(chord/np.max(self.x))
            self.saveXFOIL()
            if self.verbose:
                print('scaled airfoil to desired chord {:.3f}'.format(self.chord))

        self.x1 = None
        self.x2 = None
        self.y1 = None
        self.y2 = None
        self.z1 = z
        self.z2 = z

        # imposing required thickness
        if T_req is not None:
            self.thicken(T_req, camb)

        # cut TE
        if t > 0:
            self.cutTE_XFOIL(t, r = .5)

        if r_LE is not None:
            r_LE_current = self.LEradius()[0]
            if r_LE > r_LE_current:
                print('modifying LE radius')
                factor = r_LE / r_LE_current
                self.modify_XFOIL(1,1,factor)
                print('LE factor = {:.1f}'.format(factor))

        # twisting airfoil to required beta
        if beta is not None:
            self.rotate_XFOIL(beta, origin)

        # translating airofil to match required origin
        self.translate_XFOIL(dx - origin * self.chord, dy)

        # setting split after all airofil modifications
        self.split = split
        if self.split:
            self.splitCurve()

    def readICEM(self):
        X = np.loadtxt(self.filein, delimiter = '\t', skiprows = 1)
        self.x = X[:,0]
        self.y = X[:,1]
        self.z = self.z

    def readXFOIL(self, file = None):
        if file is None:
            X = np.loadtxt(self.filein, skiprows = 1)
        else:
            X = np.loadtxt(file, skiprows = 1)
        self.x = X[:,0]
        self.y = X[:,1]
        self.z = self.z
        self.filein = self.filebuffer

    def saveXFOIL(self):
        """  saves airfoil coords to .txt file with specified path
        """
        # close trailing edge
        # save coords to file
        if not self.split:
            with open(self.filebuffer, "w") as text_file:
                print("airfoil", file = text_file)
                for i in range(len(self.x)):
                    print("  {} {}".format(self.x[i], self.y[i]), file=text_file)
        else:
            with open(self.filebuffer, "w") as text_file:
                print("airfoil", file = text_file)
                for i in range(len(self.x2)-1,0,-1):
                    print("  {} {}".format(self.x2[i], self.y2[i]), file=text_file)
                for i in range(len(self.x1)):
                    print("  {} {}".format(self.x1[i], self.y1[i]), file=text_file)

###
###   ===================        GEOMETRY SECTION           ==========================
###

    def cutTE_XFOIL(self, t = .005, r = 0.5):
        """ modifies airfoil using xfoil to maintain camber

            t: thickness
            r: blending radius
        """
        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\ntgap '+ '{} {}'.format(t,r) + '\n\npane\n\nsave '+airfoilOUT+'\ny\n\nquit\n'
        run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)
        if self.verbose:
            print('succesfully modified TE using xfoil')
        self.readXFOIL(airfoilOUT)

    def modify_XFOIL(self, thicken = 1, camber = 1, LE_radius=1):
        """ modifies airfoil using xfoil to scale"
        thickness and camber distribution
            values below 1 decrease, above 1 increase scale (1 is no change)
        """
        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\ntfac '+ '{} {}'.format(thicken, camber) +'\nlera {} {}'.format(LE_radius, .2)+  '\n\npane\n\nsave '+airfoilOUT+'\ny\n\nquit\n'

        p = run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)
        if self.verbose:
            print('modified thickness scaled by {}'.format(thicken))
        self.readXFOIL(airfoilOUT)

    def thicken(self, req_T, camb = False):
        """ modifies thickness to required value of maximum thickness
            can also modify camber of airfoil
        """
        self.findCamberThickness()
        factor = req_T/(self.t_max*self.chord)
        print(f'{factor}')
        if camb==True:
            camb = factor
            self.modify_XFOIL(thicken = factor, camber = camb)
        else:
            camb = 1
            self.modify_XFOIL(thicken = factor, camber = camb)
        if self.verbose:
            print('modified thickness to desired value, i.e. {:.3f}, by a factor of {:.2f}'.format(req_T, factor))

    def scale_XFOIL(self, factor = 1):
        """ scales airfoil using xfoil
        """
        print('chord before modification: {:.3f}'.format(self.chord))
        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\nscal '+ '{}'.format(factor) + '\n\npane\n\nsave '+airfoilOUT+'\ny\n\nquit\n'

        p = run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)

        if self.verbose:
            print('modified chord by factor {}'.format(factor))
        self.readXFOIL(airfoilOUT)
        self.chord *= factor
        print('chord after modification: {:.3f}'.format(self.chord))

    def translate_XFOIL(self, dx = 0, dy = 0):
        """ translates airfoil by specified dx and dy
        """
        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\ntran '+ '{} {}'.format(dx, dy) + '\n\npane\n\nsave '+airfoilOUT+'\ny\n\nquit\n'

        p = run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)
        if self.verbose:
            print('airfoil translated by {:.3f} in x and {:.3f} in y'.format(dx, dy))
        self.readXFOIL(airfoilOUT)

    def rotate_XFOIL(self, angle = 0, origin  = 0):
        """ rotates airfoil using xfoil by specified angle in degrees, around (0,0), positive angle moves TE down
        """
        if origin is not 0:
            self.translate_XFOIL(dx = -origin*self.chord)

        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\nadeg '+ '{}'.format(angle) + '\n\npane\n\nsave '+airfoilOUT+'\ny\n\nquit\n'

        p = run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)
        if self.verbose:
            print('airfoil rotated by {:.2f}'.format(angle))
        self.readXFOIL(airfoilOUT)

        if origin is not 0:
            self.translate_XFOIL(dx = origin*self.chord )

    def findCamberThickness(self, plot = False, tex = False, name = ''):
        """ finds camber and thickness distributions usign xfoil  """

        self.saveXFOIL()
        airfoilIN = self.filebuffer
        airfoilOUT = self.filebuffer
        command = 'load ' + airfoilIN + '\npane\ngdes\ntcpl\ncamb\nwrtc\n{}\n\n\nquit\n'.format(self.camber_t)

        p = run([self.xfoilpath], stdout=PIPE,  input=command, encoding='ascii', shell = False)
        if p.returncode ==2:
            if self.verbose:
                print('found camber and thickness distributions')

        X = np.loadtxt(self.camber_t, skiprows = 1)
        self.camber = X[:,:2]
        self.thickness = X[:,2:]
        self.readXFOIL(airfoilOUT)
        self.t_max = 2* np.max(self.thickness[:,1])

        if plot:
            plt.figure(figsize = (6,2),dpi = 200)
            plt.rc('text', usetex=True)
            plt.rc('font', family='serif')

            plt.plot(self.camber[:,0], self.camber[:,1], 'k-',linewidth = 1.2, label = 'camber')
            plt.plot(self.thickness[:,0], self.thickness[:,1], 'k--',linewidth = 1.2, label = 'thickness')
            plt.plot(self.thickness[:,0], -self.thickness[:,1], 'k--',linewidth = 1.2)
            plt.xlabel(r'$x/c$')
            plt.ylabel(r'$y/c$',fontsize=12)
            plt.title(r"{}".format('camber and thickness distributions'), fontsize=12)
            #plt.subplots_adjust(top=0.8)
            plt.axis('equal')
            plt.legend()
            plt.tight_layout()
            plt.grid('major', linewidth = .2)
            plt.savefig(self.fileFig+'ct', dpi = 1000)
            plt.show()

        if tex:
            camberdir = self.workingdir + r'\wing3d\tex-plots\{}camber.txt'.format(name)
            thicknessdir = self.workingdir + r'\wing3d\tex-plots\{}thickness.txt'.format(name)
            np.savetxt(camberdir, self.camber)
            np.savetxt(thicknessdir, self.thickness)

    def t_x(self, x=None):
        """ finds thickness at specified x-position, x is x/c (i.e. between 0-1)

        self.t_x(0.5) returns thickness at x/c = 0.5
        if no argument passed, returns max thickness
        """
        self.findCamberThickness()
        i = 0
        if x is None:
            return self.t_max

        for i in range(len(self.thickness[:,0])):
            if self.thickness[i,0] > x :
                return 2*self.thickness[i,1]
        if self.verbose:
            print('invalid argument')

    def LEradius(self, plot = False, dpi = 500, saveFig = False):
        """ method to find leading edge radius

            buids many circles, each from 3 points from leading edge region
            lowest radius circle is chosen as le radius

            allows to plot le region to investigate le radius
        """
        def findCircle(P1, P2, P3):
            import sympy as sym

            a, b, r2 = sym.symbols('a, b, r2')
            e1 = sym.Eq((P1[0]-a)**2+(P1[1]-b)**2, r2**2)
            e2 = sym.Eq((P2[0]-a)**2+(P2[1]-b)**2, r2**2)
            e3 = sym.Eq((P3[0]-a)**2+(P3[1]-b)**2, r2**2)

            solution = sym.solve([e1, e2, e3], (a, b, r2))
            r = float(np.abs(solution[0][2]))
            x = float(np.abs(solution[0][0]))
            y = float(np.abs(solution[0][1]))
            return x,y,r

        i = np.where(self.x == min(self.x))[0][0]
        # find several circles around LE
        r = 1
        j = 1
        k = 1
        while j<5:
            while k<5:
                x_temp,y_temp,r_temp = findCircle( [self.x[i-j], self.y[i-j]], [self.x[i], self.y[i]], [self.x[i+k], self.y[i+k]]  )
                if r_temp<r:
                    r = r_temp
                    x = x_temp
                    y = y_temp
                k+=1
            j+=1

        if plot:
            an = np.linspace(0, 2*np.pi, 100)

            plt.figure(dpi = dpi)
            plt.rc('text', usetex=True)
            plt.rc('font', family='serif')
            plt.plot(self.x, self.y, 'ko-', linewidth = 1.4)
            plt.plot([x],[y],'ro')
            plt.plot(r*np.cos(an)+x, r*np.sin(an)+y, 'r-', linewidth = 1.4)
            plt.title(r"{}".format('leading edge radius close up'), fontsize=12)

            plt.axis('equal')
            plt.ylim(-r, r*3.5)

            if saveFig:
                plt.savefig(self.fileFig, dpi = 1000)
            plt.show()

            fig, ax = plt.subplots(dpi = 500)
            ax.plot(self.x, self.y, 'ko-', linewidth = 1.4)
            ax.plot([x],[y],'ro')
            ax.plot(r*np.cos(an)+x, r*np.sin(an)+y, 'r-', linewidth = 1.4)
            ax.set_xlim(-r, r*3.5)
            ax.set_ylim(-r*2, r*2)
            ax.set_title('mh117: R=2')
            ax.set_aspect(1.0)
            ax.grid(which='major', linewidth = 0.2)
            plt.show()

        return r, x , y

    def saveICEM(self, airfoilfile = None):
        """ saves points in icem format, either as a single curve of splits to upper and lower (recommended) """

        if self.y[1]>self.y[-1]:
            self.x = np.flip(self.x, axis = 0)
            self.y = np.flip(self.y, axis = 0)

        if airfoilfile is not None:
            self.fileoutICEM = airfoilfile

        if not self.split:
            self.zs = np.ones(len(self.x))*self.z
            self.fileoutICEM += '.txt'
            with open( self.fileoutICEM, 'w') as f:
                f.write('{}\t{}\n'.format(len(self.x), 1))
                for i in range(len(self.x)):
                    f.write('{}\t{}\t{}\n'.format(self.x[i]*1000, self.y[i]*1000, self.zs[i]*1000) )
        else:
            self.z1 = np.ones(len(self.x1))*self.z
            self.z2 = np.ones(len(self.x2))*self.z

            with open( self.fileoutICEM + '.0.txt', 'w') as f:
                f.write('{}\t{}\n'.format(len(self.x1), 1))
                for i in range(len(self.x1)):
                    f.write('{}\t{}\t{}\n'.format(self.x1[i]*1000, self.z1[i]*1000, self.y1[i]*1000) )

            with open( self.fileoutICEM + '.1.txt', 'w') as f:
                f.write('{}\t{}\n'.format(len(self.x2), 1))
                for i in range(len(self.x2)):
                    f.write('{}\t{}\t{}\n'.format(self.x2[i]*1000, self.z2[i]*1000, self.y2[i]*1000) )

    def saveSW(self, airfoilfile):
        """ saves points in sw format, either as a single curve of splits to upper and lower (recommended) """

        if not self.split:
            self.zs = np.ones(len(self.x))*self.z
            airfoilfile += '.txt'
            with open( airfoilfile, 'w') as f:
                for i in range(len(self.x)):
                    f.write('{}\t{}\t{}\n'.format(self.x[i]*1000, self.zs[i]*1000, self.y[i]*1000) )
        else:
            self.z1 = np.ones(len(self.x1))*self.z
            self.z2 = np.ones(len(self.x2))*self.z

            with open( airfoilfile + '.0.txt', 'w') as f:
                for i in range(len(self.x1)):
                    f.write('{}\t{}\t{}\n'.format(self.x1[i]*1000, self.z1[i]*1000, self.y1[i]*1000) )

            with open( airfoilfile + '.1.txt', 'w') as f:
                for i in range(len(self.x2)):
                    f.write('{}\t{}\t{}\n'.format(self.x2[i]*1000, self.z2[i]*1000, self.y2[i]*1000) )

###
###   ===================        ANALYSIS SECTION           ==========================
###

    def runXFOIL(self, cl=.2, alfa = None, re=1e6, m =.2, n_crit = 6, iters = 500, cp = False):
        self.saveXFOIL()
        airfoilIN = self.filebuffer

        if alfa is None:
            S = cl
            s = 'cl'
            if self.verbose:
                print('running XFOIL for: cl={}'.format(cl))
        else:
            S = alfa
            s = 'a'
            if self.verbose:
                print('running XFOIL for: aoa={}'.format(alfa))
        if not cp:
            commands = 'load ' + airfoilIN + '\npane\noper\nvpar\nn {}\n\nvisc {}'.format(n_crit, re) + '\niter '+str(iters)+'\n{} {}'.format(s, S) + '\n\nquit\n'
            p = run([self.xfoilpath], stdout=PIPE,
                    input=commands, encoding='ascii')

        else:
            commands = 'load ' + airfoilIN + '\npane\noper\nvpar\nn {}\n\nvisc {}'.format(n_crit, re) + '\niter '+str(iters)+'\n{} {} '.format(s, S) + '\ncpwr\n{}\n\nquit\n'.format(self.filecp)
            p = run([self.xfoilpath], stdout=PIPE,
                    input=commands, encoding='ascii')
            return 0

        try:
            alfa = float(p.stdout[-130:-118])
            Cl = float(p.stdout[-112:-106])
            Cd = float(p.stdout[-78:-69])
            Cm = float(p.stdout[-94:-86])

            print(alfa,Cl,Cd,Cm)
        except ValueError:
            if self.verbose:
                print('error running xfoil, try slighlty different cl/alpha') # the reason is xfoil may not converge for this particular condition but in general it converges
            if alfa is None:
                alfa, Cd, Cm, Cl = self.runXFOIL(cl = 1.01*cl, re = re, m = m, n_crit = n_crit, iters = iters)
            else:
                alfa, Cd, Cm, Cl = self.runXFOIL(alfa = .01+alfa, re = re, m = m, n_crit = n_crit, iters = iters)

            #return 1, 1, 1, 1

        return alfa, Cd, Cm, Cl

    def runPolar(self, a0=-4, a1=8, re=1e6, m=.2, n_crit = 6, plot = False):
        alfas = np.zeros(a1-a0)
        cds = np.zeros(a1-a0)
        cls = np.zeros(a1-a0)
        cms = np.zeros(a1-a0)
        i=0
        for aoa in np.arange(a0,a1,1):
            alfas[i], cds[i], cms[i], cls[i] = self.runXFOIL(alfa = aoa, re= re, m = m, n_crit = n_crit)
            i+=1

        alfas = np.delete(alfas, np.where(cds == 1))
        print (alfas)

        if plot:
            plt.figure()
            plt.plot(alfas, cls, 'o-')
            plt.xlabel(r'$ \alpha [^\circ]$')
            plt.ylabel(r'$C_L$')
            plt.show()
            plt.figure()
            plt.plot(alfas, cds, 'o-')
            plt.xlabel(r'$ \alpha[^\circ]$')
            plt.ylabel(r'$C_D$')
            plt.show()

        return alfas, cds, cms, cls

    def plotCp(self, outputtex = False, dpi = 200, name = None, saveFig = False, airfoil= True, alfa = None):
        X = np.loadtxt(self.filecp, skiprows = 3)
        x = X[:,0]
        cp = X[:,2]
        if outputtex:
            np.savetxt(self.fileCptex, X)

        if name is None:
            if alfa is not None:
                name = '$C_p$ distribution at $\alpha = {}$'.format(alfa)
            else:
                name = '$C_p$ distribution'

        plt.figure(figsize = (6,4),dpi = dpi)
        plt.rc('text', usetex=True)
        plt.rc('font', family='serif')

        plt.plot(x, -cp, 'k-',linewidth = 1)

        if airfoil:
            plt.plot(self.x/self.chord, self.y/self.chord*3-np.max(cp), 'k-',linewidth = 1)

        plt.xlabel(r'$x/c$',fontsize=12)
        plt.ylabel(r'$-C_p$',fontsize=12)
        plt.title(r"{}".format(name), fontsize=12)
        plt.subplots_adjust(top=0.8)
#        plt.axis('equal')
        plt.grid(which='major', linewidth = 0.2)
        plt.tight_layout()
#        plt.grid(True)
        if saveFig:
            plt.savefig(self.fileFig, dpi = 1000)
        plt.show()

    def runFluent(self, alfa, mach, chord,
                  rho = 1.225, T = 300, viscosity = 1.78e-5,
                  name = 'airfoil', path = None, ID = 0,
                  mesh = 'o', y1 = 0.01, n_r = 120, n_le = 30, n_top = 120,
                  model = 'kw-sst', intermittency = False, lowre = False, polar = False,
                  onlymesh = False, onlyfluent = False, mshin = None, meshunits = 'mm',
                  tt = 1, farfieldnames = ['farfield'], outletnames = [], interiornames = ['int_fluid']
                  ):
        """
        chord used to scale mesh in fluent and use for coefficients
        if using auto o-mesh, generate airfoil with unit chord and scale mesh to required value

        static method: can be applied for given mesh, without airfoil initialization

        """
        if path is None:
            path = self.workingdir + r'\fluent'

        import time
        start = time.time()
        # begin with structured mesh generation
        #
        import subprocess
        def subprocess_cmd(command):
            process = subprocess.Popen(command,stdout=subprocess.PIPE, shell=True)
            proc_stdout = process.communicate()[0].strip()
            #    print(proc_stdout)
            return proc_stdout

        if not onlyfluent:
            self.saveICEM(self.fileoutICEM)
            ICEMrun ='"C:\\Program Files\\ANSYS Inc\\v194\\icemcfd\\win64_amd\\bin\\icemcfd" -script'

                # pick mesh replay file to generate mesh
            if mesh == 'o':
                meshrpl = self.meshDir + 'omesh.rpl'
    #            ICEMscr = r'"E:\propeller\python\wing3d\rpl42.rpl"'
    #            ICEMscr = r'"E:\propeller\python\wing3d\omesh\omesh.rpl"'
                ICEMscr = f'"{meshrpl}"'
            elif mesh == 'unstructured':
                ICEMscr = r'"C:\Users\wk5521\Documents\ICEM\airfoil replays\mesh_output.rpl"'

            generateICEMScheme( y1 = y1, n_r = n_r, n_le = n_le, n_top = n_top, file = meshrpl)

            ICEM = ICEMrun + ' ' + ICEMscr
            subprocess_cmd(ICEM)

        # now having the mesh, run shceme generation, hence fluent
        if onlymesh:
            print('finished mesh')
            return 0

        fluentjournal = self.workingdir + '/fluent/journal.txt'

        casename = f'foil,{model},{alfa},{mach},{chord},{self.t}'
        if polar:
            casename = f'foil,{model},{mach},{chord},{self.t}'

        if lowre:
            casename+=',lowre'
        if intermittency:
            casename+= 'inter'

        meshin = mshin if mshin is not None else self.meshin

        generateScheme(filename = fluentjournal,
           casename = casename,
           chord = chord,
           viscosity = viscosity,
           T=T,
           alfa = alfa,
           mach = mach,
           meshin = meshin,
           meshunits = meshunits,
           farfieldnames = farfieldnames,
           outletnames = outletnames,
           interiornames = interiornames,
           path = self.fluentdir,
           model = model,
           intermittency = intermittency,
           lowre = lowre,
           polar = polar,
           tt =tt
           )

        FLUENTrun = '"C:\\Program Files\\ANSYS Inc\\v194\\fluent\\ntbin\\win64\\fluent.exe" 2d -t8 -wait -i'
        FLUENT  = FLUENTrun + ' '+ '"{}"'.format(fluentjournal)

        subprocess_cmd(FLUENT)
        end = time.time()

        showresult = False
        if showresult:
            result = np.loadtxt('{}/reports/{}.out'.format(self.fluentdir, casename), skiprows = 100)
            result = result[-10:]
            result = np.mean(result, axis = 0)
            lift = result[1]
            drag = result[2]
            moment = result[3]
            duration = end - start
            print('mesh size: {}, lift: {:.4f}, drag: {:.6f}, duration: {}'.format(2*(n_le+n_top)*n_r , lift , drag , duration))
            return 2*(n_le+n_top)*n_r , lift , drag

    def splitCurve(self):
        """ splits curve into two curves at leading edge by front-most point """
        i_min = np.where(self.x == np.amin(self.x))[0][0]
        self.split = True
        self.x1 = self.x[:i_min+1]
        self.y1 = self.y[:i_min+1]
        self.x2 = self.x[i_min:]
        self.y2 = self.y[i_min:]
        self.z1 = np.ones(len(self.x1))*self.z
        self.z2 = np.ones(len(self.x2))*self.z

    def qpropData(self, m, re, n = 12, n_crit = 5):
        """ this method finds coefficients required to define qprop input file
        returns (cl0, clalfa, cd0, clcd0, cd2u, cd2l)
        """
#        collect some data for range of angles of attack
        alfas = np.zeros(n)
        cds = np.zeros(n)
        cms = np.zeros(n)
        cls = np.zeros(n)

        j = 0
        for i in range(-6, 6, 1):
            self.cutTE_XFOIL(t = 0.005, r = .3)
            alfas[j], cds[j], cms[j], cls[j] = self.runXFOIL(alfa = i, re = re, m = m, iters = 1000, n_crit = n_crit)
            j+=1

        cl0 = cls[6]
        clalfa = (cls[-1] - cls[4]) / np.radians(7)

        # now begin drag section
        from scipy.optimize import minimize, fmin
        cd0 = cds.min()

        for index in range(len(cds)):
            if cds[index] == cd0:
                break
        clcd0 = cls[index]

        def merit(x):
            # args = (cd0, )
            merit = np.abs(  cds[index + 1] + cds[index + 2] - (cd0 + x * (cls[index + 1] - clcd0 )**2 + cd0 + x * ( cls[index + 2] - clcd0 )**2 ) )
            return  merit

        result = fmin(merit,  .1)
        cd2u = result[0]

        def merit2(x, *args):
            return  np.abs(  cds[args[0] - 1] + cds[args[0] - 2] - (cd0 + x * (cls[args[0] - 1] - clcd0 )**2 + cd0 + x * ( cls[args[0] - 2] - clcd0 )**2 ) )

        result2 = minimize(merit2, .05, args = (index))
        cd2l = result2.x[0]

        print('cl0, clalfa, cd0, clcd0 = {:.3f} {:.3f} {:.3f} {:.3f}'.format( cl0, clalfa, cd0, clcd0))
        return cl0, clalfa, cd0, clcd0, cd2u, cd2l

    def plotAirfoil(self, name=None, saveFig = False, dpi = 200, tex = False , nametex = ''):

        if name is None:
            name = 'airfoil'

        plt.figure(figsize = (6,2),dpi = dpi)
        plt.rc('text', usetex=True)
        plt.rc('font', family='serif')
        if self.split:
            plt.plot(self.x1/self.chord, self.y1/self.chord, 'k-',linewidth = 1.2)
            plt.plot(self.x2/self.chord, self.y2/self.chord, 'k-',linewidth = 1.2)
        else:
            plt.plot(self.x/self.chord, self.y/self.chord, 'k-',linewidth = 1.2)
        plt.xlabel(r'$x/c$',fontsize=12)
        plt.ylabel(r'$y/c$',fontsize=12)
        plt.title(r"{}".format(name), fontsize=12)
        plt.subplots_adjust(top=0.8)
        plt.axis('equal')
        plt.grid(which='major', linewidth = 0.2)
        plt.tight_layout()
        if saveFig:
            plt.savefig(self.fileFig, dpi = 1000)
        plt.show()

        if tex:
            X = np.append((self.x/self.chord).reshape(-1,1), (self.y/self.chord).reshape(-1,1), axis = 1 )
            savedir = self.workingdir + r'\wing3d\tex-plots\{}airfoil.txt'.format(nametex)
            np.savetxt(savedir, X)


from django.urls import path
from django.conf.urls import include
from django.contrib import admin
from app.accounts.api.v1.views import (
    UserCreatView,
    UserUpdateView,
    GetAuthToken,
    AvatarAPIView,
    ClubAPIView,
)

app_name = 'accounts'

urlpatterns = [
    path('login/', GetAuthToken.as_view(),              name='user-login'),
    path('create/', UserCreatView.as_view(),            name='user-create'),
    path('update/<int:pk>/', UserUpdateView.as_view(),  name='user-update'),
    path('avatar/<int:pk>/', AvatarAPIView.as_view(),   name='user-avatar'),

    path('club/', ClubAPIView.as_view(),                name='club-details')

]