"""
将SQL翻译成pandas的操作
SQL格式(类MySQL)：

update table_1 a
   set a.name=b.name,
       a.age=a.score+3,
       a.level=my_func(b.level, a.id)
  left join table_2 b    -- 使用join 或left join关联
    on a.id=b.id         -- 翻译成 pandas_merge 的 left_on 和 right_on
 where a.age>3              -- 支持比较操作
   and b.score<=3           -- 支持比较操作
   and a.score>b.score      -- 支持两边对比操作
   and a.id is in (1,2,3)   -- 支持 isin 操作
   and b.id not in (1,2,3)  -- 支持 not in 操作
   and a.score <> 100       -- 支持不等于操作
   and a.id is not null     -- 支持 not null
   and b.level is null      -- 支持 is null
支持操作：
1、带where条件的单表update
2、带where条件的两个表update
3、自定义python函数

不支持的操作：
1、like，比如 string like '%abc%
4、SQL中对字段进行字符串操作，比如 replace(a.name,'小明','小红')

基本假设：
1、SQL中没有对 '\n' 的操作，比如替换，因为在解析SQL时会将换行删除
2、只支持2个表的join操作，不支持3个及以上表关联
3、不对字符字段进行操作，即操作涉及到的列没有字符型，但是支持 =,!=,isin, not in
4、不支持case when 等复杂语句

"""
import re
import sqlparse
import pandas as pd

def judge_format(sql, table_1_cols, table_2_cols):
    """sql校验"""
    # 去掉换行符,多个空格只保留1个
    sql = re.compile(r'\s{2,100}', flags=re.I | re.S).subn(' ', sql.replace('\n', ' '))[0].strip()
    # 判断是否有表别名
    # update table_1 a set xxx
    tb1 = re.compile(r"update (.+?) (.+?) set", flags=re.I | re.S).findall(sql)
    if len(tb1) == 0:
        raise Exception('update表没有表别名，请检查')
    # left join table_2 b on
    if ' left join ' in sql and ' on ' in sql:
        tb2 = re.compile(r" left join (.+?) (.+?) on", flags=re.I | re.S).findall(sql)
        if len(tb2) == 0:
            raise Exception('join表没有表别名，请检查')
    elif ' join ' in sql and ' on ' in sql:
        tb2 = re.compile(r" join (.+?) (.+?) on", flags=re.I | re.S).findall(sql)
        if len(tb2) == 0:
            raise Exception('join表没有表别名，请检查')
    # 不应该有as
    if ' as ' in sql:
        raise Exception('表别名不需要as关键字，请删除')
    # 没有where的不需要SQL
    if ' where ' not in sql:
        raise Exception('没有where的sql用pandas就好了，不需要SQL这么折腾')
    # 判断是否有大写字母
    for s in sql:
        if s.isupper():
            raise Exception('SQL应该是全小写的，但是你传进来的SQL出现了大写字母')
    # 检查字段名是否带表别名
    sql_p = list(sqlparse.parse(sql)[0].flatten())
    for i in range(1, len(sql_p), 1):
        first = sql_p[i - 1]
        second = sql_p[i]
        if second._get_repr_name()=='Name' and second.value.strip() in table_1_cols and first.value!='.':
            raise Exception('字段名没有表别名，请检查，写法应该是：table_1_别名.%s'%second.value.strip())
        if second._get_repr_name()=='Name' and second.value.strip() in table_2_cols and first.value!='.':
            raise Exception('字段名没有表别名，请检查，写法应该是：table_2_别名.%s' % second.value.strip())
    return

def get_all_columns(sql):
    """解析获取所有涉及的字段名"""
    sql_p = list(sqlparse.parse(sql)[0].flatten())
    all_cols = []
    for i in range(2, len(sql_p), 1):
        first = sql_p[i - 2]
        second = sql_p[i - 1]
        third = sql_p[i]
        if first._get_repr_name() == 'Name' \
                and second._get_repr_name() == 'Punctuation' \
                and second.value == '.' \
                and third._get_repr_name() == 'Name':
            all_cols.append('%s.%s' % (first.value.strip(), third.value.strip()))
    return set(all_cols)

def tb_condition_cols_parser(sql):
    """
    从SQL中提取表名，别名，条件，涉及到的字段.
    locals是上层模块的globals，包含所有的对象，是字典类型
    """
    # 去掉换行符,多个空格只保留1个
    sql = re.compile(r'\s{2,100}', flags=re.I | re.S).subn(' ', sql.replace('\n', ' '))[0].strip()
    # 取update的表名和表别名
    tb1 = re.compile(r"update (.+?) (.+?)set", flags=re.I | re.S).findall(sql)[0]
    tb1, tb1_a = tb1[0].strip(), tb1[1].strip()
    # 提取join的表名和表别名
    if ' left join ' in sql:
        tb2 = re.compile(r"left join (.+?) (.+?)on", flags=re.I | re.S).findall(sql)[0]
        tb2, tb2_a = tb2[0].strip(), tb2[1].strip()
        join_type = 'left'
    elif ' join ' in sql:
        tb2 = re.compile(r"join (.+?) (.+?)on", flags=re.I | re.S).findall(sql)[0]
        tb2, tb2_a = tb2[0].strip(), tb2[1].strip()
        join_type = 'inner'
    else:
        tb2, tb2_a = '', ''
        join_type = ''
    # 提取set的内容
    if ' join ' not in sql:
        set_sql = re.compile(r" set(.+?) where ", flags=re.I | re.S).findall(sql)[0].strip()
    elif ' left join ' in sql:
        set_sql = re.compile(r" set(.+?) left join ", flags=re.I | re.S).findall(sql)[0].strip()
    elif ' join ' in sql:
        set_sql = re.compile(r" set(.+?) join ", flags=re.I | re.S).findall(sql)[0].strip()
    else:
        set_sql = ''
    # on 内容
    if ' join ' in sql:
        on_sql = re.compile(r" on(.+?) where ", flags=re.I | re.S).findall(sql)[0].strip()
    else:
        on_sql = ''
    # 提取where条件
    where_sql = re.compile(r" where(.*)", flags=re.I | re.S).findall(sql)[0].strip()
    # list(sqlparse.parse(where_sql)[0].get_sublists())
    # 提取a表涉及的所有列
    all_cols = get_all_columns(sql)
    tb1_cols = [col for col in all_cols if col.startswith(tb1_a + '.')]
    tb2_cols = [col for col in all_cols if col.startswith(tb2_a + '.')]
    # 组装成一个字典返回
    sql_info = {'tb1': tb1, 'tb1_a': tb1_a,
                'tb2': tb2, 'tb2_a': tb2_a,
                'join_type': join_type,
                'on_sql': on_sql,
                'set_sql': set_sql, 'where_sql': where_sql,
                'tb1_cols': tb1_cols, 'tb2_cols': tb2_cols}
    return sql_info

def tb1_join_tb2_parser(sql_info):
    """将涉及到的tb2字段全部join到tb1中，用左联接"""
    # 如果没有table2，返回空
    if not sql_info['tb2']:
        return {}
    # 取出带别名的tb1,tb2字段名
    tb1_cols = sql_info['tb1_cols']
    tb2_cols = sql_info['tb2_cols']

    # list的任意一个元素是否在string中的函数
    def list_in_string(string, list):
        for item in list:
            if item in string:
                return item

    # 找到两个表的关联字段
    left_on = []
    right_on = []
    for on_sql_sub in sql_info['on_sql'].split(' and '):
        # item=sql_info['on_sql'].split(' and ')[0]
        left_on.append(list_in_string(on_sql_sub, tb1_cols))
        right_on.append(list_in_string(on_sql_sub, tb2_cols))
    join_parser = {
        'left_on': left_on,
        'right_on': right_on
    }
    return join_parser

def tb1_filter_and_update_parser(sql_info):
    """
    前面已经将tb2 join 到 tb1 中，下面就是执行 where 和 set 操作了
    """
    # 取出set和where语句
    set_sql = sql_info['set_sql']
    where_sql = sql_info['where_sql']
    # 接下来就是转换 where_sql，使之符合pandas的语法
    where_sql_t = ''
    sub = ''
    for item in sqlparse.parse(where_sql)[0].flatten():
        item = item.value
        # 一个等号改2个等号
        if item == '=':
            sub += '=='
        elif item not in ('and', 'or'):  # 如果是断句，就应该小心了
            sub += item
        elif item in ('and', 'or'):
            # 处理and or断句
            if ' not in ' in sub:
                sub = '-' + sub
            else:
                sub = sub
            where_sql_t += sub
            where_sql_t += item
            sub = ''
    # 最后一个也放进去然后转回去
    if ' not in ' in sub:
        sub = ' -' + sub
    where_sql_t += sub
    where_sql = where_sql_t
    # in和not in，注意替换的顺序
    where_sql = where_sql.replace(' not in ', '.isin').replace(' in ', '.isin')
    # 在isin外面在嵌套一层括号
    where_sql_t = ''
    isin = False
    left_kh = False
    right_kh = False
    for item in sqlparse.parse(where_sql)[0].flatten():
        item = item.value
        if item == 'isin':
            isin = True
        if item == '(' and isin == True:
            item = '(('
            left_kh = True  # 已经提取左括号，标记
        elif item == ')' and isin == True and left_kh == True:
            item = '))'
            isin = False  # 将isin标记
            left_kh = False  # 将左括号标记
        where_sql_t += item
    # and or 转 & |
    where_sql_t = where_sql_t.replace(' and ', ') & (').replace(' or ', ') | (')
    where_sql = '(' + where_sql_t + ')'
    # null 和 notnull
    if ' is null ' in where_sql:
        where_sql = where_sql.replace(' is null ', '.isnull()')
    if ' is not null ' in where_sql:
        where_sql = where_sql.replace(' is not null ', '.notnull()')
    # <> 转 !=
    where_sql_t = ''
    for item in sqlparse.parse(where_sql)[0].flatten():
        if item._get_repr_name() == 'Comparison' and item.value.strip() == '<>':
            where_sql_t += '!='
        else:
            where_sql_t += item.value.strip()
    where_sql = where_sql_t
    # 将 a.id 写成 df['id'] 的格式
    tb1_cols = sql_info['tb1_cols']  # tb1涉及到的字段名，包含表别名
    tb2_cols = sql_info['tb2_cols']
    tb1, tb1_a = sql_info['tb1'], sql_info['tb1_a']
    for col in tb1_cols:
        col1 = col  # 此时已经包含表别名和字段名
        col2 = "%s['%s']" % (tb1, col.split('.')[-1])
        where_sql = where_sql.replace(col1, col2)
    # 需要注意的是，tb2的字段名需要增加 t_tmp_ 前缀标志
    # 注意这里是tb1，因为tb2已经join到tb1了
    for col in tb2_cols:
        col1 = col  # 此时已经包含表别名和字段名
        col2 = "%s['b_tmp_%s']" % (tb1, col.split('.')[-1])
        where_sql = where_sql.replace(col1, col2)
    # 对于set_sql要麻烦一些，要先分割，然后在替换
    # set_sql='a.age=b.b_tmp_age2+2, a.score=func(a.age, b.b_tmp_level2)'
    # set_sql = 'a.age=b.b_tmp_age2+2'
    set_sql_t = sqlparse.parse(set_sql)[0].tokens[0]
    if set_sql_t._get_repr_name() == 'Comparison':
        set_sql = [set_sql_t.value]
    if set_sql_t._get_repr_name() == 'IdentifierList':
        set_sql = [t.value for t in set_sql_t.get_sublists()]
    # 表名替换
    set_sqls2 = []
    for set_t in set_sql:
        for col in tb1_cols:
            col1 = col
            col2 = "%s.loc[wz, '%s']" % (tb1, col.split('.')[-1])
            set_t = set_t.replace(col1, col2)
        for col in tb2_cols:
            col1 = col
            # 注意，这里tb用tb1，col是tb2的，而且要加上 b_tmp_ 前缀
            col2 = "%s.loc[wz, 'b_tmp_%s']" % (tb1, col.split('.')[-1])
            set_t = set_t.replace(col1, col2)
        set_sqls2.append(set_t)
    set_sqls = set_sqls2
    #
    update_parser = {
        'where_sql': where_sql,
        'set_sqls': set_sqls  # 是一个列表
    }
    return update_parser

def update_tb2_column_name_then_join(sql_info, join_parser, table_1, table_2):
    """更新B表字段名, 然后A表关联B表，对关联后的结果校验"""
    # 重命名B表所有涉及的列
    tb2_cols_rename = []  # 其他涉及到的，不是关联key字段也要重命名
    for col in sql_info['tb2_cols']:
        col1 = col.split('.')[-1]
        col2 = 'b_tmp_' + col1
        if col1 not in table_2.columns:
            raise Exception('在%s中找不到%s字段' % (sql_info['tb2'], col1))
        table_2.rename(columns={col1: col2}, inplace=True)
        tb2_cols_rename.append(col2)
    # 重命名join on的列
    tb1_join_col = [col.split('.')[-1] for col in join_parser['left_on']]  # tb1去掉表名
    tb2_join_col = ['b_tmp_%s' % col.split('.')[-1] for col in join_parser['right_on']]  # tb2去表名再加上tmp标志
    # 关联
    len1 = len(table_1)
    table_1 = pd.merge(left=table_1,
                       right=table_2[tb2_cols_rename],
                       left_on=tb1_join_col,
                       right_on=tb2_join_col,
                       how=sql_info['join_type'])
    # 检查前后数据量变化
    len2 = len(table_1)
    print('注意，关联后数据量变化 %d --> %d' % (len1, len2)) if len1 != len2 else None
    # 检查数据类型的变化
    for col in [col for col in table_2.columns if col.startswith('b_tmp_')]:
        dtype_new = table_1.dtypes[col]
        dtype_old = table_2.dtypes[col]
        if dtype_old != dtype_new:
            print('注意，左右表关联后，原来的列 %s 的数据类型 %s -- > %s' % (col, str(dtype_old), str(dtype_new)))
    # 重命名B表所有涉及的列回原来的列名
    for col in tb2_cols_rename:
        col1 = col
        col2 = col.replace('b_tmp_', '')
        table_2.rename(columns={col1: col2}, inplace=True)
    # 最后返回
    return table_1

def update_tb1(sql_info, update_parser, table_1, func1, func2, func3):
    """根据update_parser更新table1"""
    exec('wz = ' + update_parser['where_sql'])
    for set_sql in update_parser['set_sqls']:
        exec(set_sql)
    return table_1

def delete_tb1_that_tmp_column(table_1):
    # 删除以b_tmp_开头的临时列
    for col in table_1.columns:
        if col.startswith('b_tmp_'):
            del table_1[col]
    return table_1

def sql_update_pd(sql, table_1=None, table_2=None, func1=None, func2=None, func3=None):
    """主函数"""
    # 第一步，判断SQL是否正确
    table_1_cols = table_1.columns.tolist()
    table_2_cols = table_2.columns.tolist() if isinstance(table_2, pd.DataFrame) else []
    judge_format(sql, table_1_cols, table_2_cols)
    # 第二步，解析SQL，同时再次判断SQL语法是否正确
    sql_info = tb_condition_cols_parser(sql)
    # 第三步，join关联解析
    join_parser = tb1_join_tb2_parser(sql_info)
    # 第四步，update解析
    update_parser = tb1_filter_and_update_parser(sql_info)
    # 第五步，更新B表的字段名
    # 第六步，A join B
    # 第七步，将B表的列名改回原来的
    if sql_info['tb2']:
        table_1 = update_tb2_column_name_then_join(sql_info, join_parser, table_1=table_1, table_2=table_2)
    # 第八步，update A
    table_1 = update_tb1(sql_info, update_parser, table_1 = table_1, func1=func1, func2=func2, func3=func3)
    # 第9步，删除A中临时列
    table_1 = delete_tb1_that_tmp_column(table_1 = table_1)
    # 第10步，返回A
    return table_1

def test():
    # 构造测试数据
    df1 = pd.DataFrame(data={'id': [1, 2, 3, 4], 'age': [2, 3, 4, 5], 'score': [3, 4, 7, 6], 'heigh': [5, 6, 8, 7]})
    df2 = pd.DataFrame(data={'id': [1, 2, 3, 4], 'age2': [3, 4, 5, 6], 'score2': [5, 6, 7, 7]})
    print(df1)
    print(df2)

    # 测试语法是否正确
    sql = """ update table_1 set age=id+1 """  # 没有表别名
    sql = """ update table_1 as a set age=id+1 """  # 不应该用as
    sql = """ update table_1 a set a.age = a.id + a.score"""  # 没有where条件，就没必要搞SQL这么麻烦了
    judge_format(sql)

    # 正常是写法，字段名也应该带表别名
    sql = """ update table_1 a set a.age = a.id + a.score where a.heigh>=5"""
    judge_format(sql)

    # ------------------------------------------------------------
    # 单表更新
    sql = "update table_1 a set a.age=a.id*2 where a.id>=3"
    sql_update_pd(sql, table_1 = df1)
    # ------------------------------------------------------------
    # 联合更新
    sql = "update table_1 a " \
          "set a.age=a.id+b.age2+1 " \
          "left join table_2 b " \
          "on a.id=b.id " \
          "where a.age>2 and b.age2<6"  # 更新中间两行
    new_df_1 = sql_update_pd(sql, table_1=df1, table_2=df2)

    # ------------------------------------------------------------
    # 使用自定义python函数
    def add(a, b):
        return a * b
    def add2(a,b):
        return a-b

    df1['t']=0
    sql = "update table_1 a set a.score=add(a.id, a.age), a.t = add2(a.age, a.heigh) where a.id>=3"
    new_df_1 = sql_update_pd(sql, table_1=df1, func1=add, func2=add2)

    # ------------------------------------------------------------
    # 多字段更新
    sql = "update table_1 a set a.score=a.id+2, a.heigh=a.age+2 where a.id>=3"
    new_df_1 = sql_update_pd(sql, table_1=df1)


from collections import namedtuple
from importlib.resources import Resource

from rdflib import BNode, Graph, OWL, URIRef, Literal, RDF, RDFS

from compare.compare_code.owl_helpers.restriction_classifier import get_restricting_property, \
    get_restricting_type_and_class, \
    get_restriction_modality_and_cardinality_value

RestrictionTuple = namedtuple('OntologicalType', 'restricting_property restriction_type restricted_class restriction_modality restricted_cardinality')

def identify_restriction(restriction: BNode, ontology: Graph) -> tuple:
    restricting_property = get_restricting_property(restriction=restriction, ontology=ontology)
    restricting_type_and_class = get_restricting_type_and_class(restriction=restriction, ontology=ontology)
    restriction_modality_and_cardinality_value = get_restriction_modality_and_cardinality_value(restriction=restriction, ontology=ontology)

    restriction_tuple = \
        RestrictionTuple(
            restricting_property=restricting_property,
            restriction_type=restricting_type_and_class[0],
            restricted_class=restricting_type_and_class[1],
            restriction_modality=restriction_modality_and_cardinality_value[0],
            restricted_cardinality=restriction_modality_and_cardinality_value[1])

    return restriction_tuple

def identify_datatype_restriction(datatype_restriction: BNode, ontology: Graph) -> tuple:
    restricted_datatypes = list(ontology.objects(subject=datatype_restriction, predicate=OWL.onDatatype))
    for restricted_datatype in restricted_datatypes:
        restricting_restriction = list(ontology.objects(subject=datatype_restriction, predicate=OWL.withRestrictions))
        identified_restricting_restriction = identify_resource(resource=restricting_restriction[0], ontology=ontology)
        return tuple([restricted_datatype, identified_restricting_restriction])
    resource_unions_of = list(ontology.objects(subject=datatype_restriction, predicate=OWL.unionOf))
    if len(resource_unions_of) == 1:
        identified_resource = get_listed_resources(rdf_list_object=resource_unions_of[0], ontology=ontology, rdf_list=list())
        return tuple(identified_resource)
    else:
        identified_resource = identify_collection(resource_unions_of, ontology, OWL.unionOf)
        return identified_resource

def identify_collection(collections: list, ontology: Graph, collection_type) -> tuple:
    identified_collections = list()
    for collection in collections:
        if not isinstance(collection, list):
            items = ontology.items(collection)
        identified_items = list()
        for item in items:
            identified_item = identify_resource(item, ontology)
            identified_items.append(identified_item)
        identified_collection = tuple([collection_type] + identified_items)
        identified_collections.append(identified_collection)
    return tuple(identified_collections)

def identify_resource(resource: Resource, ontology: Graph) -> object:
    identified_resource = None

    if isinstance(resource, URIRef) or isinstance(resource, Literal):
        return resource

    resource_types = set(ontology.objects(subject=resource, predicate=RDF.type))

    if OWL.Restriction in resource_types:
        identified_resource = identify_restriction(restriction=resource, ontology=ontology)
        return identified_resource

    if RDFS.Datatype in resource_types:
        identified_resource = identify_datatype_restriction(datatype_restriction=resource, ontology=ontology)
        return identified_resource

    resource_ones_of = set(ontology.objects(subject=resource, predicate=OWL.oneOf))
    if len(resource_ones_of) > 0:
        identified_resource = identify_collection(resource_ones_of, ontology, OWL.oneOf)
        return identified_resource

    resource_unions_of = set(ontology.objects(subject=resource, predicate=OWL.unionOf))
    if len(resource_unions_of) > 0:
        identified_resource = identify_collection(resource_unions_of, ontology, OWL.unionOf)
        return identified_resource

    resource_intersections_of = set(ontology.objects(subject=resource, predicate=OWL.intersectionOf))
    if len(resource_intersections_of) > 0:
        identified_resource = identify_collection(resource_intersections_of, ontology, OWL.intersectionOf)
        return identified_resource

    resource_complements_of = set(ontology.objects(subject=resource, predicate=OWL.complementOf))
    if len(resource_complements_of) > 0:
        identified_resource = identify_collection(resource_complements_of, ontology, OWL.complementOf)
        return identified_resource

    resource_subject_predicates = list(ontology.subject_predicates(object=resource))
    if len(resource_subject_predicates) > 0:
        for resource_subject_predicate in resource_subject_predicates:
            if resource_subject_predicate[1] == OWL.propertyChainAxiom:
                chained_properties = get_listed_resources(rdf_list_object=resource, ontology=ontology, rdf_list=list())
                return chained_properties
            if resource_subject_predicate[1] == OWL.withRestrictions:
                datatype_restrictions = get_listed_resources(rdf_list_object=resource, ontology=ontology, rdf_list=list())
                identified_resource = identify_collection(datatype_restrictions, ontology, OWL.withRestrictions)
                return identified_resource

    return identified_resource

def get_listed_resources(rdf_list_object: Resource, ontology: Graph, rdf_list: list) -> list:
    first_items_in_rdf_list = list(ontology.objects(subject=rdf_list_object, predicate=RDF.first))
    if len(first_items_in_rdf_list) == 0:
        return rdf_list
    rdf_list.append(first_items_in_rdf_list[0])
    rest_items_in_rdf_list = list(ontology.objects(subject=rdf_list_object, predicate=RDF.rest))
    rdf_list = get_listed_resources(rdf_list_object=rest_items_in_rdf_list[0], ontology=ontology, rdf_list=rdf_list)
    return rdf_list


import logging
import numpy as np

import torch

from pathlib import Path
from typing import Dict, Union, List, Callable

from dc_framework.data_preparation import Dataset

logger = logging.getLogger("__name__")

def init(model: torch.nn.Module, criterion: torch.nn.Module, metrics: Dict[str, Callable] = None):
    return DCFramework(model, criterion, metrics)

class DCFramework:
    def __init__(self, model: torch.nn.Module, criterion: torch.nn.Module, metrics: Dict[str, Callable] = None, lr=1e-3):
        self.model = model
        self.optimizer = torch.optim.SGD(model.parameters(), lr=lr)
        self.criterion = criterion
        if metrics:
            self.metrics = torch.nn.ModuleDict({name:func for name, func in metrics})
        else:
            self.metrics = torch.nn.ModuleDict({"mae":torch.nn.L1Loss()})

        #check if model is already on cuda
        self.on_cuda = False
        if next(model.parameters()).is_cuda:
            self.on_cuda = True

    def forward(self, feature, target):
        try:
            if self.on_cuda:
                feature = feature.cuda()
            output = self.model(feature)
        except:
            logger.warning(f"feature: {feature}")
            raise
        try:
            if self.on_cuda:
                output = output.cuda()
                target = target.cuda()

            loss = self.criterion(output, target)
            results = {}
            for name, metric in self.metrics.items():
                results[name] = metric(output, target)
            results['output'] = output
            results['loss']   = loss
        except:
            logger.warning(f"output: {output}")
            logger.warning(f"target: {target}")
            raise
        return results

    def train(self, train_data: Dict[str, np.array], val_data: Union[Dict[str, np.array], None] = None, batch_size: int = 1, epochs: int = 2):

        train_dataloader = self._prepare_loader(train_data, batch_size=batch_size)
        if val_data:
            val_dataloader = self._prepare_loader(val_data, batch_size=batch_size)

        for i in range(epochs):

            self._train_loop(train_dataloader)

            if val_data:
                self._val_loop(val_dataloader) #torch.no_grad is incorporated into the function

    @torch.no_grad()
    def val(self, val_data: Dict[str, np.array], batch_size: int = 1):
        val_dataloader = self._prepare_loader(val_data, batch_size=batch_size)
        self._val_loop(val_dataloader)

    def _train_loop(self, loader):
        for batch in loader:
            output = self.forward(*batch)
            loss = output["loss"]
            loss.backward()
            self.optimizer.step()

    @torch.no_grad()
    def _val_loop(self, loader):
        for batch in loader:
            output = self.forward(*batch)
            loss = output["loss"]

    def _prepare_loader(self, data: Dict[str, np.array], batch_size: int = 1):
        data = Dataset(data)
        loader = data.get_data_loader(batch_size=batch_size)
        return loader

    def to(self, device: Union[torch.device, str]):
        device = torch.device(device) #cpu or gpu
        self.model.to(device)
        self.metrics.to(device)
        self.criterion.to(device)

    def cpu(self):
        self.to("cpu")
        self.on_cuda = False

    def cuda(self):
        self.to("cuda")
        self.on_cuda = True

    def save(self, path: Path):
        state = {
            "model": self.model.state_dict(),
            "optimizer": self.optimizer.state_dict(),
        }
        torch.save(state, path)

    def load(self, path):
        checkpoint = torch.load(path)
        self.model.load_state_dict(checkpoint["model"])
        self.optimizer.load_state_dict(checkpoint["optimizer"])
        if self.on_cuda:
            self.to("cuda")


#!/usr/bin/env python
#title           :para_classifier.py
#description     :kernel functions for parallel computing used in Agglomerative training.
#author          :siyu zhu
#date            :Feb 3, 2015
#version         :0.1
#usage           :from para_classifier import para_adaboost
#notes           :
#python_version  :2.7

# import modules
import numpy
import scipy
from sklearn.ensemble import GradientBoostingRegressor

#--------------------BEGIN: functions used for parallel computation---------------------#
def para_adaboost(data):
	''' para_adaboost(data)

	kernel function for parallel computing adaboost classifier
	data: training data containing features and labels in a tuple
	Return: adaboost classifier model '''
	model = GradientBoostingRegressor(
		learning_rate = 1,
		n_estimators = 1000,
		max_depth = 1,
		random_state = 0
		)
	patch, label = data
	model = model.fit(patch, label)
	return model
#-----------------END: functions used for parallel computation--------------------------#


from dataclasses import dataclass
from typing import Any, Callable

from py_pdf_term.tokenizers import Term

@dataclass(frozen=True)
class PageCandidateTermList:
    """Page number and candidate terms of the page.

    Args
    ----
        page_num:
            Page number of a PDF file.
        candidates:
            Candidate terms of the page.
    """

    page_num: int
    candidates: list[Term]

    def to_nostyle_candidates_dict(
        self, to_str: Callable[[Term], str] = str
    ) -> dict[str, Term]:
        term_dict: dict[str, Term] = dict()
        for candidate in self.candidates:
            candidate_str = to_str(candidate)
            if candidate_str in term_dict:
                continue
            term_dict[candidate_str] = Term(candidate.tokens, 0.0, "", False)
        return term_dict

    def to_candidates_str_set(self, to_str: Callable[[Term], str] = str) -> set[str]:
        return {to_str(candidate) for candidate in self.candidates}

    def to_dict(self) -> dict[str, Any]:
        return {
            "page_num": self.page_num,
            "candidates": list(map(lambda term: term.to_dict(), self.candidates)),
        }

    @classmethod
    def from_dict(cls, obj: dict[str, Any]) -> "PageCandidateTermList":
        page_num, candidates = obj["page_num"], obj["candidates"]
        return cls(
            page_num,
            list(map(lambda item: Term.from_dict(item), candidates)),
        )

@dataclass(frozen=True)
class PDFCandidateTermList:
    """Path of a PDF file and candidate terms of the PDF file.

    Args
    ----
        pdf_path:
            Path of a PDF file.
        pages:
            Candidate terms of each page of the PDF file.
    """

    pdf_path: str
    pages: list[PageCandidateTermList]

    def to_nostyle_candidates_dict(
        self, to_str: Callable[[Term], str] = str
    ) -> dict[str, Term]:
        term_dict: dict[str, Term] = dict()
        for page in self.pages:
            candidate_items = page.to_nostyle_candidates_dict(to_str).items()
            for candidate_str, candidate in candidate_items:
                if candidate_str in term_dict:
                    continue
                term_dict[candidate_str] = candidate
        return term_dict

    def to_candidates_str_set(self, to_str: Callable[[Term], str] = str) -> set[str]:
        empty: set[str] = set()
        return empty.union(
            *map(lambda page: page.to_candidates_str_set(to_str), self.pages)
        )

    def to_dict(self) -> dict[str, Any]:
        return {
            "pdf_path": self.pdf_path,
            "pages": list(map(lambda page: page.to_dict(), self.pages)),
        }

    @classmethod
    def from_dict(cls, obj: dict[str, Any]) -> "PDFCandidateTermList":
        pdf_path, pages = obj["pdf_path"], obj["pages"]
        return cls(
            pdf_path,
            list(map(lambda item: PageCandidateTermList.from_dict(item), pages)),
        )

@dataclass(frozen=True)
class DomainCandidateTermList:
    """Domain name of PDF files and candidate terms of the domain.

    Args
    ----
        domain:
            Domain name. (e.g., "natural language processing")
        pdfs:
            Candidate terms of each PDF file of the domain.
    """

    domain: str
    pdfs: list[PDFCandidateTermList]

    def to_nostyle_candidates_dict(
        self, to_str: Callable[[Term], str] = str
    ) -> dict[str, Term]:
        term_dict: dict[str, Term] = dict()
        for pdf in self.pdfs:
            candidate_items = pdf.to_nostyle_candidates_dict(to_str).items()
            for candidate_str, candidate in candidate_items:
                if candidate_str in term_dict:
                    continue
                term_dict[candidate_str] = candidate
        return term_dict

    def to_candidates_str_set(self, to_str: Callable[[Term], str] = str) -> set[str]:
        empty: set[str] = set()
        return empty.union(
            *map(lambda pdf: pdf.to_candidates_str_set(to_str), self.pdfs)
        )

    def to_dict(self) -> dict[str, Any]:
        return {
            "domain": self.domain,
            "pdfs": list(map(lambda pdf: pdf.to_dict(), self.pdfs)),
        }

    @classmethod
    def from_dict(cls, obj: dict[str, Any]) -> "DomainCandidateTermList":
        domain, pdfs = obj["domain"], obj["pdfs"]
        return cls(
            domain,
            list(map(lambda item: PDFCandidateTermList.from_dict(item), pdfs)),
        )


import cv2 as cv
import numpy as np

img = cv.imread('Photos/park.jpg')
b,g,r = cv.split(img)

blank = np.zeros(img.shape[:2],dtype='uint8')

# cv.imshow('blue',b)
# cv.imshow('green',g)
# cv.imshow('red',r)

blue =cv.merge([b,blank,blank])
green = cv.merge([blank,g,blank])
red = cv.merge([blank,blank,r])

cv.imshow('blue',blue)
cv.imshow('green',green)
cv.imshow('red',red)

# merge = cv.merge([b,g,r])
# cv.imshow('merge',merge)

cv.waitKey(0)


from flask import jsonify, request, current_app
from flask_jwt_extended.utils import get_jwt_identity
from flask_jwt_extended.view_decorators import jwt_required
from app.exeptions.exeptions import InvalidZipCode
from app.models.user_model import UserModel
from app.services.verification_functions import confirm_json_format
from sqlalchemy.exc import IntegrityError

def create_user():
    data = request.json
    valid_fields = ['name', 'cell_phone', 'email', 'password', 'city', 'state', 'street', 'number', 'complement', 'zip_code']

    try:
        confirm_json_format(valid_fields, **data)

        new_user = UserModel.create_new_user(**data)

        return jsonify(new_user), 201
    except KeyError as e:
        message = e.args[0]
        return {"Error": message}, 401
    except TypeError:
        return {'Error': 'Campos inválidos'}, 409
    except ValueError:
        return {"Error": "O celular deve estar no formato (XX)xxxxx-xxxx"}, 400
    except IntegrityError:
        return {'Error': 'Email já cadastrado'}, 409
    except InvalidZipCode as e:
        return jsonify(e.args[0]), 400

def user_login():
    data = request.get_json()
    valid_fields = ['name', 'password']

    try:
        confirm_json_format(valid_fields, **data)

        access_token = UserModel.login(**data)

        return {"access_token": access_token}, 200

    except KeyError as e:
        return {"Error": "Campos de login inválidos, envie apenas o nome e senha do usuário."}, 400
    except IndexError:
        return {"Error": "usuário não encontrado."}, 404
    except ValueError as e:
        message = e.args[0]
        return {"Error": message}, 401
    except TypeError:
        return {'Error': 'Campos inválidos'}, 409

def get_user_by_id(id: int):

    user = UserModel.query.get(id)

    if not user:
        return  {"Error": "Usuário não encontrado."}, 404

    return jsonify(user), 200

@jwt_required()
def create_user_address():
    data = request.get_json()
    valid_fields = []
    user = get_jwt_identity()

    try:
        confirm_json_format(valid_fields, **data)

        user_found = UserModel.query.filter(UserModel.name == user.get('name')).first()

        if not user_found:
            raise IndexError

        new_address = UserModel.create_new_address(**data)
        address_id = {'adress_id': new_address.id}

        UserModel.update_info(user_found.name, **address_id)

        address_response = {
            'city': new_address.city,
            'state': new_address.state,
            'street': new_address.street,
            'number': new_address.number,
            'complement': new_address.complement,
            'cep': new_address.cep
        }

        return jsonify(address_response), 201
    except IndexError:
        return {"Error": "usuário não encontrado."}, 404
    except KeyError as e:
        message = e.args[0]
        return {"Error": message}, 400


import csv
from os import path
from typing import List
import numpy as np
import random
import datetime
from skimage import io
import os
from algorithms.Util import Util

WHITE=255

class NoisyImageGenerator(object):
    def __init__(self, salt_pepper_ratio:float, max_distance_between_points:float,output_folder:str,width:float, height:float,max_lines:int):
        self._salt_pepper_ratio=salt_pepper_ratio
        self._max_distance_between_points=max_distance_between_points
        self._output_folder=output_folder
        self._image_width=width
        self._image_height=height
        self._max_lines=max_lines
        self._noisy_image:np.ndarray=None
        random.seed(datetime.datetime.now())

    @property
    def filename(self):
        """Generates a filename based on parameters supplied in the constructore."""
        return ("%s-LINECOUNT-%d-SP-%.3f-MAXD-%.1f.png") % ("NoisyImage", self._max_lines,self._salt_pepper_ratio,self._max_distance_between_points)

    @property
    def count_of_blackpixels(self):
        """Returns the  count of blackpixels in the generatd image."""
        all_white_indices=np.where(self._noisy_image == WHITE)
        count_of_white=len(all_white_indices[0])
        count_of_black=self._noisy_image.shape[0]*self._noisy_image.shape[1] - count_of_white
        return count_of_black

    @property
    def total_pixel_count(self):
        """The total number of pixels in the image."""
        return self._noisy_image.shape[0]*self._noisy_image.shape[1]

    @total_pixel_count.setter
    def total_pixel_count(self, value):
        self._total_pixel_count = value

    def create_noisy_image(self):
        noisy_image=Util.generate_noisy_image(width=self._image_width, height=self._image_height,salt_pepper=self._salt_pepper_ratio)

        for line_index in range(0,self._max_lines):
            #alternate the lines left edge to right edge, top edge to bottom edge
            if (line_index%2 ==0):
                random_x_first=(self._image_width)*random.random()
                random_x_second=self._image_width - random_x_first
                bottom_edge=0
                top_edge=self._image_height
                noisy_image=Util.superimpose_straight_line_between_2_points(noisy_image,start_x=random_x_first, start_y=bottom_edge, end_x=random_x_second, end_y=top_edge, max_distance=self._max_distance_between_points)
            else:
                left_edge=0
                right_edge=self._image_width
                random_y_first=(self._image_height)*random.random()
                random_y_second=self._image_height -random_y_first
                noisy_image=Util.superimpose_straight_line_between_2_points(noisy_image,start_x=left_edge, start_y=random_y_first, end_x=right_edge, end_y=random_y_second, max_distance=self._max_distance_between_points)

        absolute_filepath=os.path.join(self._output_folder,self.filename)
        io.imsave(absolute_filepath,noisy_image)
        self._noisy_image=noisy_image
        pass


import pandas as pd
from termcolor import colored
import sys
import os
import logging
import json
import math
import numpy as np
from typing import Tuple

from ampyutils import amutils
from glab import glab_constants as glc
from GNSS import wgs84

__author__ = 'amuls'

def statistics_glab_outfile(df_outp: pd.DataFrame, logger: logging.Logger) -> Tuple[dict, dict]:
    """
    splitStatusFile splits the statistics file into the POS, SAT, CLK & VELACC parts
    """
    cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')

    logger.info('{func:s}: calculating statistics of OUTPUT messages'.format(func=cFuncName))

    # dictionary containing the statistics
    dStats = {}
    dStats['dop_bin'] = statistics_dopbin(df_dop_enu=df_outp[glc.dgLab['OUTPUT']['XDOP'] + glc.dgLab['OUTPUT']['dENU'] + glc.dgLab['OUTPUT']['sdENU']], logger=logger)
    dStats['crd'] = statistics_coordinates(df_crd=df_outp[glc.dgLab['OUTPUT']['llh'] + glc.dgLab['OUTPUT']['dENU'] + glc.dgLab['OUTPUT']['sdENU'] + glc.dgLab['OUTPUT']['UTM']], logger=logger)

    # print(dStats.keys())
    # print(dStats['crd'].keys())
    # print(dStats['crd']['dN0'].keys())
    # create the dNEU information to store in the glabng output database
    dDB_crd = {}
    for crd in glc.dgLab['OUTPUT']['dENU']:
        dDB_crd[crd] = '{crd:s},'.format(crd=crd)
        dDB_crd[crd] += '{mean:+.3f},'.format(mean=dStats['crd'][crd]['mean'])
        dDB_crd[crd] += '{std:+.3f},'.format(std=dStats['crd'][crd]['std'])
        dDB_crd[crd] += '{max:+.3f},'.format(max=dStats['crd'][crd]['max'])
        dDB_crd[crd] += '{min:+.3f},'.format(min=dStats['crd'][crd]['min'])
        # print('crd from glc = {crd:s} - {info:s}'.format(crd=crd, info=dDB_crd[crd]))

        for i, (dop_min, dop_max) in enumerate(zip(glc.dop_bins[:-1], glc.dop_bins[1:])):
            bin_interval = 'bin{:d}-{:.0f}'.format(dop_min, dop_max)
            # print(bin_interval)

            dDB_crd[crd] += '{mean:+.3f},'.format(mean=dStats['dop_bin'][bin_interval][crd]['mean'])
            dDB_crd[crd] += '{std:+.3f},'.format(std=dStats['dop_bin'][bin_interval][crd]['std'])
            dDB_crd[crd] += '{max:+.3f},'.format(max=dStats['dop_bin'][bin_interval][crd]['max'])
            dDB_crd[crd] += '{min:+.3f}'.format(min=dStats['dop_bin'][bin_interval][crd]['min'])

            if i < len(glc.dop_bins[:-1]) - 1:
                dDB_crd[crd] += ','

    # create the llh information to store in the glabng output database
    for i, crd in enumerate(glc.dgLab['OUTPUT']['llh']):
        dDB_crd[crd] = '{crd:s},'.format(crd=crd)
        if crd == 'ellh':
            dDB_crd[crd] += '{mean:+.3f},'.format(mean=dStats['crd'][crd]['mean'])
            dDB_crd[crd] += '{std:+.3f},'.format(std=dStats['crd'][crd]['std'])
            dDB_crd[crd] += '{max:+.3f},'.format(max=dStats['crd'][crd]['max'])
            dDB_crd[crd] += '{min:+.3f}'.format(min=dStats['crd'][crd]['min'])
        else:
            dDB_crd[crd] += '{mean:+.9f},'.format(mean=dStats['crd'][crd]['mean'])
            dDB_crd[crd] += '{std:+.9f},'.format(std=dStats['crd'][crd]['std'])
            dDB_crd[crd] += '{max:+.9f},'.format(max=dStats['crd'][crd]['max'])
            dDB_crd[crd] += '{min:+.9f}'.format(min=dStats['crd'][crd]['min'])

    # create the llh information to store in the glabng output database
    for i, crd in enumerate(glc.dgLab['OUTPUT']['UTM']):
        dDB_crd[crd] = '{crd:s},'.format(crd=crd)
        dDB_crd[crd] += '{mean:+.3f},'.format(mean=dStats['crd'][crd]['mean'])
        dDB_crd[crd] += '{std:+.3f},'.format(std=dStats['crd'][crd]['std'])
        dDB_crd[crd] += '{max:+.3f},'.format(max=dStats['crd'][crd]['max'])
        dDB_crd[crd] += '{min:+.3f}'.format(min=dStats['crd'][crd]['min'])

    # print('crd from glc = {crd:s} - {info:s}'.format(crd=crd, info=dDB_crd[crd]))

    return dStats, dDB_crd

def statistics_dopbin(df_dop_enu: pd.DataFrame, logger: logging.Logger) -> dict:
    """
    statistics_dopbin calculates the xDOP statistics for each xDOP bin
    """
    cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')

    logger.info('{func:s}: calculating statistics of xDOP'.format(func=cFuncName))

    dStats_dop = {}

    amutils.printHeadTailDataFrame(df=df_dop_enu, name='df_dop_enu')

    # go over all PDOP bins and plot according to the markersBin defined
    for i in range(len(glc.dop_bins) - 1):
        bin_PDOP = 'bin{:d}-{:.0f}'.format(glc.dop_bins[i], glc.dop_bins[i + 1])
        logger.debug('{func:s}: bin_PDOP = {bin!s}'.format(bin=bin_PDOP, func=cFuncName))

        # create the dict for this PDOP interval
        dStats_dop[bin_PDOP] = {}

        # find the indices within this bin
        index4Bin = (df_dop_enu['PDOP'] > glc.dop_bins[i]) & (df_dop_enu['PDOP'] <= glc.dop_bins[i + 1])

        dStats_dop[bin_PDOP]['perc'] = index4Bin.mean()
        dStats_dop[bin_PDOP]['count'] = int(index4Bin.count() * index4Bin.mean())

        for dENU, sdENU in zip(glc.dgLab['OUTPUT']['dENU'], glc.dgLab['OUTPUT']['sdENU']):
            dENU_stats = {}

            dENU_stats['wavg'] = amutils.wavg(df_dop_enu.loc[index4Bin], dENU, sdENU)
            dENU_stats['sdwavg'] = amutils.stddev(df_dop_enu.loc[index4Bin, dENU], dENU_stats['wavg'])
            dENU_stats['mean'] = df_dop_enu.loc[index4Bin, dENU].mean()
            dENU_stats['median'] = df_dop_enu.loc[index4Bin, dENU].median()
            dENU_stats['std'] = df_dop_enu.loc[index4Bin, dENU].std()
            dENU_stats['min'] = df_dop_enu.loc[index4Bin, dENU].min()
            dENU_stats['max'] = df_dop_enu.loc[index4Bin, dENU].max()

            # add for this crd dENU
            dStats_dop[bin_PDOP][dENU] = dENU_stats

            logger.debug('{func:s}: in {bin:s} statistics for {crd:s} are {stat!s}'.format(func=cFuncName, bin=bin_PDOP, crd=dENU, stat=dENU_stats))

    # report to the user
    logger.info('{func:s}: dStats_dop =\n{json!s}'.format(func=cFuncName, json=json.dumps(dStats_dop, sort_keys=False, indent=4, default=amutils.DT_convertor)))

    return dStats_dop

def statistics_coordinates(df_crd: pd.DataFrame, logger: logging.Logger) -> dict:
    """
    statistics_coordinates calculates the coordinate statistics
    """
    cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')

    logger.info('{func:s}: calculating coordinate statistics'.format(func=cFuncName))

    # init class WGS84
    wgs_84 = wgs84.WGS84()

    amutils.printHeadTailDataFrame(df=df_crd, name='df_crd', index=False)
    dStat = {}
    for crd in (glc.dgLab['OUTPUT']['llh'] + glc.dgLab['OUTPUT']['dENU'] + glc.dgLab['OUTPUT']['UTM']):
        dStat[crd] = {}

    # make sure to have the wavg for latitude since it is used for converting the stddev of geodetic coordinates into meter
    dStat['lat']['wavg'] = amutils.wavg(df_crd, 'lat', 'dN0')
    for llh, sdENU in zip(glc.dgLab['OUTPUT']['llh'], glc.dgLab['OUTPUT']['sdENU']):
        dStat[llh]['wavg'] = amutils.wavg(df_crd, llh, sdENU)
        if llh == 'lat':
            dStat[llh]['sdwavg'] = math.radians(amutils.stddev(df_crd[llh], dStat['lat']['wavg'])) * wgs_84.a
        elif llh == 'lon':
            dStat[llh]['sdwavg'] = math.radians(amutils.stddev(df_crd[llh], dStat['lat']['wavg'])) * wgs_84.a * math.cos(math.radians(dStat['lat']['wavg']))
        else:
            dStat[llh]['sdwavg'] = amutils.stddev(df_crd[llh], dStat[llh]['wavg'])

    for dENU, sdENU in zip(glc.dgLab['OUTPUT']['dENU'], glc.dgLab['OUTPUT']['sdENU']):
        dStat[dENU]['wavg'] = amutils.wavg(df_crd, dENU, sdENU)
        dStat[dENU]['sdwavg'] = amutils.stddev(df_crd[dENU], dStat[dENU]['wavg'])

    for dUTM, sdENU in zip(glc.dgLab['OUTPUT']['UTM'], glc.dgLab['OUTPUT']['sdENU'][:2]):
        dStat[dUTM]['wavg'] = amutils.wavg(df_crd, dUTM, sdENU)
        dStat[dUTM]['sdwavg'] = amutils.stddev(df_crd[dUTM], dStat[dENU]['wavg'])

    # calculate statistics for the nuùeric values
    for crd in (glc.dgLab['OUTPUT']['llh'] + glc.dgLab['OUTPUT']['dENU'] + glc.dgLab['OUTPUT']['UTM']):
        dStat[crd]['mean'] = df_crd[crd].mean()
        dStat[crd]['median'] = df_crd[crd].median()
        dStat[crd]['std'] = df_crd[crd].std()
        dStat[crd]['max'] = df_crd[crd].max()
        dStat[crd]['min'] = df_crd[crd].min()

        # results of gLAB kalman filter
        dStat[crd]['kf'] = df_crd[crd].iloc[-1]
        try:
            dStat[crd]['sdkf'] = df_crd['s{:s}'.format(crd[:2])].iloc[-1]
        except KeyError:
            dStat[crd]['sdkf'] = np.nan

    logger.info('{func:s}: OUTPUT statistics information =\n{json!s}'.format(func=cFuncName, json=json.dumps(dStat, sort_keys=False, indent=4, default=amutils.DT_convertor)))

    return dStat


import sys
from io import StringIO
import unittest
import logging
logging.basicConfig(level=logging.DEBUG)

def resolve():
    s = input()
    m = input()
    check = True
    for i in range(len(s)):
        if s[i] == m[i]:
            next
        elif s[i] == "@" and m[i] in "atcoder":
            next
        elif m[i] == "@" and s[i] in "atcoder":
            next
        else:
            check = False
    if check:
        print("You can win")
    else:
        print("You will lose")

class TestClass(unittest.TestCase):
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

    def test_input_1(self):
        print("test_input_1")
        input = """ch@ku@ai
choku@@i"""
        output = """You can win"""
        self.assertIO(input, output)

    def test_input_2(self):
        print("test_input_2")
        input = """aoki
@ok@"""
        output = """You will lose"""
        self.assertIO(input, output)

    def test_input_3(self):
        print("test_input_3")
        input = """arc
abc"""
        output = """You will lose"""
        self.assertIO(input, output)

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-
from rest_framework import generics
from rest_framework.response import Response
from rest_framework.views import APIView
from .models import Product, Lesson, LessonViewed
from .serializers import LessonSerializer, ProductStatDetailSerializer
from django.contrib.auth.models import User
from . import models
from django.db.models import Sum

class UserLessonsListView(generics.ListAPIView):
    serializer_class = LessonSerializer

    def get_queryset(self):
        user_id = self.request.user.id
        return Lesson.objects.filter(products__owner=user_id)

class LessonByProductView(generics.ListAPIView):
    serializer_class = LessonSerializer

    def get_queryset(self):
        user_id = self.request.user.id
        product_id = self.kwargs['pk']
        print(f"user_id: {user_id}, product_id: {product_id}")
        queryset = Lesson.objects.filter(products__owner=user_id, products__id=product_id)
        print(f"Queryset: {queryset}")
        return queryset

    # def get_queryset(self):
    #     user_id = self.request.user.id
    #     product_id = self.kwargs['pk']
        # return Lesson.objects.filter(products__owner=user_id, products__id=product_id)
class ProductStatDetailView(APIView):
    def get(self, request):
        products = Product.objects.all()
        stat_data = []

        for product in products:
            total_lessons_viewed = LessonViewed.objects.filter(lesson__products=product).count()
            total_time_spent = LessonViewed.objects.filter(lesson__products=product).aggregate(total_time_spent=Sum('viewed_time_seconds'))['total_time_spent']
            total_students = LessonViewed.objects.filter(lesson__products=product).values('user').distinct().count()
            total_users = User.objects.all().count()
            acquisition_percentage = (total_students / total_users) * 100

            stat_data.append({
                'product': {
                    'id': product.id,
                    'name': product.name
                },
                'total_lessons_viewed': total_lessons_viewed,
                'total_time_spent': total_time_spent,
                'total_students': total_students,
                'acquisition_percentage': acquisition_percentage
            })

        serializer = ProductStatDetailSerializer(stat_data, many=True)
        return Response(serializer.data)


# coding=utf-8
"""Shared utility functions"""

import collections
import os
import re
import subprocess
import sys
import threading
import unicodedata
from typing import Any, BinaryIO, Iterable, List, Optional, TextIO, Union

from wcwidth import wcswidth

from . import constants

def strip_ansi(text: str) -> str:
    """Strip ANSI escape codes from a string.

    :param text: string which may contain ANSI escape codes
    :return: the same string with any ANSI escape codes removed
    """
    return constants.ANSI_ESCAPE_RE.sub('', text)

def ansi_safe_wcswidth(text: str) -> int:
    """
    Wraps wcswidth to make it compatible with colored strings

    :param text: the string being measured
    """
    # Strip ANSI escape codes since they cause wcswidth to return -1
    return wcswidth(strip_ansi(text))

def is_quoted(arg: str) -> bool:
    """
    Checks if a string is quoted
    :param arg: the string being checked for quotes
    :return: True if a string is quoted
    """
    return len(arg) > 1 and arg[0] == arg[-1] and arg[0] in constants.QUOTES

def quote_string_if_needed(arg: str) -> str:
    """ Quotes a string if it contains spaces and isn't already quoted """
    if is_quoted(arg) or ' ' not in arg:
        return arg

    if '"' in arg:
        quote = "'"
    else:
        quote = '"'

    return quote + arg + quote

def strip_quotes(arg: str) -> str:
    """ Strip outer quotes from a string.

     Applies to both single and double quotes.

    :param arg:  string to strip outer quotes from
    :return: same string with potentially outer quotes stripped
    """
    if is_quoted(arg):
        arg = arg[1:-1]
    return arg

def namedtuple_with_defaults(typename: str, field_names: Union[str, List[str]],
                             default_values: collections.Iterable = ()):
    """
    Convenience function for defining a namedtuple with default values

    From: https://stackoverflow.com/questions/11351032/namedtuple-and-default-values-for-optional-keyword-arguments

    Examples:
        >>> Node = namedtuple_with_defaults('Node', 'val left right')
        >>> Node()
        Node(val=None, left=None, right=None)
        >>> Node = namedtuple_with_defaults('Node', 'val left right', [1, 2, 3])
        >>> Node()
        Node(val=1, left=2, right=3)
        >>> Node = namedtuple_with_defaults('Node', 'val left right', {'right':7})
        >>> Node()
        Node(val=None, left=None, right=7)
        >>> Node(4)
        Node(val=4, left=None, right=7)
    """
    T = collections.namedtuple(typename, field_names)
    # noinspection PyProtectedMember,PyUnresolvedReferences
    T.__new__.__defaults__ = (None,) * len(T._fields)
    if isinstance(default_values, collections.Mapping):
        prototype = T(**default_values)
    else:
        prototype = T(*default_values)
    T.__new__.__defaults__ = tuple(prototype)
    return T

def cast(current: Any, new: str) -> Any:
    """Tries to force a new value into the same type as the current when trying to set the value for a parameter.

    :param current: current value for the parameter, type varies
    :param new: new value
    :return: new value with same type as current, or the current value if there was an error casting
    """
    typ = type(current)
    orig_new = new

    if typ == bool:
        try:
            return bool(int(new))
        except (ValueError, TypeError):
            pass
        try:
            new = new.lower()
            if (new == 'on') or (new[0] in ('y', 't')):
                return True
            if (new == 'off') or (new[0] in ('n', 'f')):
                return False
        except AttributeError:
            pass
    else:
        try:
            return typ(new)
        except (ValueError, TypeError):
            pass
    print("Problem setting parameter (now {}) to {}; incorrect type?".format(current, orig_new))
    return current

def which(editor: str) -> Optional[str]:
    """Find the full path of a given editor.

    Return the full path of the given editor, or None if the editor can
    not be found.

    :param editor: filename of the editor to check, ie 'notepad.exe' or 'vi'
    :return: a full path or None
    """
    try:
        editor_path = subprocess.check_output(['which', editor], stderr=subprocess.STDOUT).strip()
        editor_path = editor_path.decode()
    except subprocess.CalledProcessError:
        editor_path = None
    return editor_path

def is_text_file(file_path: str) -> bool:
    """Returns if a file contains only ASCII or UTF-8 encoded text.

    :param file_path: path to the file being checked
    :return: True if the file is a text file, False if it is binary.
    """
    import codecs

    expanded_path = os.path.abspath(os.path.expanduser(file_path.strip()))
    valid_text_file = False

    # Check if the file is ASCII
    try:
        with codecs.open(expanded_path, encoding='ascii', errors='strict') as f:
            # Make sure the file has at least one line of text
            # noinspection PyUnusedLocal
            if sum(1 for line in f) > 0:
                valid_text_file = True
    except OSError:  # pragma: no cover
        pass
    except UnicodeDecodeError:
        # The file is not ASCII. Check if it is UTF-8.
        try:
            with codecs.open(expanded_path, encoding='utf-8', errors='strict') as f:
                # Make sure the file has at least one line of text
                # noinspection PyUnusedLocal
                if sum(1 for line in f) > 0:
                    valid_text_file = True
        except OSError:  # pragma: no cover
            pass
        except UnicodeDecodeError:
            # Not UTF-8
            pass

    return valid_text_file

def remove_duplicates(list_to_prune: List) -> List:
    """Removes duplicates from a list while preserving order of the items.

    :param list_to_prune: the list being pruned of duplicates
    :return: The pruned list
    """
    temp_dict = collections.OrderedDict()
    for item in list_to_prune:
        temp_dict[item] = None

    return list(temp_dict.keys())

def norm_fold(astr: str) -> str:
    """Normalize and casefold Unicode strings for saner comparisons.

    :param astr: input unicode string
    :return: a normalized and case-folded version of the input string
    """
    return unicodedata.normalize('NFC', astr).casefold()

def alphabetical_sort(list_to_sort: Iterable[str]) -> List[str]:
    """Sorts a list of strings alphabetically.

    For example: ['a1', 'A11', 'A2', 'a22', 'a3']

    To sort a list in place, don't call this method, which makes a copy. Instead, do this:

    my_list.sort(key=norm_fold)

    :param list_to_sort: the list being sorted
    :return: the sorted list
    """
    return sorted(list_to_sort, key=norm_fold)

def try_int_or_force_to_lower_case(input_str: str) -> Union[int, str]:
    """
    Tries to convert the passed-in string to an integer. If that fails, it converts it to lower case using norm_fold.
    :param input_str: string to convert
    :return: the string as an integer or a lower case version of the string
    """
    try:
        return int(input_str)
    except ValueError:
        return norm_fold(input_str)

def natural_keys(input_str: str) -> List[Union[int, str]]:
    """
    Converts a string into a list of integers and strings to support natural sorting (see natural_sort).

    For example: natural_keys('abc123def') -> ['abc', '123', 'def']
    :param input_str: string to convert
    :return: list of strings and integers
    """
    return [try_int_or_force_to_lower_case(substr) for substr in re.split(r'(\d+)', input_str)]

def natural_sort(list_to_sort: Iterable[str]) -> List[str]:
    """
    Sorts a list of strings case insensitively as well as numerically.

    For example: ['a1', 'A2', 'a3', 'A11', 'a22']

    To sort a list in place, don't call this method, which makes a copy. Instead, do this:

    my_list.sort(key=natural_keys)

    :param list_to_sort: the list being sorted
    :return: the list sorted naturally
    """
    return sorted(list_to_sort, key=natural_keys)

def unquote_redirection_tokens(args: List[str]) -> None:
    """
    Unquote redirection tokens in a list of command-line arguments
    This is used when redirection tokens have to be passed to another command
    :param args: the command line args
    """
    for i, arg in enumerate(args):
        unquoted_arg = strip_quotes(arg)
        if unquoted_arg in constants.REDIRECTION_TOKENS:
            args[i] = unquoted_arg

def find_editor() -> str:
    """Find a reasonable editor to use by default for the system that the cmd2 application is running on."""
    editor = os.environ.get('EDITOR')
    if not editor:
        if sys.platform[:3] == 'win':
            editor = 'notepad'
        else:
            # Favor command-line editors first so we don't leave the terminal to edit
            for editor in ['vim', 'vi', 'emacs', 'nano', 'pico', 'gedit', 'kate', 'subl', 'geany', 'atom']:
                if which(editor):
                    break
    return editor

class StdSim(object):
    """
    Class to simulate behavior of sys.stdout or sys.stderr.
    Stores contents in internal buffer and optionally echos to the inner stream it is simulating.
    """
    def __init__(self, inner_stream, echo: bool = False,
                 encoding: str = 'utf-8', errors: str = 'replace') -> None:
        """
        Initializer
        :param inner_stream: the emulated stream
        :param echo: if True, then all input will be echoed to inner_stream
        :param encoding: codec for encoding/decoding strings (defaults to utf-8)
        :param errors: how to handle encoding/decoding errors (defaults to replace)
        """
        self.inner_stream = inner_stream
        self.echo = echo
        self.encoding = encoding
        self.errors = errors
        self.pause_storage = False
        self.buffer = ByteBuf(self)

    def write(self, s: str) -> None:
        """Add str to internal bytes buffer and if echo is True, echo contents to inner stream"""
        if not isinstance(s, str):
            raise TypeError('write() argument must be str, not {}'.format(type(s)))

        if not self.pause_storage:
            self.buffer.byte_buf += s.encode(encoding=self.encoding, errors=self.errors)
        if self.echo:
            self.inner_stream.write(s)

    def getvalue(self) -> str:
        """Get the internal contents as a str"""
        return self.buffer.byte_buf.decode(encoding=self.encoding, errors=self.errors)

    def getbytes(self) -> bytes:
        """Get the internal contents as bytes"""
        return bytes(self.buffer.byte_buf)

    def read(self) -> str:
        """Read from the internal contents as a str and then clear them out"""
        result = self.getvalue()
        self.clear()
        return result

    def readbytes(self) -> bytes:
        """Read from the internal contents as bytes and then clear them out"""
        result = self.getbytes()
        self.clear()
        return result

    def clear(self) -> None:
        """Clear the internal contents"""
        self.buffer.byte_buf.clear()

    def isatty(self) -> bool:
        """StdSim only considered an interactive stream if `echo` is True and `inner_stream` is a tty."""
        if self.echo:
            return self.inner_stream.isatty()
        else:
            return False

    def __getattr__(self, item: str):
        if item in self.__dict__:
            return self.__dict__[item]
        else:
            return getattr(self.inner_stream, item)

class ByteBuf(object):
    """
    Used by StdSim to write binary data and stores the actual bytes written
    """
    def __init__(self, std_sim_instance: StdSim) -> None:
        self.byte_buf = bytearray()
        self.std_sim_instance = std_sim_instance

    def write(self, b: bytes) -> None:
        """Add bytes to internal bytes buffer and if echo is True, echo contents to inner stream."""
        if not isinstance(b, bytes):
            raise TypeError('a bytes-like object is required, not {}'.format(type(b)))
        if not self.std_sim_instance.pause_storage:
            self.byte_buf += b
        if self.std_sim_instance.echo:
            self.std_sim_instance.inner_stream.buffer.write(b)

class ProcReader(object):
    """
    Used to captured stdout and stderr from a Popen process if any of those were set to subprocess.PIPE.
    If neither are pipes, then the process will run normally and no output will be captured.
    """
    def __init__(self, proc: subprocess.Popen, stdout: Union[StdSim, BinaryIO, TextIO],
                 stderr: Union[StdSim, BinaryIO, TextIO]) -> None:
        """
        ProcReader initializer
        :param proc: the Popen process being read from
        :param stdout: the stream to write captured stdout
        :param stderr: the stream to write captured stderr
        """
        self._proc = proc
        self._stdout = stdout
        self._stderr = stderr

        self._out_thread = threading.Thread(name='out_thread', target=self._reader_thread_func,
                                            kwargs={'read_stdout': True})

        self._err_thread = threading.Thread(name='out_thread', target=self._reader_thread_func,
                                            kwargs={'read_stdout': False})

        # Start the reader threads for pipes only
        if self._proc.stdout is not None:
            self._out_thread.start()
        if self._proc.stderr is not None:
            self._err_thread.start()

    def send_sigint(self) -> None:
        """Send a SIGINT to the process similar to if <Ctrl>+C were pressed."""
        import signal
        if sys.platform.startswith('win'):
            signal_to_send = signal.CTRL_C_EVENT
        else:
            signal_to_send = signal.SIGINT
        self._proc.send_signal(signal_to_send)

    def terminate(self) -> None:
        """Terminate the process"""
        self._proc.terminate()

    def wait(self) -> None:
        """Wait for the process to finish"""
        if self._out_thread.is_alive():
            self._out_thread.join()
        if self._err_thread.is_alive():
            self._err_thread.join()

        # Handle case where the process ended before the last read could be done.
        # This will return None for the streams that weren't pipes.
        out, err = self._proc.communicate()

        if out:
            self._write_bytes(self._stdout, out)
        if err:
            self._write_bytes(self._stderr, err)

    def _reader_thread_func(self, read_stdout: bool) -> None:
        """
        Thread function that reads a stream from the process
        :param read_stdout: if True, then this thread deals with stdout. Otherwise it deals with stderr.
        """
        if read_stdout:
            read_stream = self._proc.stdout
            write_stream = self._stdout
        else:
            read_stream = self._proc.stderr
            write_stream = self._stderr

        # The thread should have been started only if this stream was a pipe
        assert read_stream is not None

        # Run until process completes
        while self._proc.poll() is None:
            # noinspection PyUnresolvedReferences
            available = read_stream.peek()
            if available:
                read_stream.read(len(available))
                self._write_bytes(write_stream, available)

    @staticmethod
    def _write_bytes(stream: Union[StdSim, BinaryIO, TextIO], to_write: bytes) -> None:
        """
        Write bytes to a stream
        :param stream: the stream being written to
        :param to_write: the bytes being written
        """
        try:
            if hasattr(stream, 'buffer'):
                stream.buffer.write(to_write)
            else:
                stream.write(to_write)
        except BrokenPipeError:
            # This occurs if output is being piped to a process that closed
            pass

class ContextFlag(object):
    """A context manager which is also used as a boolean flag value within the default sigint handler.

    Its main use is as a flag to prevent the SIGINT handler in cmd2 from raising a KeyboardInterrupt
    while a critical code section has set the flag to True. Because signal handling is always done on the
    main thread, this class is not thread-safe since there is no need.
    """
    def __init__(self) -> None:
        # When this flag has a positive value, it is considered set.
        # When it is 0, it is not set. It should never go below 0.
        self.__count = 0

    def __bool__(self) -> bool:
        return self.__count > 0

    def __enter__(self) -> None:
        self.__count += 1

    def __exit__(self, *args) -> None:
        self.__count -= 1
        if self.__count < 0:
            raise ValueError("count has gone below 0")

class RedirectionSavedState(object):
    """Created by each command to store information about their redirection."""

    def __init__(self, self_stdout: Union[StdSim, BinaryIO, TextIO], sys_stdout: Union[StdSim, BinaryIO, TextIO],
                 pipe_proc_reader: Optional[ProcReader]) -> None:
        # Used to restore values after the command ends
        self.saved_self_stdout = self_stdout
        self.saved_sys_stdout = sys_stdout
        self.saved_pipe_proc_reader = pipe_proc_reader

        # Tells if the command is redirecting
        self.redirecting = False

        # If the command created a process to pipe to, then then is its reader
        self.pipe_proc_reader = None


import asyncio, threading, sys, time, json

from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory

from lib.EventBus import EventBus
from lib.Topics import Topics

class MyClientProtocol(WebSocketClientProtocol):

    def __init__(self, retryCallback, eventBus):
        super().__init__()

        self.eventBus = eventBus
        self.retryCallback = retryCallback
        self.expectingPong = False
        self.terminated = False
        self.eventBus.on(Topics.wsReceivedMessageJSON, self._sendMessage)

    def onConnect(self, response):
        print("Server connected: {0}".format(response.peer))

    def onConnecting(self, transport_details):
        print("Connecting; transport details: {}".format(transport_details))
        return None  # ask for defaults

    def _sendMessage(self, jsonData):
        self.sendMessage(json.dumps(jsonData))

    def onOpen(self):
        print("WebSocket connection open.")

        def pingPong():
            if self.expectingPong:
                if not self.terminated:
                    self.retryCallback()
                    return
            self.sendMessage(u"ping".encode('utf8'))
            self.expectingPong = True
            self.factory.loop.call_later(1, pingPong)

        # start sending messages every second ..
        pingPong()

    def onMessage(self, payload, isBinary):
        if isBinary:
            print("Binary message received: {0} bytes".format(len(payload)))
        else:
            textMessage = payload.decode('utf8')
            if textMessage == 'pong':
                self.expectingPong = False
            else:
                data = json.load(textMessage)
                self.eventBus.emit(Topics.wsReceivedMessageJSON, data)

    def onClose(self, wasClean, code, reason):
        print("WebSocket connection closed: {0}".format(reason))
        self.terminated = True
        self.retryCallback()

class WSClient(threading.Thread):

    def __init__(self, eventBus):
        self.clientActive = False
        self.socketActive = False

        self.factory = WebSocketClientFactory(u"ws://127.0.0.1:9000")
        self.factory.protocol = lambda: MyClientProtocol(self.retry, eventBus)
        self.loop = asyncio.get_event_loop()
        self.closingLoop = asyncio.new_event_loop()

        threading.Thread.__init__(self)

    def __del__(self):
        print("cleaning up")
        self.cleanup()

    def run(self):
        self.startWebsocket()

    def cleanup(self):
        self.clientActive = False
        self.socketActive = False

    def retry(self):
        self.socketActive = False

    def startWebsocket(self):
        self.clientActive = True
        try:
            self.loop.run_until_complete(self.attempt())
        except KeyboardInterrupt as e:
            self.cleanup()

    async def attempt(self):
        while self.clientActive:
            self.socketActive = True
            await self.tryToConnect()
            await asyncio.sleep(0.5)

    async def tryToConnect(self):
        try:
            await self.loop.create_connection(self.factory, '127.0.0.1', 9000)
            await self.waitToFinish()
        except ConnectionRefusedError as err:
            pass

    async def waitToFinish(self):
        while self.socketActive:
            await asyncio.sleep(0.1)

if __name__ == "__main__":
    eventBus = EventBus()
    client = WSClient(eventBus)
    client.start()


#! /usr/bin/env python3

import argparse
import set

parser = argparse.ArgumentParser("How many cards need to be dealt until there's a set?")
parser.add_argument('-n', default=100, type=int, help='Number of shuffles/deals to simulate')
parser.add_argument('-v', action='store_true', help="Verbose")
parser.add_argument('-i', default=10000, type=int, help='Interval for printing intermediate histograms')
parser.add_argument('-o', default=None, type=str, help='File to write all counts to')
args = parser.parse_args()

file = None
if args.o:
    file = open(args.o, mode="w")

print(args.n)

counts = [0]*82       # 0 not used, 1-81 might(?) get populated
for i in range(args.n):
    cards = set.deck()
    deal = []
    while set.find_sets(deal, force_last=True, quit_after_one=True) is None:
        deal.append(cards.pop())

    counts[len(deal)] += 1
    if  file:
        file.write(str(len(deal))+'\n')

    if args.v:
        ijks = set.find_sets(deal)
        print(len(deal),"cards dealt, set(s) found:",len(ijks))
        for ijk in ijks:
            print("Set found:")
            deal[ijk[0]].describe()
            deal[ijk[1]].describe()
            deal[ijk[2]].describe()

    if i>0 and i%args.i == 0:
        print("Simulations: ", i)
        print(set.histogram(counts))

print(set.histogram(counts))


#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""counts.py

methods for counting and scoring items.

"""
import logging
import subprocess
from collections import Counter
from io import StringIO
from tempfile import NamedTemporaryFile

# requirements
from association_measures import measures
from pandas import DataFrame, MultiIndex, read_csv

# part of module
from .cl import Corpus as Crps
from .utils import fold_df, time_it

logger = logging.getLogger(__name__)

def count_items(items, names, tuples=True):
    """Get type frequency table of items.

    :param list items: list of values or tuples
    :param list names: name(s) for the attributes to count
    :param bool tuples: treat each item as a tuple?

    :return: frequency counts of items
    :rtype: FreqFrame
    """

    logger.info(f"counting {len(items)} items")
    counts = Counter(items)
    df_counts = DataFrame.from_dict(counts, orient='index', columns=['freq'])

    # transform index
    if tuples:
        df_counts.index = MultiIndex.from_tuples(df_counts.index, names=names)
        df_counts = df_counts.reset_index()
        df_counts['item'] = df_counts[names].agg(' '.join, axis=1)
        df_counts = df_counts.set_index('item')
    else:
        df_counts = df_counts.reset_index()
        df_counts.columns = names + ['freq']
        df_counts = df_counts.set_index(names[0], drop=False)
        df_counts.index.name = 'item'

    df_counts = df_counts[['freq'] + names]

    df_counts = df_counts.sort_values(by=['freq', 'item'], ascending=False)

    return df_counts

def read_freq_list(path, min_freq=2, columns=None):
    """Read frequency list, e.g. output of lexdecode (see below)

    :param str path: path to read from
    :param int min_freq: drop everything that doesn't appear at least this often

    :return: frequency list (index: item, columns: freq) and original size
    :rtype: tuple(DataFrame, int)

    """

    # read data
    logger.info('reading frequency list ...')
    df = read_csv(path, sep="\t", header=None, quoting=3,
                  keep_default_na=False)
    logger.info(f'reading frequency list ... {df.shape[0]} items')

    # get corpus size
    R = df[0].sum()

    # apply frequency threshold
    logger.info('applying frequency threshold ...')
    df = df.loc[df[0] >= min_freq]
    logger.info(f'applying frequency threshold ... {df.shape[0]} items')

    # indexing
    logger.info('combining relevant columns ...')
    col = list(df.columns[1:]) if columns is None else columns
    df.columns = ['freq'] + col
    if len(col) > 1:
        df['item'] = df[col].agg(' '.join, axis=1)
    else:
        df['item'] = df[col]
    df = df.set_index('item')
    logger.info(
        'combining relevant columns ... item="%s"' % " ".join([str(c) for c in col])
    )

    df = df.sort_values(['freq', 'item'], ascending=False)

    return df, R

def cwb_lexdecode(corpus_name, registry_dir,
                  p_att='word', cmd='cwb-lexdecode', min_freq=2):
    """Run cwb-lexdecode: create frequency list of p-attribute.
    CLI: cwb-lexdecode -f -s -P lemma CORPUS_NAME

    :param str corpus_name:
    :param str registry_dir:
    :param str p_att:
    :param str cmd: path to binary
    :param int min_freq:

    :return: frequency list of p-attribute values and corpus size
    :rtype: tuple(DataFrame, int)

    """

    logger.info("running cwb-lexdecode ...")
    command = [cmd, '-f', '-P', p_att, '-r', registry_dir, corpus_name]
    lexdecode = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    ret = lexdecode.communicate()[0].decode()
    df_counts, R = read_freq_list(StringIO(ret), min_freq=min_freq, columns=[p_att])

    return df_counts, R

def cwb_scan_corpus(corpus_name, registry_dir, path=None,
                    p_atts=['word'], cmd='cwb-scan-corpus', min_freq=2):
    """Run cwb-scan-corpus: create frequency list of p-attribute(s) at corpus positions.
    CLI: cwb-scan-corpus [-R path] CORPUS_NAME p_atts

    :return: counts of the p-attribute values (or their combinations)
             at the given positions (if any)
    :param str corpus_name:
    :param str registry_dir:
    :param str p_att:
    :param str cmd: path to binary
    :param int min_freq:

    :return: frequency list of p-attribute values and corpus size
    :rtype: tuple(DataFrame, int)

    """

    logger.info("running cwb-scan-corpus ...")
    command = [cmd, '-r', registry_dir]
    if path is not None:
        command += ['-R', path]
    command += [corpus_name] + p_atts
    scan = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    ret = scan.communicate()[0].decode()
    df_counts, R = read_freq_list(StringIO(ret), min_freq=min_freq, columns=p_atts)

    return df_counts, R

class Counts:
    """All methods return a FreqFrame

    == (item)* freq, p_atts[0], ...  ==

    where item = " ".join(p_atts)

    If index is not split, MWUs are " "-joined.

    TODO: counting with group?

    """
    def __init__(self, corpus_name,
                 registry_dir='/usr/local/share/cwb/registry/'):

        self.corpus_name = corpus_name
        self.registry_dir = registry_dir
        self.attributes = Crps(
            self.corpus_name,
            registry_dir=registry_dir
        )

    def _cpos2patts(self, cpos, p_atts=['word'], ignore=True):
        """Retrieve p-attributes of corpus position.

        :param int cpos: corpus position to fill
        :param list p_atts: p-attribute(s) to fill position with
        :param bool ignore: whether to return (None, .*) for -1

        :return: p-attribute(s) at cpos
        :rtype: tuple

        """

        if cpos == -1 and ignore:
            token = [None] * len(p_atts)
        else:
            token = [
                self.attributes.attribute(p_att, 'p')[cpos] for p_att in p_atts
            ]

        return tuple(token)

    def cpos(self, cpos_list, p_atts=['word']):
        """Create a frequency table for the p-attribute values of the
        cpos-list.

        - strategy: split   /YES; flags  ; combo x

        :param list cpos_list: corpus positions to fill
        :param list p_atts: p-attribute (combinations) to count

        :return: counts of the p_attribute (combinations) of the positions
        :rtype: FreqFrame

        """
        items = [self._cpos2patts(p, p_atts=p_atts) for p in cpos_list]
        df_counts = count_items(items, p_atts)
        return df_counts

    @time_it
    def dump(self, df_dump, start='match', end='matchend',
             p_atts=['word'], split=False, strategy=2):
        """Count tokens in [start .. end] (columns or index columns in df_dump).

        - strategy 1: split NO/YES; flags  ; combo x
        - strategy 2: split   /YES; flags  ; combo x

        :param list df_dump: corpus positions to fill
        :param str start: column name where to start counting
        :param str end: column name where to end counting
        :param list p_atts: p-attribute (combinations) to count
        :param bool split: token-based count? (default: MWU)
        :param int strategy: strategy 2 (cwb-scan-corpus) is faster,
                             does not support MWU counts though

        :return: counts of the p_att (combin.) in the spans of two columns of the dump
        :rtype: FreqFrame

        """

        # choose strategy
        if strategy == 2 and not split:
            logger.warning("dump: cannot use cwb-scan-corpus for MWUs")
            strategy = 1
        logger.info(f"dump: strategy {strategy}")

        # for working with match, matchend
        df_dump = df_dump.reset_index()

        if strategy == 1:

            logger.info("... extracting tokens")
            ls = df_dump.apply(
                lambda x: [self._cpos2patts(cpos, p_atts) for cpos in range(
                    x[start], x[end] + 1
                )], axis=1
            ).values            # list of list of tuples (p_att_1, p_att_2, ...)

            logger.info("... splitting")
            if split:
                tokens = [token for tokens in ls for token in tokens]
            else:
                tokens = [
                    tuple([" ".join(m) for m in zip(*mwu_list)]) for mwu_list in ls
                ]

            df_counts = count_items(tokens, p_atts)

        elif strategy == 2:
            df_dump = df_dump.reset_index()
            with NamedTemporaryFile(mode="wt") as f:
                logger.info("... writing dump temporarily to disk")
                df_dump[[start, end]].to_csv(f.name, sep="\t", header=None, index=False)
                df_counts, R = cwb_scan_corpus(
                    self.corpus_name, self.registry_dir, f.name, p_atts
                )

        df_counts = df_counts.sort_values(by=['freq', 'item'], ascending=False)

        return df_counts

    @time_it
    def matches(self, cqp, name, p_atts=["word"], split=False, flags=None, strategy=3):
        """Counts tokens in [match .. matchend] of named subcorpus defined in
        running cqp.

        - strategy 1: split NO/   ; flags x; combo
        - strategy 2: split NO/YES; flags x; combo
        - strategy 3: split   /YES; flags  ; combo x

        :param CQP cqp: running cqp process
        :param list name: name of the subcorpus
        :param list p_atts: p-attribute(-combinations) to count
        :param bool split: token-based count? (default: MWU)
        :param str flags: %c, %d, %cd

        :return: counts of the p_attribute (combinations) of the matches
        :rtype: FreqFrame

        """

        # choose strategy
        combo = len(p_atts) > 1

        #    s f c
        # 1: - - -
        # 1: - x -
        # 2: - - -
        # 2: - x -
        # 2: x - -
        # 2: x x -
        # 3: x - -
        # 3: x - x

        # implemented:
        #    - - - 1,2
        #    - x - 1,2
        #    x - - 2,3
        #    x x - 2
        #    x - x 3

        # not implemented:
        #    - - x
        #    - x x
        #    x x x

        if combo:
            if flags or (not flags and not split):
                raise NotImplementedError(
                    "matches does not support parameter combination:",
                    str(" ".join(['x' if x else '-' for x in [
                        split, len(flags) > 0, combo
                        ]]))
                )

        if strategy == 1:
            if split or combo:
                logger.warning(
                    "matches: cannot use cqp-count"
                )
                strategy = 2
        if strategy == 2:
            if combo:
                logger.warning(
                    "matches: cannot use cqp-tabulate"
                )
                strategy = 3
        if strategy == 3:
            if flags or not split:
                logger.warning(
                    "matches: cannot use cwb-scan-corpus"
                )
                strategy = 2
        logger.info(f"matches: strategy {strategy}")

        if strategy == 1:
            # split NO; flags NO/YES; combo NO
            # generally slow
            logger.info("... cqp is counting")
            cqp_return = cqp.Exec(f'count {name} by {p_atts[0]} {flags};')
            df_counts = read_csv(
                StringIO(cqp_return), sep="\t", header=None,
                names=["freq", "unknown", p_atts[0]]
            )
            df_counts = df_counts.set_index(p_atts[0], drop=False)
            df_counts.index.name = 'item'
            df_counts = df_counts.sort_values(['freq', 'item'], ascending=False)

        elif strategy == 2:
            # split NO/YES; flags NO/YES; combo NO
            # generally faster
            logger.info("... cqp is tabulating")
            cqp_return = cqp.Exec(f'tabulate {name} match .. matchend {p_atts[0]} {flags};')
            logger.info("... splitting tokens")
            if split:           # split strings into tokens
                cqp_return = cqp_return.replace(" ", "\n")
            tokens = cqp_return.split("\n")
            df_counts = count_items(tokens, names=[p_atts[0]], tuples=False)

        elif strategy == 3:
            # split YES; flags NO; combo YES
            # generally fastest
            with NamedTemporaryFile(mode="wt") as f:
                logger.info("... writing dump temporarily to disk")
                cqp.Exec(f'dump {name} > "{f.name}";')
                df_counts, R = cwb_scan_corpus(
                    self.corpus_name, self.registry_dir, f.name, p_atts
                )

        return df_counts

    @time_it
    def mwus(self, cqp, queries, p_atts=None, fill_missing=True, strategy=1):
        """Calculates frequencies for MWU queries in activated subcorpus.
        queries are a list of valid CQP queries, e.g.

        '[lemma="Angela"%cd & pos="NE"] [lemma="Merkel"%cd & pos="NE"]?'

        - strategy 1: split NO| - ; flags x; combo x; mwu NO
        - strategy 2: split NO| - ; flags x; combo x; mwu YES
        - strategy 3: split NO| - ; flags x; combo  ; mwu YES

        caveat: strategy 1 does not yield breakdown in attributes
        this implies also different indexing

        Strategies:
        - Strategy 1: for each item
          1. run query
          2. get size of NQR via CQP
        - Strategy 2:
          1. run query for all items at the same time
          2. dump df
          3. count_dump()
        - Strategy 3:
          1. run query for all items at the same time
          2. count_matches()

        :param CQP cqp: running cqp process
        :param set queries: set of query strings to get frequency breakdown for
        :param bool fill_missing: count 0 for missing items?
        :param int strategy: strategy to use (see below)

        :return: counts of the queries (strategy 1) or the items in the queries
        :rtype: FreqFrame

        """

        # subcorpus name to use
        name = 'Tmp'

        # choose strategy
        if strategy == 1:
            if p_atts:
                logger.warning(
                    "mwus: cannot get frequency breakdown when not inspecting dump"
                )
                strategy = 2

        # necessary for strategies 2 & 3:
        p_atts = ['word'] if not p_atts else p_atts

        if strategy == 3 and len(p_atts) > 1:
            logger.warning(
                "mwus: cannot combine query when looking at several p-attributes"
            )
            strategy = 2
        logger.info(f"mwus: strategy {strategy}")

        # count
        if strategy == 1:
            logger.info("... running each query")
            freqs = list()
            for query in queries:
                cqp.Exec(f'{name}={query};')
                freq = cqp.Exec(f'size {name};')
                freqs.append(freq)
            df = DataFrame(data=freqs, index=queries, columns=['freq'])
            df.index.name = 'item'
            df['freq'] = df['freq'].astype(int)
            df = df.sort_values(by=['freq', 'item'], ascending=False)

        elif strategy == 2:
            query = "|".join(queries)
            cqp.Exec(f'{name}={query};')
            df_dump = cqp.Dump(name)
            df = self.dump(df_dump, start='match', end='matchend',
                           p_atts=p_atts, split=False, strategy=1)

        elif strategy == 3:
            query = "|".join(queries)
            cqp.Exec(f'{name}={query};')
            df = self.matches(cqp, name, p_atts=p_atts,
                              split=False, flags=None, strategy=2)

        # post-process dataframe
        df["freq"] = df["freq"].astype(int)
        # df = df.loc[df["freq"] != 0]

        return df

def score_counts(df, order='log_likelihood', cut_off=1000,
                 flags=None, ams=None, digits=6, vocab=None):
    """score counts in DataFrame.

    :param DataFrame df: DataFrame with reasonably-named columns, index 'item'

    :param str order: association measure for sorting (in descending order)
    :param int cut_off: number of items to retrieve
    :param str flags: '%c' / '%d' / '%cd' (cwb-ccc algorithm)
    :param list ams: association measures to calculate (None=all)
    :param int digits: round dataframe

    :return: scored counts
    :rtype: ScoreFrame

    """

    logger.info(f'scoring {len(df)} counts')

    # post-processing: fold items
    df = fold_df(df, flags)

    # calculate associations
    df = measures.score(df, measures=ams, freq=True, per_million=True,
                        digits=digits, boundary='poisson', vocab=vocab)

    # sort
    df = df.sort_values(by=[order, 'item'], ascending=[False, True])

    # apply cut-off
    df = df.head(cut_off) if cut_off is not None else df

    return df


from PyQt5.QtCore import QDate, QFile, Qt, QTextStream
from PyQt5.QtGui import (QFont, QIcon, QKeySequence, QTextCharFormat,
        QTextCursor, QTextTableFormat)
from PyQt5.QtPrintSupport import QPrintDialog, QPrinter
from PyQt5.QtWidgets import (QAction, QApplication, QDialog, QDockWidget,
        QFileDialog, QListWidget, QMainWindow, QMessageBox, QTextEdit)

import dockwidgets_rc

class MainWindow(QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()

        self.textEdit = QTextEdit()
        self.setCentralWidget(self.textEdit)

        self.createActions()
        self.createMenus()
        self.createToolBars()
        self.createStatusBar()
        self.createDockWindows()

        self.setWindowTitle("Dock Widgets")

        self.newLetter()

    def newLetter(self):
        self.textEdit.clear()

        cursor = self.textEdit.textCursor()
        cursor.movePosition(QTextCursor.Start)
        topFrame = cursor.currentFrame()
        topFrameFormat = topFrame.frameFormat()
        topFrameFormat.setPadding(16)
        topFrame.setFrameFormat(topFrameFormat)

        textFormat = QTextCharFormat()
        boldFormat = QTextCharFormat()
        boldFormat.setFontWeight(QFont.Bold)
        italicFormat = QTextCharFormat()
        italicFormat.setFontItalic(True)

        tableFormat = QTextTableFormat()
        tableFormat.setBorder(1)
        tableFormat.setCellPadding(16)
        tableFormat.setAlignment(Qt.AlignRight)
        cursor.insertTable(1, 1, tableFormat)
        cursor.insertText("The Firm", boldFormat)
        cursor.insertBlock()
        cursor.insertText("321 City Street", textFormat)
        cursor.insertBlock()
        cursor.insertText("Industry Park")
        cursor.insertBlock()
        cursor.insertText("Some Country")
        cursor.setPosition(topFrame.lastPosition())
        cursor.insertText(QDate.currentDate().toString("d MMMM yyyy"),
                textFormat)
        cursor.insertBlock()
        cursor.insertBlock()
        cursor.insertText("Dear ", textFormat)
        cursor.insertText("NAME", italicFormat)
        cursor.insertText(",", textFormat)
        for i in range(3):
            cursor.insertBlock()
        cursor.insertText("Yours sincerely,", textFormat)
        for i in range(3):
            cursor.insertBlock()
        cursor.insertText("The Boss", textFormat)
        cursor.insertBlock()
        cursor.insertText("ADDRESS", italicFormat)

    def print_(self):
        document = self.textEdit.document()
        printer = QPrinter()

        dlg = QPrintDialog(printer, self)
        if dlg.exec_() != QDialog.Accepted:
            return

        document.print_(printer)

        self.statusBar().showMessage("Ready", 2000)

    def save(self):
        filename, _ = QFileDialog.getSaveFileName(self,
                "Choose a file name", '.', "HTML (*.html *.htm)")
        if not filename:
            return

        file = QFile(filename)
        if not file.open(QFile.WriteOnly | QFile.Text):
            QMessageBox.warning(self, "Dock Widgets",
                    "Cannot write file %s:\n%s." % (filename, file.errorString()))
            return

        out = QTextStream(file)
        QApplication.setOverrideCursor(Qt.WaitCursor)
        out << self.textEdit.toHtml()
        QApplication.restoreOverrideCursor()

        self.statusBar().showMessage("Saved '%s'" % filename, 2000)

    def undo(self):
        document = self.textEdit.document()
        document.undo()

    def insertCustomer(self, customer):
        if not customer:
            return
        customerList = customer.split(', ')
        document = self.textEdit.document()
        cursor = document.find('NAME')
        if not cursor.isNull():
            cursor.beginEditBlock()
            cursor.insertText(customerList[0])
            oldcursor = cursor
            cursor = document.find('ADDRESS')
            if not cursor.isNull():
                for i in customerList[1:]:
                    cursor.insertBlock()
                    cursor.insertText(i)
                cursor.endEditBlock()
            else:
                oldcursor.endEditBlock()

    def addParagraph(self, paragraph):
        if not paragraph:
            return
        document = self.textEdit.document()
        cursor = document.find("Yours sincerely,")
        if cursor.isNull():
            return
        cursor.beginEditBlock()
        cursor.movePosition(QTextCursor.PreviousBlock, QTextCursor.MoveAnchor,
                2)
        cursor.insertBlock()
        cursor.insertText(paragraph)
        cursor.insertBlock()
        cursor.endEditBlock()

    def about(self):
        QMessageBox.about(self, "About Dock Widgets",
                "The <b>Dock Widgets</b> example demonstrates how to use "
                "Qt's dock widgets. You can enter your own text, click a "
                "customer to add a customer name and address, and click "
                "standard paragraphs to add them.")

    def createActions(self):
        self.newLetterAct = QAction(QIcon(':/images/new.png'), "&New Letter",
                self, shortcut=QKeySequence.New,
                statusTip="Create a new form letter", triggered=self.newLetter)

        self.saveAct = QAction(QIcon(':/images/save.png'), "&Save...", self,
                shortcut=QKeySequence.Save,
                statusTip="Save the current form letter", triggered=self.save)

        self.printAct = QAction(QIcon(':/images/print.png'), "&Print...", self,
                shortcut=QKeySequence.Print,
                statusTip="Print the current form letter",
                triggered=self.print_)

        self.undoAct = QAction(QIcon(':/images/undo.png'), "&Undo", self,
                shortcut=QKeySequence.Undo,
                statusTip="Undo the last editing action", triggered=self.undo)

        self.quitAct = QAction("&Quit", self, shortcut="Ctrl+Q",
                statusTip="Quit the application", triggered=self.close)

        self.aboutAct = QAction("&About", self,
                statusTip="Show the application's About box",
                triggered=self.about)

        self.aboutQtAct = QAction("About &Qt", self,
                statusTip="Show the Qt library's About box",
                triggered=QApplication.instance().aboutQt)

    def createMenus(self):
        self.fileMenu = self.menuBar().addMenu("&File")
        self.fileMenu.addAction(self.newLetterAct)
        self.fileMenu.addAction(self.saveAct)
        self.fileMenu.addAction(self.printAct)
        self.fileMenu.addSeparator()
        self.fileMenu.addAction(self.quitAct)

        self.editMenu = self.menuBar().addMenu("&Edit")
        self.editMenu.addAction(self.undoAct)

        self.viewMenu = self.menuBar().addMenu("&View")

        self.menuBar().addSeparator()

        self.helpMenu = self.menuBar().addMenu("&Help")
        self.helpMenu.addAction(self.aboutAct)
        self.helpMenu.addAction(self.aboutQtAct)

    def createToolBars(self):
        self.fileToolBar = self.addToolBar("File")
        self.fileToolBar.addAction(self.newLetterAct)
        self.fileToolBar.addAction(self.saveAct)
        self.fileToolBar.addAction(self.printAct)

        self.editToolBar = self.addToolBar("Edit")
        self.editToolBar.addAction(self.undoAct)

    def createStatusBar(self):
        self.statusBar().showMessage("Ready")

    def createDockWindows(self):
        dock = QDockWidget("Customers", self)
        dock.setAllowedAreas(Qt.LeftDockWidgetArea | Qt.RightDockWidgetArea)
        self.customerList = QListWidget(dock)
        self.customerList.addItems((
            "John Doe, Harmony Enterprises, 12 Lakeside, Ambleton",
            "Jane Doe, Memorabilia, 23 Watersedge, Beaton",
            "Tammy Shea, Tiblanka, 38 Sea Views, Carlton",
            "Tim Sheen, Caraba Gifts, 48 Ocean Way, Deal",
            "Sol Harvey, Chicos Coffee, 53 New Springs, Eccleston",
            "Sally Hobart, Tiroli Tea, 67 Long River, Fedula"))
        dock.setWidget(self.customerList)
        self.addDockWidget(Qt.RightDockWidgetArea, dock)
        self.viewMenu.addAction(dock.toggleViewAction())

        dock = QDockWidget("Paragraphs", self)
        self.paragraphsList = QListWidget(dock)
        self.paragraphsList.addItems((
            "Thank you for your payment which we have received today.",
            "Your order has been dispatched and should be with you within "
                "28 days.",
            "We have dispatched those items that were in stock. The rest of "
                "your order will be dispatched once all the remaining items "
                "have arrived at our warehouse. No additional shipping "
                "charges will be made.",
            "You made a small overpayment (less than $5) which we will keep "
                "on account for you, or return at your request.",
            "You made a small underpayment (less than $1), but we have sent "
                "your order anyway. We'll add this underpayment to your next "
                "bill.",
            "Unfortunately you did not send enough money. Please remit an "
                "additional $. Your order will be dispatched as soon as the "
                "complete amount has been received.",
            "You made an overpayment (more than $5). Do you wish to buy more "
                "items, or should we return the excess to you?"))
        dock.setWidget(self.paragraphsList)
        self.addDockWidget(Qt.RightDockWidgetArea, dock)
        self.viewMenu.addAction(dock.toggleViewAction())

        self.customerList.currentTextChanged.connect(self.insertCustomer)
        self.paragraphsList.currentTextChanged.connect(self.addParagraph)

if __name__ == '__main__':

    import sys

    app = QApplication(sys.argv)
    mainWin = MainWindow()
    mainWin.show()
    sys.exit(app.exec_())