from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import Permission, Group
from django.utils.translation import gettext_lazy as _

from apps.users.forms import LedgerModelForm, UserChangeForm
from apps.users.models import UserDocument, WorkSite, TimeSheetMonthlyRecord, Ledger, User

from django.contrib.admin.options import flatten_fieldsets
from django.contrib.admin.templatetags.admin_modify import register
from django.contrib.admin.templatetags.admin_modify import submit_row as original_submit_row

class ReadOnlyModelAdmin(admin.ModelAdmin):
    """
    ModelAdmin class that prevents modifications through the admin.
    The changelist and the detail view work, but a 403 is returned
    if one actually tries to edit an object.
    Source: https://gist.github.com/aaugustin/1388243
    """
    actions = None

    # We cannot call super().get_fields(request, obj) because that method calls
    # get_readonly_fields(request, obj), causing infinite recursion. Ditto for
    # super().get_form(request, obj). So we  assume the default ModelForm.
    def get_readonly_fields(self, request, obj=None):
        if request.user.is_superuser:
            return []
        return self.fields or [f.name for f in self.model._meta.fields]

    def has_change_permission(self, request, obj=None):
        return True if request.user.is_superuser else (request.method in ['GET', 'HEAD'] and
                                                       request.user.is_staff)

    def has_module_permission(self, request):
        return request.user.is_superuser or request.user.is_staff

    def get_readonly_fields(self, request, obj=None):
        user = request.user
        if user.is_staff and not user.is_superuser:
            if self.fieldsets:
                return flatten_fieldsets(self.fieldsets or [])
            else:
                return list(set(
                    [field.name for field in self.opts.local_fields] +
                    [field.name for field in self.opts.local_many_to_many]))

        return super().get_readonly_fields(request, obj)

    @register.inclusion_tag('admin/submit_line.html', takes_context=True)
    def submit_row(context):
        ctx = original_submit_row(context)

        user = context['request'].user
        if user.is_staff and not user.is_superuser:
            ctx.update({
                'show_save_and_add_another': False,
                'show_save_and_continue': False,
                'show_save': False,
                'show_save_as_new': False,
            })
        return ctx

    def has_view_permission(self, request, obj=None):
        return request.user.is_staff

class UserDocumentAdmin(ReadOnlyModelAdmin):
    list_display = ('user', 'document_type', 'name', 'description', 'issued_by', 'issued_date', 'expiry_date',
                    'image_tag')

    list_filter = ('user', 'document_type')

    class Meta:
        model = UserDocument

    def get_queryset(self, request):
        queryset = super(UserDocumentAdmin, self).get_queryset(request)
        if not request.user.is_superuser:
            queryset = queryset.filter(user=request.user)
        return queryset

    def get_list_filter(self, request):
        if not request.user.is_superuser:
            return []
        return self.list_filter

class EmployeeAdmin(ReadOnlyModelAdmin, UserAdmin):
    form = UserChangeForm

    list_display = ('id', '_name', 'profile_pic_tag', '_total_expenses', '_total_earning', '_current_balance')

    def get_queryset(self, request):
        queryset = super(EmployeeAdmin, self).get_queryset(request)
        if not request.user.is_superuser:
            queryset = queryset.filter(pk=request.user.pk)
        # queryset2 = queryset.annotate(total_expenses=Sum('user__ledgers__amount'))
        return queryset

    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'password1', 'password2')
        }),
    )

    fieldsets = (
        (None, {'fields': ('username', 'password')}),
        (_('Personal info'), {'fields': ('first_name', 'last_name', 'email', 'profile_pic')}),
        (_('Permissions'), {
            'fields': ('is_active', 'is_staff', 'is_superuser', 'groups', 'user_permissions'),
        }),
        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
    )

    class Meta:
        model = User

    @staticmethod
    def _name(obj):
        return obj.get_full_name() or obj.username
    _name.short_description = "Name"

    def get_list_filter(self, request):
        if not request.user.is_superuser:
            return []
        return self.list_filter

    def get_readonly_fields(self, request, obj=None):
        readonly_fields = list(super().get_readonly_fields(request, obj))
        if not request.user.is_superuser:
            readonly_fields.extend(
                ['is_superuser', 'is_active', 'is_staff', 'is_superuser', 'groups', 'user_permissions',
                 ])
            readonly_fields = tuple(readonly_fields)
        return readonly_fields

    def get_form(self, request, obj=None, **kwargs):
        form = super().get_form(request, obj, **kwargs)
        is_superuser = request.user.is_superuser
        disabled_fields = set()  # type: Set[str]

        if not is_superuser:
            disabled_fields |= {
                'username',
                'is_superuser',
                'date_joined',
            }

        for f in disabled_fields:
            if f in form.base_fields:
                form.base_fields[f].disabled = True

        return form

class WorkSiteAdmin(ReadOnlyModelAdmin):
    list_display = ('id', 'name', 'location')

    class Meta:
        model = WorkSite

    def has_module_permission(self, request):
        return True if request.user.is_superuser else False

class LedgerAdmin(ReadOnlyModelAdmin):
    list_display = ['user', 'type', 'expense_date', 'amount', 'notes', 'time_sheet_record', 'hours', 'hourly_rate',
                    'trade']

    list_filter = ['type', 'expense_date']

    form = LedgerModelForm

    class Meta:
        model = Ledger

    def get_queryset(self, request):
        queryset = super(LedgerAdmin, self).get_queryset(request)
        if not request.user.is_superuser:
            queryset = queryset.filter(user=request.user)
        return queryset

    def get_list_filter(self, request):
        list_filter = self.list_filter
        if request.user.is_superuser:
            self.list_filter.append('user')
        return list_filter

class MonthlySheetAdmin(ReadOnlyModelAdmin):

    list_display = ['work_month', 'time_sheet_file', 'work_site', 'notes']

    class Meta:
        model = TimeSheetMonthlyRecord

admin.site.unregister(Group)

admin.site.register(User, EmployeeAdmin)
admin.site.register(UserDocument, UserDocumentAdmin)
admin.site.register(WorkSite, WorkSiteAdmin)
admin.site.register(TimeSheetMonthlyRecord, MonthlySheetAdmin)
admin.site.register(Ledger, LedgerAdmin)


import sys
import threading
import time
from collections import namedtuple
import logging
import queue

# LOGGER
LOGGER = logging.getLogger(__name__)

# Computer job format
Job = namedtuple('Job', ['chain', 'work', "data"])

class PriorityItem:
    """Priority queue item.

    This is helper class of priority queue.
    """

    def __init__(self, elem_to_wrap, priority):
        """Init."""
        self.job = elem_to_wrap
        self.priority = priority
        self.time = time.time()

    def __lt__(self, other):
        """Comparison order."""
        if self.priority < other.priority:
            return True

        if self.time < other.time:
            return True

        return False

class Computer(threading.Thread):
    """Class Computer.

    init:
        Computer(COMPUTER_DEF)

    COMPUTER_DEF = {
        'compute_model': Oneshot(DeepLSTM(300, 300),
                           optimizers.adam(),
                           loss_func, acc_func)
        'callbacks':{
            'result': callback_func(result)
        }
    }
    """

    def __init__(self, compute_model, callbacks=None):
        """Init."""
        super(Computer, self).__init__()
        self.setDaemon(True)

        # model is required
        self.compute_model = compute_model

        # computer internal variable
        self.__datacnt = 0

        # queue
        self.__q = queue.PriorityQueue()

        # callbacks
        self.callbacks = {}
        if isinstance(callbacks, dict):
            for name in callbacks:
                if not self.set_event(name, callbacks[name]):
                    raise ValueError("registration callback is failed.")

        self._trigger_event("load")

    @staticmethod
    def _handler(func, *args):
        return func(*args)

    def _trigger_event(self, event, *args):
        if event not in self.callbacks:
            return False

        return self._handler(self.callbacks[event], *args)

    def _compute(self, job):
        return getattr(self.compute_model, job.work)(job.data)

    def _print_message(self, process_time, addtext=""):
        iops = 1.0 / process_time
        message = ("\r({0}/{1}) in {2:.2f}s ({3:.2f} IOPS) {4} "
                   .format(self.__datacnt, self.__q.qsize() + self.__datacnt,
                           process_time, iops, addtext))
        sys.stdout.write(message)
        sys.stdout.flush()
        return message

    def _exec_queue(self, job):
        # datacnt copy before process
        self.__datacnt += 1
        datacnt = self.__datacnt

        # compute
        processstart = time.time()
        output = self._compute(job)
        process_time = time.time() - processstart

        # append info
        machine_info = {
            "datacnt": datacnt,
            "process_time": process_time,
            "work": job.work,
            "chain": job.chain,
        }

        # post massage
        self._print_message(process_time)

        if output is None:
            return (None, machine_info)

        # loop event
        if job.work == "loop":
            if output is False:
                return (None, machine_info)

            self.add_queue(job.data, job.work, job.chain, 10)

        return (output, machine_info)

    def _check_job(self, data, work):
        if work is not None:
            if not isinstance(work, str):
                return False

            if work.startswith("_"):
                return False

            if not callable(getattr(self.compute_model, work, None)):
                return False

        return True

    def set_event(self, event, callback, force=False):
        """Set callback event."""
        if not callable(callback):
            return False
        if event in self.callbacks and force is False:
            return False

        self.callbacks[event] = callback

        return True

    def add_queue(self, data, work, chain, priority=3):
        """Add_queue, when check_job() return True."""
        if self._check_job(data, work):
            job = Job(chain, work, data)
            self.__q.put(PriorityItem(job, priority))
            self._trigger_event("accept")
            return True

        self._trigger_event("reject")
        return False

    def run(self):
        """Thread loop."""

        # loop start
        if callable(getattr(self.compute_model, "setup", None)):
            self.add_queue(None, "setup", "setup", priority=1)

        if callable(getattr(self.compute_model, "loop", None)):
            self.add_queue(None, "loop", "loop", priority=2)

        while True:
            next_queue = self.__q.get(block=True).job
            self._trigger_event("fetch", next_queue)
            output, machine_info = self._exec_queue(next_queue)
            if output is not None:
                self._trigger_event("writeback", output, machine_info)
            self.__q.task_done()


from __future__ import print_function, division

__author__ = 'amrit'

import sys

sys.dont_write_bytecode = True
import pandas as pd
from random import seed, shuffle
import numpy as np
from ML import *
import time
from demos import cmd
from sklearn.model_selection import StratifiedKFold
import pickle
from collections import OrderedDict
import os

learners_class=[run_dectreeclas,run_rfclas,run_svmclas]
learners_reg=[run_dectreereg, run_rfreg,run_svmreg]
learners_para_dic=[OrderedDict(), OrderedDict(), OrderedDict()]
metrics=['accuracy','recall','precision','false_alarm','times','features']

ROOT=os.getcwd()
metric='recall'

def _test(res=''):
    seed(1)
    np.random.seed(1)

    file_names = []
    for _, _, files in os.walk(ROOT + "/../data/preprocessed_data/"):
        for file in files:
            if file.startswith(res):
                file_names.append(file.split(".csv")[0])

    if res == 'dataset1_math' or res == 'dataset1_portuguese':
        class_flag = False
    else:
        class_flag = True

    for f in file_names:
        df = pd.read_csv("../data/preprocessed_data/" + f + ".csv")
        corpus, labels = np.array(df[df.columns[:-1]].values.tolist()), np.array(df[df.columns[-1]].values.tolist())
        columns=df.columns[:-1].values.tolist()
        ranges = range(0, len(labels))
        temp = {}
        for num,i in enumerate(learners_class):

            temp[i.__name__] = {}
            temp['cols'] = columns

            if class_flag:
                for m in metrics:
                    temp[i.__name__][m] = []
                _, whole = learners_class[num](OrderedDict(learners_para_dic[num]), corpus
                                            , labels, corpus, labels, metric)
                temp[i.__name__]['whole']=whole[1]

                for _ in range(5):
                    shuffle(ranges)
                    corpus,labels=corpus[ranges],labels[ranges]
                    skf = StratifiedKFold(n_splits=5)
                    for train_index, test_index in skf.split(corpus, labels):
                        train_data, train_labels = corpus[train_index], labels[train_index]
                        test_data, test_labels = corpus[test_index], labels[test_index]

                        start_time=time.time()
                        _,val=learners_class[num](OrderedDict(learners_para_dic[num]),train_data
                                 , train_labels,test_data,test_labels,metric)
                        temp[i.__name__]['times'].append(time.time() - start_time)
                        for x in val[0]:
                            temp[i.__name__][x].append(val[0][x])
                        temp[i.__name__]['features'].append(val[1])

            else:
                ## Whole Model being checked
                _ , whole = learners_reg[num](OrderedDict(learners_para_dic[num]), corpus
                                          ,labels, corpus, labels)
                temp[i.__name__]['times'] = []
                temp[i.__name__]['MSE'] = []
                temp[i.__name__]['features'] = []
                temp[i.__name__]['whole'] = whole[1]

                for k in range(25):
                    shuffle(ranges)
                    train_data, train_labels, test_data , test_labels = corpus[ranges[:int(0.8*len(ranges))]]\
                        , labels[ranges[:int(0.8 * len(ranges))]]\
                        ,corpus[ranges[int(0.8*len(ranges)):]],labels[ranges[int(0.8*len(ranges)):]]
                    start_time=time.time()
                    t, val = learners_reg[num](OrderedDict(learners_para_dic[num]), train_data
                                            , train_labels, test_data, test_labels)
                    temp[i.__name__]['times'].append(time.time() - start_time)
                    temp[i.__name__]['MSE'].append(t)
                    temp[i.__name__]['features'].append(val[1])

        with open('../dump/'+f+'_untuned.pickle', 'wb') as handle:
            pickle.dump(temp, handle)

if __name__ == '__main__':
    eval(cmd())


from sys import stdin
from collections import deque
input = stdin.readline

N, K = map(int, input().split())

dp = deque()
ysps_list = []
for i in range(N):
    dp.append(i+1)

while dp:
    for _ in range(K-1):
        dp.append(dp.popleft())

    ysps_list.append(dp.popleft())

print(str(ysps_list).replace('[', '<').replace(']', '>'))


import sys, os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname( __file__ ), '../..')))
import connection_oep as coep
import pandas as pd
import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker
import yaml
import helpers

def define_tables(engine, metadata):
    r"""
    Define hardcoded tables

    Returns
    -------
    tables : dict
        Dictionary containing tables.
    """
    tables = {}
    tables['input_param_table'] = sa.Table(
        'oemof_heat_system_b_input_parameter',
        metadata,
        sa.Column('component', sa.String(50)),
        sa.Column('var_name', sa.String(50)),
        sa.Column('var_value', sa.Float()),
        sa.Column('var_unit', sa.String(10)),
        sa.Column('reference', sa.String(10)),
        sa.Column('comment', sa.String(70)),
        sa.Column('tags', sa.String(50)),
        schema='sandbox')

    tables['timeseries_table'] = sa.Table(
        'oemof_heat_system_b_timeseries_temperature',
        metadata,
        sa.Column('timestamp', sa.String(50)),
        sa.Column('T', sa.Float()),
        schema='sandbox')

    return tables

def upload_data_to_oep(tables, engine, metadata):
    r"""
    Upload data to oep

    Parameters
    ----------
    tables : dict
        Dictionary containing tables
        to download.

    engine : sqlalchemy.Engine

    metadata : sqlalchemy.MetaData

    Returns
    -------
    engine : sqlalchemy.Engine

    metadata : sqlalchemy.MetaData
    """
    abs_path = os.path.dirname(os.path.abspath(os.path.join(__file__, '..')))

    input_parameters = pd.read_csv(os.path.join(abs_path, 'data_raw/oep_data/input_parameter.csv'))
    timeseries = pd.read_csv(os.path.join(abs_path, 'data_raw/oep_data/weather_data.csv'))

    coep.upload_to_oep(input_parameters, tables['input_param_table'], engine, metadata)
    coep.upload_to_oep(timeseries, tables['timeseries_table'], engine, metadata)

    return engine, metadata

def download_data_from_oep(tables, engine, metadata):
    r"""
    Gets data from oep.

    Parameters
    ----------
    tables : dict
        Dictionary containing tables
        to download.

    engine : sqlalchemy.Engine

    metadata : sqlalchemy.MetaData

    Returns
    -------
    data : dict
        Dictionary containing dataframes.
    """
    abs_path = os.path.dirname(os.path.abspath(os.path.join(__file__, '..')))

    # download
    data = {}
    for table_name, table in tables.items():
        data[table_name] = coep.get_df(engine, table)

    # save
    for key, value in data.items():
        save_as = os.path.join(abs_path, 'data_raw/oep_data',f'{key}.csv')
        value.to_csv(save_as)
        print('Saved as ', save_as)

    return data

def connect_to_oep(config_path, results_dir):
    with open(config_path, 'r') as ymlfile:
        cfg = yaml.load(ymlfile)

    if 'oep_download' in cfg and cfg['oep_download']:
        with open('../oep_cred.yml', 'r') as oep_cred:
            cred = yaml.load(oep_cred)

        engine, metadata = coep.connect_oep(cred['username'], cred['token'])
        print('Connection established')
        tables = define_tables(engine, metadata)
        import time
        start = time.time()
        upload_data_to_oep(tables, engine, metadata)
        end = time.time()
        print('time', end-start)
        download_data_from_oep(tables, engine, metadata)

if __name__ == '__main__':
    config_path, results_dir = helpers.setup_experiment()
    connect_to_oep(config_path=config_path, results_dir=results_dir)


import pandas as pd
import numpy as np
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import mean_absolute_error
import time
import warnings
warnings.filterwarnings('ignore')

# 导入数据
tr_data = pd.read_csv('xfdata/房屋租赁价格预测挑战赛公开数据/train.csv')
te_data = pd.read_csv('xfdata/房屋租赁价格预测挑战赛公开数据/test.csv')

data = pd.concat([tr_data, te_data], ignore_index=True)

print("#"*10 + "进行特征工程..." + "#"*10)
# 时间处理
data["建成年份"] = data["建成年份"].apply(lambda x: int(x) if 1811 <= x <= 2020 else np.nan)
data["最后翻新年份"] = data["最后翻新年份"].apply(lambda x : int(x) if 1990<= x <= 2020 else np.nan)
data["上传年份"] = data["上传日期"].replace({"Feb20": 2020, "May19": 2019,"Oct19": 2019,"Sep18": 2018})   # 2019年有两个月份
data["上传月份"] = data["上传日期"].replace({"Feb20": 3, "May19": 6, "Oct19": 10,"Sep18": 9})

data["上传-建成年差"] = data["上传年份"] - data["建成年份"]
data["最后翻新-建成年差"] = data["最后翻新年份"] - data["建成年份"]
data["上传-最后翻新年差"] = data["上传年份"] - data["最后翻新年份"]

# 处理负值的上传年差：
data["上传-建成年差"] = data["上传-建成年差"].apply(lambda x: x if x >=0 else np.nan)
data["最后翻新-建成年差"] = data["最后翻新-建成年差"].apply(lambda x: x if x >=0 else np.nan)
data["上传-最后翻新年差"] = data["上传-最后翻新年差"].apply(lambda x: x if x >=0 else np.nan)

# 费用、房间面积
data["房间数量"] = data["房间数量"].apply(lambda x: int(x) if 1 <= x <= 8 else 8)

data["供暖费用"] = data["供暖费用"].apply(lambda x : x if x <= 198 else np.nan)
data["服务费"] = data["服务费"].apply(lambda x : x if x <= 1000 else np.nan)

data["log_供暖费用"] = np.log(data["供暖费用"] + 1)         # # 供暖费相关性要比log_供暖费强
data["log_服务费"] = np.log(data["服务费"]+1)         # 服务费相关性要比log_服务费强

# 居住面积有0，这是问题的，但是test中也有，总不能删掉test吧？
data["居住面积"] = data["居住面积"].apply(lambda x: x if 10<=x<=500 else np.nan)  # 再议
data["log_居住面积"] = np.log(data["居住面积"])         # 居住面积的异常值感觉要处理，7/15 去除+1，已经没有0了

# 处理房屋租金
data.drop(data[(data["房屋租金"]>=10000)|(data["房屋租金"]<=100)].index, inplace=True)
data["log_房屋租金"] = np.log(data["房屋租金"]+1)   # 还有必要+1吗？

data["单位log面积服务费"] = data["服务费"] / data["log_居住面积"]
data["单位面积服务费"] = data["服务费"] / data["居住面积"]
data["单位房间服务费"] = data["服务费"] / data["房间数量"]

data["单位log面积上传图片数"] = data["上传图片数"] / data["log_居住面积"]
data["单位面积上传图片数"] = data["上传图片数"] / data["居住面积"]
data["单位房间上传图片数"] = data["上传图片数"] / data["房间数量"]

# 楼层处理
def judge_use_GB50096(x):
    # 使用住宅设计规范判定楼层：低层、多层、中高层、高层，外加地下室
    if x == 10086:   # 缺失值
        return np.nan
    elif x <= 0:
        return 0
    elif 1<=x and x<=3:
        return 1
    elif 4<=x and x<=6:
        return 2
    elif 7<=x and x<=9:      # & 是位运算符，and才是对于的逻辑运算符，python中建议常用逻辑运算符
        return 3
    else:
        return 4

data["所处楼层"] = data["所处楼层"].apply(lambda x: int(x) if x <= 31 else np.nan) # 31  # 由异常值函数得到下限10
data["建筑楼层"] = data["建筑楼层"].apply(lambda x : int(x) if x <= 54 else np.nan) # 54 # 由函数得到下限11
data["楼层占比"] = data["所处楼层"] / data["建筑楼层"]   # -1 地下一层
data["楼层占比"] = data["楼层占比"].apply(lambda x: x if np.abs(x) <= 1 else np.nan)
# 划分等级
data["所处楼层划分"] = data["所处楼层"].fillna(10086).apply(lambda x: judge_use_GB50096(x))
data["建筑楼层划分"] = data["建筑楼层"].fillna(10086).apply(lambda x: judge_use_GB50096(x))

# 电梯效应，所在楼层大于7楼没电梯，影响比1楼没电梯大
# <=11, 12<=18 19<=33(31)
# data["电梯效应"] = data.apply(lambda x: x["所处楼层"] if np.exp(x["有电梯"]+1) else -np.exp(x["所处楼层"]+1), axis=1)   # axis=1将行传入apply

# 其他
data["可带宠物_2"] = data["可带宠物"].replace({"negotiable": 2, "no": 1, "yes": 3})

# 类型特征编码
# 有缺失值的特征不适于one-hot，会重叠其中一类和缺失值
data = pd.get_dummies(data, columns=['上传月份'], drop_first=True)

# 特征统计量 ["区域1",'区域2', '区域3', '街道']
# 服务费         6385
# 所处楼层       47743
# 建筑楼层       90806
# 价格趋势        1699
# 房间数量           0
# 居住面积     本无缺失
temp = data.groupby("区域1")['服务费'].agg([('区域1_服务费_mean','mean'), ('区域1_服务费_std','std')])   # 这么高级的吗？
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域1', how='left')

temp = data.groupby("区域2")['服务费'].agg([('区域2_服务费_mean','mean'), ('区域2_服务费_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域2', how='left')

temp = data.groupby("区域3")['服务费'].agg([('区域3_服务费_mean','mean'), ('区域3_服务费_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域3', how='left')

temp = data.groupby("街道")['服务费'].agg([('街道_服务费_mean','mean'), ('街道_服务费_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='街道', how='left')

# 增加居住面积
temp = data.groupby("区域1")['居住面积'].agg([('区域1_居住面积_mean','mean'), ('区域1_居住面积_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域1', how='left')

temp = data.groupby("区域2")['居住面积'].agg([('区域2_居住面积_mean','mean'), ('区域2_居住面积_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域2', how='left')

temp = data.groupby("区域3")['居住面积'].agg([('区域3_居住面积_mean','mean'), ('区域3_居住面积_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域3', how='left')

temp = data.groupby("街道")['居住面积'].agg([('街道_居住面积_mean','mean'), ('街道_居住面积_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='街道', how='left')

# 增加更多特征统计量
temp = data.groupby("区域1")['所处楼层'].agg([('区域1_所处楼层_mean','mean'), ('区域1_所处楼层_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域1', how='left')

temp = data.groupby("区域1")['价格趋势'].agg([('区域1_价格趋势_mean','mean'), ('区域1_价格趋势_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域1', how='left')

temp = data.groupby("区域1")['房间数量'].agg([('区域1_房间数量_mean','mean'), ('区域1_房间数量_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域1', how='left')
# 1
temp = data.groupby("区域2")['所处楼层'].agg([('区域2_所处楼层_mean','mean'), ('区域2_所处楼层_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域2', how='left')

temp = data.groupby("区域2")['价格趋势'].agg([('区域2_价格趋势_mean','mean'), ('区域2_价格趋势_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域2', how='left')

temp = data.groupby("区域2")['房间数量'].agg([('区域2_房间数量_mean','mean'), ('区域2_房间数量_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域2', how='left')
# 2
temp = data.groupby("区域3")['所处楼层'].agg([('区域3_所处楼层_mean','mean'), ('区域3_所处楼层_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域3', how='left')

temp = data.groupby("区域3")['价格趋势'].agg([('区域3_价格趋势_mean','mean'), ('区域3_价格趋势_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域3', how='left')

temp = data.groupby("区域3")['房间数量'].agg([('区域3_房间数量_mean','mean'), ('区域3_房间数量_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='区域3', how='left')
# 3
temp = data.groupby("街道")['所处楼层'].agg([('街道_所处楼层_mean','mean'), ('街道_所处楼层_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='街道', how='left')

temp = data.groupby("街道")['价格趋势'].agg([('街道_价格趋势_mean','mean'), ('街道_价格趋势_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='街道', how='left')

temp = data.groupby("街道")['房间数量'].agg([('街道_房间数量_mean','mean'), ('街道_房间数量_std','std')])
temp.fillna(0, inplace=True)
data = data.merge(temp, on='街道', how='left')

l = data.columns.to_list()
ft = ["ID","房屋租金","log_供暖费用",
      "没有停车位","log_房屋租金",
      "上传日期","可带宠物","log_服务费",'log_居住面积','单位面积服务费','单位面积上传图片数']

features = [i for i in l if i not in ft]
category_fea = []
for i in ['房屋状况','内饰质量','加热类型','房屋类型','所处楼层划分', '建筑楼层划分','可带宠物_2']:
    category_fea.append(features.index(i))

train_data = data[data["房屋租金"].notnull()].reset_index(drop=True)
test_data = data[data["房屋租金"].isnull()].reset_index(drop=True)
x_train_temp = train_data[features]
y_train = train_data["log_房屋租金"]
x_test_temp = test_data[features]

# 处理种类基数较多的分类变量
from category_encoders import TargetEncoder
import pandas as pd
enc = TargetEncoder(cols=["区域1",'区域2', '区域3', '街道',"邮政编码"])
# transform the datasets
x_train = enc.fit_transform(x_train_temp, y_train)
x_test = enc.transform(x_test_temp)

# 保存x_test，用于预测
x_test.to_csv("user_data/tmp_data/x_test.csv", index=False)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
print(l)
print("\n")
print(features)
print("all_len: {}, fea_len: {}".format(len(l),len(features)))
print("train_data_num: {}, test_data_num: {}".format(len(train_data),len(test_data)))
print("category_fea: ", category_fea)
print("#"*10 + "特征工程已完成！" + "#"*10)
print("\n\n")
print("#"*10 + "进行训练模型..." + "#"*10)
############################################################
# 训练模型
def cv_model(clf, train_x, train_y, test_x, clf_name):
    folds = 10
    seed = 2022
    kf = KFold(n_splits=folds, shuffle=True, random_state=seed)

    train = np.zeros(train_x.shape[0])
#     test = np.zeros(test_x.shape[0])

    cv_scores = []

    for i, (train_index, valid_index) in enumerate(kf.split(train_x, train_y)):
        print('************************************ {} ************************************'.format(str(i+1)))
        print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
        trn_x, trn_y, val_x, val_y = train_x.iloc[train_index], train_y[train_index], train_x.iloc[valid_index], train_y[valid_index]

        if clf_name == "lgb":
            train_matrix = clf.Dataset(trn_x, label=trn_y)
            valid_matrix = clf.Dataset(val_x, label=val_y)

            params = {
                'boosting_type': 'gbdt',
                'objective': 'regression',
                'metric': 'l1',          # mae mean_absolute_error

                'seed': 2022,
                'n_jobs':-1,

                'bagging_fraction': 0.9426917523072508,
                'bagging_freq': 3,
                'feature_fraction': 0.6495602688979487,
                'lambda_l1': 0.02052223766043643,
                'lambda_l2': 3.7224143113186443,
                'learning_rate': 0.03319994978090327,
                'max_bin': 158,
                'min_data_in_leaf': 26,
                'min_sum_hessian_in_leaf': 8.865519169470296,
                'num_leaves': 89,
            }

            # category_fea
            model = clf.train(params, train_matrix, 90000, valid_sets=[train_matrix, valid_matrix],
                              categorical_feature=category_fea, verbose_eval=3000, early_stopping_rounds=1000)
            val_pred = model.predict(val_x, num_iteration=model.best_iteration)
#             test_pred = model.predict(test_x, num_iteration=model.best_iteration)
            print(list(sorted(zip(features, model.feature_importance("gain")), key=lambda x: x[1], reverse=True)))
        # 保存模型
        model.save_model("user_data/model_data/lgb_model_{}.txt".format(str(i+1)))
        train[valid_index] = val_pred
#         test += test_pred / kf.n_splits     # 这里必有加号
        cv_scores.append(mean_absolute_error(val_y, val_pred))
        print(cv_scores)

    print("%s_scotrainre_list:" % clf_name, cv_scores)
    print("%s_score_mean:" % clf_name, np.mean(cv_scores))
    print("%s_score_std:" % clf_name, np.std(cv_scores))
    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
    return train

lgb_train = cv_model(lgb, x_train, y_train, x_test, "lgb")
# lgb_train.to_csv("user_data/tmp_data/lgb_train.csv", index=False)   # 可用于stacking，但本方案并未采取模型融合
print("#"*10 + "模型训练已完成！" + "#"*10)


#!/bin/python

from ortools.graph import pywrapgraph

def main():
    # The problem data. A directed edge (u, v) has a capacity c.
    u = [0, 0, 0, 1, 1, 2, 2, 3, 3]
    v = [1, 2, 3, 2, 4, 3, 4, 2, 4]
    c = [20, 30, 10, 40, 30, 10, 20, 5, 20]

    assert len(u) == len(v)
    assert len(v) == len(c)

    max_flow = pywrapgraph.SimpleMaxFlow()
    for i in range(len(u)):
        max_flow.AddArcWithCapacity(u[i], v[i], c[i])

    # Solve the problem, find max flow between nodes 0 and 1.
    status = max_flow.Solve(0, 4)
    if status == max_flow.OPTIMAL:
        print(f'Max flow: {max_flow.OptimalFlow()}')
        print()
        print('  Arc  Flow / Capacity')
        for i in range(max_flow.NumArcs()):
            print(f'{max_flow.Tail(i)} -> {max_flow.Head(i)} {max_flow.Flow(i)} / {max_flow.Capacity(i)}')

        print(f'Source side min-cut: {max_flow.GetSourceSideMinCut()}')
        print(f'Sink side min-cut: {max_flow.GetSinkSideMinCut()}')
    else:
        print('Unable to compute max-flow.')

if __name__ == '__main__':
    main()


import cv2
import torch
from torch.utils.data import Dataset
import json
import os
from PIL import Image
import numpy as np

import transforms as T

def get_faster_rcnn_model():
    model = fasterrcnn_resnet50_fpn(pretrained=True)
    return model

def get_transform(train):
    transforms = []
    transforms.append(T.ToTensor())
    if train:
        pass
        #transforms.append(T.RandomHorizontalFlip(0.5))
    return T.Compose(transforms)

class DetectionDataset(Dataset):
    """
    A PyTorch Dataset class to be used in a PyTorch DataLoader to create batches.
    """

    def __init__(self, data_dir, split, keep_difficult=False, transforms=None):
        """
        :param data_dir: dir where data files are stored
        :param split: split, one of 'TRAIN' or 'TEST'
        :param keep_difficult: keep or discard objects that are considered difficult to detect?
        """
        self.split = split

        assert self.split in {'train', 'val', 'test'}

        self.data_dir = data_dir
        self.keep_difficult = keep_difficult
        self.transforms = transforms

        # Read data files
        self.image_files = sorted(os.listdir(os.path.join(data_dir, f'{self.split}/images')))
        self.image_files = [os.path.join(data_dir, f'{self.split}/images/{i}') for i in self.image_files]
        self.annot_files = sorted(os.listdir(os.path.join(data_dir, f'{self.split}/annotations')))
        self.annot_files = [os.path.join(data_dir, f'{self.split}/annotations/{i}') for i in self.annot_files]
        assert len(self.image_files) == len(self.annot_files)

    def __getitem__(self, idx):
        # Read image
        image = Image.open(self.image_files[idx]).convert("RGB")
        # NOTE: opencv load image
        #image = cv2.imread(self.image_files[idx])
        #image = image.transpose((2,0,1))
        #image = torch.FloatTensor(image)

        # Read objects in this image (bounding boxes, labels, difficulties)
        objects = self.preprocess_annotation(self.annot_files[idx])
        boxes = torch.FloatTensor(objects[0])  # (n_objects, 4)
        obj_score = torch.FloatTensor(objects[1])  # (n_objects)
        labels = torch.LongTensor([int(i) for i in objects[2]])  # (n_objects)
        truncation = torch.FloatTensor(objects[3])  # (n_objects)
        #truncation = torch.as_tensor(objects[3], dtype=torch.uint8)  # (n_objects)
        occlusion = torch.as_tensor(objects[4], dtype=torch.uint8)  # (n_objects)
        #occlusion = torch.FloatTensor(objects[4])  # (n_objects)
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])

        targets = {}
        targets['boxes'] = boxes
        targets['obj_score'] = obj_score
        targets['labels'] = labels
        targets['truncation'] = truncation
        targets['iscrowd'] = occlusion
        targets['image_id'] = torch.tensor(idx)
        targets['area'] = area

        if self.transforms is not None:
            image, targets = self.transforms(image, targets)

        return image, targets

    def __len__(self):
        return len(self.image_files)

    def preprocess_annotation(self, annot_file_path):
        objects = []
        objectness_score = []
        labels = []
        truncation = []
        occlusion = []

        with open(annot_file_path, 'r') as f:
            for line in f.readlines():
                annots = line.strip().split(',')
                annots = [float(i) for i in annots if i != '']
                assert len(annots) == 8
                if annots[0] >= annots[0] + annots[2] or annots[1] >= annots[1] + annots[3]:
                    print(annot_file_path, line)
                    continue
                objects.append([int(annots[0]), int(annots[1]), int(annots[0]+annots[2]), int(annots[1]+annots[3])]) # (N, 4)
                objectness_score.append(annots[4]) # (N)
                labels.append(annots[5]) # (N)
                truncation.append(annots[6]) # (N)
                occlusion.append(annots[7]) # (N)

        return objects, objectness_score, labels, truncation, occlusion

    def remove_invalid_annotations(self):
        """remove annotations with invalid formats"""
        pass

    def collate_fn(self, batch):
        """
        Since each image may have a different number of objects, we need a collate function (to be passed to the DataLoader).

        This describes how to combine these tensors of different sizes. We use lists.

        Note: this need not be defined in this Class, can be standalone.

        :param batch: an iterable of N sets from __getitem__()
        :return: a tensor of images, lists of varying-size tensors of bounding boxes, labels, and difficulties
        """
        images = []
        targets = []
        for i in batch:
            if i[1]['boxes'].size(0) != 0:
                images.append(i[0])
                targets.append(i[1])

        return images, targets

if __name__ == '__main__':
    image_dataset = DetectionDataset('/home/welcome/uav-object-detection/data', 'train', transforms=get_transform(True))
    dataloader = torch.utils.data.DataLoader(
                                image_dataset,
                                batch_size=32,
                                shuffle=False,
                                num_workers=4,
                                pin_memory=True,
                                collate_fn=image_dataset.collate_fn,
                                )
    #for i in dataloader:
    #    pass
    print(image_dataset[0])


import logging
from pathlib import Path

import torch
import torch.nn as nn
import yaml

from gluonts.core.component import validated
from gluonts.dataset.common import load_datasets
from gluonts.dataset.field_names import FieldName
from gluonts.dataset.loader import PseudoShuffled, TrainDataLoader
from gluonts.dataset.repository.datasets import get_dataset
from gluonts.nursery.ContextFIDExperiment.FID import calculate_fid
from gluonts.torch.batchify import batchify
from gluonts.torch.model.predictor import PyTorchPredictor
from gluonts.transform import (
    Chain,
    ExpectedNumInstanceSampler,
    InstanceSplitter,
)

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s %(message)s",
    datefmt="[%Y-%m-%d %H:%M:%S]",
)
logger = logging.getLogger(__name__)

class ExperimentSetUp:
    @validated()
    def __init__(
        self,
        gan_model_path: Path,
        shuffle_buffer_length: int = 5,
        train_data: bool = False,
    ) -> None:
        self.gan_model_path = gan_model_path
        self.yaml_data = self._load_yaml_data()
        self.pytorch_predictor = self._load_PytorchPredictor()
        self.pseudo_shuffle_DL = self._set_PseudoShuffledDL(
            shuffle_buffer_length, train_data
        )

    def _convert_ds_name(
        self,
        ds_name: str,
    ):
        ds_to_ds = {
            "m4_hourly": "/home/ec2-user/SageMaker/gluon-ts-gan/scaled_dataset/m4_hourly",
            "electricity": "/home/ec2-user/SageMaker/gluon-ts-gan/scaled_dataset/electricity_nips_scaled",
            "solar-energy": "/home/ec2-user/SageMaker/gluon-ts-gan/scaled_dataset/solar_nips_scaled",
            "exchange_rate": "/home/ec2-user/SageMaker/gluon-ts-gan/scaled_dataset/exchange_rate_nips_scaled",
            "traffic": "/home/ec2-user/SageMaker/gluon-ts-gan/scaled_dataset/traffic_nips_scaled",
        }
        return Path(ds_to_ds[ds_name])

    def _load_yaml_data(
        self,
    ):
        parsed_yaml_file = yaml.load(
            open(self.gan_model_path / "data.yml"), Loader=yaml.FullLoader
        )
        return parsed_yaml_file

    def _load_PytorchPredictor(
        self,
    ):
        pp = PyTorchPredictor.deserialize(
            self.gan_model_path, device=torch.device("cpu")
        )
        return pp

    def _load_dataset(
        self,
    ):
        if self.yaml_data["scaling"] == "NoScale":
            ds_path = self._convert_ds_name(self.yaml_data["dataset"])
            dataset = load_datasets(
                metadata=ds_path / "metadata",
                train=ds_path / "train",
                test=ds_path / "test",
            )
            return dataset
        else:
            return get_dataset(self.yaml_data["dataset"])

    def _change_transformation(self):
        small_t = [
            k
            for k in self.pytorch_predictor.input_transform.transformations[
                :-1
            ]
        ]
        IS = InstanceSplitter(
            target_field=FieldName.TARGET,
            is_pad_field=FieldName.IS_PAD,
            start_field=FieldName.START,
            forecast_start_field=FieldName.FORECAST_START,
            instance_sampler=ExpectedNumInstanceSampler(
                num_instances=1,
                min_future=self.yaml_data["target_len"],
            ),
            past_length=self.yaml_data["target_len"],
            future_length=self.yaml_data["target_len"],
            time_series_fields=[
                FieldName.FEAT_TIME,
                FieldName.OBSERVED_VALUES,
            ],
        )
        small_t.append(IS)
        return Chain(small_t)

    def _load_TrainDataLoader(
        self,
        train_data: bool = False,
    ):
        dataset = self._load_dataset()
        TrainDL = TrainDataLoader(
            dataset=dataset.train if train_data else dataset.test,
            transform=self._change_transformation(),
            batch_size=self.pytorch_predictor.batch_size,
            stack_fn=lambda data: batchify(
                data, self.pytorch_predictor.device
            ),
        )
        return TrainDL

    def _set_PseudoShuffledDL(
        self, shuffle_buffer_length: int = 5, train_data: bool = False
    ):
        TrainDL = iter(self._load_TrainDataLoader(train_data))
        PseudoShuffleTrainDL = PseudoShuffled(TrainDL, shuffle_buffer_length)
        return PseudoShuffleTrainDL

class Experiment:
    @validated()
    def __init__(
        self,
        model_similarity_score: nn.Module,
        gan_model_path: Path,
        shuffle_buffer_length: int = 5,
        train_data: bool = False,
    ) -> None:
        self.model_similarity_score = model_similarity_score.eval()
        self.experiment_setup = ExperimentSetUp(
            gan_model_path=gan_model_path,
            shuffle_buffer_length=shuffle_buffer_length,
            train_data=train_data,
        )
        self.pytorch_predictor = self.experiment_setup.pytorch_predictor
        self.yaml_data = self.experiment_setup.yaml_data
        self.pseudo_shuffle_DL = self.experiment_setup.pseudo_shuffle_DL

    def _sample_ts(self, PseudoTrainDL: PseudoShuffled):
        batch = next(PseudoTrainDL)
        with torch.no_grad():
            synthetic_ts = self.pytorch_predictor.prediction_net(
                past_target=batch["past_target"],
                time_features=batch["future_time_feat"],
                feat_static_cat=batch["feat_static_cat"],
            ).detach()
            if self.yaml_data["which_model"] == "TIMEGAN":
                synthetic_ts = synthetic_ts.permute(0, 2, 1)
            synthetic_ts = synthetic_ts.squeeze(1)
        real_ts = batch["future_target"]
        return real_ts, synthetic_ts

    def _embed_ts(self, ts: torch.Tensor):
        with torch.no_grad():
            embed_ts = self.model_similarity_score(ts.unsqueeze(1)).detach()
        return embed_ts

    def _calculate_FID(self, PseudoTrainDL: PseudoShuffled):
        real_ts, synthetic_ts = self._sample_ts(PseudoTrainDL)
        real_embed, synthetic_embed = (
            self._embed_ts(real_ts),
            self._embed_ts(synthetic_ts),
        )
        real_fid = calculate_fid(synthetic_ts, real_ts)
        embed_fid = calculate_fid(synthetic_embed, real_embed)
        return real_fid, embed_fid

    def run_FID(
        self,
        nb_run: int = 10,
    ):
        logger.info("Starting experiment for the Transformer")
        fid_ts = torch.empty(nb_run)
        fid_embed = torch.empty(nb_run)
        PseudoTrainDL = iter(self.pseudo_shuffle_DL)

        for k in range(nb_run):
            logger.info(f"Running experiment number : {k}")
            fid_ts[k], fid_embed[k] = self._calculate_FID(PseudoTrainDL)

        logger.info(
            f"FID on time series: Mean = {torch.mean(fid_ts).item()}, Std = {torch.std(fid_ts).item()}"
        )
        logger.info(
            f"FID on embedding: Mean = {torch.mean(fid_embed).item()}, Std = {torch.std(fid_embed).item()}"
        )
        return fid_ts, fid_embed


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 22 15:35:47 2020

@author: Thomas
"""

"""
So, what does this do ?

Same file as db_maker.py, except that creating a db is now a function.
Why did I do that ? Laziness.
"""

import model
import numpy as np
import cv2
import json
import pickle
from os import listdir

np_load_old = np.load
np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True)

def processAndSave(fileName, data):
    #with open('image.p', 'rb') as infile:
    #    imageArray = pickle.load(infile)
    imageArray = data
    img = cv2.imread("all/bad/"+fileName)
    imgg = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    imgg = cv2.GaussianBlur(imgg,(5,5),cv2.BORDER_DEFAULT)
    imgg = cv2.threshold(imgg, 252, 255, cv2.THRESH_BINARY_INV)[1]
    imageArray[fileName] = imgg
    return imageArray

def createDb():
    allPics = listdir("all/bad")
    data = {}
    for picture in allPics:
        data = processAndSave(picture, data)
    with open('image.p', 'wb') as outfile:
        pickle.dump(data, outfile, protocol=pickle.HIGHEST_PROTOCOL)


import random, datetime
from datetime import datetime

def bulletin_picker():

    text = bulletins()
    month = datetime.now().month

    if month == 1:
        color = 'red'
    if month == 2:
        color = 'purple'
    if month == 3:
        color = 'aqua'
    if month == 4:
        color = 'yellow'
    if month == 5:
        color = 'green'
    if month == 6:
        color = 'navy'
    if month == 7:
        color = 'red'
    if month == 8:
        color = 'orange'
    if month == 9:
        color = 'blue'
    if month == 10:
        color = 'brown'
    if month == 11:
        color = 'violet'
    if month == 12:
        color = 'garnet'

    return (text, color)

def bulletins():
    bulletin_board = ['Sidewalk Film Festival: Aug. 2-6', 'Concert Tonight: Carrie Underwood', 'Hot Dog Eating Contest: Five Points',
                      'Sloss Furnace Music Festival', ' Pepper Place Market: Every Saturday', 'Play: Wonderland Wives', 'Birmingham Restaurant Week',
                      'Barons Game: Tonight', 'Magic City Brewfest', 'Birimingham Innovation Week', 'Rock and Roll Bingo: Tonight',
                      'The Three Musketeers Live Show: Brookwood Village', 'Free Art Friday: July 13th', 'McWayne Science Center Celebration',
                      'Birmingham TACO FEST: Avondale', 'Cooking Class at Railroad Park: Sunday', 'Sloss Furnace: Metal Casting Class']
    bulletin_board.append(_visitor_bulletin())
    bulletin_choice = random.choice(bulletin_board)
    return bulletin_choice

def _read_arduino_visitors():
    with open('smartpark/arduino/output.txt') as f:
        peeps = f.readline().strip()
        doggos = f.readline().strip()
    # leaving as strings. We aren't crunching this!
    num_peeps = peeps.split(':')[1].strip()
    num_doggos = doggos.split(':')[1].strip()
    return (num_peeps, num_doggos)

def _visitor_bulletin():
    visitors = _read_arduino_visitors()
    peeps = visitors[0]
    doggos = visitors[1]
    # This is a terrible way to write strings #sorrynotsorry
    bulletin = "We've had " + peeps + " visitors and " + doggos + " canine visitors today!"
    return bulletin


from slack_sdk.webhook import WebhookClient
import os
import requests

# Add in from Telegram
CHATID = "CHANGEME"
BOTKEY = "CHANGEME"

TGURL = f"https://api.telegram.org/bot{BOTKEY}/sendMessage"

# Chance to match slack webhook
slackurl = "https://hooks.slack.com/services/VALUE1/VALUE2"
webhook = WebhookClient(slackurl)
HOSTNM = os.popen('hostname').read()
IP = os.popen('curl -s http://whatismyip.akamai.com/').read()
LOGININFO=os.popen('last -1 -i | head -n 1 | cut -d " " -f 1').read()

tgheaders = {
    'Content-Type': 'application/json',
}

tgdata = '{"chat_id": "{}", "text": "Log in to: {}\\n{}\\nfrom: {}", "disable_notification": false}'.format(CHATID,HOSTNM,IP,LOGININFO)

tgresponse = requests.post(f'https://api.telegram.org/bot{BOTKEY}/sendMessage', headers=tgheaders, data=tgdata)

response = webhook.send(text='Log in to: {}\n from {}\n by\n User: {} \n'.format(HOSTNM,IP,LOGININFO))
assert response.status_code == 200
assert response.body == "ok"


import pandas as pd
import matplotlib.pyplot as plt

#import a subset of the modules
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score

#read from the csv file and return a Pandas DataFrame.
# print the column names
# nba is the variable that has the whole data set imported from the Excel
print("\n", original_headers)
nba = pd.read_csv('NBAstats.csv')

##print the first three rows.
#print(nba[0:3])
## "(pos)" is the class attribute we are predicting.
class_column = 'Pos'

##The dataset contains attributes such as player name and team name.
##We know that they are not useful for classification and thus do not
##include them as features.
feature_columns = ['Age', 'G', 'GS', 'MP', 'FG', 'FGA', 'FG%', '3P', '3PA', \
   '3P%', '2P', '2PA', '2P%', 'eFG%', 'FT', 'FTA', 'FT%', 'ORB', 'DRB', \
   'TRB', 'AST', 'STL', 'BLK', 'TOV', 'PF', 'PS/G']
#
##Pandas DataFrame allows you to select columns.
##We use column selection to split the data into features and class.
nba_feature = nba[feature_columns]
nba_class = nba[class_column]

train_feature, test_feature, train_class, test_class = \
    train_test_split(nba_feature, nba_class, stratify=nba_class, \
    train_size=0.75, test_size=0.25)

training_accuracy = []
test_accuracy = []

tree = DecisionTreeClassifier(max_depth=6, random_state=0)
tree.fit(train_feature, train_class)
print("Training set score: {:.3f}".format(tree.score(train_feature, train_class)))
print("Test set score: {:.3f}".format(tree.score(test_feature, test_class)))

prediction = tree.predict(test_feature)
print("Confusion matrix:")
print(pd.crosstab(test_class, prediction, rownames=['True'], colnames=['Predicted'], margins=True))

train_class_df = pd.DataFrame(train_class,columns=[class_column])
train_data_df = pd.merge(train_class_df, train_feature, left_index=True, right_index=True)
train_data_df.to_csv('train_data.csv', index=False)

temp_df = pd.DataFrame(test_class,columns=[class_column])
temp_df['Predicted Pos']=pd.Series(prediction, index=temp_df.index)
test_data_df = pd.merge(temp_df, test_feature, left_index=True, right_index=True)
test_data_df.to_csv('test_data.csv', index=False)


import datetime
import fcntl
import time
import logging
import os
import shutil
import tempfile
from collections import OrderedDict

import numpy as np
import torch
import torch.nn as nn

def set_hyperparameter(config):
    """Load/set hyperparameter settings based on predefined config"""

    if config.hp_setting == "in_metanas":  # setting for MetaNAS
        config.task_train_steps = 5
        config.n_train = 15
        config.batch_size = 20
        config.batch_size_test = 5 if config.n == 1 else 15  # reptile paper A1
        config.meta_batch_size = 10
        config.w_lr = 0.001
        config.alpha_lr = 0.001
        config.w_meta_lr = 1.0
        config.a_meta_lr = 0.6
        config.a_meta_anneal = 0
        config.a_task_anneal = 0
        config.w_meta_anneal = 0
        config.w_task_anneal = 0

    elif config.hp_setting == "og_metanas":  # setting for MetaNAS
        config.task_train_steps = 5
        config.n_train = 15
        config.batch_size = 20
        config.batch_size_test = 10
        config.meta_batch_size = 10
        config.w_lr = 0.005
        config.alpha_lr = 0.005
        config.w_meta_lr = 1.0
        config.a_meta_lr = 0.6
        config.a_meta_anneal = 0
        config.a_task_anneal = 0
        config.w_meta_anneal = 0
        config.w_task_anneal = 0

    elif config.hp_setting == "og":  # default setting from REPTILE paper
        print("Using 'og' hp setting")
        config.task_train_steps = 10
        config.n_train = 10
        config.batch_size = 20
        config.batch_size_test = 10
        config.meta_batch_size = 5
        config.w_lr = 0.0005
        config.alpha_lr = 0.0
        config.w_meta_lr = 1.0
        config.a_meta_lr = 0.0
        config.a_meta_anneal = 0
        config.a_task_anneal = 0
        config.w_meta_anneal = 1
        config.w_task_anneal = 0

    elif config.hp_setting == "in":  # default setting from REPTILE paper
        print("Using 'in' hp setting")
        config.task_train_steps = 8
        config.n_train = 15
        config.batch_size = 10
        config.batch_size_test = 5 if config.n == 1 else 15  # reptile paper A1
        config.meta_batch_size = 5
        config.w_lr = 0.001
        config.alpha_lr = 0.0
        config.w_meta_lr = 1.0
        config.a_meta_lr = 0.0
        config.a_meta_anneal = 0
        config.a_task_anneal = 0
        config.w_meta_anneal = 1
        config.w_task_anneal = 0

    else:
        raise RuntimeError(f"Unrecognized hp_setting {config.hp_setting}")

    # compatibility with older versions
    if not hasattr(config, "batch_size_test"):
        config.batch_size_test = config.batch_size
    return config

def get_logger(file_path):
    """ Make python logger """
    logger = logging.getLogger("darts")
    log_format = "%(asctime)s | %(message)s"
    formatter = logging.Formatter(log_format, datefmt="%m/%d %I:%M:%S %p")
    file_handler = logging.FileHandler(file_path)
    file_handler.setFormatter(formatter)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)

    logger.addHandler(file_handler)
    logger.addHandler(stream_handler)
    logger.setLevel(logging.INFO)

    return logger

def get_experiment_path(config):
    # Write experiment output (logging, parameters, tensorboardX, ..) to
    # experiments/<EXPERIMENT_GROUP>/<DATE>_<NAME>_<UNIQUE_ID>/
    current_date = datetime.datetime.today().strftime(f"%m-%d")
    current_time = datetime.datetime.now().time()
    experiment_group_dir = os.path.join("experiments", config.experiment_group)
    os.makedirs(experiment_group_dir, exist_ok=True)
    experiment_name = f"{current_date}_{config.name}_"

    if config.job_id:
        experiment_name = f"{experiment_name}{config.job_id}"
        experiment_path = os.path.join(experiment_group_dir, experiment_name)
        if not os.path.exists(experiment_path):
            os.makedirs(experiment_path)
        else:
            print("----------------------------------")
            print("Warning: Dir already exists. Will overwrite.")
            print("----------------------------------")
    else:
        experiment_path = tempfile.mkdtemp(
            prefix=experiment_name, dir=experiment_group_dir
        )
    return experiment_path

def parse_gpus(gpus):
    if gpus == "all":
        return list(range(torch.cuda.device_count()))
    else:
        return [int(s) for s in gpus.split(",")]

def print_config_params(config, prtf=print):
    prtf("")
    prtf("Parameters:")
    for attr, value in sorted(vars(config).items()):
        prtf(f"{attr.upper()}={value}")
    prtf("")

def param_size(model):
    """ Compute parameter size in MB """
    n_params = sum(
        np.prod(v.size())
        for k, v in model.named_parameters()
        if not k.startswith("aux_head")
    )
    return n_params / 1024.0 / 1024.0

class AverageMeter:
    """ Computes and stores the average and current value """

    def __init__(self):
        self.val, self.avg, self.sum, self.count = (0.0, 0.0, 0.0, 0.0)
        self.reset()

    def reset(self):
        """ Reset all statistics """
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        """ Update statistics """
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

class EMAMeter:
    """Computes and stores an exponential moving average

    Attributes:
        avg: The current EMA
        alpha: The degree of weight decrease (a higher alpha discounts older observations faster)
    """

    def __init__(self, alpha=0.01):
        self.alpha = alpha
        self.avg = 0.0

    def update(self, val, _):
        self.avg = self.alpha * val + (1 - self.alpha) * self.avg

    def reset(self):
        self.avg = 0.0

def accuracy(output, target, topk=(1,)):
    """ Computes the precision@k for the specified values of k """
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    # one-hot case
    if target.ndimension() > 1:
        target = target.max(1)[1]

    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(1.0 / batch_size))

    return res

def save_checkpoint(state, ckpt_dir, is_best=False):
    filename = os.path.join(ckpt_dir, "checkpoint.pth.tar")
    torch.save(state, filename)
    if is_best:
        best_filename = os.path.join(ckpt_dir, "best.pth.tar")
        shutil.copyfile(filename, best_filename)

def count_params(net):
    return sum(p.numel() for p in net.parameters())

def save_state(
    meta_model,
    meta_optimizer,
    task_optimizer,
    path: str,
    epoch: int = None,
    job_id: str = None,
):
    """Save the model and optimizer states using pytorch

    Args:
        meta_model:
        meta_optimizer:
        task_optimizer:
        path: The path where the model is stored
        epoch: Epoch that is appended to the file name "meta_state".
        job_id: String that is used to save a temporary file with the
            job_id appended to the state file name. This file is rename
            after saving to the regular name.
    """

    epochpath = os.path.join(path, f"e{epoch}_") if epoch is not None else path

    # save the model (to temporary path if job_id is specified then then rename)
    model_file = epochpath + "meta_state"
    model_file_tmp = model_file if job_id is None else model_file + f"_{job_id}"
    torch.save(
        {
            "meta_model": meta_model.state_dict(),
            "w_meta_optim": meta_optimizer.w_meta_optim.state_dict(),
            "a_meta_optim": meta_optimizer.a_meta_optim.state_dict(),
            "w_task_optim": task_optimizer.w_optim.state_dict(),
            "a_task_optim": task_optimizer.a_optim.state_dict(),
        },
        model_file_tmp,
    )
    if model_file_tmp != model_file:
        os.rename(model_file_tmp, model_file)

def load_state(
    meta_model,
    meta_optimizer,
    task_optimizer,
    path,
    filename="meta_state",
):

    meta_state = torch.load(os.path.join(path, filename))
    meta_model.load_state_dict(meta_state["meta_model"])
    if meta_optimizer is not None:
        meta_optimizer.w_meta_optim.load_state_dict(meta_state["w_meta_optim"])
        meta_optimizer.a_meta_optim.load_state_dict(meta_state["a_meta_optim"])
    if task_optimizer is not None:
        task_optimizer.w_optim.load_state_dict(meta_state["w_task_optim"])
        task_optimizer.a_optim.load_state_dict(meta_state["a_task_optim"])

def load_model_from_state(meta_model, path, strict, filename="meta_state"):
    meta_state = torch.load(os.path.join(path, filename))
    meta_model.load_state_dict(meta_state["meta_model"], strict=strict)

def get_genotype_from_model_ckpt(path, model_instance):
    meta_state = torch.load(path)
    model_instance.load_state_dict(meta_state["meta_model"])
    return model_instance.genotype()


#!/usr/bin/env python3
'''
    Solitaire
'''
from GameView import animation, log, draw, init_view, wait, check_touch
from GameView import touch_deck, touch_waste, touch_button, touch_pile, select_box
import Model as model
import pygame
import sys

#
# Controller
#

def move(dest, source, index):
    'move card(s) from source to destination'
    length = len(dest.cards)
    animation.set(dest=dest, cards=model.get_cards_from(source, index))
    log.append(("move", (dest, source, length)))
    # animation loop
    while animation.in_motion():
        draw()
        wait(animation.speed)
        pygame.event.get()
        # while(not pygame.event.get()): pass   # single step through animation
    # place card(s) in final position then update view
    for card in animation.cards:
        animation.dest.cards.append(card)
    draw()
    print(model.get_state())
    model.countSteps += 1

def check_move_to_pile(atouch):
    'verify if a card is compatible with a pile'
    if not atouch:
        return None
    card = atouch.source.cards[touch.index]
    apile = card.match_pile()
    if apile != None:
        move(dest=apile, source=atouch.source, index=atouch.index)
    return apile

def check_move_to_foundation(atouch):
    'verify if a card is compatible with a foundation'
    if not atouch:
        return None
    if len(atouch.source.cards[touch.index:]) > 1:
        return None
    card = atouch.source.cards[touch.index]
    foundation = card.match_foundation()
    if foundation != None:
        move(dest=foundation, source=atouch.source, index=atouch.index)
    return foundation

def undo():
    'undo last operation logged'
    if log.list:
        action, params = log.pop()
        if action == "move":
            dest, source, index = params
            move(source, dest, index)      # animate the reverse of the last move
            log.pop()                       # avoid logging the undo action
        elif action == "turn":
            card = params.top()
            undo()                          # move / animate
            card.face = 'D'                 # turn back down
        elif action == "recycle":
            model.undo_recycle()
        draw()
        print(model.get_state())

def user_input():
    'returns a touch tuple, intercepts QUIT'
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        if event.type == pygame.MOUSEBUTTONUP:
            return check_touch(pygame.mouse.get_pos())
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_q:
                sys.exit()
            if event.key == pygame.K_m:
                return touch_waste()
            if event.key == pygame.K_SPACE:
                return touch_deck()
            if event.key == pygame.K_COMMA:
                return touch_button(0)

def start_game():
    'set up the intial game configuration'
    model.init()
    # check if a configuration restore is required for debugging
    if len(sys.argv) > 1:
        model.restore_state(sys.argv[1])
        del(sys.argv[1])
    else:
        model.shuffle()
        model.deal()
    model.countSteps = 0
    log.clear()

#----------------------------------------------------------------------

if __name__ == "__main__":
    init_view()
    auto = False
    victory = False
    start_game()
    draw()
    # print state for debugging purposes
    print(model.get_state())

    # main loop
    while True:

        wait(animation.speed)
        touch = user_input()

        if victory:
            choice = select_box(["victory", "Start a new Game", "Quit"])
            if choice == 0:
                auto = False
                victory = False
                start_game()
                continue
            else:
                sys.exit()

        # auto mode activated
        elif auto:
            # try to use the top card from the waste
            if model.WASTE:
                # print ('check Waste')
                # try to move the card automatically to the first matching foundation
                if check_move_to_foundation(touch_waste()):
                    victory = model.check_finished()

            # try with one of the tips
            for pile in model.PILES:
                if pile:
                    # print ('check Pile', pile)
                    if check_move_to_foundation(touch_pile(pile)):
                        victory = model.check_finished()

        # else respond to user input
        elif touch:
            if touch.source.name == "deck":
                if model.DECK.cards:
                    # get the top card down into the waste
                    move(dest=model.WASTE, source=model.DECK, index=-1)
                else: # if deck is empty
                    model.recycle()
                    draw()
                    print(model.get_state())
                    log.append(("recycle", 0))

            elif touch.source.name == "pile":
                if not check_move_to_foundation(touch):
                    check_move_to_pile(touch)

            elif touch.source.name == "tail":
                check_move_to_pile(touch)

            elif touch.source.name == "foundation":
                check_move_to_pile(touch)

            elif touch.source.name == "waste":
                if not check_move_to_foundation(touch):
                    check_move_to_pile(touch)

            elif touch.source.name == "button":
                if touch.index == 0:  # undo
                    undo()

                elif touch.index == 1:  # restart
                    model.restart()
                    model.count_steps = 0
                    log.clear()

                elif touch.index == 2:  # re-shuffle and deal a new one
                    auto = False
                    victory = False
                    start_game()
                draw()
                # print state for debugging purposes
                print(model.get_state())

            # check if auto can be activated
            if not auto:
                auto = (model.count_hidden() < 1)


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from shutil import which
import chromedriver_binary
from fake_useragent import UserAgent
import time
from scrapy.selector import Selector
from selenium.webdriver.common.action_chains import ActionChains
import csv
from selenium.webdriver.support.ui import Select
import re
from datetime import datetime
import requests
from requests.structures import CaseInsensitiveDict
import sys
class kijihi():

    """
    proxies = [
        ""
        ]

    """
    count = 0
    links =[]

    def start(self):
        options = Options()
        options.add_experimental_option("detach", True)
        # options.add_argument("--headless")
        # options.add_argument("--no-sandbox")
        # options.add_argument('--disable-gpu')
        ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
        options.add_argument(f'user-agent={ua}')
        options.add_argument("start-maximized")
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)
        options.add_argument("--disable-blink-features=AutomationControlled")
        self.driver = webdriver.Chrome(options=options)
        self.driver.get('https://google.com')
        time.sleep(2)

    def target_url(self,url):
        self.driver.get(url)
        time.sleep(60)
        self.next_page()

    def result_scrap(self):
        html = self.driver.page_source
        resp = Selector(text=html)
        results = resp.xpath("//div[contains(@class,'ProductTile')]")
        for ri in results:
            li = ri.xpath(".//div/h3[@class='product-brand-name']/a/@href").extract_first()
            li = str(li)
            li = li.strip()
            li = f"https://www.liquorland.com.au{li}"
            self.links.append(li)

    def next_page(self):
        try:
            self.result_scrap()
            next_button = self.driver.find_element_by_xpath("//button[contains(@title,'next')]")
            if next_button:
                next_button.click()
                time.sleep(20)
                self.next_page()
        except:
            None

    def header(self,out_name):
        with open(out_name, 'w', newline='') as file:
            writer = csv.writer(file)
            writer.writerow(["scrape_date","website_url","Product_ID","GTIN","Brand","product","price","pack_size","multibuy_deal_quantity","multibuy_deal_price","undiscounted_price","delivery_available","review_rating","review_count","style","type","packaging","standard_drinks","alcohol_content","organic","origin"])

    def scrap(self,scrape_date,out_name):
        self.header(out_name)
        for ulo in self.links:
            self.driver.get(ulo)
            try:
                WebDriverWait(self.driver, 30).until(EC.presence_of_element_located(
                    (By.XPATH, "//span[contains(text(),'Product ID')]/following-sibling::span")))
            except:
                None

            time.sleep(7)
            html = self.driver.page_source
            resp = Selector(text=html)

            website_url = str(ulo)
            Product_ID = resp.xpath("normalize-space(//span[contains(text(),'Product ID')]/following-sibling::span/text())").extract_first()
            GTIN = resp.xpath("normalize-space(//span[contains(text(),'GTIN')]/following-sibling::span/text())").extract_first()

            Brand = resp.xpath("//span[contains(text(),'Brand')]/following-sibling::span/a/text()").extract_first()
            product = resp.xpath("(//div[@class='product-name'])[1]/text()").extract_first()
            price = resp.xpath("normalize-space((//span[@class='PriceTag zero-cents current primary']//span[@class='dollarAmount'])[1]/text())").extract_first()

            pack_size = resp.xpath("normalize-space((//span[@class='unitOfMeasure'])[1]/text())").extract_first()

            multibuy_deal_quantity = resp.xpath("normalize-space(//div[@class='callout callout-text' and contains(text(),' for ')]/text())").extract_first()
            multibuy_deal_quantity = str(multibuy_deal_quantity)
            m_deal_quantity = re.findall(r"^([\w\-]+)",multibuy_deal_quantity)
            m_deal_quantity = str(m_deal_quantity)
            m_deal_quantity = m_deal_quantity.replace("'","")
            m_deal_quantity = m_deal_quantity.replace("[","")
            m_deal_quantity = m_deal_quantity.replace("]","")
            m_deal_quantity = m_deal_quantity.strip()

            multibuy_deal_quantity = multibuy_deal_quantity.replace(m_deal_quantity,'')
            multibuy_deal_quantity = multibuy_deal_quantity.replace('for','')
            multibuy_deal_quantity = multibuy_deal_quantity.replace('$','')
            multibuy_deal_price = multibuy_deal_quantity.strip()

            undiscounted_price = resp.xpath("(//span[contains(text(),'Price reduced from')])[1]/following-sibling::div/span[@class='dollarAmount']/text()").extract_first()

            delivery_available = resp.xpath("(//span[contains(text(),'Delivery')])[last()-3]/text()").extract_first()
            if delivery_available == "Delivery":
                delivery_available = True
            else:
                delivery_available = False

            review_rating = resp.xpath("//div[contains(@class,'bv_avgRating_component_container')]/text()").extract_first()
            review_count = resp.xpath("//div[contains(@class,'bv_numReviews_text')]/text()").extract_first()
            review_count = str(review_count)
            review_count = review_count.replace('-','')
            review_count = review_count.replace('(','')
            review_count = review_count.replace(')','')
            review_count = review_count.strip()
            style = resp.xpath("//span[contains(text(),'Style')]/following-sibling::span/text()").extract_first()
            type = resp.xpath("//span[contains(text(),'Type')]/following-sibling::span/text()").extract_first()
            packaging = resp.xpath("//span[contains(text(),'Packaging')]/following-sibling::span/text()").extract_first()
            standard_drinks = resp.xpath("//span[contains(text(),'Standard Drinks')]/following-sibling::span/text()").extract_first()
            alcohol_content = resp.xpath("//span[contains(text(),'Alcohol Content')]/following-sibling::span/text()").extract_first()
            organic = resp.xpath("//span[contains(text(),'Organic')]/following-sibling::span/text()").extract_first()
            origin = resp.xpath("//span[contains(text(),'Origin')]/following-sibling::span/text()").extract_first()

            print()
            print("scrape_date",scrape_date)
            print("website_url:",website_url)
            print("Product_ID:",Product_ID)
            print("GTIN:",GTIN)
            print("Brand:",Brand)
            print("product:",product)
            print("price:",price)
            print("pack_size:",pack_size)
            print("multibuy_deal_quantity:",m_deal_quantity)
            print("multibuy_deal_price:",multibuy_deal_price)
            print("undiscounted_price:",undiscounted_price)
            print("delivery_available:",delivery_available)
            print("review_rating:",review_rating)
            print("review_count:",review_count)
            print("style:",style)
            print("type:",type)
            print("packaging:",packaging)
            print("standard_drinks:",standard_drinks)
            print("alcohol_content:",alcohol_content)
            print("organic:",organic)
            print("origin:",origin)
            print()

            with open(out_name,'a',newline='',encoding='utf-8') as file:
                writer = csv.writer(file)
                writer.writerow([scrape_date,website_url,Product_ID,GTIN,Brand,product,price,pack_size,m_deal_quantity,multibuy_deal_price,undiscounted_price,delivery_available,review_rating,review_count,style,type,packaging,standard_drinks,alcohol_content,organic,origin])
                self.count = self.count + 1
                print("Data saved in excel: ",self.count)

    def close(self):
        self.driver.close()

if __name__ == '__main__':

    out_name = sys.argv[1]
    scrape_date = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
    scraper = kijihi()
    scraper.start()
    url = "https://www.liquorland.com.au/beer"

    scraper.target_url(url)
    scraper.scrap(scrape_date,out_name)
    scraper.close()


from PyQt6 import Qsci, QtGui

# These values are just used for testing, you can delete them:
VALUE_JSON = """{
    "asdf": "123",
    "array": [
        {
            "hello": "world",
            "age": 123
        }
    ]
}"""

VALUE = """<!DOCTYPE html>
<html lang="en-US">
 <head>
  <!-- Global site tag (gtag.js) - Google Analytics -->
  <script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-38714717-1">
  </script>
  <script>
   window.dataLayer = window.dataLayer || [];
        function gtag(){dataLayer.push(arguments);}
        gtag('js', new Date());
        gtag('config', 'UA-38714717-1');
  </script>
  <!-- Google Optimize -->
  <script src="https://www.googleoptimize.com/optimize.js?id=OPT-56H9SXM">
  </script>
  <!-- Google Tag Manager -->
  <script>
   (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
            new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
        j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
        'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
    })(window,document,'script','dataLayer','GTM-W82WNBG');
  </script>
  <!-- End Google Tag Manager -->
  <!-- META DATA -->
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1.0" name="viewport"/>
  <!--[if IE]><meta http-equiv="cleartype" content="on" /><![endif]-->
  <!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge" /><![endif]-->
  <link href="http://gmpg.org/xfn/11" rel="profile"/>
  <link href="https://www.synack.com/xmlrpc.php" rel="pingback"/>"""

VALUE_JS = """(function() {
    var didInit = false;
    function initMunchkin() {
        if(didInit === false) {
            didInit = true;
            Munchkin.init('738-OEX-476');
        }
    }
    var s = document.createElement('script');
    s.type = 'text/javascript';
    s.async = true;
    s.src = '//munchkin.marketo.net/munchkin.js';
    s.onreadystatechange = function() {
        if (this.readyState == 'complete' || this.readyState == 'loaded') {
            initMunchkin();
        }
    };
    s.onload = initMunchkin;
    document.getElementsByTagName('head')[0].appendChild(s);
})();"""

class DarkTheme:
    default_bg = "#1E1E1E"
    default_color = "#EEFFFF"
    darker_color = "#545454"
    key_color = "#C792EA"
    value_color = "#C3E88D"
    number_color = "#FF5370"
    operator_color = "#569CD6"
    invalid_color = "#f14721"
    selected_secondary_color = "#4E5256"
    other_color = "#FFCB6B"

    bg_dark = "#252526"
    bg_input = "#404040"
    bg_input_hover = "#3A3A3A"

    @classmethod
    def new_html_lexer(cls):
        lexer = Qsci.QsciLexerHTML()

        lexer.setDefaultPaper(QtGui.QColor(cls.default_bg))
        lexer.setDefaultColor(QtGui.QColor(cls.default_color))

        tags = [
            Qsci.QsciLexerHTML.SGMLDefault,
            Qsci.QsciLexerHTML.SGMLCommand,
            Qsci.QsciLexerHTML.SGMLParameter,
            Qsci.QsciLexerHTML.SGMLDoubleQuotedString,
            Qsci.QsciLexerHTML.SGMLSingleQuotedString,
            Qsci.QsciLexerHTML.SGMLError,
            Qsci.QsciLexerHTML.SGMLSpecial,
            Qsci.QsciLexerHTML.SGMLEntity,
            Qsci.QsciLexerHTML.SGMLComment,
            Qsci.QsciLexerHTML.SGMLParameterComment,
            Qsci.QsciLexerHTML.SGMLBlockDefault,
        ]
        for tag in tags:
            lexer.setColor(QtGui.QColor(cls.other_color), tag)
            lexer.setPaper(QtGui.QColor(cls.default_bg), tag)

        lexer.setColor(QtGui.QColor(cls.number_color), Qsci.QsciLexerHTML.Tag)
        lexer.setColor(QtGui.QColor(cls.number_color), Qsci.QsciLexerHTML.UnknownTag)
        lexer.setColor(QtGui.QColor(cls.key_color), Qsci.QsciLexerHTML.Attribute)
        lexer.setColor(QtGui.QColor(cls.key_color), Qsci.QsciLexerHTML.UnknownAttribute)

        lexer.setColor(QtGui.QColor(cls.operator_color), Qsci.QsciLexerHTML.XMLStart)
        lexer.setColor(QtGui.QColor(cls.operator_color), Qsci.QsciLexerHTML.XMLEnd)

        lexer.setColor(QtGui.QColor(cls.value_color), Qsci.QsciLexerHTML.OtherInTag)
        lexer.setColor(QtGui.QColor(cls.value_color), Qsci.QsciLexerHTML.HTMLNumber)
        lexer.setColor(QtGui.QColor(cls.value_color), Qsci.QsciLexerHTML.HTMLDoubleQuotedString)
        lexer.setColor(QtGui.QColor(cls.value_color), Qsci.QsciLexerHTML.HTMLSingleQuotedString)

        lexer.setPaper(QtGui.QColor(cls.default_bg), Qsci.QsciLexerHTML.JavaScriptDefault)
        lexer.setColor(QtGui.QColor(cls.default_color), Qsci.QsciLexerHTML.JavaScriptDefault)
        lexer.setPaper(QtGui.QColor(cls.default_bg), Qsci.QsciLexerHTML.Script)
        lexer.setColor(QtGui.QColor(cls.default_color), Qsci.QsciLexerHTML.Script)

        lexer.setColor(QtGui.QColor(cls.darker_color), Qsci.QsciLexerHTML.HTMLComment)
        lexer.setColor(QtGui.QColor(cls.default_color), Qsci.QsciLexerHTML.Default)
        lexer.setPaper(QtGui.QColor(cls.default_bg), Qsci.QsciLexerHTML.Default)

        # Javascript colours
        colours = {
            Qsci.QsciLexerHTML.JavaScriptDefault: cls.default_color,
            Qsci.QsciLexerHTML.JavaScriptSymbol: cls.default_color,
            Qsci.QsciLexerHTML.JavaScriptKeyword: cls.operator_color,

            Qsci.QsciLexerHTML.JavaScriptWord: cls.key_color,
            Qsci.QsciLexerHTML.JavaScriptKeyword: cls.key_color,

            Qsci.QsciLexerHTML.JavaScriptDoubleQuotedString: cls.value_color,
            Qsci.QsciLexerHTML.JavaScriptSingleQuotedString: cls.value_color,
        }
        for key, colour in colours.items():
            lexer.setColor(QtGui.QColor(colour), key)
            lexer.setPaper(QtGui.QColor(cls.default_bg), key)

        lexer.setFont(cls.get_font())
        return lexer

    @classmethod
    def new_json_lexer(cls):
        lexer = Qsci.QsciLexerJSON()

        lexer.setHighlightEscapeSequences(False)

        lexer.setDefaultPaper(QtGui.QColor(cls.default_bg))
        lexer.setDefaultColor(QtGui.QColor(cls.default_color))

        lexer.setPaper(QtGui.QColor(cls.default_bg), Qsci.QsciLexerJSON.Error)
        lexer.setColor(QtGui.QColor(cls.invalid_color), Qsci.QsciLexerJSON.Error)
        lexer.setPaper(QtGui.QColor(cls.default_bg), Qsci.QsciLexerJSON.UnclosedString)
        lexer.setColor(QtGui.QColor(cls.invalid_color), Qsci.QsciLexerJSON.UnclosedString)

        lexer.setColor(QtGui.QColor(cls.key_color), Qsci.QsciLexerJSON.Property)
        lexer.setColor(QtGui.QColor(cls.value_color), Qsci.QsciLexerJSON.String)
        lexer.setColor(QtGui.QColor(cls.number_color), Qsci.QsciLexerJSON.Number)
        lexer.setColor(QtGui.QColor(cls.default_color), Qsci.QsciLexerJSON.Operator)

        lexer.setFont(cls.get_font())
        return lexer

    @classmethod
    def new_js_lexer(cls):
        lexer = Qsci.QsciLexerJavaScript()

        lexer.setHighlightEscapeSequences(False)

        lexer.setDefaultPaper(QtGui.QColor(cls.default_bg))
        lexer.setDefaultColor(QtGui.QColor(cls.default_color))

        colours = {
            Qsci.QsciLexerCPP.Default: cls.default_color,
            Qsci.QsciLexerCPP.Operator: cls.default_color,
            Qsci.QsciLexerCPP.InactiveOperator: cls.operator_color,
            Qsci.QsciLexerCPP.Identifier: cls.operator_color,

            Qsci.QsciLexerCPP.Keyword: cls.key_color,

            Qsci.QsciLexerCPP.DoubleQuotedString: cls.value_color,
            Qsci.QsciLexerCPP.SingleQuotedString: cls.value_color,
            Qsci.QsciLexerCPP.InactiveDoubleQuotedString: cls.value_color,
            Qsci.QsciLexerCPP.InactiveSingleQuotedString: cls.value_color,
        }
        for key, colour in colours.items():
            lexer.setColor(QtGui.QColor(colour), key)

        lexer.setFont(cls.get_font())
        return lexer

    @classmethod
    def get_font(cls) -> QtGui.QFont:
        # fonts = QtGui.QFontDatabase.families()
        # TODO: Search through the fonts and find one that matches
        font = QtGui.QFontDatabase.font('Menlo', 'Regular', 11)
        return font


import pandas as pd
import numpy as np
import os
import sys
import time
from scipy import stats
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster, fclusterdata, maxdists
import matplotlib.pyplot as plt

def shan_entropy(c):
  c_normalized = c / float(np.sum(c))
  c_normalized = c_normalized[np.nonzero(c_normalized)]
  H = -sum(c_normalized* np.log2(c_normalized))
  return H

def NMI(label_true, label_pred, Ks):

  H_x = 0
  for K in range(Ks):
    if np.sum(label_pred == K) > 0:
      p = np.mean(label_pred == K)
      H_x += -p * np.log2(p)

  H_y = 0
  for K in range(Ks):
    if np.sum(label_true == K) > 0:
      p = np.mean(label_true == K)
      H_y += -p * np.log2(p)

  I_xy = 0
  for K in range(Ks):
    if np.sum(label_pred == K) > 0:
      for label in range(10):
        if np.sum(label_true == label) > 0:
          p_xy = np.mean((label_pred == K) & (label_true == label))
          if p_xy > 0:
            p_x = np.mean(label_pred == K)
            p_y = np.mean(label_true == label)
            I_xy += p_xy * np.log2(p_xy/(p_x*p_y))

  NMI = I_xy/(H_x + H_y)
  return NMI

def SC(data, centers, K):

  sum_s = np.array([0])
  for center in range(K):
    in_c = data[np.where(data[:,0] == center)]
    out_c = data[np.where(data[:,0] != center)]
    in_cc = np.delete(in_c, 0, 1)
    out_cc = np.delete(out_c, 0, 1)
    size_in = np.size(in_cc, 0)
    size_out = np.size(out_cc, 0)
    if size_in == 1: continue
    #A
    new_in = np.repeat(in_cc, size_in, 0)
    new_inin = np.tile(in_cc, (size_in, 1))
    self_re = np.linalg.norm(new_in - new_inin, axis = 1)
    new_self = np.sum(np.reshape(self_re, (-1, size_in))/size_in, axis = 1)
    #B
    new_out = np.repeat(in_cc, size_out, 0)
    new_outt = np.tile(out_cc, (size_in, 1))
    other_re = np.linalg.norm(new_out - new_outt, axis = 1)
    new_other = np.sum(np.reshape(other_re, (-1, size_out))/size_out, axis = 1)
    #S
    sis = (new_other-new_self)/np.maximum(new_self, new_other)
    sum_s = np.concatenate((sum_s, sis), axis=None)

  final = np.sum(sum_s)/(sum_s.size-1)
  return final

def WC_SSD(data, centers, K):

  total = 0
  for center in range(K):
    nodes = data[np.where(data[:,0] == center)]
    pro_nodes = np.delete(nodes, 0, 1)
    dists = np.linalg.norm(pro_nodes-centers[center], axis = 1)
    sub_sum = np.dot(dists, dists.T)
    total = total + sub_sum
  return total

def classify(data, centers, it, K):

  sizes = np.size(data, 0)
  new_data = np.repeat(data, K, axis = 0)
  new_cent = np.tile(centers, (sizes, 1))
  result = np.linalg.norm(new_data-new_cent, axis = 1)
  new_result = np.reshape(result, (-1, K))
  mins_pos = np.argmin(new_result, axis = 1) #each node's new classification
  matrix = np.reshape(mins_pos, (1,sizes)) #to matrix
  labeled_data = np.concatenate((matrix.T, data), axis = 1)
  # print(labeled_data)
  if it > 0:
    for center in range(K):
      nodes = labeled_data[np.where(labeled_data[:,0] == center)]
      if np.size(nodes,0) == 0: continue
      cut = np.delete(nodes, 0, 1)
      new_center = np.mean(cut, axis = 0)
      centers[center] = new_center
    return classify(data, centers, it-1, K)
  else:
    return labeled_data, centers

def k_means(data, K, max_it, dist):

  N = np.size(data, 0)
  rows = np.random.randint(0, N, size=K)  #get K random centers
  x = np.take(data[:,2], rows)
  y = np.take(data[:,3], rows)
  centers = np.column_stack((x,y))  #construct K centers
  new_data = data[:,2:]
  classfied, new_centers = classify(new_data, centers, max_it-1, K)
  total_WC = WC_SSD(classfied, new_centers, K)
  total_SC = SC(classfied, new_centers, K)
  total_NMI = NMI(data[:,1], classfied[:,0], K)

  return total_WC, total_SC, total_NMI

def painter(WC, SC, K, num):

  fig, ax = plt.subplots()
  ax.scatter(SC, WC)
  ax.plot(SC,WC)

  for i, txt in enumerate(K):
    ax.annotate(txt, (SC[i], WC[i]))

  plt.xlabel('SC')
  plt.ylabel('WC-SSD')
  plt.title('Datasets'+str(num))
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num)+'.png']))
  # plt.show()

def paint(data, y, K, num, cases):

  plt.scatter(K, data)
  plt.plot(K,data)
  plt.xlabel('Number of K')
  plt.ylabel(y)
  plt.title(y +' of Dataset '+str(cases))
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num) +'.png']))
  plt.clf()

def miner1(data, Ks, num, check):

  wcs = np.zeros(Ks.size)
  scs = np.zeros(Ks.size)
  it = 0
  for K in Ks:
    WC, SC, NMI = k_means(data, K, 50, 'Euclidean')
    wcs[it] = WC
    scs[it] = SC
    it += 1
  if check:
    paint(wcs, 'WC', Ks, num, num)
    paint(scs, 'SC', Ks, num+3, num)

  return wcs, scs

def deal_with_2(WC, SC, Ks):

  #Find Knee of Curve
  values = list(WC)
  nPoints = len(values)
  allCoord = np.vstack((range(nPoints), values)).T
  firstPoint = allCoord[0]
  lineVec = allCoord[-1] - allCoord[0]
  lineVecNorm = lineVec / np.sqrt(np.sum(lineVec**2))
  vecFromFirst = allCoord - firstPoint
  scalarProduct = np.sum(vecFromFirst * np.matlib.repmat(lineVecNorm, nPoints, 1), axis=1)
  vecFromFirstParallel = np.outer(scalarProduct, lineVecNorm)
  vecToLine = vecFromFirst - vecFromFirstParallel
  distToLine = np.sqrt(np.sum(vecToLine ** 2, axis=1))
  idxOfBestPoint = np.argmax(distToLine)
  WC_best = Ks[idxOfBestPoint]

  #Find Max of Curve
  otherBest = np.argmax(SC)
  SC_best = Ks[otherBest]

  if WC_best == SC_best:
    return SC_best
  else:
    return WC_best

def deal_with_1(data, data2, data3, Ks, item):

  np.random.seed(item)
  wcs, scs = miner1(data, Ks, 1, True)
  K1 = deal_with_2(wcs, scs, Ks)
  wcs2, scs2 = miner1(data2, Ks, 2, True)
  K2 = deal_with_2(wcs2, scs2, Ks)
  wcs3, scs3 = miner1(data3, Ks, 3, True)
  K3 = deal_with_2(wcs3, scs3, Ks)
  return K1, K2, K3

def paint_STD(data, std, y, K, num, cases):

  plt.scatter(K, data, color = 'blue')
  plt.plot(K,data, color = 'blue')
  plt.errorbar(K, data, yerr = std, color = 'blue')
  plt.xlabel('Number of K')
  plt.ylabel(y)
  plt.title(y +' of Dataset '+str(cases))
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num) +'.png']))
  plt.clf()

def minor3(data, Ks, seeds, num):

  re_wc = np.zeros((Ks.size,seeds))
  re_sc = np.zeros((Ks.size,seeds))

  ix = iy = 0
  for K in Ks:
    for item in range(seeds):
      np.random.seed(item+1)
      WC, SC, NMI = k_means(data, K, 50, 'Euclidean')
      re_wc[ix,iy] = WC
      re_sc[ix,iy] = SC
      iy += 1
    iy = 0
    ix += 1

  mean_wc = np.mean(re_wc, axis = 1)
  mean_sc = np.mean(re_sc, axis = 1)
  std_wc = np.std(re_wc, axis = 1)
  std_sc = np.std(re_sc, axis = 1)
  paint_STD(mean_wc, std_wc, 'Mean and Std of WC', Ks, 1+num*6, num)
  paint_STD(mean_sc, std_sc, 'Mean and Std of SC', Ks, 2+num*6, num)
  # paint(std_wc, 'Std of WC', Ks, 3+num*6, num)
  # paint(std_sc, 'Std of SC', Ks, 4+num*6, num)
  # return mean_wc, mean_sc, std_wc, std_sc

def deal_with_3(data, data2, data3, Ks, seed):

  minor3(data, Ks, seed, 1)
  minor3(data2, Ks, seed, 2)
  minor3(data3, Ks, seed, 3)

def add_k_means(data, K, max_it, dist):

  N = np.size(data, 0)
  rows = np.random.randint(0, N, size=K)  #get K random centers
  x = np.take(data[:,2], rows)
  y = np.take(data[:,3], rows)
  centers = np.column_stack((x,y))  #construct K centers
  new_data = data[:,2:]
  classfied, new_centers = classify(new_data, centers, max_it-1, K)
  total_NMI = NMI(data[:,1], classfied[:,0], K)

  return classfied[:,0], total_NMI

def minor4(data, K, num):

  clusters, NMI = add_k_means(data, K, 50, 'Euclidean')
  rows = np.size(data, 0)
  row_index = np.random.randint(0, rows, 1000)
  label = np.take(clusters, row_index)
  chosed_x = np.take(data[:,2], row_index)
  chosed_y = np.take(data[:,3], row_index)
  pic = plt.scatter(chosed_x, chosed_y, c = label)
  plt.title('Dataset:' + str(num) + ', NMI:'+ str(round(NMI, 3)))
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num+100) +'.png']))
  plt.clf()

def deal_with_4(data, data2, data3, item, k1, k2, k3):

  np.random.seed(item)
  minor4(data, k1, 1)
  minor4(data2, k2, 2)
  minor4(data3, k3, 3)

def analysis(data, K):

  #2.1
  np.random.seed(0)
  WC, SC, NMI = k_means(data, K, 50, 'Euclidean')
  print('WC-SSD:',WC)
  print('SC:',SC)
  print('NMI:',NMI)

  # #2.2~2.4
  Ks = np.array([2, 4, 8, 16, 32])
  data2 = data[np.isin(data[:,1], np.array([2,4,6,7]))]
  data3 = data2[np.isin(data2[:,1], np.array([6,7]))]

  k1, k2, k3 = deal_with_1(data, data2, data3, Ks, 0)

  print('Chosen K for Dataset1 using K-means:', k1)
  print('Chosen K for Dataset2 using K-means:', k2)
  print('Chosen K for Dataset3 using K-means:', k3)

  deal_with_3(data, data2, data3, Ks, 10)
  deal_with_4(data, data2, data3, 0, k1, k2, k3)

def deal_with_hc(data, link_type, num):

  matrix = linkage(data, link_type)
  fig = plt.figure(figsize=(25, 10))
  dn = dendrogram(matrix)
  plt.title('Dataset 1 with '+ link_type +' linkage')
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num)+'.png']))
  plt.clf()
  return matrix

def HC_paint(data, K, y, types, num):

  plt.scatter(K, data)
  plt.plot(K,data)
  plt.xlabel('Number of K')
  plt.ylabel(y)
  plt.title(y + ' of '+ types + ' linkage')
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(num) +'.png']))
  plt.clf()

def calc_WC_SC(matrix, group, Ks, types, num):

  scale = np.size(group, 0)
  re_wc = np.zeros(Ks.size)
  re_sc = np.zeros(Ks.size)
  it = 0

  for K in Ks:
    total_WC = 0
    total_SC = 0
    tmp_label = fcluster(matrix, t=K, criterion='maxclust')
    for diff in range(K):
      data = group[tmp_label == diff+1]
      if data.size == 0: continue
      center = np.mean(data, axis = 0)
      total_WC += np.sum(np.square(data-center))

    for each in range(scale):
      center = tmp_label[each]
      in_A = group[tmp_label == center]
      in_B = group[tmp_label != center]
      A = np.linalg.norm(group[each]-in_A)
      B = np.linalg.norm(group[each]-in_B)
      SC = (B-A)/np.max([A,B]) #sometimes B-A < 0
      if np.size(in_A, 0) == 1: SC = 0
      total_SC += SC

    re_wc[it] = total_WC
    re_sc[it] = total_SC/scale
    it += 1

  HC_paint(re_wc, Ks, 'WC', types, num*100)
  HC_paint(re_sc, Ks, 'SC', types, num*101)
  return re_wc, re_sc

def preprocessed(data):

  np.random.seed(0)
  group = np.array([[0,0]])
  for index in range(10):
    new_data = data[np.where(data[:,1] == index)]
    rows = np.size(new_data, 0)
    new_data[:,0] = np.arange(rows)
    selected = np.random.randint(0, rows, 10)
    chosed = new_data[selected,2:]
    group = np.concatenate((group, chosed), axis=0)

  sub_group = np.delete(group, np.array([[0,0]]), axis = 0)
  centers = np.repeat(np.arange(10), 10)
  return sub_group, centers

def calc_NMI(matrix, K, labels, num):

  pred_label = fcluster(matrix, t=K, criterion='maxclust')
  result = NMI(labels, pred_label-1, K)
  return result

def Hi_Clustering(data):

  #3.1~3.2
  group, centers = preprocessed(data)
  matrix_s = deal_with_hc(group, 'single', 1000)
  matrix_c = deal_with_hc(group, 'complete', 1001)
  matrix_a = deal_with_hc(group, 'average', 1002)

  #3.3
  Ks = np.arange(2,101)
  wc1, sc1 = calc_WC_SC(matrix_s, group, Ks, 'single', 11)
  wc2, sc2 = calc_WC_SC(matrix_c, group, Ks, 'complete', 12)
  wc3, sc3 = calc_WC_SC(matrix_a, group, Ks, 'average', 13)
  #3.4
  k1 = deal_with_2(wc1, sc1, Ks)
  k2 = deal_with_2(wc2, sc2, Ks)
  k3 = deal_with_2(wc3, sc3, Ks)

  print('Chosen K for Dataset1 using single linkage:', k1)
  print('Chosen K for Dataset1 using complete linkage:', k2)
  print('Chosen K for Dataset1 using average linkage:', k3)

  #3.5
  NMI_1 = calc_NMI(matrix_s, k1, centers, 1)
  NMI_2 = calc_NMI(matrix_c, k2, centers, 2)
  NMI_3 = calc_NMI(matrix_a, k3, centers, 3)

  print('NMI of '+str(k1)+' for Dataset1 using single linkage:', NMI_1)
  print('NMI of '+str(k2)+' for Dataset1 using complete linkage:', NMI_2)
  print('NMI of '+str(k3)+' for Dataset1 using average linkage:', NMI_3)

def main():

  input_df = sys.argv[1]
  input_k = int(sys.argv[2])
  df = pd.read_csv(input_df)
  data = df.values
  start = time.time()
  analysis(data, input_k)
  Hi_Clustering(data)
  end = time.time()
  print('time:', end - start)

if __name__== "__main__":

  main()


import numpy as np
from numpy import genfromtxt
import pickle
from fuzzywuzzy import fuzz

with open('./pickles/movie_to_idx.pickle', 'rb') as f:
    # Dump the dictionary into the file using pickle
    movie_to_idx = pickle.load(f)

with open('./pickles/m_dist.pickle', 'rb') as f:
    # Dump the dictionary into the file using pickle
    m_dist = pickle.load(f)

with open('./pickles/movie_dict.pickle', 'rb') as f:
    # Dump the dictionary into the file using pickle
    movie_dict = pickle.load(f)

item_vecs = genfromtxt('csv/item_vecs.csv', delimiter=',')

def fuzzy_matching(fav_movie, verbose=True):
    print('inside of fuzzy matching')
    """
    return the closest match via fuzzy ratio. If no match found, return None

    Parameters
    ----------
    mapper: dict, map movie title name to index of the movie in data

    fav_movie: str, name of user input movie

    verbose: bool, print log if True

    Return
    ------
    index of the closest match
    """
    match_tuple = []
    # get match
    for title, idx in movie_to_idx.items():
        ratio = fuzz.ratio(title.lower(), fav_movie.lower())
        if ratio >= 60:
            match_tuple.append((title, idx, ratio))
    # sort
    match_tuple = sorted(match_tuple, key=lambda x: x[2])[::-1]
    if not match_tuple:
        print('Oops! No match is found')
        return
    if verbose:
        print('Found possible matches in our database: {0}\n'.format([x[0] for x in match_tuple]))
    return match_tuple[0][1]

def make_pred(fav_movie, n_recommandations = 10):
    id_listesi = []
    title_listesi = []
    idx= fuzzy_matching(fav_movie)
    for i in np.argsort(m_dist[idx])[::1][:n_recommandations]:
        movid = int(item_vecs[i,0])
        id_listesi.append(movid)
        title_listesi.append(movie_dict[item_vecs[i,0]]['title'])

    return id_listesi, title_listesi


import cv2
import pyzbar.pyzbar as pyzbar

def decodeDisplay(image):
    barcodes = pyzbar.decode(image)
    for barcode in barcodes:
        # get location

        (x, y, w, h) = barcode.rect
        cv2.rectangle(image, (x, y), (x + w, y + h), (225, 225, 225), 2)

        # to string
        barcodeData = barcode.data.decode("utf-8")
        barcodeType = barcode.type

        # get image type
        text = "{} ({})".format(barcodeData, barcodeType)
        cv2.putText(image, text, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX,
                    .5, (225, 225, 225), 2)

        # show code image type
        print("[INFO] Found {} barcode: {}".format(barcodeType, barcodeData))
    return image

def detect():
    camera = cv2.VideoCapture(0)

    while True:
        # get frame
        ret, frame = camera.read()
        # change color
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        im = decodeDisplay(gray)

        cv2.waitKey(5)
        cv2.imshow("camera", im)
        # quit
        if cv2.waitKey(10) & 0xFF == ord('q'):
            break
    camera.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    detect()