# -*- coding: utf8 -*-

import os
import sys
import re
import requests
import shutil
from bs4 import BeautifulSoup
# from datetime import datetime, timedelta
import libs.common as CMN
g_logger = CMN.WSL.get_web_scrapy_logger()

@CMN.CLS.Singleton
class WebScrapyWorkdayCanlendar(object):

    def __init__(self, date_start=None, date_end=None):
        self.source_url_parsing_cfg = CMN.DEF.DEF_SOURCE_URL_PARSING[CMN.DEF.DEF_WORKDAY_CANLENDAR_DATA_SOURCE_REFERENCE_INDEX]
        self.url_format = self.source_url_parsing_cfg["url_format"]
        self.encoding = self.source_url_parsing_cfg["url_encoding"]
        self.select_flag = self.source_url_parsing_cfg["url_css_selector"]
###############################################################################################
# Caution: The types of the following member variabiles are FinanceDate
        self.date_start = CMN.CLS.FinanceDate(CMN.DEF.DEF_START_DATE_STR) if date_start is None else date_start
        self.date_end = CMN.FUNC.get_last_url_data_date(CMN.DEF.DEF_TODAY_MARKET_DATA_EXIST_HOUR, CMN.DEF.DEF_TODAY_MARKET_DATA_EXIST_MINUTE) if date_end is None else date_end
# The start/end time of scrapying data from the web
        self.date_start_from_web = self.date_start
        self.date_end_from_web = self.date_end
# The first/last workday
        self.date_first = None
        self.date_last = None
###############################################################################################
        self.update_from_web_flag = False # Should be updated for just one time
        self.date_start_year = self.date_start.year
        self.workday_canlendar = None
        self.workday_year_list = None

    def initialize(self):
        # import pdb; pdb.set_trace()
        self.update_workday_canlendar()

    def is_workday(self, date_cur):
        return self.__is_workday(date_cur.year, date_cur.month, date_cur.day)

    def __is_workday(self, year, month, day):
        if self.workday_canlendar is None:
            raise RuntimeError("Workday Canlendar is NOT initialized")
        date_check = CMN.CLS.FinanceDate(year, month, day)
        if not self.is_in_range(date_check):
            raise RuntimeError("The check date[%s] is OUT OF RANGE[%s %s]" % (date_check, self.date_start, self.date_end))
        return day in self.workday_canlendar[year][month - 1]

    def update_workday_canlendar(self):
        # import pdb; pdb.set_trace()
# Update data from the file
        need_update_from_web = self.__update_workday_from_file()
# It's required to update the new data
        if need_update_from_web:
# Update data from the web
            self.__update_workday_from_web()
# Write the result into the config file
            self.__write_workday_canlendar_to_file()
# Copy the config file to the finance_analyzer/finance_recorder_java project
            self.__copy_workday_canlendar_config_file()

    def __copy_workday_canlendar_config_file(self):
        current_path = os.path.dirname(os.path.realpath(__file__))
        [working_folder, project_name, lib_folder] = current_path.rsplit('/', 2)
        dst_folderpath_list = [
            "%s/%s/%s" % (working_folder, CMN.DEF.DEF_COPY_CONF_FILE_DST_PROJECT_NAME1, CMN.DEF.DEF_CONF_FOLDER),
            "%s/%s/%s" % (working_folder, CMN.DEF.DEF_COPY_CONF_FILE_DST_PROJECT_NAME2, CMN.DEF.DEF_CONF_FOLDER),
        ]
        src_filepath = "%s/%s/%s/%s" % (working_folder, project_name, CMN.DEF.DEF_CONF_FOLDER, CMN.DEF.DEF_WORKDAY_CANLENDAR_CONF_FILENAME)
        for dst_folderpath in dst_folderpath_list:
            if os.path.exists(dst_folderpath):
                g_logger.debug("Copy the file[%s] to %s" % (CMN.DEF.DEF_WORKDAY_CANLENDAR_CONF_FILENAME, dst_folderpath))
                shutil.copy2(src_filepath, dst_folderpath)

    def __update_workday_from_file(self):
        need_update_from_web = True
        conf_filepath = CMN.FUNC.get_config_filepath(CMN.DEF.DEF_WORKDAY_CANLENDAR_CONF_FILENAME)
        g_logger.debug("Try to Acquire the Workday Canlendar data from the file: %s......" % conf_filepath)
        if not os.path.exists(conf_filepath):
            g_logger.warn("The Workday Canlendar config file does NOT exist")
            return need_update_from_web
        try:
            date_range_str = None
            with open(conf_filepath, 'r') as fp:
                for line in fp:
                    if date_range_str is None:
                        date_range_str = line
                        mobj = re.search("([\d]{4}-[\d]{2}-[\d]{2}) ([\d]{4}-[\d]{2}-[\d]{2})", date_range_str)
                        if mobj is None:
                            raise RuntimeError("Unknown format[%s] of date range in Workday Canlendar config file", line)
                        date_start_from_file = CMN.CLS.FinanceDate(mobj.group(1))
                        date_end_from_file = CMN.CLS.FinanceDate(mobj.group(2))
                        g_logger.debug("Original Date Range in Workday Canlendar config file: %s, %s" % (mobj.group(1), mobj.group(2)))
                        if date_start_from_file > self.date_start:
                            raise RuntimeError("Incorrect start date; File: %s, Expected: %s", date_start_from_file, self.date_start)
# Check the time range in the file and adjust the range of scrapying data from the web
                        if date_end_from_file >= self.date_end:
                            g_logger.debug("The last data has already existed in the file. It's no need to scrap data from the web")
                            need_update_from_web = False
                        else:
                            self.date_start_from_web = date_end_from_file + 1
                            g_logger.debug("Adjust the time range of web scrapy to %s %s" % (self.date_start_from_web, self.date_end_from_web))
                    else:
# Obtain the "year" value
                        mobj = re.search("\[([\d]{4})\]", line)
                        if mobj is None:
                            raise RuntimeError("Incorrect year format in Workday Canlendar config file: %s", line)
                        year = int(mobj.group(1))
                        # self.workday_canlendar[year] = self.__init_canlendar_each_year_list()
                        tmp_data_list = line.split(']')
                        if len(tmp_data_list) != 2:
                            raise RuntimeError("Incorrect data format in Workday Canlendar config file: %s", line)
# Parse the workday in each month
                        year_workday_list = tmp_data_list[1].rstrip("\n").split(";")
                        for year_workday_list_per_month in year_workday_list:
                            tmp_data1_list = year_workday_list_per_month.split(':')
                            if len(tmp_data1_list) != 2:
                                raise RuntimeError("Incorrect per month data format in Workday Canlendar config file: %s", line)
                            month = int(tmp_data1_list[0])
                            workday_list_str = tmp_data1_list[1]
                            workday_list = [int(workday_str) for workday_str in workday_list_str.split(",")]
# Check if there are nay duplicate elements in the list
                            if len(workday_list) != len(set(workday_list)):
                                # import pdb; pdb.set_trace()
                                raise RuntimeError("Duplicate element in [%d, %d]: %s", year, month, workday_list)
                            self.__set_canlendar_each_month(year, month, workday_list)
        except Exception as e:
            g_logger.error("Error occur while parsing Workday Canlendar config, due to %s" % str(e))
            raise e

        self.workday_year_list = sorted(self.workday_canlendar.keys())
        return need_update_from_web

    def __update_workday_from_web(self):
        if not self.update_from_web_flag:
            self.update_from_web_flag = True
        else:
            raise RuntimeError("Update workday from web should NOT be called more than once")
        # import pdb; pdb.set_trace()
        g_logger.debug("Try to Acquire the Workday Canlendar [%s:%s] data from the web......" % (self.date_start_from_web, self.date_end_from_web))
        if self.date_start_from_web.year == self.date_end_from_web.year and self.date_start_from_web.month == self.date_end_from_web.month:
            self.__update_workday_from_web_by_month(self.date_start_from_web.year, self.date_start_from_web.month, self.date_start_from_web.day, self.date_end_from_web.day)
        else:
            self.__update_workday_from_web_by_month(self.date_start_from_web.year, self.date_start_from_web.month, start_day=self.date_start_from_web.day)
            year = self.date_start_from_web.year
            month = self.date_start_from_web.month
            year_end = self.date_end_from_web.year
            month_end = self.date_end_from_web.month
            def get_next_month(cur_year, cur_month):
                next_year = cur_year + 1 if cur_month == 12 else cur_year
                next_month = 1 if cur_month == 12 else cur_month + 1
                return (next_year, next_month)
            while True:
                (year, month) = get_next_month(year, month)
                if year == year_end and month == month_end:
                    break
                self.__update_workday_from_web_by_month(year, month)
            self.__update_workday_from_web_by_month(self.date_end_from_web.year, self.date_end_from_web.month, end_day=self.date_end_from_web.day)
        self.workday_year_list = sorted(self.workday_canlendar.keys())
        # for year in self.workday_year_list:
        #     workday_month_list = self.workday_canlendar[year]
        #     print "%d: %s" % (year, workday_month_list)

    def __update_workday_from_web_by_month(self, year, month, start_day=None, end_day=None):
# Check if scrapying the whole month data
        # import pdb; pdb.set_trace()
        whole_month_data = False
#***************************************************
# # Check if the whole month data is required
#         if start_day is None and end_day is None:
#             whole_month_data = True
#         elif end_day is None and start_day == 1:
#             whole_month_data = True
#         elif start_day is None and end_day == CMN.FUNC.get_month_last_day(year, month):
#             whole_month_data = True
# # Assemble the URL
#         url = self.url_format.format(*(year, month))
#***************************************************
# Setup the start/end date if it's None
        if start_day is None: start_day = 1
        if end_day is None: end_day = CMN.FUNC.get_month_last_day(year, month)
        if start_day == 1 and end_day == CMN.FUNC.get_month_last_day(year, month):
            whole_month_data = True
        url = self.url_format.format(*(year, month, start_day, end_day))
# Scrap the web data
        # try:
        #     # g_logger.debug("Try to Scrap data [%s]" % url)
        #     res = CMN.FUNC.request_from_url_and_check_return(url)
        # except requests.exceptions.Timeout as e:
        #     # g_logger.debug("Try to Scrap data [%s]... Timeout" % url)
        #     fail_to_scrap = False
        #     for index in range(self.SCRAPY_RETRY_TIMES):
        #         time.sleep(randint(1,3))
        #         try:
        #             res = requests.get(url, timeout=CMN.DEF.DEF_SCRAPY_WAIT_TIMEOUT)
        #         except requests.exceptions.Timeout as ex:
        #             fail_to_scrap = True
        #         if not fail_to_scrap:
        #             break
        #     if fail_to_scrap:
        #         g_logger.error("Fail to scrap workday list data even retry for %d times !!!!!!" % self.SCRAPY_RETRY_TIMES)
        #         raise e
        req = CMN.FUNC.try_to_request_from_url_and_check_return(url)
# Select the section we are interested in
        req.encoding = self.encoding
        # print res.text
        soup = BeautifulSoup(req.text)
        web_data = soup.select(self.select_flag)
        workday_list = []
        web_data_len = len(web_data)
# Parse the web data and obtain the workday list
        if web_data_len != 0:
        # print "len: %d" % data_len
#***************************************************
            # for tr in web_data[2:]:
#***************************************************
            for tr in web_data[web_data_len - 1 : 0 : -1]:
                td = tr.select('td')
                date_list = td[0].text.split('/')
                if len(date_list) != 3:
                    raise RuntimeError("The date format is NOT as expected: %s", date_list)
                cur_day = int(date_list[2])
                if not whole_month_data and cur_day < start_day:
# Caution: It's NO need to consider the end date since the last data is always today
                    continue
                workday_list.append(cur_day)
        # import pdb; pdb.set_trace()
        self.__set_canlendar_each_month(year, month, workday_list)
# # Find the workday list
#         allday_list = []
#         if whole_month_data:
#             allday_list = range(1, CMN.get_month_last_day(year, month) + 1)
#         else:
#             list_start_date = 1 if start_day is None else start_day
#             list_end_date = CMN.get_month_last_day(year, month) if end_day is None else end_day
#             allday_list = range(list_start_date, list_end_date + 1)
# # Get the non-workday list and set to the data structure
#         non_workday_list = list(set(allday_list) - set(workday_list))
#         self.__set_canlendar_each_month(year, month, non_workday_list)

    def __set_canlendar_each_month(self, year, month, workday_list):
        if not workday_list:
            return
        if self.workday_canlendar is None:
            self.workday_canlendar = {}
        if year not in self.workday_canlendar:
            self.workday_canlendar[year] = self.__init_canlendar_each_year_list()
        self.workday_canlendar[year][month - 1].extend(workday_list)
        self.workday_canlendar[year][month - 1].sort()

    def __init_canlendar_each_year_list(self):
        workday_canlendar_year_list = []
        for index in range(12):
            workday_canlendar_year_list.append([])
        return workday_canlendar_year_list

    def __write_workday_canlendar_to_file(self):
        # import pdb; pdb.set_trace()
        conf_filepath = CMN.FUNC.get_config_filepath(CMN.DEF.DEF_WORKDAY_CANLENDAR_CONF_FILENAME)
        g_logger.debug("Write the Workday Canlendar data to the file: %s......" % conf_filepath)
        try:
            date_range_str = None
            with open(conf_filepath, 'w') as fp:
                g_logger.debug("Start to write workday into %s", CMN.DEF.DEF_WORKDAY_CANLENDAR_CONF_FILENAME)
                fp.write("%s %s\n" % (self.date_start, self.date_end))
                # year_list = sorted(self.workday_canlendar)
                for year in self.workday_year_list:
                    workday_month_list = self.workday_canlendar[year]
                    # print "%d: %s" % (year, workday_month_list)
                    # sys.stderr.write("%s" % ("[%04d]" % year + ";".join(["%d:%s" % (month + 1, ",".join([str(workday) for workday in workday_month_list[month]])) for month in range(12)]) + "\n"))
                    workday_each_year_str = "[%04d]" % year + ";".join(["%d:%s" % (month + 1, ",".join([str(workday) for workday in workday_month_list[month]])) for month in range(12) if len(workday_month_list[month]) != 0]) + "\n"
                    # for month in range(12):
                    #     workday_each_year_str += "%d:%s;" % (month + 1, ",".join([str(workday) for workday in workday_month_list[month]]))
                    # workday_each_year_str += "\n"
                    g_logger.debug("Write data: %s" % workday_each_year_str)
                    fp.write(workday_each_year_str)
                    # print workday_each_year_str
        except Exception as e:
            g_logger.error("Error occur while writing Workday Canlendar into config file, due to %s" % str(e))
            raise e

    def get_first_workday(self):
        if self.date_first is None:
            if self.workday_canlendar is None:
                raise RuntimeError("Incorrect Operation: self.workday_canlendar is None")
            year = self.workday_year_list[0]
            for month in range(12):
                if len(self.workday_canlendar[year][month]) != 0:
                    self.date_first = CMN.CLS.FinanceDate(year, month + 1, self.workday_canlendar[year][month][0])
                    break
            if self.date_first is None:
                raise RuntimeError("Fail to find the frst workday in the canlendar")
        return self.date_first

    def get_last_workday(self):
        if self.date_last is None:
            if self.workday_canlendar is None:
                raise RuntimeError("Incorrect Operation: self.workday_canlendar is None")
            # import pdb; pdb.set_trace()
            # workday_year_list = sorted(self.workday_canlendar.keys())
            year = self.workday_year_list[-1]
            for month in range(12):
                if len(self.workday_canlendar[year][month]) == 0:
                    self.date_last = CMN.CLS.FinanceDate(year, month, self.workday_canlendar[year][month - 1][-1])
                    break
            if self.date_last is None:
                month = 12
                self.date_last = CMN.CLS.FinanceDate(year, month, self.workday_canlendar[year][month - 1][-1])
        return self.date_last

    def is_in_range(self, date_cur):
        date_first = self.get_first_workday()
        date_last = self.get_last_workday()
        result = (self.get_first_workday() <= date_cur <= self.get_last_workday())
        if not result:
            g_logger.debug("Date[%s] is NOT in range[%s, %s]" % (date_cur, self.get_first_workday(), self.get_last_workday()))
        return result

    def find_index(self, date_cur):
        # import pdb; pdb.set_trace()
        if self.workday_canlendar is None:
            raise RuntimeError("Incorrect Operation: self.workday_canlendar is None")
        year_index = None
        month_index = None
        day_index = None
        try:
            year_index = self.workday_year_list.index(date_cur.year)
        except ValueError as e:
            return None
        year = self.workday_year_list[year_index]
        if len(self.workday_canlendar[year][date_cur.month - 1]) == 0:
            return None
        month_index = date_cur.month - 1
        try:
            day_index = self.workday_canlendar[year][month_index].index(date_cur.day)
        except ValueError as e:
            return None

        # g_logger.debug("The date[%s] index: %d %d %d" % (CMN.to_date_only_str(date_cur), year_index, month_index, day_index))
        return (year_index, month_index, day_index)

    def find_nearest_next_index(self, date_cur):
        if self.workday_canlendar is None:
            raise RuntimeError("Incorrect Operation: self.workday_canlendar is None")
        year_index = None
        month_index = None
        day_index = None
        is_workday = False
        if not self.is_in_range(date_cur):
            if date_cur > self.get_last_workday():
                return None
            else:
                date_cur = self.get_first_workday()
                is_workday = True
        else:
            is_workday = self.is_workday(date_cur)

        if is_workday:
            (year_index, month_index, day_index) = self.find_index(date_cur)
        else:
            if not self.is_in_range(date_cur):
                return None
            try:
                year_index = self.workday_year_list.index(date_cur.year)
            except ValueError as e:
                return None
            # year = self.workday_year_list[year_index]
            month_index = date_cur.month - 1
            day_list = self.workday_canlendar[date_cur.year][month_index]
            for index, day in enumerate(day_list):
                if day > date_cur.day:
                    day_index = index
                    break
            if day_index is None:
                if month_index == 11:
# A new year
                    year_index += 1
                    month_index = 0
                    day_index = 0
                else:
# A new month
                    month_index += 1
                    day_index = 0
        # g_logger.debug("The date[%s] index: %d %d %d" % (date_cur, year_index, month_index, day_index))
        return (year_index, month_index, day_index)

    def get_nearest_next_workday(self, date_cur):
        index_tuple = self.find_nearest_next_index(date_cur)
        if index_tuple is None:
            raise RuntimeError("Fail to find the nearest next workday from the date: %s" % date_cur)
        (year_index, month_index, day_index) = index_tuple
        year = self.workday_year_list[year_index]
        date_nearest_next = CMN.CLS.FinanceDate(year, month_index + 1, self.workday_canlendar[year][month_index][day_index])
        if not self.is_workday(date_cur):
            g_logger.debug("Find the nearest next workday[%s]: %s" % (date_cur, date_nearest_next))
        return date_nearest_next

    def find_nearest_prev_index(self, date_cur):
        if self.workday_canlendar is None:
            raise RuntimeError("Incorrect Operation: self.workday_canlendar is None")
        year_index = None
        month_index = None
        day_index = None
        is_workday = False
        if not self.is_in_range(date_cur):
            if date_cur < self.get_first_workday():
                return None
            else:
                date_cur = self.get_last_workday()
                is_workday = True
        else:
            is_workday = self.is_workday(date_cur)

        if is_workday:
            (year_index, month_index, day_index) = self.find_index(date_cur)
        else:
            try:
                year_index = self.workday_year_list.index(date_cur.year)
            except ValueError as e:
                return None
            # year = self.workday_year_list[year_index]
            month_index = date_cur.month - 1
            # import pdb; pdb.set_trace()
            day_list = self.workday_canlendar[date_cur.year][month_index]
            for index, day in reversed(list(enumerate(day_list))):
                if day < date_cur.day:
                    day_index = index
                    break
            if day_index is None:
                if month_index == 1:
# An old year
                    year_index -= 1
                    month_index = 11
                    day_index = len(self.workday_canlendar[date_cur.year - 1][month_index]) - 1
                else:
# An old month
                    month_index -= 1
                    day_index = len(self.workday_canlendar[date_cur.year - 1][month_index]) - 1
        # g_logger.debug("The date[%s] index: %d %d %d" % (CMN.to_date_only_str(date_cur), year_index, month_index, day_index))
        return (year_index, month_index, day_index)

    def get_nearest_prev_workday(self, date_cur):
        # import pdb; pdb.set_trace()
        index_tuple = self.find_nearest_prev_index(date_cur)
        if index_tuple is None:
            raise RuntimeError("Fail to find the nearest previous workday from the date: %s" % date_cur)
        (year_index, month_index, day_index) = index_tuple
        year = self.workday_year_list[year_index]
        date_nearest_prev = CMN.CLS.FinanceDate(year, month_index + 1, self.workday_canlendar[year][month_index][day_index])
        if not self.is_workday(date_cur):
            g_logger.debug("Find the nearest previous workday[%s]: %s" % (date_cur, date_nearest_prev))
        return date_nearest_prev

####################################################################################################

class WorkdayIterator(object):

    def __init__(self, date_start=None, date_end=None, ExceptionWhenStartLaterThanEnd=False):
        self.workday_canlendar_obj = WebScrapyWorkdayCanlendar.Instance()
        self.workday_canlendar = self.workday_canlendar_obj.workday_canlendar
        self.workday_year_list = self.workday_canlendar_obj.workday_year_list

        if date_start is None:
            date_start = self.workday_canlendar_obj.get_first_workday()
        assert (isinstance(date_start, CMN.CLS.FinanceDate)), "The type of start time[%s] is NOT FinanceDate" % date_start
        if date_end is None:
            date_end = self.workday_canlendar_obj.get_last_workday()
        assert (isinstance(date_end, CMN.CLS.FinanceDate)), "The type of end time[%s] is NOT FinanceDate" % date_end

        self.time_to_stop = False
        self.cur_year_index = None
        self.cur_month_index = None
        self.cur_day_index = None
        if date_start > date_end:
            if ExceptionWhenStartLaterThanEnd:
                raise ValueError("The start date[%s] should NOT be later than the end date[%s]" % (date_start, date_end))
            else:
                g_logger.debug("The start date[%s] is later than the end date[%s]" % (date_start, date_end))
            self.time_to_stop = True
        else:
            start_tuple = self.workday_canlendar_obj.find_index(date_start)
            if start_tuple is None:
                raise ValueError("The start date[%s] is NOT a workday" % date_start)
            (start_year_index, start_month_index, start_day_index) = start_tuple
            g_logger.debug("The start date[%s] index: %d %d %d" % (date_start, start_year_index, start_month_index, start_day_index))
            end_tuple = self.workday_canlendar_obj.find_index(date_end)
            if end_tuple is None:
                raise ValueError("The end date[%s] is NOT a workday" % date_end)
            (self.end_year_index, self.end_month_index, self.end_day_index) = end_tuple
            g_logger.debug("The end date[%s] index: %d %d %d" % (date_end, self.end_year_index, self.end_month_index, self.end_day_index))
            self.cur_year_index = start_year_index
            self.cur_month_index = start_month_index
            self.cur_day_index = start_day_index

    def __iter__(self):
        return self

    def next(self):
        # import pdb; pdb.set_trace()
        if self.time_to_stop:
            raise StopIteration
        if self.cur_year_index == self.end_year_index and self.cur_month_index == self.end_month_index and self.cur_day_index == self.end_day_index:
            self.time_to_stop = True
        year = self.workday_year_list[self.cur_year_index]
        date_cur = CMN.CLS.FinanceDate(year, self.cur_month_index + 1, self.workday_canlendar[year][self.cur_month_index][self.cur_day_index])
# Find the next element
        if self.cur_month_index == 11:
            if self.cur_day_index == len(self.workday_canlendar[year][self.cur_month_index]) - 1:
# A new year
                self.cur_year_index += 1
                self.cur_month_index = 0
                self.cur_day_index = 0
            else:
                self.cur_day_index += 1
        else:
            if self.cur_day_index == len(self.workday_canlendar[year][self.cur_month_index]) - 1:
# A new month
                self.cur_month_index += 1
                self.cur_day_index = 0
            else:
                self.cur_day_index += 1
        return date_cur

####################################################################################################

class WorkdayNearestIterator(WorkdayIterator):

    def __init__(self, date_start=None, date_end=None, ExceptionWhenStartLaterThanEnd=False):
        # import pdb;pdb.set_trace()
        workday_canlendar_obj = WebScrapyWorkdayCanlendar.Instance()
        date_nearest_start = workday_canlendar_obj.get_nearest_next_workday(date_start) if date_start is not None else None
        date_nearest_end = workday_canlendar_obj.get_nearest_prev_workday(date_end) if date_end is not None else None
        super(WorkdayNearestIterator, self).__init__(date_nearest_start, date_nearest_end, ExceptionWhenStartLaterThanEnd)


# -*- coding: utf-8 -*-
"""
テスト用
"""
import sys
import subprocess

# pythonバージョン
print(sys.version_info)

# シェル、コマンド実行
subprocess.call("echo 'Hello World.'", shell=True)

# シェル、コマンド実行の結果を受け取る
ret = subprocess.check_output("ls")
print(ret)

# 入力待ち
data = input()

if __name__ == '__main__':
    print("__main__")


import os
os.chdir('..')

import sys
sys.path.append(".")

if os.path.exists('pywikibot.lwp'):
    os.remove('pywikibot.lwp') #pepega

import pywikibot

import azurlane.load_src
import azurlane.equip
import azurlane.wiki

equip_stats_srcs = azurlane.load_src.load_sharecfg('equip_data_statistics')

site = pywikibot.Site('azurlane')  # The site we want to run our bot on
site.login()
category = pywikibot.Category(site, 'Submarine Torpedo')

insert_after = ['WepRange ', 'ProjRange']
insert_before = []
keys = ['ProjSpeed']
edit_message = 'Bot: set ' + ', '.join(keys)
print(edit_message)

for page in category.articles():
    print(page.title())
    equip_wikis = page.text.split('|-|')

    for equip_wiki_index, equip_wiki in enumerate(equip_wikis):
        equip_id = int(azurlane.wiki.get_template_value(equip_wiki, 'BaseID'))
        if equip_id not in equip_stats_srcs:
            print('No data for equip id', equip_id)
            continue
        equip = azurlane.equip.Equip(equip_id)

        inserts = []

        for key in keys:
            old_value = azurlane.wiki.get_template_value(equip_wiki, key)
            new_value = equip.wiki_value(key)
            inserts.append((key, new_value))
            if new_value != old_value:
                print(' ', equip_wiki_index+1, ':', key, ':', old_value, '->', new_value)

        if len(inserts) > 0:
            equip_wiki = azurlane.wiki.set_template_values(equip_wiki, inserts, insert_after, insert_before)
            equip_wikis[equip_wiki_index] = equip_wiki

    new_text = '|-|'.join(equip_wikis)
    if new_text != page.text:
        page.text = new_text
        #print(page.text)
        page.save(edit_message)
    else:
        print('  no change, skipped')


from flask_restful import Resource
from bson.json_util import dumps
import json
from flask import jsonify
import pymongo

'''
API endpoints to get servers summary
'''

class compute_summary_all(Resource):
    '''
    GET compute summary
    '''
    def __init__(self, **kwargs):
        self.DB = kwargs['DB']
        self.env = kwargs['env']

    def get(self):
        # Get all the machines (rage against)
        summaries = self.DB['compute_summary'].find().sort("hostname", pymongo.ASCENDING)

        return(jsonify(json.loads(dumps(summaries))))

class compute_summary_detail(Resource):
    '''
    GET compute summary detail (disks/nics/software)
    '''
    def __init__(self, **kwargs):
        self.DB = kwargs['DB']
        self.env = kwargs['env']

    def get(self, hostname):
        summary = self.DB['compute_summary'].find_one({'hostname' : hostname})
        disks = self.DB['compute_disks'].find({'hostname' : hostname}).sort("hostname", pymongo.ASCENDING)
        nics = self.DB['compute_network'].find({'hostname' : hostname}).sort("hostname", pymongo.ASCENDING)

        result = {
            'summary' : summary,
            'disks' : list(disks),
            'nics' : list(nics)
        }

        return(jsonify(json.loads(dumps(result))))


import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

state_polygons = pd.read_csv("../data/state-polygons-cleaned.csv")

def plot_state(state):
    """
    Plots state as a polygon

    Arguments:
        state - name of state to plot
    """
    polygon = state_polygons[state_polygons["state"] == state]
    plt.plot("lon", "lat", data=polygon)
    plt.axis(option = "equal");

def write_to_utils(function):
    """
    Writes function to utils.py file

    Arguments:
        function - name of function to be written
    """
    func = inspect.getsource(function)
    with open("utils.py", "r+") as f:
        if func.split(" ")[1] not in f.read():
            with open("utils.py", "a") as f:
                f.write(func + "\n")

def get_state(state):
    """
    Returns portion of `state_polygons` df for specific state

    Arguments:
        state - name of state to select
    """
    return state_polygons[state_polygons["state"] == state]


#Package avec toutes les fonctions nécessaires pour effectuer la
#retropropagation du gradient pour UNE SEULE couche cachee

from random import gauss
from math import *
import numpy as np
import listeOperation as lo
import exploitation as explo

""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !!!!  On prendra n et Nc en parametres a chaque fois que necessaire !!!!!
    !!!!!!  car ils seront connus dans l'algo de retropropagation       !!!!!
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"""

"""Fonction initialisant la matrice des parametres (poids synaptiques)
param : n -> nombre de variables
        Nc -> nombre de neurones de LA couche cachée
retour : W -> vecteur des paramètres du réseau. """

def initParam(n,Nc):
    W = []
    #Connexions entre les variables et la premiere couche
    for i in range(Nc):
        #Initialement la composante du biais est nulle
        W.append(0)
        for j in range(1,n+1):
            W.append(gauss(0,1/(n-1)))
    #Connexions entre la dernière couche et la sortie
    W.append(0)  #Composante nulle selon le biais en sortie
    for j in range(1,Nc+1):
        W.append(gauss(0,1/(n-1)))
    return W

#print(initParam(2,3)[9])

""" Fonction calculant la sortie d'un reseau de n variables (+un biais), UNE SEULE couche
cachee et UNE SEULE sortie
param : x -> exemple prit pour le calcul
        n -> nombre de variables par exemples (ici nombre d'harmonique par
             enregistrement
        Nc -> nombre de neurones de LA couche cachée
        W1 -> poids entre les variables et la couche cachee
        W2 -> poids entre la couche cachee et la sortie
sortie : prediction du modele pour l'exemple x et tableau des potentiels des
        differents neurones du reseau """

def g(x,n,Nc,W):
    modele = W[(n+1)*Nc]
    #Le tableau des potentiels est initialisé avec le biais de la couche cachee
    potentiels = [1]
    for i in range(Nc):
        #Calcul du potentiel du neurone i
        p = 0
        for j in range(n+1):
            p += W[i*(n+1)+j]*x[j]
        potentiels.append(p)
        modele += W[Nc*(n+1)+i]*tanh(p)
    pS = 0
    for j in range(Nc+1):
        pS += W[Nc*(n+1)+j]*potentiels[j]
    potentiels.append(pS)
    return modele,potentiels

""" Fonction calculant l'erreur quadratique moyenne entre deux vecteurs y et g
precondition : y et g sont de même taille. """

def EQM(y,g):
    err = 0
    errk = []
    n = len(g)
    if (n != len(y)):
        print("***EQM***")
        print("len(y) = " + str(len(y)) + " != len(g) = " +str(n))
    for i in range(n):
        r = y[i] - g[i]
        errk.append(r)
        err += r**2
    err = 0.5*sqrt(err)
    return err, errk

""" Fonction calculant la fraction d'exemples mal classés en classification. """

def errClass(y,g):
    err = 0
    errk = []
    for i in range(len(y)):
        r = -y[i]*g[i]
        if (r >= 0):
            err += 1
            errk.append(1)
        else:
            errk.append(0)
    return err, errk

""" Fonction calculant les "delta" de l'algorithme de retropropagation.
param : Nc -> nombre de neurones de la couche cachee
        y -> sortie pour l'exemple considere
        g -> sortie fournie par le reseau pour l'exemple considere
        potentiel -> vecteur des potentiels de chacun des neurones
        W2 -> poids des connexions entre la couche cachee et la sortie
sortie : delta pour les neurones cachés dans les premières cases et celui du neuronne de sortie dans la derniere case
du tableau """

def retro(n,Nc,y,g,potentiel,W):

    #Construction du tableau de derivee : en tout il y a Nc+1 neurones
    delta = []
    #Calcul pour le neurone de sortie
    deltaS = -2*(y-g)*(1/(cosh(potentiel[Nc+1])**2))

    #Calcul des autres poids
    for i in range(Nc+1):
        der = (potentiel[Nc+1]*W[(n+1)*Nc+i])/(cosh(potentiel[i])**2)
        delta.append(der)

    #Ajout de la derivee de la sortie
    delta.append(deltaS)

    return delta

""" Fonction calculant le gradient au point représenté par W. """

def gradient(nbExemples,n,Nc,y,sortiesEx,potentielsEx,Z,W):
    delta = []
    derg = [[] for i in range(nbExemples)]
    for k in range(nbExemples):
        d = retro(n,Nc,y[k],sortiesEx[k],potentielsEx[k],W)
        delta.append(d)
    deriveesW = []
    for i in range(1,Nc+1):
        for j in range(n+1):
            der = 0
            for k in range(nbExemples):
                d = delta[k][i]*Z[k][j]
                derg[k].append(d/(-2*(y[k]-sortiesEx[k])))
                der += d
            deriveesW.append(der)
    for i in range(Nc+1):
        der = 0
        for k in range(nbExemples):
            d = delta[k][Nc+1]*tanh(potentielsEx[k][i])
            derg[k].append(d/(-2*(y[k]-sortiesEx[k])))
            der += d
        deriveesW.append(der)

    return deriveesW, derg

""" Fonction calculant le pas de Levenberg-Marquardt.
param : mu -> damping step
        J  -> matrice jacobienne de G(w)."""

def pasLM(mu,J,nbPoids):
    Z = np.array(J).transpose().dot(np.array(J))
    eigenValues = np.linalg.eigvals(Z)
    diago = np.array(lo.diag(eigenValues))
    H = Z + mu*np.eye(nbPoids,nbPoids)
    L = np.linalg.cholesky(np.array(H))
    invL = np.linalg.inv(L)
    pas = invL.transpose()*invL
    return pas

""" Fonction modifiant les poids par la méthode de Levenberg (-Marquardt)
param : W1 -> poids entre les variables et la couche cachee
        W2 -> poids entre la couche cachee et la sortie
        invH -> matrice inverse intervenant dans le pas de LM
        derW1 -> derivees de la fonction coût par rapport aux param. de la 1ere couche
        derW2 -> derivees de la fonction coût par rapport aux param. de la 2eme couche
"""

def pasPoids(invH, derW):
    nW = []
    for i in range(len(invH)):
        nW.append(lo.ps(invH[i],derW))
    return list(nW)

"""Fonction effectuant le calcul des parametres du reseau par retropropagation du
gradient pour UNE SEULE couche cachee
param : n -> nombre de variables par exemples (ici nombre d'harmonique par
             enregistrement
        Nc -> nombre de neurones de LA couche cachée
        seuil -> seuil en deça duquel on s'arrête de minimiser la fonction coût
        l -> pas pour effectuer la modification des poids
        nbIterRechercheMax -> nombre d'itérations maximal pour la méthode de LM
        r -> facteur d'échelle pour mu
retour : tableau des parametres du reseau apres apprentissage"""

def retropropagation(chemin,n,Nc,c1,c2,mu_0,nbIterMax,nbIterRechercheMax,r,y=[],Z=[]):

    #Liste des variables créées
    var = []

    #Tableau flag pour la recherche du pas dans Levengerb
    flag = [0,0]

    #Recuperation des spectres et du vecteur de sortie associe
    if (y == []): #on suppose implicitement qu'alors Z est également vide (on ne précise pas l'un sans preciser l'autre)
        y,Z,nbDesc = explo.getDataEx(chemin)
        var.append(y)
        var.append(Z)

    #Initialisation des parametres
    W = initParam(n,Nc)
    nbPoids = (n+1)*Nc + Nc+1 #=len(W)
    var.append(W)

    nbExemples = len(Z)
    var.append(nbExemples)

    #Propagation de tous les exemples avec récuperation des potentiels
    potentielsEx = []
    sortiesEx = []
    var.append(potentielsEx)
    var.append(sortiesEx)
    for i in range(nbExemples):
        sortie,potentiels = g(Z[i],n,Nc,W)
        sortiesEx.append(sortie)
        potentielsEx.append(potentiels)

    #Calcul de l'erreur quadratique moyenne associee
    erreur, errk = EQM(y,sortiesEx)
    var.append(erreur)
    var.append(errk)

    #Norme du gradient au point de depart
    deriveesW, deriveesg = gradient(nbExemples,n,Nc,y,sortiesEx,potentielsEx,Z,W)
    var.append(deriveesW)
    normG0 = lo.norm2(deriveesW)
    #print("nomrG0 = " + str(normG0))
    normG = 10**(10)
    W_prec = [10**(10) for i in range(nbPoids)]

    print("-------- DEBUT --------" + str(lo.distriDistUn(y,sortiesEx)) + " <-> " + str(erreur) + " <-> " + str(normG))

    nbIter = 1
    tableErreur = [erreur]

    #Tant que l'erreur n'est pas suffisamment petite
    while (normG > c1*normG0) & (nbIter <= nbIterMax) & ((lo.norm2(lo.sous(W,W_prec))) > c2*lo.norm2(W_prec)):
        print("              Itération : " + str(nbIter))
        ##### MODIFICATION DES POIDS ######
        mu = mu_0
        #Modification de mu (et donc du pas) tant que l'on a pas accepte la modification
        accepte = False
        nbIterRecherche = 1
        #Tableaux pour retenir les mu et les erreurs associées
        mu_calc = []
        erreur_calc = []
        W_prec = W
        while ((not accepte) & (nbIterRecherche <= nbIterRechercheMax)):

            #Calcul de l'inverse du pas du second ordre pour LM (avec l'identité et pas diag(H))
            invH = pasLM(mu,deriveesg,nbPoids)
            if (nbIter ==1) & (nbIterRecherche==1):
                var.append(invH)
            W_prec = W
            erreur_prec = erreur

            #Modification des poids avec une constante valant mu
            delta = pasPoids(invH,deriveesW)
            W = lo.add(W,delta)

            #Comparaison de l'erreur commise avec ces paramètres et de l'ancienne et decision
            sortiesEx = []
            for i in range(nbExemples):
                sortie,potentiels = g(Z[i],n,Nc,W)
                sortiesEx.append(sortie)
            erreur,errk = EQM(y,sortiesEx)

            if (erreur < erreur_prec):
                accepte = True
                mu = mu/r
                flag[0] += 1
            else :
                mu_calc.append(mu)
                erreur_calc.append(erreur)
                mu = mu*r
                W = W_prec
                erreur = erreur_prec

            nbIterRecherche += 1

        #Si on est sorti sans trouver de mu convenable, il faut quand même modifier les poids avce la dernière valeur de mu trouvee
        if (nbIterRecherche > nbIterRechercheMax) & (not accepte):
            #On regarde quelle erreur était la plus petite parmis celles recontrees, et on recalcule les poids en conséquence
            indErrMin = lo.indMin(erreur_calc)
            mu = mu_calc[indErrMin]
            invH = pasLM(mu,deriveesg,nbPoids)
            #Modification des poids avec une constante valant mu
            delta = pasPoids(invH,deriveesW)
            W = lo.add(W,delta)
            flag[1] += 1
        #Propagation des exemples et recalcule des potentiels
        potentielsEx = []
        sortiesEx = []
        for i in range(nbExemples):
            sortie,potentiels = g(Z[i],n,Nc,W)
            sortiesEx.append(sortie)
            potentielsEx.append(potentiels)
        print("              " + str(lo.distriDistUn(y,sortiesEx)) + " <-> " + str(erreur) + " <-> " + str(normG))

        #Recalculer l'erreur quadratique moyenne
        erreur,errk = EQM(y,sortiesEx)
        tableErreur.append(erreur)

        #Retropropagation des exemples
        deriveesW, deriveesg = gradient(nbExemples,n,Nc,y,sortiesEx,potentielsEx,Z,W)
        normG = lo.norm2(deriveesW)

        nbIter += 1

    if (normG <= c1*normG0):
        flag.append("Décroissance gradient.")
    elif (nbIter > nbIterMax):
        flag.append("Iteration max")
    print(' ')
    print("              " + str(flag))
    print("              " + str(lo.distriDistUn(y,sortiesEx)) + " <-> " + str(erreur) + " <-> " + str(normG))
    print(' ')

    #Nettoyage memoire
    explo.clear(var)

    #Trace
    return tableErreur


import math
def distance(point):
        return (math.sqrt(pow(point[0],2) + pow(point[1], 2)))

def kpoints(points, n):
    dist = []
    for each in points:
        dist.append((each, distance(each)))

    dist = sorted(dist, key= lambda x: x[1])
    ans = []
    for each in dist[:n]:
        ans.append(each[0])
    print (ans)

kpoints([[3,3],[5,-1],[-2,4]], 2)


# The MIT License (MIT)
# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
This module provides general utility operations that wrap specific ``xarray`` functions.

The intention is to make available the ``xarray`` API as a set of general, domain-independent
utility functions.

All operations in this module are tagged with the ``"utility"`` tag.

"""

import pandas as pd
import xarray as xr

from cate.core.op import op, op_input, op_return
from cate.core.types import DatasetLike, PointLike, TimeLike, DictLike, Arbitrary, Literal
from cate.util.monitor import Monitor

@op(tags=['utility', 'internal'])
@op_input('ds', data_type=DatasetLike)
@op_input('point', data_type=PointLike, units='degree')
@op_input('time', data_type=TimeLike)
@op_input('indexers', data_type=DictLike)
@op_input('method', value_set=['nearest', 'ffill', 'bfill'])
def sel(ds: DatasetLike.TYPE,
        point: PointLike.TYPE = None,
        time: TimeLike.TYPE = None,
        indexers: DictLike.TYPE = None,
        method: str = 'nearest') -> xr.Dataset:
    """
    Return a new dataset with each array indexed by tick labels along the specified dimension(s).

    This is a wrapper for the ``xarray.sel()`` function.

    For documentation refer to xarray documentation at
    http://xarray.pydata.org/en/stable/generated/xarray.Dataset.sel.html#xarray.Dataset.sel

    :param ds: The dataset from which to select.
    :param point: Optional geographic point given by longitude and latitude
    :param time: Optional time
    :param indexers: Keyword arguments with names matching dimensions and values given by scalars,
           slices or arrays of tick labels. For dimensions with multi-index, the indexer may also be
           a dict-like object with keys matching index level names.
    :param method: Method to use for inexact matches:
           * None: only exact matches
           * ``pad`` / ``ffill``: propagate last valid index value forward
           * ``backfill`` / ``bfill``: propagate next valid index value backward
           * ``nearest`` (default): use nearest valid index value
    :return: A new Dataset with the same contents as this dataset, except each variable and dimension
             is indexed by the appropriate indexers. In general, each variable's data will be a view of the
             variable's data in this dataset.
    """
    ds = DatasetLike.convert(ds)
    point = PointLike.convert(point)
    time = TimeLike.convert(time)
    indexers = DictLike.convert(indexers)
    indexers = dict(indexers or {})
    if point is not None:
        indexers.setdefault('lon', point.x)
        indexers.setdefault('lat', point.y)
    if time is not None:
        indexers.setdefault('time', time)
    # Filter out non-existent coordinates
    indexers = {name: value for name, value in indexers.items() if name in ds.coords}
    return ds.sel(method=method, **indexers)

@op(tags=['utility'])
def from_dataframe(df: pd.DataFrame) -> xr.Dataset:
    """
    Convert the given dataframe to an xarray dataset.

    This is a wrapper for the ``xarray.from_dataframe()`` function.

    For documentation refer to xarray documentation at
    http://xarray.pydata.org/en/stable/generated/xarray.Dataset.from_dataframe.html#xarray.Dataset.from_dataframe

    :param df: Dataframe to convert
    :return: A dataset created from the given dataframe
    """
    return xr.Dataset.from_dataframe(df)

@op(tags=['utility'])
@op_input('value', data_type=Arbitrary)
@op_return(data_type=Arbitrary)
def identity(value: Arbitrary.TYPE) -> Arbitrary.TYPE:
    """
    Return the given value.
    This operation can be useful to create constant resources to be used as input for other operations.

    :param value: An arbitrary (Python) value.
    """
    return value

@op(tags=['utility'])
@op_input('value', data_type=Literal)
@op_return(data_type=Arbitrary)
def literal(value: Literal.TYPE) -> Arbitrary.TYPE:
    """
    Return the given value.
    This operation can be useful to create constant resources to be used as input for other operations.

    :param value: An arbitrary (Python) literal.
    """
    return Literal.convert(value)

@op(tags=['utility'])
@op_input('step_duration', units='seconds')
def no_op(num_steps: int = 10,
          step_duration: float = 0.5,
          fail_before: bool = False,
          fail_after: bool = False,
          monitor: Monitor = Monitor.NONE) -> bool:
    """
    An operation that basically does nothing but spending configurable time.
    It may be useful for testing purposes.

    :param num_steps: Number of steps to iterate.
    :param step_duration: How much time to spend in each step in seconds.
    :param fail_before: If the operation should fail before spending time doing nothing.
    :param fail_after: If the operation should fail after spending time doing nothing.
    :param monitor: A progress monitor.
    :return: Always True
    """
    import time
    monitor.start('Computing nothing', num_steps)
    if fail_before:
        raise ValueError('Intentionally failed before doing anything.')
    for i in range(num_steps):
        time.sleep(step_duration)
        monitor.progress(1.0, 'Step %s of %s doing nothing' % (i + 1, num_steps))
    if fail_after:
        raise ValueError('Intentionally failed after doing nothing.')
    monitor.done()
    return True

@op(tags=['utility', 'internal'])
@op_input('method', value_set=['backfill', 'bfill', 'pad', 'ffill'])
def pandas_fillna(df: pd.DataFrame,
                  value: float=None,
                  method: str=None,
                  limit: int=None,
                  **kwargs) -> pd.DataFrame:
    """
    Return a new dataframe with NaN values filled according to the given value
    or method.

    This is a wrapper for the ``pandas.fillna()`` function For additional
    keyword arguments and information refer to pandas documentation at
    http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.fillna.html

    :param df: The dataframe to fill
    :param value: Value to fill
    :param method: Method according to which to fill NaN. ffill/pad will
           propagate the last valid observation to the next valid observation.
           backfill/bfill will propagate the next valid observation back to the last
           valid observation.
    :param limit: Maximum number of NaN values to forward/backward fill.
    :return: A dataframe with nan values filled with the given value or according to the given method.
    """
    # The following code is needed, because Pandas treats any kw given in kwargs as being set, even if just None.
    kwargs = dict(kwargs)
    if value:
        kwargs.update(value=value)
    if method:
        kwargs.update(method=method)
    if limit:
        kwargs.update(limit=limit)

    return df.fillna(**kwargs)


# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import utool as ut
import six
import itertools as it
from ibeis import _ibeis_object
from ibeis.control.controller_inject import make_ibs_register_decorator
(print, rrr, profile) = ut.inject2(__name__, '[annot]')

CLASS_INJECT_KEY, register_ibs_method = make_ibs_register_decorator(__name__)

BASE_TYPE = type

@register_ibs_method
def annots(ibs, aids=None, **kwargs):
    """ Makes an Annots object """
    if aids is None:
        aids = ibs.get_valid_aids()
    elif aids.__class__.__name__ == 'Annots':
        return aids
    aids = ut.ensure_iterable(aids)
    return Annots(aids, ibs, **kwargs)

@register_ibs_method
def _annot_groups(ibs, aids_list=None, config=None):
    annots_list = [ibs.annots(aids, config=config) for aids in aids_list]
    return AnnotGroups(annots_list, ibs)

ANNOT_BASE_ATTRS = [
    'aid',
    'parent_aid',

    'uuids', 'hashid_uuid', 'visual_uuids', 'hashid_visual_uuid',
    'semantic_uuids', 'hashid_semantic_uuid', 'verts', 'thetas',
    'bboxes', 'bbox_area',
    'species_uuids', 'species', 'species_rowids', 'species_texts', 'yaw_texts',
    'yaws', 'qualities', 'quality_texts', 'exemplar_flags',
    # Images
    # 'image_rowids',
    'gids', 'image_uuids', 'image_gps', 'image_unixtimes_asfloat',
    'image_datetime_str', 'image_contributor_tag',
    # Names
    'nids', 'names', 'name_uuids',
    # Inferred from context attrs
    'contact_aids', 'num_contact_aids', 'groundfalse', 'groundtruth',
    'num_groundtruth', 'has_groundtruth', 'otherimage_aids',
    # Image Set
    'imgset_uuids', 'imgsetids', 'image_set_texts',
    # Occurrence / Encounter
    'encounter_text', 'occurrence_text', 'primary_imageset',
    # Tags
    'all_tags',
    'case_tags',
    'annotmatch_tags', 'notes',
    # Processing State
    'reviewed', 'reviewed_matching_aids', 'has_reviewed_matching_aids',
    'num_reviewed_matching_aids', 'detect_confidence',
]

ANNOT_SETTABLE_ATTRS = [
    'age_months_est_max', 'age_months_est_min',
    'bboxes', 'thetas', 'verts',
    'quality_texts', 'yaw_texts', 'yaws'
    'sex', 'sex_texts', 'species',
    'exemplar_flags',
    'multiple',
    'detect_confidence', 'reviewed',
    'name_texts', 'names',
    'notes',
    'parent_rowid',
]

class _AnnotPropInjector(BASE_TYPE):
    """
    Example:
        >>> from ibeis import _ibeis_object
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> objname = 'annot'
        >>> blacklist = ['annot_pair']
        >>> _ibeis_object._find_ibeis_attrs(ibs, objname, blacklist)
    """
    def __init__(metaself, name, bases, dct):
        super(_AnnotPropInjector, metaself).__init__(name, bases, dct)
        metaself.rrr = rrr

        attrs = ANNOT_BASE_ATTRS

        settable_attrs = ANNOT_SETTABLE_ATTRS

        configurable_attrs = [
            # Chip
            'chip_dlensqrd', 'chip_fpath', 'chip_sizes', 'chip_thumbpath',
            'chip_thumbtup', 'chips',
            # Feat / FeatWeight / Kpts / Desc
            'feat_rowids', 'num_feats', 'featweight_rowids', 'fgweights',
            'fgweights_subset', 'kpts', 'kpts_distinctiveness', 'vecs',
            'vecs_cache', 'vecs_subset',
        ]
        #misc = [
        #    'gar_rowids', 'alrids', 'alrids_oftype', 'lblannot_rowids',
        #    'lblannot_rowids_oftype', 'lblannot_value_of_lbltype', 'rows',
        #    'instancelist', 'lazy_dict', 'lazy_dict2', 'missing_uuid',
        #    'been_adjusted', 'class_labels',
        #]
        #extra_attrs = [
        #    # Age / Sex
        #    'age_months_est', 'age_months_est_max', 'age_months_est_max_texts',
        #    'age_months_est_min', 'age_months_est_min_texts',
        #    'age_months_est_texts', 'sex', 'sex_texts',

        #    # Stats
        #    'stats_dict', 'per_name_stats', 'qual_stats', 'info', 'yaw_stats',
        #    'intermediate_viewpoint_stats',
        #]
        #inverse_attrs = [
        #    # External lookups via superkeys
        #    'aids_from_semantic_uuid',
        #    'aids_from_uuid',
        #    'aids_from_visual_uuid',
        #    'rowids_from_partial_vuuids',
        #]

        depcache_attrs = [
            ('hog', 'hog'),
        ]

        objname = 'annot'
        _ibeis_object._inject_getter_attrs(metaself, objname, attrs,
                                           configurable_attrs, 'depc_annot',
                                           depcache_attrs, settable_attrs)

        # TODO: incorporate dynamic setters
        #def set_case_tags(self, tags):
        #    self._ibs.append_annot_case_tags(self._rowids, tags)
        #fget = metaself.case_tags.fget
        #fset = set_case_tags
        #setattr(metaself, 'case_tags', property(fget, fset))

@ut.reloadable_class
@six.add_metaclass(_AnnotPropInjector)
class Annots(_ibeis_object.ObjectList1D):
    """
    Represents a group of annotations. Efficiently accesses properties from a
    database using lazy evaluation.

    CommandLine:
        python -m ibeis.annots Annots --show

    Example:
        >>> from ibeis.annots import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> aids = ibs.get_valid_aids()
        >>> a = self = annots = Annots(aids, ibs)
        >>> a.preload('vecs', 'kpts', 'nids')
        >>> print(ut.depth_profile(a.vecs))
        >>> print(a)
        <Annots(num=13)>

    """
    #def __init__(self, aids, ibs, config=None, caching=False):
    #    super(Annots, self).__init__(aids, ibs, config, caching)

    @property
    def aids(self):
        return self._rowids

    #@property
    def get_speeds(self):
        #import vtool as vt
        edges = self.get_aidpairs()
        speeds = self._ibs.get_annotpair_speeds(edges)
        #edges = vt.pdist_indicies(len(annots))
        #speeds = self._ibs.get_unflat_annots_speeds_list([self.aids])[0]
        edge_to_speed = dict(zip(edges, speeds))
        return edge_to_speed

    def get_aidpairs(self):
        aids = self.aids
        aid_pairs = list(it.combinations(aids, 2))
        return aid_pairs

    def get_am_rowids(self):
        ibs = self._ibs
        edges = self.get_aidpairs()
        aids1 = ut.take_column(edges, 0)
        aids2 = ut.take_column(edges, 1)
        ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2)
        ams = ut.filter_Nones(ams)
        return ams

    def get_am_rowids_and_pairs(self):
        ibs = self._ibs
        aid_pairs = self.get_aidpairs()
        aids1 = ut.take_column(aid_pairs, 0)
        aids2 = ut.take_column(aid_pairs, 1)
        ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2)
        flags = ut.not_list(ut.flag_None_items(ams))
        ams = ut.compress(ams, flags)
        aid_pairs = ut.compress(aid_pairs, flags)
        return ams, aid_pairs

    def get_am_aidpairs(self):
        ibs = self._ibs
        ams = self.get_am_rowids()
        aids1 = ibs.get_annotmatch_aid1(ams)
        aids2 = ibs.get_annotmatch_aid2(ams)
        aid_pairs = list(zip(aids1, aids2))
        return aid_pairs

    @property
    def hog_img(self):
        from ibeis import core_annots
        return [core_annots.make_hog_block_image(hog) for hog in self.hog_hog]

    def append_tags(self, tags):
        self._ibs.append_annot_case_tags(self._rowids, tags)

    def remove_tags(self, tags):
        self._ibs.remove_annot_case_tags(self._rowids, tags)

class _AnnotGroupPropInjector(BASE_TYPE):
    def __init__(metaself, name, bases, dct):
        super(_AnnotGroupPropInjector, metaself).__init__(name, bases, dct)
        metaself.rrr = rrr

        #attrs = ANNOT_BASE_ATTRS
        #objname = 'annot'

        # TODO: move to ibeis object as a group call
        def _make_unflat_getter(objname, attrname):
            ibs_funcname = 'get_%s_%s' % (objname, attrname)
            def ibs_unflat_getter(self, *args, **kwargs):
                ibs_callable = getattr(self._ibs, ibs_funcname)
                rowids = self._rowids_list
                ibs = self._ibs
                return ibs.unflat_map(ibs_callable, rowids, *args, **kwargs)
            ut.set_funcname(ibs_unflat_getter, 'unflat_' + ibs_funcname)
            return ibs_unflat_getter

        for attrname in ANNOT_BASE_ATTRS:
            if hasattr(metaself, attrname):
                print('Cannot inject annot group attrname = %r' % (attrname,))
                continue
            ibs_unflat_getter = _make_unflat_getter('annot', attrname)
            setattr(metaself, '_unflat_get_' + attrname, ibs_unflat_getter)
            setattr(metaself, attrname, property(ibs_unflat_getter))

@ut.reloadable_class
@six.add_metaclass(_AnnotGroupPropInjector)
class AnnotGroups(ut.NiceRepr):
    """ Effciently handle operations on multiple groups of annotations """
    def __init__(self, annots_list, ibs):
        self._ibs = ibs
        self.annots_list = annots_list
        self._rowids_list = [a._rowids for a in self.annots_list]

    def __nice__(self):
        import numpy as np
        len_list = ut.lmap(len, self.annots_list)
        num = len(self.annots_list)
        mean = np.mean(len_list)
        std = np.std(len_list)
        if six.PY3:
            nice = '(n=%r, μ=%.1f, σ=%.1f)' % (num, mean, std)
        else:
            nice = '(n=%r, m=%.1f, s=%.1f)' % (num, mean, std)
        return nice

    @property
    def aids(self):
        return [a.aids for a in self.annots_list]

    @property
    def images(self, config=None):
        return self._ibs.images(self.gids, config)

    @property
    def match_tags(self):
        """ returns pairwise tags within the annotation group """
        ams_list = self._ibs.get_unflat_am_rowids(self.aids)
        tags = self._ibs.unflat_map(self._ibs.get_annotmatch_case_tags, ams_list)
        return tags

if __name__ == '__main__':
    r"""
    CommandLine:
        python -m ibeis.annot
        python -m ibeis.annot --allexamples
    """
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()


import os
import sys
import json
import copy
import requests
import datetime
import glob
import csv
import string

class baseInfo:
    postURL = "$POSTURL"

    shipmentEventBase = {
    "associatedAssetSize": None,
    "associatedUnitId": None,
    "billOfLadingNumber": None,
    "bookingNumber": None,
    "bookingOffice": None,
    "carrierCode": None,
    "carrierName": None,
    "city": None,
    "codeType": None,
    "consigneeName": None,
    "containerBookingNumber": None,
    "country": None,
    "createdBy": None,
    "customerOrderReferenceNumber": None,
    "destinationCity": None,
    "destinationSPLC": None,
    "destinationState": None,
    "eventCode": None,
    "eventName": None,
    "eventTime": None,
    "houseBill": None,
    "importReferenceNumber": None,
    "latitude": 0,
    "location": None,
    "longitude": 0,
    "masterBill": None,
    "notes": None,
    "onHandNumber": None,
    "originSPLC": None,
    "originatorCode": None,
    "originatorId": 0,
    "originatorName": None,
    "postalCode": None,
    "purchaseOrderReferenceNumber": None,
    "railBillingNumber": None,
    "reasonCode": None,
    "reasonName": None,
    "receiverCode": None,
    "receiverName": None,
    "reportSource": None,
    "reportedBy": None,
    "resolvedEventId": 0,
    "resolvedEventOriginalId": 0,
    "resolvedEventSource": None,
    "resolvedEventStatus": None,
    "sealNumber": None,
    "shipmentReferenceNumber": None,
    "signedBy": None,
    "state": None,
    "stopType": None,
    "terminalCode": None,
    "terminalFunction": None,
    "unitId": None,
    "unitSize": 0,
    "unitState": None,
    "unitTypeCode": None,
    "vessel": None,
    "voyageNumber": None,
    "workOrderNumber": None
    }

    StatusMapHyundai = {
        "Gate out empty": "OA",
        "Gate in empty": "I",
        "Vessel departed": "VD",
        "Vessel arrived": "VA",
        "Loaded": "AE",
        "Discharged": "UV"
    }

def HyundaiCodeToName(code):
    if(code == "AE"):
        return "Loaded on Vessel"
    elif(code == "VD"):
        return "Vessel Departure"
    elif(code == "VA"):
        return "Vessel Arrival"
    elif(code == "UV"):
        return "Unloaded From Vessel"
    elif(code == "OA"):
        return "OUTGATE"
    elif(code == "I"):
        return "INGATE"
    return None

def HyundaiEventTranslate(event):
    for key, value in baseInfo.StatusMapHyundai.items():
        if(event.find(key) != -1):
            return value, HyundaiCodeToName(value)
    return (None, None)

def HyundaiPost(container, path):
    if(os.path.isfile(path+"ContainerInformation\\"+container+".csv")):
        with open(path+"ContainerInformation\\"+container+".csv") as containerInfo:
            reader = csv.reader(containerInfo)
            next(reader) # skip first row
            for row in reader:
                postJson = copy.deepcopy(baseInfo.shipmentEventBase)
                postJson["unitId"] = container
                postJson["location"] = row[1].split("\n")[0]
                postJson["city"] = postJson["location"].split(",")[0]
                if(row[2].strip() == "" or row[3].strip() == ""):
                    continue
                postJson["eventTime"] = datetime.datetime.strptime(row[2]+" "+row[3], '%Y-%m-%d %H:%M').strftime('%m-%d-%Y %H:%M:%S')
                postJson["vessel"] = row[11]
                postJson["voyageNumber"] = row[12]
                postJson["workOrderNumber"] = row[14]
                postJson["billOfLadingNumber"] = row[13]
                postJson["eventCode"], postJson["eventName"] = HyundaiEventTranslate(row[0])
                postJson["resolvedEventSource"] = "Hyundai RPA"
                postJson["codeType"] = "UNLOCODE"
                postJson["reportSource"] = "OceanEvent"
                print(json.dumps(postJson))
                if(postJson["eventCode"] == None):
                    continue
                headers = {'content-type':'application/json'}
                r = requests.post(baseInfo.postURL, data = json.dumps(postJson), headers = headers, verify = False)
                print(r)
    return

def main(containerList, cwd):
    path=""
    for x in cwd.split("\\"):
        path+=x+"\\\\"
    for container in containerList:
        HyundaiPost(container, path)

if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2])


if __name__ == "__main__":

    import ROOT
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('--weight', default="up", choices=['up', 'down'], help="Which H pT reweighting?")
    options = parser.parse_args()

    suffix=""
    if options.scale=="up":
       suffix="_CMS_ggHinUp"
    if options.scale=="down":
       suffix="_CMS_ggHinDown"

    fggH=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH125.root","r")

#    fggH110=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH110.root","r")
#    fggH115=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH115.root","r")
#    fggH120=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH120.root","r")
#    fggH130=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH130.root","r")
#    fggH135=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH135.root","r")
#    fggH140=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH140.root","r")
#    fggH145=ROOT.TFile("weighted_files_ggHin"+options.scale+"/ggH145.root","r")

    fSM=ROOT.TFile("htt_tt_sm_cecile"+suffix+".root","recreate")
    rep1=fSM.mkdir("tauTau_1jet_high_mediumhiggs")
    rep2=fSM.mkdir("tauTau_1jet_high_highhiggs")
    rep3=fSM.mkdir("VBF")
    rep=[rep1,rep2,rep3]

    ggH1=fggH.Get("Cat1")
    ggH2=fggH.Get("Cat2")
    ggH3=fggH.Get("Cat3")
    ggH=[ggH1,ggH2,ggH3]

#    ggH110_1=fggH110.Get("Cat1")
#    ggH110_2=fggH110.Get("Cat2")
#    ggH110_3=fggH110.Get("Cat3")
#    ggH110=[ggH110_1,ggH110_2,ggH110_3]
#    ggH115_1=fggH115.Get("Cat1")
#    ggH115_2=fggH115.Get("Cat2")
#    ggH115_3=fggH115.Get("Cat3")
#    ggH115=[ggH115_1,ggH115_2,ggH115_3]
#    ggH120_1=fggH120.Get("Cat1")
#    ggH120_2=fggH120.Get("Cat2")
#    ggH120_3=fggH120.Get("Cat3")
#    ggH120=[ggH120_1,ggH120_2,ggH120_3]
#    ggH130_1=fggH130.Get("Cat1")
#    ggH130_2=fggH130.Get("Cat2")
#    ggH130_3=fggH130.Get("Cat3")
#    ggH130=[ggH130_1,ggH130_2,ggH130_3]
#    ggH135_1=fggH135.Get("Cat1")
#    ggH135_2=fggH135.Get("Cat2")
#    ggH135_3=fggH135.Get("Cat3")
#    ggH135=[ggH135_1,ggH135_2,ggH135_3]
#    ggH140_1=fggH140.Get("Cat1")
#    ggH140_2=fggH140.Get("Cat2")
#    ggH140_3=fggH140.Get("Cat3")
#    ggH140=[ggH140_1,ggH140_2,ggH140_3]
#    ggH145_1=fggH145.Get("Cat1")
#    ggH145_2=fggH145.Get("Cat2")
#    ggH145_3=fggH145.Get("Cat3")
#    ggH145=[ggH145_1,ggH145_2,ggH145_3]
#
    for i in range(0,3):
       ggH[i].SetName("ggH125"+suffix)
       rep[i].cd()
       ggH[i].Write()
#       ggH110[i].SetName("ggH110"+suffix)
#       ggH115[i].SetName("ggH115"+suffix)
#       ggH120[i].SetName("ggH120"+suffix)
#       ggH130[i].SetName("ggH130"+suffix)
#       ggH135[i].SetName("ggH135"+suffix)
#       ggH140[i].SetName("ggH140"+suffix)
#       ggH145[i].SetName("ggH145"+suffix)
#       ggH110[i].Write()
#       ggH115[i].Write()
#       ggH120[i].Write()
#       ggH130[i].Write()
#       ggH135[i].Write()
#       ggH140[i].Write()
#       ggH145[i].Write()


"""Simple Actor network."""

import gin
import tensorflow as tf  # pylint: disable=g-explicit-tensorflow-version-import

from tf_agents.networks import network
from tf_agents.networks import utils

@gin.configurable
class LinearActorNetwork(network.Network):
  """Creates an actor network with linear activation in
  the final layer."""

  def __init__(self,
               input_tensor_spec,
               output_tensor_spec,
               fc_layer_params=None,
               dropout_layer_params=None,
               conv_layer_params=None,
               activation_fn=tf.keras.activations.relu,
               kernel_initializer=None,
               last_kernel_initializer=None,
               name='LinearActorNetwork'):
    """Creates an instance of `LinearActorNetwork`.

    Args:
      input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the
        inputs.
      output_tensor_spec: A nest of `tensor_spec.BoundedTensorSpec` representing
        the outputs.
      fc_layer_params: Optional list of fully_connected parameters, where each
        item is the number of units in the layer.
      dropout_layer_params: Optional list of dropout layer parameters, each item
        is the fraction of input units to drop or a dictionary of parameters
        according to the keras.Dropout documentation. The additional parameter
        `permanent`, if set to True, allows to apply dropout at inference for
        approximated Bayesian inference. The dropout layers are interleaved with
        the fully connected layers; there is a dropout layer after each fully
        connected layer, except if the entry in the list is None. This list must
        have the same length of fc_layer_params, or be None.
      conv_layer_params: Optional list of convolution layers parameters, where
        each item is a length-three tuple indicating (filters, kernel_size,
        stride).
      activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ...
      kernel_initializer: kernel initializer for all layers except for the value
        regression layer. If None, a VarianceScaling initializer will be used.
      last_kernel_initializer: kernel initializer for the value regression
         layer. If None, a RandomUniform initializer will be used.
      name: A string representing name of the network.

    Raises:
      ValueError: If `input_tensor_spec` or `action_spec` contains more than one
        item, or if the action data type is not `float`.
    """

    super().__init__(
      input_tensor_spec=input_tensor_spec,
      state_spec=(),
      name=name)

    if len(tf.nest.flatten(input_tensor_spec)) > 1:
      raise ValueError('Only a single observation is supported by this network')

    flat_action_spec = tf.nest.flatten(output_tensor_spec)
    if len(flat_action_spec) > 1:
      raise ValueError('Only a single action is supported by this network')
    self._single_action_spec = flat_action_spec[0]

    if self._single_action_spec.dtype not in [tf.float32, tf.float64]:
      raise ValueError('Only float actions are supported by this network.')

    if kernel_initializer is None:
      kernel_initializer = tf.compat.v1.keras.initializers.VarianceScaling(
        scale=1. / 3., mode='fan_in', distribution='uniform')
    if last_kernel_initializer is None:
      last_kernel_initializer = tf.keras.initializers.RandomUniform(
        minval=-0.003, maxval=0.003)

    self._mlp_layers = utils.mlp_layers(
      conv_layer_params,
      fc_layer_params,
      dropout_layer_params,
      activation_fn=activation_fn,
      kernel_initializer=kernel_initializer,
      name='input_mlp')

    self._mlp_layers.append(
      tf.keras.layers.Dense(
        flat_action_spec[0].shape.num_elements(),
        activation=tf.keras.activations.linear,
        kernel_initializer=last_kernel_initializer,
        name='action'))

    self._output_tensor_spec = output_tensor_spec

  def call(self, observations, step_type=(), network_state=(), training=False):
    del step_type  # unused.
    observations = tf.nest.flatten(observations)
    output = tf.cast(observations[0], tf.float32)
    for layer in self._mlp_layers:
      output = layer(output, training=training)

    actions = tf.cast(
      tf.reshape(output, [-1] + self._single_action_spec.shape.as_list()),
      self._single_action_spec.dtype)
    output_actions = tf.nest.pack_sequence_as(self._output_tensor_spec,
                                              [actions])

    return output_actions, network_state


dataset = [32, 94, 128, 1286, 6, 71]

def xy_cmp(x, y):
    if x+y < y+x:
        return 1
    elif x+y > y+x:
        return -1
    else:
        return 0

def number_join(dataset):
    dataset = list(map(str, dataset))
    dataset.sort(key=cmp_to_key(xy_cmp))
    return "".join(dataset)


#!/usr/bin/env python2
#
# The MIT License (MIT)
#
# Copyright (c) 2014 Fam Zheng <fam@euphon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import subprocess
import tempfile

class GitRepo(object):
    def __init__(self, directory, stdout=None, stderr=None):
        self._dir = directory
        self._stdout = stdout
        self._stderr = stderr

    def __call__(self, *args):
        p = subprocess.Popen(["git"] + list(args), cwd=self._dir,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        p.wait()
        self._stdout.write(p.stdout.read())
        self._stderr.write(p.stderr.read())
        return p.returncode

    def check_merged(self, msg_ids):
        cmd = ["log", "--oneline"]
        for i in msg_ids:
            cmd += ["--grep", "Message-id: " + i]
        try:
            out = subprocess.check_output(['git'] + list(cmd))
            if len(out.splitlines()) == len(msg_ids):
                return True
        except:
            pass
        return False

    def apply_patches(self, patches):
        cmd = ["am"] + list(patches)
        return self(*cmd) == 0


from PyCamellia import *
from ConditionParser import *
import unittest

Points = [0.,1.,2.,-1.,-2.]

class TestConditionParser(unittest.TestCase):

    """Test Only X Equals"""
    def test_OnlyXEquals(self):
        filter = parseCondition("x=0")
        for i in Points:
            for j in Points:
                x = (i == 0.0)
                self.assertEqual(x, filter.matchesPoint(i,j))

    """Test Only X Greater"""
    def test_OnlyXGreater(self):
        filter = parseCondition("x>0")
        for i in Points:
            for j in Points:
                x = (i >= 0.0)
                self.assertEqual(x, filter.matchesPoint(i,j))

    """Test Only X Less"""
    def test_OnlyXLess(self):
        filter = parseCondition("x<0")
        for i in Points:
            for j in Points:
                x = (i <= 0.0)
                self.assertEqual(x, filter.matchesPoint(i,j))

    """Test Only Y Equals"""
    def test_OnlyYEquals(self):
        filter = parseCondition("y=0")
        for i in Points:
            for j in Points:
                y = (j == 0.0)
                self.assertEqual(y, filter.matchesPoint(i,j))

    """Test Only Y Greater"""
    def test_OnlyYGreater(self):
        filter = parseCondition("y>0")
        for i in Points:
            for j in Points:
                y = (j >= 0.0)
                self.assertEqual(y, filter.matchesPoint(i,j))

    """Test Only Y Less"""
    def test_OnlyYLess(self):
        filter = parseCondition("y<0")
        for i in Points:
            for j in Points:
                y = (j <= 0.0)
                self.assertEqual(y, filter.matchesPoint(i,j))

    """Test XY"""
    def test_XY(self):
        filter = parseCondition("x=0,y<0")
        for i in Points:
            for j in Points:
                x = (i == 0.0)
                y = (j <= 0.0)
                self.assertEqual(x and y, filter.matchesPoint(i,j))

    """Test YX"""
    def test_YX(self):
        filter = parseCondition("y=0,x>0")
        for i in Points:
            for j in Points:
                x = (i >= 0.0)
                y = (j == 0.0)
                self.assertEqual(x and y, filter.matchesPoint(i,j))

    """Test Doubles"""
    def test_Doubles(self):
        filter = parseCondition("x>0.0,y>0.0")
        for i in Points:
            for j in Points:
                x = (i >= 0.0)
                y = (j >= 0.0)
                self.assertEqual(x and y, filter.matchesPoint(i,j))

    """Test Half-Doubles"""
    def test_HalfDoubles(self):
        filter = parseCondition("x>0.,y>0.")
        for i in Points:
            for j in Points:
                x = (i >= 0.0)
                y = (j >= 0.0)
                self.assertEqual(x and y, filter.matchesPoint(i,j))

    """Test on many arguments"""
    def test_ManyArgs(self):
        filter = parseCondition("y=0,x>0,y=3")
        for i in Points:
            for j in Points:
                x = (i >= 0.0)
                y1 = (j == 0.0)
                y2 = (j == 3.0)
                self.assertEqual(x and y1 and y2, filter.matchesPoint(i,j))

    """Test Out of Order input"""
    def test_OutOfOrder(self):
        error = False
        try:
            filter = parseCondition("0=x")
        except ValueError:
            error = True
        self.assertEqual(True,error)

    """Test No Commas between arguments"""
    def test_NoComma(self):
        error = False
        try:
            filter = parseCondition("x=0 y<2")
        except ValueError:
            error = True
        self.assertEqual(True,error)

    """Multiple arguments"""
    def test_MultipleArgs(self):
        filter = parseCondition("x>0,x<2,y<2")
        for i in Points:
            for j in Points:
                x1 = (i >= 0.0)
                x2 = (i <= 2.0)
                y = (j <= 2.0)
                self.assertEqual(x1 and x2 and y, filter.matchesPoint(i,j))

    if __name__ == '__main__':
        unittest.main()


import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
import warnings
import random

class RF_Iter_Missing(object):

	def __init__(self):
		pass

	def Preprocessing_Iterative(self, data):
		# # prepare for the data
		# data = pd.read_csv('missing0.3.csv', header = 0, index_col = 0)
		missing = [x for x in data.columns if data[x].isnull().any()]
		# the index of tuple with some missing features
		missing_lines = {x:np.where(data[x].isnull())[0] for x in missing}
		xx, yy = np.where(data.isnull())
		l = len(xx)
		# ground_truth = data.iloc(xx[1])
		ground_data = pd.read_csv('iris.csv', header = 0)
		ground_truth = {}
		for x in missing:
			temp = []
			lines = np.where(data[x].isnull())[0]
			for l in lines:
				temp.append(ground_data.at[l, x])
			ground_truth[x] = temp
		# get the missing part of the y part(ground truth)
		return data, missing, missing_lines, ground_truth

	def RF(self, data, missing_feature, real_missing_lines, missing_lines, type, num):
		# data of the tuple without missing feature
		# data = pd.get_dummies(data)
		left_features = list(set(data.columns) - set(missing_feature))
		# divide the not missing part into x and y
		training_part = data.drop(missing_lines, axis = 0)
		train_x = training_part[left_features]; train_y = training_part[missing_feature]
		# get the missing part of the x part
		predict_x = data.loc[missing_lines, left_features]
		if type == 'float':
			# model = RandomForestRegressor(oob_score=True, random_state = 1)
			model = RandomForestRegressor(random_state = 1)
			model.fit(train_x, train_y.values.ravel())
			predict_y = model.predict(predict_x)
			# fill the missing parts
			data.loc[missing_lines, missing_feature] = predict_y.round(2)
		else:
			model = RandomForestClassifier(random_state = 1)
			model.fit(train_x, train_y.values.ravel())
			predict_y = model.predict(predict_x)
			data.loc[missing_lines, missing_feature] = predict_y
			if len(predict_y) > 0:
				predict_y = list(num.inverse_transform(predict_y))
		m = dict(zip(missing_lines, predict_y))
		fake = list(set(missing_lines) - set(real_missing_lines))
		fake_prediction = [m[x] for x in fake]
		real_prediction = [m[x] for x in real_missing_lines]
		return data, real_prediction, fake_prediction
		# get the precision, recall and f1-measure of the result

	def Create_Fake_Missing(self, data, missing_count, missing_all):
		xx, yy = np.where(data.notnull())
		length = len(xx) * 0.1
		fake_missing_count = {k:int(length * v /missing_all) for k, v in missing_count.items()}
		cat2num = dict(zip(data.columns, range(len(data.columns))))
		fake_missing = {}
		fake_ground_truth = {}
		for c, n in fake_missing_count.items():
			pos = np.where(yy == cat2num[c])[0]
			x = pos[random.sample([i for i in range(len(pos))], n)]
			fake_ground_truth[c] = data.loc[xx[x], c]
			data.loc[xx[x], c] = None
			fake_missing[c] = x
		return data, fake_missing, fake_ground_truth, fake_missing_count

	def cal_precison(self, ground_truth, prediction, limitation, dt):
			x, y = len(ground_truth), len(prediction)
			if x != y or x == 0:
				print('Length of the two array does not match')
				return -1
			correct = 0
			if dt == 'float':
				for i in range(x):
					if ground_truth[i] - prediction[i] < limitation:
						correct += 1
			else:
				for i in range(x):
					if ground_truth[i] == prediction[i]:
						correct += 1
			return float(correct/x)

	def RF_Missing_Iterative(self, D):
		data, missing, real_missing_lines, real_ground_truth = self.Preprocessing_Iterative(D)
		# ransform the catogory label into numeric label
		num = preprocessing.LabelEncoder()
		real_missing_count = {x:data[x].isnull().value_counts()[1] for x in missing}
		real_missing_count = dict(sorted(real_missing_count.items(), key = lambda x:x[1], reverse = False))
		real_missing_all = sum([v for k, v in real_missing_count.items()])
		# add some missing data artifically
		data, fake_missing, fake_ground_truth, fake_missing_count = self.Create_Fake_Missing(data, real_missing_count, real_missing_all)
		fake_missing_all = sum([v for k, v in fake_missing_count.items()])
		not_missing_lines = {x:np.where(data[x].notnull())[0] for x in missing}
		column_type = {x:data[x].dtype for x in data.columns}
		missing_lines = {x:np.where(data[x].isnull())[0] for x in missing}
		missing_count = {x:data[x].isnull().value_counts()[1] for x in missing}
		missing_count = dict(sorted(missing_count.items(), key = lambda x:x[1], reverse = False))
		missing_all = sum([v for k, v in missing_count.items()])
		num2Cat = {}
		# fill all the missing data with mean
		for x in missing:
			if data[x].dtype == 'float':
				x_mean = data.loc[not_missing_lines[x], x].mean()
				data[x].fillna(x_mean, inplace=True)
			else:
				x_mode = data.loc[not_missing_lines[x], x].mode()
				data[x].fillna(x_mode[0], inplace=True)
		for x in data.columns:
			if data[x].dtype != 'float':
				data[x] = num.fit_transform(data[x])
				num2Cat[x] = {i:num.classes_[i] for i  in range(len(num.classes_))}
		count = 0
		real_precision = {x:0 for x in missing}
		fake_precision = {x:0 for x in missing}
		limitation = 0.5
		while True:
			for x in missing_count:
				data, real_prediction, fake_prediction = self.RF(data, x, real_missing_lines[x], missing_lines[x], column_type[x], num)
				# data, prediction = XGB(data, x, missing_lines[x], column_type[x], num)
				real_precision[x] = self.cal_precison(real_ground_truth[x], real_prediction, limitation, column_type[x])
				fake_precision[x] = self.cal_precison(list(fake_ground_truth[x]), fake_prediction, limitation, column_type[x])
			real_p = sum([real_precision[x] * real_missing_count[x] for x in missing])/real_missing_all
			fake_p = sum([fake_precision[x] * fake_missing_count[x] for x in missing])/fake_missing_all
			print('missing_for_real:%f' %real_p)
			print('missing_for_fake:%f' %fake_p)
			print('**********')
			count += 1
			if fake_p > 0.9 or count > 10:
				break
		print(real_precision)
		# recover all numeric attributes to category attributes
		for x in data.columns:
			if data[x].dtype != 'float':
				data[x] = data[x].map(lambda t: num2Cat[x][t])
		return data

if __name__ == '__main__':
	warnings.filterwarnings(action='ignore', category=DeprecationWarning)
	data = pd.read_csv('missing0.3.csv', header = 0, index_col = 0)
	test = RF_Iter_Missing()
	new_data = test.RF_Missing_Iterative(data)
	new_data.to_csv('recover_data.csv')


import numpy as np
from sklearn import svm
import datetime
from random import *

print("Loading y")
y = np.loadtxt('Labels.csv')

print("Loading A")
A = np.loadtxt('DataMatrix.csv',delimiter=",",usecols=range(45849))

rows = y.shape

validation_indeces = sample(range(rows[0]), int(rows[0]/3))
training_indeces = [index for index in range(rows[0]) if index not in validation_indeces]

training_matrix = np.array([A[i,:] for i in training_indeces])
training_y = np.array([y[i] for i in training_indeces])

validation_matrix = np.array([A[i,:] for i in validation_indeces])
validation_y = np.array([y[i] for i in validation_indeces])

def KernalizedSVM(ker,reg,cache,deg):
    #classifier = svm.SVC(kernel=ker,C=reg, cache_size=cache, degree = deg) #For standard kernelized SVM
    classifier = svm.NuSVC(kernel=ker,cache_size=cache, degree = deg) #For NuSVM
    print("starting SVM at " + str(datetime.datetime.now()))
    classifier.fit(training_matrix,training_y)
    hams = 0
    spams = 0
    misclass = 0
    false_pos = 0
    false_neg = 0
    for index,value in enumerate(validation_y):
        output = classifier.predict(validation_matrix[index,:].reshape(1,-1))
        if value == 1:
            hams +=1
        else:
            spams +=1
        if value != output[0]:
            misclass +=1
            if value == 1 and output[0] == -1:
                false_pos +=1
            else:
                false_neg += 1

    print("Of the data used to validate this neural net, there were ",hams," hams and ",spams," spams.")
    print("There were ",misclass," misclassifications.")
    print("This equates to a misclassification rate of ",round(float(misclass)/(hams+spams) * 100,3), " percent.")
    print("There were ",false_pos," false positives (ham incorrectly labeled spam) and ",false_neg," false negatives (spam labeled as ham).")
    print("This means ", round(false_neg/spams * 100,3), " percent of spams made it through the filter.")
    print("Of all the misclassifications, ",round(false_pos/misclass * 100,3)," percent were false positives.")
    print("Of all the misclassifications, ",round(false_neg/misclass * 100,3)," percent were false negatives.")

    with open(str(ker)+"-"+str(reg) + "-" +str(deg) +".txt",'w') as f:
        f.write("Of the data used to validate this neural net, there were " + str(hams) + " hams and " + str(spams) + " spams." + "\n")
        f.write("There were "+ str(misclass) + " misclassifications."+ "\n")
        f.write("This equates to a misclassification rate of " + str(round(float(misclass)/(hams+spams) * 100,3)) + " percent."+ "\n")
        f.write("There were " + str(false_pos) + " false positives (ham incorrectly labeled spam) and " + str(false_neg) + " false negatives (spam labeled as ham)."+ "\n")
        f.write("This means " + str(round(false_neg/spams * 100,3)) + " percent of spams made it through the filter."+ "\n")
        f.write("Of all the misclassifications, " + str(round(false_pos/misclass * 100,3)) + " percent were false positives."+ "\n")
        f.write("Of all the misclassifications, " + str(round(false_neg/misclass * 100,3)) + " percent were false negatives."+ "\n")
        f.close

KernalizedSVM("rbf",0.000001,1000,0)
KernalizedSVM("rbf",0.0001,1000,0)
KernalizedSVM("rbf",0.001,1000,0)
KernalizedSVM("rbf",0.1,1000,0)