import csv
import datetime
import json
import logging
import operator
import sys
import time
from collections import defaultdict, namedtuple
from time import sleep
from urllib.parse import parse_qs, urlparse

import facebook
import psycopg2
import psycopg2.extras
from OpenSSL import SSL

from db_functions import DBInterface
from slack_notifier import notify_slack
import config_utils

DEFAULT_MINIMUM_EXPECTED_NEW_ADS = 10000
DEFAULT_MINIMUM_EXPECTED_NEW_IMPRESSIONS = 10000

#data structures to hold new ads
AdRecord = namedtuple(
    "AdRecord",
    [
        "ad_creation_time",
        "ad_creative_body",
        "ad_creative_link_caption",
        "ad_creative_link_description",
        "ad_creative_link_title",
        "ad_delivery_start_time",
        "ad_delivery_stop_time",
        "ad_snapshot_url",
        "ad_status",
        "archive_id",
        "country_code",
        "currency",
        "first_crawl_time",
        "funding_entity",
        "impressions__lower_bound",
        "impressions__upper_bound",
        "page_id",
        "page_name",
        "publisher_platform",
        "spend__lower_bound",
        "spend__upper_bound",
    ],
)
PageRecord = namedtuple("PageRecord", ["id", "name"])
SnapshotRegionRecord = namedtuple(
    "SnapshotRegionRecord",
    [
        "archive_id",
        "region",
        "spend_percentage",
        "min_impressions",
        "max_impressions",
        "min_spend",
        "max_spend",
    ],
)
SnapshotDemoRecord = namedtuple(
    "SnapshotDemoRecord",
    [
        "archive_id",
        "age_range",
        "gender",
        "spend_percentage",
        "min_impressions",
        "max_impressions",
        "min_spend",
        "max_spend",
    ],
)
SearchRunnerParams = namedtuple(
        'SearchRunnerParams',
        ['country_code',
         'facebook_access_token',
         'sleep_time',
         'request_limit',
         'max_requests',
         'soft_max_runtime_in_seconds'
         ])

FIELDS_TO_REQUEST = [
    "ad_creation_time",
    "ad_creative_body",
    "ad_creative_link_caption",
    "ad_creative_link_description",
    "ad_creative_link_title",
    "ad_delivery_start_time",
    "ad_delivery_stop_time",
    "ad_snapshot_url",
    "currency",
    "demographic_distribution",
    "funding_entity",
    "impressions",
    "page_id",
    "page_name",
    "publisher_platform",
    "region_distribution",
    "spend",
]

class SearchRunner():

    def __init__(self, crawl_date, connection, db, search_runner_params):
        self.crawl_date = crawl_date
        self.country_code = search_runner_params.country_code
        self.connection = connection
        self.db = db
        self.fb_access_token = search_runner_params.facebook_access_token
        self.sleep_time = search_runner_params.sleep_time
        self.request_limit = search_runner_params.request_limit
        self.max_requests = search_runner_params.max_requests
        self.new_ads = set()
        self.new_funding_entities = set()
        self.new_pages = set()
        self.new_regions = set()
        self.new_impressions = set()
        self.new_ad_region_impressions = list()
        self.new_ad_demo_impressions = list()
        self.existing_pages = set()
        self.existing_funding_entities = set()
        self.existing_ads_to_end_time_map = dict()
        self.total_ads_added_to_db = 0
        self.total_impressions_added_to_db = 0
        self.graph_error_counts = defaultdict(int)
        self.stop_time = None
        if search_runner_params.soft_max_runtime_in_seconds:
            start_time = time.monotonic()
            soft_deadline = search_runner_params.soft_max_runtime_in_seconds
            self.stop_time = start_time + soft_deadline
            logging.info(
                'Will cease execution after %d seconds.', soft_deadline)

    def num_ads_added_to_db(self):
        return self.total_ads_added_to_db

    def num_impressions_added_to_db(self):
        return self.total_impressions_added_to_db

    def get_ad_from_result(self, result):
        url_parts = urlparse(result['ad_snapshot_url'])
        archive_id = int(parse_qs(url_parts.query)['id'][0])
        ad_status = 1
        if  'ad_delivery_stop_time' in result:
            ad_status = 0
        curr_ad = AdRecord(
            ad_creation_time=result.get('ad_creation_time', None),
            ad_creative_body=result.get('ad_creative_body', None),
            ad_creative_link_caption=result.get('ad_creative_link_caption', None),
            ad_creative_link_description=result.get('ad_creative_link_description', None),
            ad_creative_link_title=result.get('ad_creative_link_title', None),
            ad_delivery_start_time=result.get('ad_delivery_start_time', None),
            ad_delivery_stop_time=result.get('ad_delivery_stop_time', self.crawl_date),
            ad_snapshot_url=result.get('ad_snapshot_url', None),
            ad_status=ad_status,
            archive_id=archive_id,
            country_code=self.country_code,
            currency=result.get('currency', None),
            first_crawl_time=self.crawl_date,
            funding_entity=result.get('funding_entity', None),
            impressions__lower_bound=result.get('impressions', dict()).get('lower_bound', '0'),
            impressions__upper_bound=result.get('impressions', dict()).get('upper_bound', '0'),
            page_id=result.get('page_id', None),
            page_name=result.get('page_name', None),
            publisher_platform=result.get('publisher_platform', 'NotProvided'),
            spend__lower_bound=result.get('spend', dict()).get('lower_bound', '0'),
            spend__upper_bound=result.get('spend', dict()).get('upper_bound', '0'),
        )
        return curr_ad

    def process_funding_entity(self, ad):
        if ad.funding_entity not in self.existing_funding_entities:
            # We use tuples because it makes db updates simpler
            self.new_funding_entities.add((ad.funding_entity,))

    def process_page(self, ad):
            if int(ad.page_id) not in self.existing_pages:
                self.new_pages.add(PageRecord(ad.page_id, ad.page_name))
                self.existing_pages.add(int(ad.page_id))

    def process_ad(self, ad):
        if ad.archive_id not in self.existing_ads_to_end_time_map:
            self.new_ads.add(ad)
            self.existing_ads_to_end_time_map[ad.archive_id] = ad.ad_delivery_stop_time

    def process_impressions(self, ad):
        self.new_impressions.add(ad)

    def process_demo_impressions(self, demographic_distribution, curr_ad):
        if not demographic_distribution:
            logging.info("no demo impression information for: %s", curr_ad.archive_id)

        for demo_result in demographic_distribution:
            try:
                self.new_ad_demo_impressions.append(SnapshotDemoRecord(
                    curr_ad.archive_id,
                    demo_result['age'],
                    demo_result['gender'],
                    demo_result['percentage'],
                    float(demo_result['percentage']) * int(curr_ad.impressions__lower_bound),
                    float(demo_result['percentage']) * int(curr_ad.impressions__upper_bound),
                    float(demo_result['percentage']) * int(curr_ad.spend__lower_bound),
                    float(demo_result['percentage']) * int(curr_ad.spend__upper_bound)))
            except KeyError as key_error:
                logging.warning(
                        '%s error while processing ad archive ID %s demographic_distribution: %s',
                        key_error, curr_ad.archive_id, demo_result)

    def process_region_impressions(self, region_distribution, curr_ad):
        if not region_distribution:
            logging.info("no region impression information for: %s", curr_ad.archive_id)

        regions = set()
        for region_result in region_distribution:
            # If we get the same region more than once for an ad, the second occurance
            # This is a data losing proposition but can't be helped till FB fixes the results
            # They provide on the API
            try:
                if region_result['region'] in regions:
                    continue
                else:
                    regions.add(region_result['region'])
                    self.new_ad_region_impressions.append(SnapshotRegionRecord(
                    curr_ad.archive_id,
                    region_result['region'],
                    region_result['percentage'],
                    float(region_result['percentage']) * int(curr_ad.impressions__lower_bound),
                    float(region_result['percentage']) * int(curr_ad.impressions__upper_bound),
                    float(region_result['percentage']) * int(curr_ad.spend__lower_bound),
                    float(region_result['percentage']) * int(curr_ad.spend__upper_bound)))
            except KeyError as key_error:
                logging.warning(
                        '%s error while processing ad archive ID %s region_distribution: %s',
                        key_error, curr_ad.archive_id, region_result)

    def run_search(self, page_id=None, page_name=None):
        self.crawl_date = datetime.date.today()

        #cache of ads/pages/regions/demo_groups we've already seen so we don't reinsert them
        self.existing_ads_to_end_time_map = self.db.existing_ads()
        self.existing_pages = self.db.existing_pages()
        self.existing_funding_entities = self.db.existing_funding_entities()

        #get ads
        graph = facebook.GraphAPI(access_token=self.fb_access_token)
        has_next = True
        next_cursor = ""
        backoff_multiplier = 1
        logging.info(datetime.datetime.now())
        logging.info("page_id = %s", page_id)
        logging.info("page_name = %s", page_name)
        request_count = 0
        # TODO: Remove the request_count limit
        #LAE - this is more of a conceptual thing, but perhaps we should be writing to DB more frequently? In cases where we query by the empty string, we are high stakes succeeding or failing.
        curr_ad = None
        while (has_next and request_count < self.max_requests and
               self.allowed_execution_time_remaining()):
            #structures to hold all the new stuff we find
            self.new_ads = set()
            self.new_ad_sponsors = set()
            self.new_funding_entities = set()
            self.new_pages = set()
            self.new_regions = set()
            self.new_impressions = set()
            self.new_ad_region_impressions = list()
            self.new_ad_demo_impressions = list()
            request_count += 1
            total_ad_count = 0
            try:
                results = None
                if page_name is not None:
                    logging.info(f"making search term request for {page_name}")
                    logging.info(f"making request {request_count}")
                    results = graph.get_object(
                        id='ads_archive',
                        ad_reached_countries=self.country_code,
                        ad_type='POLITICAL_AND_ISSUE_ADS',
                        ad_active_status='ALL',
                        limit=self.request_limit,
                        search_terms=page_name,
                        fields=",".join(FIELDS_TO_REQUEST),
                        after=next_cursor)
                else:
                    logging.info(f"making page_id request for {page_id}")
                    logging.info(f"making request {request_count}")
                    results = graph.get_object(
                        id='ads_archive',
                        ad_reached_countries=self.country_code,
                        ad_type='POLITICAL_AND_ISSUE_ADS',
                        ad_active_status='ALL',
                        limit=self.request_limit,
                        search_page_ids=page_id,
                        fields=",".join(FIELDS_TO_REQUEST),
                        after=next_cursor)
                backoff_multiplier = 1
            except facebook.GraphAPIError as e:
                logging.error("Graph Error")
                logging.error(e.code)
                logging.error(e)
                self.graph_error_counts[e.code] += 1
                logging.error('Error code %d has occured %d times so far', e.code,
                              self.graph_error_counts[e.code])
                if results:
                    logging.error(results)
                else:
                    logging.error("No results")

                # Error 4 is application level throttling
                # Error 613 is "Custom-level throttling" "Calls to this api have exceeded the rate limit."
                # https://developers.facebook.com/docs/graph-api/using-graph-api/error-handling/
                if e.code == 4 or e.code == 613:
                    backoff_multiplier *= 4
                    logging.info('Rate liimit exceeded, back off multiplier is now %d.',
                                 backoff_multiplier)
                else:
                    backoff_multiplier += 1

                logging.info("resetting graph")
                graph = facebook.GraphAPI(access_token=self.fb_access_token)
                continue

            except OSError as e:
                logging.error("OS error: {0}".format(e))
                logging.error(datetime.datetime.now())
                # Reset backoff multiplier since this is a local OS issue and not an API issue.
                backoff_multiplier = 1
                logging.info("resetting graph")
                graph = facebook.GraphAPI(access_token=self.fb_access_token)
                continue

            except SSL.SysCallError as e:
                logging.error(e)
                backoff_multiplier += backoff_multiplier
                logging.error("resetting graph")
                graph = facebook.GraphAPI(access_token=self.fb_access_token)
                continue

            finally:
                sleep_time = self.sleep_time * backoff_multiplier
                logging.info(f"waiting for {sleep_time} seconds before next query.")
                sleep(sleep_time)

            for result in results['data']:
                total_ad_count += 1
                curr_ad = self.get_ad_from_result(result)
                self.process_ad(curr_ad)
                self.process_funding_entity(curr_ad)
                self.process_page(curr_ad)
                self.process_impressions(curr_ad)

                # Update impressions
                self.process_impressions(curr_ad)
                self.process_demo_impressions(result.get('demographic_distribution', []), curr_ad)
                self.process_region_impressions(result.get('region_distribution', []), curr_ad)

            #we finished parsing all ads in the result
            self.write_results()
            self.refresh_state()

            if "paging" in results and "next" in results["paging"]:
                next_cursor = results["paging"]["cursors"]["after"]
            else:
                has_next = False

    def allowed_execution_time_remaining(self):
        # No deadline configured.
        if self.stop_time is None:
            return True

        if time.monotonic() >= self.stop_time:
            logging.info('Allowed execution time has elapsed. quiting.')
            return False

        return True

    def write_results(self):
        #write new pages, regions, and demo groups to self.db first so we can update our caches before writing ads
        self.db.insert_funding_entities(self.new_funding_entities)
        self.db.insert_pages(self.new_pages)

        #write new ads to our database
        num_new_ads = len(self.new_ads)
        logging.info("writing %d new ads to db", num_new_ads)
        self.db.insert_new_ads(self.new_ads)
        self.total_ads_added_to_db += num_new_ads

        #write new impressions to our database
        num_new_impressions = len(self.new_impressions)
        logging.info("writing %d impressions to db", num_new_impressions)
        self.db.insert_new_impressions(self.new_impressions)
        self.total_impressions_added_to_db += num_new_impressions

        logging.info("writing self.new_ad_demo_impressions to db")
        self.db.insert_new_impression_demos(self.new_ad_demo_impressions)

        logging.info("writing self.new_ad_region_impressions to db")
        self.db.insert_new_impression_regions(self.new_ad_region_impressions)
        self.connection.commit()

    def refresh_state(self):
        # We have to reload these since we rely on the row ids from the database for indexing
        self.existing_funding_entities = self.db.existing_funding_entities()
        self.connection.commit()

    def get_formatted_graph_error_counts(self, delimiter='\n'):
        """Get GraphAPI error counts (sorted by count descending) string with specified delimiter.

        Args:
            delimiter: str, used to separate 'error: count' tokens.
        Returns:
            str 'error: count' joined by specified delimiter.
        """
        if not self.graph_error_counts:
            return ''

        count_msgs = [
            '%s: %d' % (error, count) for error, count in sorted(self.graph_error_counts.items(),
                                                                 key=operator.itemgetter(1),
                                                                 reverse=True)]
        return 'GraphAPI error counts %s' % delimiter.join(count_msgs)

#get page data
def get_page_data(connection, config):
    page_ids = {}
    input_TYPE = config['INPUT']['TYPE']
    if input_TYPE == 'file':
        input_FILES = config['INPUT']['FILES']
        logging.info(input_FILES)
        file_list = json.loads(input_FILES)
        for file_name in file_list:
            with open(file_name) as input:
                for row in input:
                    page_ids[row.strip()] = 0
    else:
        cursor = connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
        page_ads_query = "select page_id, count(*) as ad_count from ads group by page_id"
        cursor.execute(page_ads_query)
        for row in cursor:
            page_ids[row['page_id']] = row['ad_count']

    return page_ids

def get_pages_from_archive(archive_path):
    page_ads = {}
    if not archive_path:
        return page_ads
    with open(archive_path) as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            if row["\ufeffPage ID"] in page_ads:
                page_ads[row["\ufeffPage ID"]] += row["Number of Ads in Library"]
            else:
                page_ads[row["\ufeffPage ID"]] = row["Number of Ads in Library"]

    return page_ads

def send_completion_slack_notification(
        slack_url, country_code, completion_status, start_time, end_time,
        num_ads_added, num_impressions_added, min_expected_new_ads,
        min_expected_new_impressions, graph_error_count_string):
    duration_minutes = (end_time - start_time).seconds / 60
    slack_msg_error_prefix = ''
    if (num_ads_added < min_expected_new_ads or
            num_impressions_added < min_expected_new_impressions):
        error_log_msg = (
            f"Minimum expected records not met! Ads expected: "
            f"{min_expected_new_ads} added: {num_ads_added}, "
            f"impressions expected: {min_expected_new_impressions} added: "
            f"{num_impressions_added} ")
        logging.error(error_log_msg)
        slack_msg_error_prefix = (
            ":rotating_light: :rotating_light: :rotating_light: "
            f" {error_log_msg} "
            ":rotating_light: :rotating_light: :rotating_light: ")

    completion_message = (
        f"{slack_msg_error_prefix}Collection started at{start_time} for "
        f"{country_code} completed in {duration_minutes} minutes. Added "
        f"{num_ads_added} ads, and {num_impressions_added} impressions. "
        f"Completion status {completion_status}. {graph_error_count_string}")
    notify_slack(slack_url, completion_message)

def main(config):
    logging.info("starting")

    slack_url = config.get('LOGGING', 'SLACK_URL', fallback='')

    if 'MINIMUM_EXPECTED_NEW_ADS' in config['SEARCH']:
        min_expected_new_ads = int(config['SEARCH']['MINIMUM_EXPECTED_NEW_ADS'])
    else:
        min_expected_new_ads = DEFAULT_MINIMUM_EXPECTED_NEW_ADS
    logging.info('Expecting minimum %d new ads.', min_expected_new_ads)

    if 'MINIMUM_EXPECTED_NEW_IMPRESSIONS' in config['SEARCH']:
        min_expected_new_impressions = int(config['SEARCH']['MINIMUM_EXPECTED_NEW_IMPRESSIONS'])
    else:
        min_expected_new_impressions = DEFAULT_MINIMUM_EXPECTED_NEW_IMPRESSIONS
    logging.info('Expecting minimum %d new impressions.', min_expected_new_impressions)

    search_runner_params = SearchRunnerParams(
        country_code=config['SEARCH']['COUNTRY_CODE'],
        facebook_access_token=config_utils.get_facebook_access_token(config),
        sleep_time=config.getint('SEARCH', 'SLEEP_TIME'),
        request_limit=config.getint('SEARCH', 'LIMIT'),
        max_requests=config.getint('SEARCH', 'MAX_REQUESTS'),
        soft_max_runtime_in_seconds=config.getint('SEARCH', 'SOFT_MAX_RUNIME_IN_SECONDS',
                                                  fallback=None))

    connection = config_utils.get_database_connection_from_config(config)
    logging.info('Established conneciton to %s', connection.dsn)
    db = DBInterface(connection)
    search_runner = SearchRunner(
        datetime.date.today(),
        connection,
        db,
        search_runner_params)
    page_ids = get_pages_from_archive(config['INPUT']['ARCHIVE_ADVERTISERS_FILE'])
    page_string = page_ids or 'all pages'
    start_time = datetime.datetime.now()
    country_code_uppercase = search_runner_params.country_code.upper()
    notify_slack(slack_url,
                 f"Starting UNIFIED collection at {start_time} for "
                 f"{country_code_uppercase} for {page_string}")
    completion_status = 'Failure'
    try:
        if page_ids:
            curr_page_ids = get_page_data(connection, config)
            for page_id, ad_count in page_ids.items():
                if page_id in curr_page_ids:
                    curr_ad_count = curr_page_ids[page_id]
                    if ad_count > curr_ad_count:
                        page_delta[page_id] = ad_count - curr_ad_count
                else:
                    page_delta[page_id] = ad_count

            prioritized_page_ids = [x for x in sorted(page_delta, key=d.get, reverse=True)]
            #LAE - alter this to work for up to 10 page ids at a time
            for page_id in prioritized_page_ids:
                search_runner.run_search(page_id=page_id)
        else:
            search_runner.run_search(page_name="''")
        completion_status = 'Success'
    except Exception as e:
        completion_status = f'Uncaught exception: {e}'
        logging.error(completion_status, exc_info=True)
    finally:
        connection.close()
        end_time = datetime.datetime.now()
        num_ads_added = search_runner.num_ads_added_to_db()
        num_impressions_added = search_runner.num_impressions_added_to_db()
        logging.info(search_runner.get_formatted_graph_error_counts())
        send_completion_slack_notification(
            slack_url, country_code_uppercase, completion_status, start_time,
            end_time, num_ads_added, num_impressions_added,
            min_expected_new_ads, min_expected_new_impressions,
            search_runner.get_formatted_graph_error_counts())

if __name__ == '__main__':
    config = config_utils.get_config(sys.argv[1])
    country_code = config['SEARCH']['COUNTRY_CODE'].lower()

    config_utils.configure_logger(f"{country_code}_fb_api_collection.log")
    if len(sys.argv) < 2:
        exit(f"Usage:python3 {sys.argv[0]} generic_fb_collector.cfg")
    main(config)


import sys
import copy
import Pyro4
import Pyro4.errors
import Pyro4.configuration

# ensure requests are dealt with sequentially (message ordering)
Pyro4.config.SERVERTYPE = 'multiplex'

NAME = ''

@Pyro4.behavior(instance_mode='single')
class Backend():
    def __init__(self):
        # init server data
        self._name = NAME
        self._backups = []
        self._clientID = 1
        self._responses = {}
        self._orders = {}
        self._db = [
            ['General Store', [['Potato', 0.24, 300],
                               ['Carrot', 0.05, 5000],
                               ['Pizza', 4, 50],
                               ['Cake', 7, 100],
                               ['Loaf of Bread', 0.5, 10000]]],
            ['Secret Shop', [['Mango', 0.85, 1000],
                             ['Enchanted Mango', 500, 2],
                             ['Magic Beans', 200, 3],
                             ['Dragon Fruit', 2, 300]]]
        ]

    def find_backups(self):
        try:
            with Pyro4.locateNS() as ns:
                # search for backup servers
                for backup, backup_uri in ns.list(prefix='justHungry.backend.').items():
                    if backup != f'justHungry.backend.{self._name}':
                        backup_proxy = Pyro4.Proxy(backup_uri)
                        exists = False
                        for entry in self._backups:
                            if backup == entry[0]:
                                exists = True
                        if exists:
                            continue
                        try:
                            # pass current state of primary to backup
                            res = backup_proxy.initBackup(self._db, self._responses, self._orders, self._clientID)
                            if not res:
                                try:
                                    # if backup dead de register it from nameserver
                                    with Pyro4.locateNS() as ns:
                                        ns.remove(name=backup)
                                except Pyro4.errors.NamingError:
                                    pass
                            else:
                                # add to the listof good backups
                                self._backups.append([backup, backup_proxy])
                        except Pyro4.errors.PyroError:
                            try:
                                # remove dead backup from nameserver
                                with Pyro4.locateNS() as ns:
                                    ns.remove(name=backup)
                            except Pyro4.errors.NamingError:
                                pass
        except Pyro4.errors.NamingError:
            pass

    @Pyro4.expose
    def initBackup(self, db, resp, new_orders, new_client_id):
        self._responses = copy.deepcopy(resp)
        self._db = db[::]
        self._orders = copy.deepcopy(new_orders)
        self._clientID = new_client_id
        return True

    @Pyro4.expose
    def propogate(self, u_id, resp, new_db, new_orders, new_client_id):
        self._responses[u_id] = resp
        self._db = new_db[::]
        self._orders = copy.deepcopy(new_orders)
        self._clientID = new_client_id
        return True

    @Pyro4.expose
    def getItem(self, store, order_item, u_id):
        # look for any new backup servers
        self.find_backups()
        if u_id in self._responses:
            item = self._responses[u_id]
        else:
            try:
                item = self._db[store][1][order_item]
            except:
                item = []
            self._responses[u_id] = item
        return item

    @Pyro4.expose
    def getOrders(self, client_id, u_id):
        self.find_backups()
        if u_id in self._responses:
            orders = self._responses[u_id]
        else:
            try:
                orders = self._orders[client_id]
            except:
                orders = []
            self._responses[u_id] = orders
        return orders

    @Pyro4.expose
    def getItemName(self, store, order_item, u_id):
        self.find_backups()
        if u_id in self._responses:
            item_name = self._responses[u_id]
        else:
            try:
                item_name = self._db[store][1][order_item][0]
            except:
                item_name = ''
            self._responses[u_id] = item_name
        return item_name

    @Pyro4.expose
    def getItems(self, store, u_id):
        self.find_backups()
        if u_id in self._responses:
            items = self._responses[u_id]
        else:
            try:
                items = self._db[store][1]
            except:
                items = ['ERROR']
            self._responses[u_id] = items
        return items

    @Pyro4.expose
    def getStoreName(self, store, u_id):
        self.find_backups()
        if u_id in self._responses:
            store_name = self._responses[u_id]
        else:
            try:
                store_name = self._db[store][0]
            except:
                store_name = ''
            self._responses[u_id] = store_name
        return store_name

    @Pyro4.expose
    def getStores(self, u_id):
        self.find_backups()
        if u_id in self._responses:
            stores = self._responses[u_id]
        else:
            stores = []
            for store in self._db:
                stores.append(store[0])
            self._responses[u_id] = stores
        return stores

    @Pyro4.expose
    def placeOrder(self, store, order_item, quant, u_id):
        self.find_backups()
        if u_id in self._responses:
            valid = self._responses[u_id]
        else:
            valid = (0 < quant <= self._db[store][1][order_item][2])
            self._responses[u_id] = valid
        return valid

    @Pyro4.expose
    def finaliseOrder(self, store, order_item, quant, address, client_id, u_id):
        self.find_backups()
        resp = False
        if u_id in self._responses:
            resp = self._responses[u_id]
        else:
            resp = (0 < quant <= self._db[store][1][order_item][2])
            if not resp:
                return False
            # process order
            self._responses[u_id] = resp
            self._db[store][1][order_item][2] -= quant
            if client_id not in self._orders:
                self._orders[client_id] = []
            self._orders[client_id].append(
                [self._db[store][0], self._db[store][1][order_item][0], quant, quant*self._db[store][1][order_item][1], address])
            # go through all backup servers
            for backup in self._backups[::]:
                try:
                    # try to propogate order
                    res = backup[1].propogate(
                        u_id, client_id, self._db, self._orders, self._clientID)
                    if not res:
                        try:
                            # if failed remove that backup
                            with Pyro4.locateNS() as ns:
                                ns.remove(name=backup[0])
                                self._backups.remove(backup)
                        # on comm error undo the order and return error
                        except Pyro4.errors.NamingError:
                            self._db[store][1][order_item][2] += quant
                            self._responses[u_id] = False
                            self._orders[client_id].pop()
                            return False
                # remove dead backups
                except Pyro4.errors.PyroError:
                    try:
                        with Pyro4.locateNS() as ns:
                            ns.remove(name=backup[0])
                            self._backups.remove(backup)
                    # on comm error undo order and return error
                    except Pyro4.errors.NamingError:
                        self._db[store][1][order_item][2] += quant
                        self._responses[u_id] = False
                        self._orders[client_id].pop()
                        return False
        return resp

    @Pyro4.expose
    def clientId(self, u_id):
        self.find_backups()
        client_id = None
        if u_id in self._responses:
            client_id = self._responses[u_id]
        else:
            client_id = self._clientID
            self._clientID += 1
            self._responses[u_id] = client_id
            for backup in self._backups[::]:
                try:
                    res = backup[1].propogate(
                        u_id, client_id, self._db, self._orders, self._clientID)
                    if not res:
                        try:
                            with Pyro4.locateNS() as ns:
                                ns.remove(name=backup[0])
                                self._backups.remove(backup)
                        except Pyro4.errors.NamingError:
                            self._clientID -= 1
                            self._responses[u_id] = None
                            return None
                except Pyro4.errors.PyroError:
                    try:
                        with Pyro4.locateNS() as ns:
                            ns.remove(name=backup[0])
                            self._backups.remove(backup)
                    except Pyro4.errors.NamingError:
                        self._clientID -= 1
                        self._responses[u_id] = None
                        return None
        return client_id

    @Pyro4.expose
    def notifyPrimary(self):
        return True

# parse args
if len(sys.argv) < 2 or len(sys.argv) > 2:
    sys.exit('backend should be called with 1 argument, a unique id for the server')
NAME = sys.argv[1]

# attempt to register with nameserver
try:
    with Pyro4.Daemon() as daemon:
        backend_uri = daemon.register(Backend)
        try:
            with Pyro4.locateNS() as ns:
                ns.register('justHungry.backend.'+NAME, backend_uri)
        except Pyro4.errors.NamingError:
            sys.exit('Could not find nameserver. EXITING')
        daemon.requestLoop()
except Pyro4.errors.DaemonError:
    sys.exit('Pyro daemon has crashed. EXITING')


import pymongo
from bson.objectid import ObjectId
import os

class DLExMongoRecorder():

    def __init__(self, db, pid, expid):
        self.db = db
        self.pid = pid
        self.expid = expid

    def get_exp_info(self, setting_name):

        col = self.db.experiments
        query = {"_id":ObjectId(self.expid)}
        # new_vlues = {"$set": {setting_name: setting_value}}
        # col.update_one(query, new_vlues, upsert=True)
        x =col.find(query)
        return x.next()[setting_name]

    def get_hyper_params(self):
        print("started get hyper param")
        col = self.db.hyperparam
        query = {"expid": self.expid}
        x = col.find(query, {"_id": 0, "expid": 0})
        # print("xxx next=", x.next())
        return x.next()

    def get_train_settings(self):
        col = self.db.train_settings
        query = {"expid": self.expid}
        output = col.find_one(query, {"_id": 0, "expid": 0})
        return output

    # Recording training stat values.
    def record_train_stat(self, iteration, stat_name, stat_value ):
        col = self.db.trainstats
        query = {"expid":self.expid, "iteration": iteration }
        new_values = {"$set": {stat_name: stat_value}}
        col.update_one(query, new_values, upsert=True)

    # record information about experiment (to track progress of the experiment)
    def record_exp_info(self, info_name, info_value):
        col = self.db.experiments
        query = {"_id":ObjectId(self.expid) }
        new_value = {"$set": {info_name: info_value}}
        col.update_one(query, new_value, upsert=True)

    def set_exp_state(self, status):
        col = self.db.experiments
        query = {"_id":ObjectId(self.expid)}
        new_value = { "$set": { "status": status } }

        col.update_one(query, new_value, upsert=True)

    # function to record total epochs
    def add_total_epoch(self):
        current_epoch = self.get_exp_info("total_epochs")
        self.record_exp_info("total_epochs", current_epoch +1 )

    # function to read current total epoch number
    def read_total_epoch(self):
        current_epoch = self.get_exp_info("total_epochs")
        return current_epoch

    # function to record iterations
    def record_iters(self, iter_num):
        self.record_exp_info("iters", iter_num)

    # save dataloader size
    def record_dataloader_size(self, len_dataloader):
        self.record_exp_info("dataloader_size", len_dataloader)

    # Handling models

    def generate_checkpoint_path(self, checkpoint_iter, checkpoint_type):

        col = self.db.models
        # checkpoint_type: "BATCH", "EPOCH".. etc.
        query = {"pid": self.pid, "expid": self.expid, "iter": checkpoint_iter,  "type": checkpoint_type }
        x = col.insert_one(query)

        print("x=",str(x.inserted_id))

        col_exp = self.db.experiments
        query_exp = {"_id": ObjectId(self.expid), "pid": self.pid}
        model_dir_path = col_exp.find_one(query_exp)["models_path"]
        print("model dir path:", model_dir_path)

        # new model path
        model_path = os.path.join(model_dir_path, str(x.inserted_id) + ".tar")

        # update models table back
        new_value = {"$set": {"path": model_path}}

        x1 = col.update(query, new_value, upsert=True)

        print(model_path)

        return model_path

    def load_checkpoint_path(self, checkpoint_iter, checkpoint_type):

        col = self.db.models
        query = {"pid": self.pid, "expid": self.expid, "iter": checkpoint_iter,  "type": checkpoint_type }

        out_dict = col.find_one(query)
        print("chekpoint path =", out_dict["path"])
        return out_dict["path"]

    # handlign img paths

    def add_image(self,  datatype, **kw): # type: INPUTDATA, GENDATA, --> return a path to image

        # get new id for new data
        col = self.db.outputdata
        query ={"expid": self.expid, "type": datatype}
        query.update(kw)
        print("add image updated query:", query)
        x = col.insert_one(query) # x.inserted_id

        col_exp = self.db.experiments
        query_exp = {"_id": ObjectId(self.expid)}
        exp_path = col_exp.find(query_exp, {"_id":0,"output_path":1})

        # new image path
        imgpath = exp_path.next()["output_path"] + "/" + str(x.inserted_id) + ".png"

        # update outputdata collection
        new_value = {"$set": {"imgpath": imgpath}}
        x1 = col.update(query, new_value, upsert=True)

        print(x.inserted_id)
        #  print(exp_path.next())
        return imgpath


# -*- coding:utf-8 -*-
from pyquery import PyQuery
from collections import OrderedDict
from urlparse import urlparse
from json import dumps

# COSTANTS
URL = "https://news.ycombinator.com/newest?"
POSTS_EACH_PAGE = 30
# --*---
# LOADS
# adding a method to PyQuery class in order to create a list of elements without id
fn = lambda: this.map(lambda i, el: PyQuery(this).outerHtml())
PyQuery.fn.listOuterHtml = fn
# --*--

def sanitize_url(host, sub_url):
    # converts relative uri into absolute uri
    parsed_sub_url = urlparse(sub_url)
    parsed_host = urlparse(host)
    if not parsed_sub_url.scheme:
        parsed_sub_url = parsed_sub_url._replace(scheme=parsed_host.scheme)
    if not parsed_sub_url.netloc:
        parsed_sub_url = parsed_sub_url._replace(netloc=parsed_host.netloc)
    return parsed_sub_url.geturl()

def scrape_top(fragment_str):
    # parse the content of the '.athing' class
    # getting the fields id,title,uri,rank
    s = PyQuery(fragment_str)
    post_id = s.attr('id')
    title = s(".storylink").text()[:256]
    title = "None" if not title else title
    uri = sanitize_url(URL, s(".storylink").attr("href"))
    try:
        rank = int(s(".rank").text()[:-1])
    except:
        rank = 0
    return post_id, title, uri, rank

def scrape_subtitle(fragment_str):
    # parse the content of the '.subtext' class
    # getting the fields author, post, comments
    s = PyQuery(fragment_str)
    author = s(".hnuser").text()[:256]
    author = "None" if not author else author
    try:
        points = int(s(".score").text().split()[0])
    except:
        points = 0
    try:
        comments = int(PyQuery(s("a").listOuterHtml()[3]).text()[:-9])
    except:
        comments = 0
    return author, points, comments

def scrape_article(top_str, bot_str):
    # assemble and returns the fields gotten by calling the functions for the top and
    # the bottom part of the post label
    static_dict = OrderedDict()
    post_id, title, uri, rank = scrape_top(top_str)
    author, points, comments = scrape_subtitle(bot_str)
    static_dict["title"] = title
    static_dict["uri"] = uri
    static_dict["author"] = author
    static_dict["points"] = points
    static_dict["comments"] = comments
    static_dict["rank"] = rank
    return post_id, static_dict

def acquire_hakernews_list(n):
    # takes as input the number of posts to acquire and goes page to page getting 30 elements
    # each time , then return the list of static dict each containing a post with its fields
    post_list = []
    path = ""
    for z in xrange(n / POSTS_EACH_PAGE + 1):
        if (n - (z * POSTS_EACH_PAGE)) > 0:
            # gets the page and init PyQuery
            s = PyQuery(url=URL + path)
            # creates two different lists, the "top" contains the upper part of each post
            top = s('.athing').listOuterHtml()
            # creates two different lists, the "sub_title" contains the lower part of each post
            sub_title = s('.subtext').listOuterHtml()
            # Saves the id of first in order to be sure to have the previous n posts from the current first post
            # then create the path the get the posts in the following page during next acquisition
            id_post, message = scrape_article(top[0], sub_title[0])
            if z == 0:
                id_str = id_post
            path = "next=" + id_str + "&n=" + str((z + 1) * POSTS_EACH_PAGE + 1)
            # Appends the first message of the page to the post_list and as well
            # acquire and append the remaining posts to acuire inside the page
            post_list.append(message)
            for i in xrange(1, min(POSTS_EACH_PAGE, n - z * POSTS_EACH_PAGE)):
                _, message = scrape_article(top[i], sub_title[i])
                post_list.append(message)
    return post_list


from bs4 import BeautifulSoup
from selenium import webdriver
import time ##페이지가 열릴 때까지 기다려 주는 명령을 위해
import sys

query_txt = input('구글에 검색할 키워드는 무엇입니까?: ')
f_name = input('검색 결과를 저장할 파일경로와 이름을 지정하세요(예:c:\\data\\test.txt): ')

print(type(query_txt))
path = "C:\PythonSkill_Training\PythonSkill_Training\Crawling\chromedriver_win32\chromedriver.exe" # 셀레늄path
driver = webdriver.Chrome(path)
driver.get("https://www.google.co.kr")
time.sleep(2)
element = driver.find_element_by_name("q")
element.send_keys(query_txt)
time.sleep(2)
driver.find_element_by_xpath('//*[@id="tsf"]/div[2]/div[1]/div[2]/div[2]/div[2]/center/input[1]').click()

#######분석
time.sleep(1)
full_html = driver.page_source
soup = BeautifulSoup(full_html, 'html.parser')
content_list = soup.find('div',class_='srg')
#######
for i in content_list:
    print(i.text.strip())
    print("\n")

orig_stdout = sys.stdout
f = open(f_name , 'a' , encoding='UTF-8')
sys.stdout = f ##출력 방향을 윗줄로 바꿈
time.sleep(1)

html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')
content_list = soup.find('div',class_='srg')

for i in content_list:
    print(i.text.strip())
    print("\n")

sys.stdout = orig_stdout
f.close()

print(" 요청하신 데이터 수집 작업이 정상적으로 완료되었습니다")


"""Collection of types and type generators."""

import re
import operator
from enum import Enum
from typing import Dict, List, Tuple, Any, Union, Optional, Type, Pattern
from .util import Path, _issubclass

__all__ = [
    'registered_types',
    'restricted_number_type',
    'restricted_string_type',
    'path_type',
    'PositiveInt',
    'NonNegativeInt',
    'PositiveFloat',
    'NonNegativeFloat',
    'ClosedUnitInterval',
    'OpenUnitInterval',
    'NotEmptyStr',
    'Email',
    'Path_fr',
    'Path_fc',
    'Path_dw',
]

_operators1 = {
    operator.gt: '>',
    operator.ge: '>=',
    operator.lt: '<',
    operator.le: '<=',
    operator.eq: '==',
    operator.ne: '!=',
}
_operators2 = {v: k for k, v in _operators1.items()}
_schema_operator_map = {
    operator.gt: 'exclusiveMinimum',
    operator.ge: 'minimum',
    operator.lt: 'exclusiveMaximum',
    operator.le: 'maximum',
}

registered_types = {}  # type: Dict[Tuple, Any]

def restricted_number_type(
    name: Optional[str],
    base_type: Type,
    restrictions: Union[Tuple, List[Tuple]],
    join: str = 'and',
    docstring: Optional[str] = None,
) -> Type:
    """Creates or returns an already registered restricted number type class.

    Args:
        name: Name for the type or None for an automatic name.
        base_type: One of {int, float}.
        restrictions: Tuples of pairs (comparison, reference), e.g. ('>', 0).
        join: How to combine multiple comparisons, one of {'or', 'and'}.
        docstring: Docstring for the type class.

    Returns:
        The created or retrieved type class.
    """
    if base_type not in {int, float}:
        raise ValueError('Expected base_type to be one of {int, float}.')
    if join not in {'or', 'and'}:
        raise ValueError("Expected join to be one of {'or', 'and'}.")

    restrictions = [restrictions] if isinstance(restrictions, tuple) else restrictions
    if not isinstance(restrictions, list) or \
       not all(isinstance(x, tuple) and len(x) == 2 for x in restrictions) or \
       not all(x[0] in _operators2 and x[1] == base_type(x[1]) for x in restrictions):
        raise ValueError('Expected restrictions to be a list of tuples each with a comparison operator '
                         '(> >= < <= == !=) and a reference value of type '+base_type.__name__+'.')

    register_key = (tuple(sorted(restrictions)), base_type, join)
    if register_key in registered_types:
        registered_type = registered_types[register_key]
        if name is not None and registered_type.__name__ != name:
            raise ValueError('Same type already registered with a different name: '+registered_type.__name__+'.')
        return registered_type

    restrictions = [(_operators2[x[0]], x[1]) for x in restrictions]
    expression = (' '+join+' ').join(['v'+_operators1[op]+str(ref) for op, ref in restrictions])

    class RestrictedNumber:

        _restrictions = restrictions
        _expression = expression
        _type = base_type
        _join = join

        def __new__(cls, v):
            def within_restriction(cls, v):
                check = [comparison(v, ref) for comparison, ref in cls._restrictions]
                if (cls._join == 'and' and not all(check)) or \
                   (cls._join == 'or' and not any(check)):
                    return False
                return True

            v = cls._type(v)
            if not within_restriction(cls, v):
                raise ValueError('invalid value, '+str(v)+' does not conform to restriction '+cls._expression)
            return super().__new__(cls, v)

    if name is None:
        name = base_type.__name__
        for num, (comparison, ref) in enumerate(restrictions):
            name += '_'+join+'_' if num > 0 else '_'
            name += comparison.__name__ + str(ref).replace('.', '')

    restricted_type = type(name, (RestrictedNumber, base_type), {})
    if docstring is not None:
        restricted_type.__doc__ = docstring
    register_type(register_key, restricted_type)

    return restricted_type

def restricted_string_type(
    name: str,
    regex: Union[str, Pattern],
    docstring: Optional[str] = None,
) -> Type:
    """Creates or returns an already registered restricted string type class.

    Args:
        name: Name for the type or None for an automatic name.
        regex: Regular expression that the string must match.
        docstring: Docstring for the type class.

    Returns:
        The created or retrieved type class.
    """
    if isinstance(regex, str):
        regex = re.compile(regex)
    expression = 'matching '+regex.pattern

    register_key = (expression, str)
    if register_key in registered_types:
        registered_type = registered_types[register_key]
        if registered_type.__name__ != name:
            raise ValueError('Same type already registered with a different name: '+registered_type.__name__+'.')
        return registered_type

    class RestrictedString:

        _regex = regex
        _expression = expression
        _type = str

        def __new__(cls, v):
            v = str(v)
            if not cls._regex.match(v):
                raise ValueError('invalid value, "'+v+'" does not match regular expression '+cls._expression)
            return super().__new__(cls, v)

    restricted_type = type(name, (RestrictedString, str), {})
    if docstring is not None:
        restricted_type.__doc__ = docstring
    register_type(register_key, restricted_type)

    return restricted_type

def path_type(
    mode: str,
    docstring: Optional[str] = None,
) -> Type:
    """Creates or returns an already registered path type class.

    Args:
        mode: The required type and access permissions among [fdrwxcuFDRWX].
        docstring: Docstring for the type class.

    Returns:
        The created or retrieved type class.
    """
    name = 'Path_'+mode

    register_key = ('path '+''.join(sorted(mode)), str)
    if register_key in registered_types:
        return registered_types[register_key]

    class PathType(Path):

        _expression = name
        _mode = mode
        _type = str

        def __init__(self, v):
            super().__init__(v, mode=self._mode)

        def __repr__(self):
            return self.rel_path

    restricted_type = type(name, (PathType, str), {})
    if docstring is not None:
        restricted_type.__doc__ = docstring
    register_type(register_key, restricted_type)

    return restricted_type

def register_type(register_key, new_type):
    assert register_key not in registered_types
    if new_type.__name__ in globals():
        raise ValueError('Type name "'+new_type.__name__+'" clashes with name already defined in jsonargparse.typing.')
    globals()[new_type.__name__] = new_type
    registered_types[register_key] = new_type

PositiveInt        = restricted_number_type('PositiveInt',        int, ('>', 0),
                                            docstring='int restricted to be >0')
NonNegativeInt     = restricted_number_type('NonNegativeInt',     int, ('>=', 0),
                                            docstring='int restricted to be ≥0')
PositiveFloat      = restricted_number_type('PositiveFloat',      float, ('>', 0),
                                            docstring='float restricted to be >0')
NonNegativeFloat   = restricted_number_type('NonNegativeFloat',   float, ('>=', 0),
                                            docstring='float restricted to be ≥0')
ClosedUnitInterval = restricted_number_type('ClosedUnitInterval', float, [('>=', 0), ('<=', 1)],
                                            docstring='float restricted to be ≥0 and ≤1')
OpenUnitInterval   = restricted_number_type('OpenUnitInterval',   float, [('>', 0), ('<', 1)],
                                            docstring='float restricted to be >0 and <1')

NotEmptyStr = restricted_string_type('NotEmptyStr', r'^.*[^ ].*$',
                                     docstring=r'str restricted to not-empty pattern ^.*[^ ].*$')
Email       = restricted_string_type('Email', r'^[^@ ]+@[^@ ]+\.[^@ ]+$',
                                     docstring=r'str restricted to the email pattern ^[^@ ]+@[^@ ]+\.[^@ ]+$')

Path_fr = path_type('fr',
                    docstring='str pointing to a file that exists and is readable')
Path_fc = path_type('fc',
                    docstring='str pointing to a file that can be created if it does not exist')
Path_dw = path_type('dw',
                    docstring='str pointing to a directory that exists and is writeable')

def is_optional(annotation, ref_type):
    """Checks whether a type annotation is an optional for one type class."""
    return hasattr(annotation, '__origin__') and \
        annotation.__origin__ == Union and \
        len(annotation.__args__) == 2 and \
        any(type(None) == a for a in annotation.__args__) and \
        any(_issubclass(a, ref_type) for a in annotation.__args__)

def annotation_to_schema(annotation) -> Optional[Dict[str, str]]:
    """Generates a json schema from a type annotation if possible.

    Args:
        annotation: The type annotation to process.

    Returns:
        The json schema or None if an unsupported type.
    """
    schema = None
    if issubclass(annotation, (int, float)) and \
       hasattr(annotation, '_join') and annotation._join == 'and' and \
       hasattr(annotation, '_restrictions') and \
       all(x[0] in _schema_operator_map for x in annotation._restrictions):
        schema = {'type': 'integer' if issubclass(annotation, int) else 'number'}
        for comparison, ref in annotation._restrictions:
            schema[_schema_operator_map[comparison]] = ref
    elif issubclass(annotation, str) and hasattr(annotation, '_regex'):
        schema = {'type': 'string', 'pattern': annotation._regex.pattern}
    return schema

def type_in(obj, types_set):
    return obj in types_set or (hasattr(obj, '__origin__') and obj.__origin__ in types_set)

def type_to_str(obj):
    if _issubclass(obj, (bool, int, float, str, Enum)):
        if hasattr(obj, '_expression'):
            return obj._type.__name__ + ' ' + obj._expression
        else:
            return obj.__name__
    elif obj is not None:
        return re.sub(r'[a-z_.]+\.', '', str(obj)).replace('NoneType', 'null')


import logging
import voluptuous as vol

from homeassistant.helpers import (
    template as template_helper,
    config_validation as cv
)

from homeassistant.const import (
    CONF_VALUE_TEMPLATE,
    ATTR_TEMPERATURE,
)

from homeassistant.components import climate, mqtt
from homeassistant.components.climate import (
    PLATFORM_SCHEMA as CLIMATE_PLATFORM_SCHEMA,
    SUPPORT_TARGET_TEMPERATURE,
    SUPPORT_PRESET_MODE,
    HVAC_MODE_HEAT,
    HVAC_MODE_OFF,
)

from homeassistant.components.climate.const import (
    FAN_LOW,
    DEFAULT_MIN_TEMP,
    DEFAULT_MAX_TEMP,
    ATTR_HVAC_MODE,
)

from homeassistant.components.mqtt.climate import (
    MqttClimate,
    async_setup_entry as mqtt_async_setup_entry,
    TOPIC_KEYS,
    TEMPLATE_KEYS,
    CONF_MODE_LIST,
    CONF_HOLD_LIST,
    CONF_PAYLOAD_ON,
    CONF_PAYLOAD_OFF,
    CONF_SEND_IF_OFF,
    CONF_TEMP_INITIAL,
    CONF_TEMP_MIN,
    CONF_TEMP_MAX,
    CONF_TEMP_STEP,
    CONF_HOLD_COMMAND_TOPIC,
    CONF_HOLD_STATE_TOPIC,
    CONF_MODE_COMMAND_TOPIC,
    CONF_MODE_STATE_TOPIC,
    CONF_MODE_STATE_TEMPLATE,
    CONF_TEMP_COMMAND_TOPIC,
    CONF_TEMP_STATE_TOPIC,
    PLATFORM_SCHEMA as MQTT_PLATFORM_SCHEMA,
)

from homeassistant.components.mqtt.discovery import MQTT_DISCOVERY_NEW, clear_discovery_hash

from homeassistant.components.mqtt import (
    ATTR_DISCOVERY_HASH,
    CONF_QOS,
    CONF_RETAIN,
    CONF_UNIQUE_ID,
    MQTT_BASE_PLATFORM_SCHEMA,
    MqttAttributes,
    MqttAvailability,
    MqttDiscoveryUpdate,
    MqttEntityDeviceInfo,
    subscription
)

from homeassistant.helpers.dispatcher import async_dispatcher_connect
from homeassistant.helpers.typing import ConfigType, HomeAssistantType

from . import (
    DATA_ZIPATO_CONFIG,
    CONF_TOPIC
)

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = "Zipato Thermostat"

CONF_NAME = "name"

CONF_ATTR_HEATING_LEVEL_ACTUAL = "attribute_heating_level_actual"
CONF_ATTR_HEATING_LEVEL_DISABLED = "attribute_heating_level_disabled"
CONF_ATTR_HEATING_LEVEL_STATE = "attribute_heating_level_state"
CONF_ATTR_HEATING_LEVEL_TARGET = "attribute_heating_level_target"

CONF_ATTR_MASTER_CONTROL_MODE = "attribute_master_control_mode"
CONF_ATTR_MASTER_CONTROL_VALUE = "attribute_master_control_value"
CONF_ATTR_MASTER_CONTROL_PRESET = "attribute_master_control_preset"
CONF_ATTR_MASTER_CONTROL_HOLD_UNTIL = "attribute_master_control_hold_until"

SCHEMA_BASE = CLIMATE_PLATFORM_SCHEMA.extend(MQTT_BASE_PLATFORM_SCHEMA.schema)
PLATFORM_SCHEMA = (
    SCHEMA_BASE.extend({
        vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string,
        vol.Optional(CONF_RETAIN, default=mqtt.DEFAULT_RETAIN): cv.boolean,
        vol.Optional(CONF_TEMP_INITIAL, default=21): cv.positive_int,
        vol.Optional(CONF_TEMP_MIN, default=DEFAULT_MIN_TEMP): vol.Coerce(float),
        vol.Optional(CONF_TEMP_MAX, default=DEFAULT_MAX_TEMP): vol.Coerce(float),
        vol.Optional(CONF_TEMP_STEP, default=1.0): vol.Coerce(float),
        vol.Required(CONF_ATTR_HEATING_LEVEL_ACTUAL): cv.string,
        vol.Required(CONF_ATTR_HEATING_LEVEL_DISABLED): cv.string,
        vol.Optional(CONF_ATTR_HEATING_LEVEL_STATE): cv.string,
        vol.Optional(CONF_ATTR_HEATING_LEVEL_TARGET): cv.string,
        vol.Required(CONF_ATTR_MASTER_CONTROL_MODE): cv.string,
        vol.Required(CONF_ATTR_MASTER_CONTROL_VALUE): cv.string,
        vol.Optional(CONF_ATTR_MASTER_CONTROL_PRESET): cv.string,
        vol.Optional(CONF_ATTR_MASTER_CONTROL_HOLD_UNTIL): cv.string,
    })
    .extend(mqtt.MQTT_AVAILABILITY_SCHEMA.schema)
    .extend(mqtt.MQTT_JSON_ATTRS_SCHEMA.schema)
)

MODE_REQUEST_TOPIC = "mode_request_topic"
TEMP_REQUEST_TOPIC = "temp_request_topic"
HOLD_REQUEST_TOPIC = "hold_request_topic"

COMMAND_TEMPLATE = "command_template"
MODE_COMMAND_TEMPLATE = "mode_command_template"
TEMP_COMMAND_TEMPLATE = "temp_command_template"
HOLD_COMMAND_TEMPLATE = 'hold_command_template'

DEFAULT_CONFIG = {
    CONF_SEND_IF_OFF: True,
    CONF_MODE_LIST: [HVAC_MODE_OFF, HVAC_MODE_HEAT],
    CONF_HOLD_LIST: {
        "PROGRAM",
        "HOLD_UNTIL",
        "HOLD_PERIOD",
        "HOLD_PERMANENT",
    },
    CONF_PAYLOAD_ON: "ON",
    CONF_PAYLOAD_OFF: "OFF",
    CONF_VALUE_TEMPLATE: template_helper.Template('{{value_json.value}}'),
    CONF_MODE_STATE_TEMPLATE: template_helper.Template('{% if value_json.value %}off{% else %}heat{% endif %}'),
    COMMAND_TEMPLATE: template_helper.Template('{"value": "{{value}}" }'),
    TEMP_COMMAND_TEMPLATE: template_helper.Template('{"value": "{{value}}" }'),
    MODE_COMMAND_TEMPLATE: template_helper.Template('{% if value == "off" %}{"value": true}{% else %}{"value": false}{% endif %}'),
}

COMMAND_TEMPLATE_KEYS = (
    MODE_COMMAND_TEMPLATE,
    TEMP_COMMAND_TEMPLATE,
    HOLD_COMMAND_TEMPLATE
)

REQUEST_KEYS = {
    MODE_REQUEST_TOPIC,
    TEMP_REQUEST_TOPIC,
    HOLD_REQUEST_TOPIC
}

STATE_TOPIC_WIREFRAME = "%s/attributes/%s/#"
COMMAND_TOPIC_WIREFRAME = "%s/request/attributes/%s/value"
REQUEST_TOPIC_WIREFRAME = "%s/request/attributes/%s/getValue"

async def async_setup_platform(
    hass: HomeAssistantType, config: ConfigType, async_add_entities, discovery_info=None
):
    """Set up MQTT climate device through configuration.yaml."""
    await _async_setup_entity(hass, config, async_add_entities)

async def async_setup_entry(hass, config_entry, async_add_entities):
    """Set up MQTT climate device dynamically through MQTT discovery."""

    await mqtt_async_setup_entry(hass, config_entry, async_add_entities)

async def _async_setup_entity(
    hass, config, async_add_entities, config_entry=None, discovery_hash=None
):
    """Set up the MQTT climate devices."""
    async_add_entities([ZipatoClimate(hass, config, config_entry, discovery_hash)])

class ZipatoClimate(MqttClimate):
    """Representation of a Zipato climate device."""

    def __init__(self, hass, config, config_entry, discovery_hash):
        self._zipato_config = hass.data[DATA_ZIPATO_CONFIG]
        self._topic_prefix = self._zipato_config.get(CONF_TOPIC)

        config.update(DEFAULT_CONFIG)
        MqttClimate.__init__(self, hass, config, config_entry, discovery_hash)

    def _setup_from_config(self, config):
        """(Re)Setup the entity."""
        prefix = self._topic_prefix

        # set to None in non-optimistic mode
        self._target_temp = (
            self._current_fan_mode
        ) = self._current_operation = self._current_swing_mode = None
        self._target_temp_low = None
        self._target_temp_high = None

        self._topic = {}

        for key in TOPIC_KEYS:
            self._topic[key] = None

        self._topic.update({
            CONF_TEMP_STATE_TOPIC: STATE_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_VALUE]),
            CONF_TEMP_COMMAND_TOPIC: COMMAND_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_VALUE]),
            TEMP_REQUEST_TOPIC: REQUEST_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_VALUE]),
            CONF_MODE_STATE_TOPIC: STATE_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_HEATING_LEVEL_DISABLED]),
            CONF_MODE_COMMAND_TOPIC: COMMAND_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_HEATING_LEVEL_DISABLED]),
            MODE_REQUEST_TOPIC: REQUEST_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_HEATING_LEVEL_DISABLED]),
            CONF_HOLD_STATE_TOPIC: STATE_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_MODE]),
            CONF_HOLD_COMMAND_TOPIC: COMMAND_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_MODE]),
            HOLD_REQUEST_TOPIC: REQUEST_TOPIC_WIREFRAME % (prefix, config[CONF_ATTR_MASTER_CONTROL_MODE]),
        })

        self._target_temp = config[CONF_TEMP_INITIAL]

        self._target_temp_low = config[CONF_TEMP_INITIAL]
        self._target_temp_high = config[CONF_TEMP_INITIAL]

        self._current_fan_mode = FAN_LOW
        self._current_swing_mode = HVAC_MODE_OFF
        self._current_operation = HVAC_MODE_OFF

        self._action = None
        self._away = False
        self._hold = None
        self._aux = False

        value_templates = {}
        for key in TEMPLATE_KEYS:
            value_templates[key] = lambda value: value
        if CONF_VALUE_TEMPLATE in config:
            value_template = config.get(CONF_VALUE_TEMPLATE)
            value_template.hass = self.hass
            value_templates = {
                key: value_template.async_render_with_possible_json_value
                for key in TEMPLATE_KEYS
            }
        for key in TEMPLATE_KEYS & config.keys():
            tpl = config[key]
            value_templates[key] = tpl.async_render_with_possible_json_value
            tpl.hass = self.hass

        self._value_templates = value_templates

        command_templates = {}
        for key in COMMAND_TEMPLATE_KEYS:
            command_templates[key] = lambda value: value
        if COMMAND_TEMPLATE in config:
            command_template = config.get(COMMAND_TEMPLATE)
            command_template.hass = self.hass
            command_templates = {
                key: command_template.async_render_with_possible_json_value
                for key in COMMAND_TEMPLATE_KEYS
            }
        for key in COMMAND_TEMPLATE_KEYS & config.keys():
            tpl = config[key]
            command_templates[key] = tpl.async_render_with_possible_json_value
            tpl.hass = self.hass

        self._command_templates = command_templates

    async def _subscribe_topics(self):
        """(Re)Subscribe to topics."""
        await super(ZipatoClimate, self)._subscribe_topics()

        for request in REQUEST_KEYS:
            self._publish(request, '')

    @property
    def supported_features(self):
        """Return the list of supported features."""
        return SUPPORT_TARGET_TEMPERATURE | SUPPORT_PRESET_MODE

    @property
    def preset_modes(self):
        """Return preset modes."""
        return self._config[CONF_HOLD_LIST]

    async def async_set_temperature(self, **kwargs):
        """Set new target temperatures."""
        if kwargs.get(ATTR_HVAC_MODE) is not None:
            operation_mode = kwargs.get(ATTR_HVAC_MODE)
            await self.async_set_hvac_mode(operation_mode)

        temp_command = self._command_templates[TEMP_COMMAND_TEMPLATE]
        self._set_temperature(
            temp_command(kwargs.get(ATTR_TEMPERATURE)), CONF_TEMP_COMMAND_TOPIC,
            CONF_TEMP_STATE_TOPIC, '_target_temp')

        # Always optimistic?
        self.async_write_ha_state()

    async def async_set_hvac_mode(self, hvac_mode) -> None:
        """Set new operation mode."""
        mode_command = self._command_templates[MODE_COMMAND_TEMPLATE]
        self._publish(CONF_MODE_COMMAND_TOPIC,
            mode_command(hvac_mode))

        if self._topic[CONF_MODE_STATE_TOPIC] is None:
            self._current_operation = hvac_mode
            self.async_write_ha_state()

        self._publish(TEMP_REQUEST_TOPIC, '')
        self._publish(HOLD_REQUEST_TOPIC, '')

    def _set_hold_mode(self, hold_mode):
        """Set hold mode.

        Returns if we should optimistically write the state.
        """
        hold_command = self._command_templates[HOLD_COMMAND_TEMPLATE]
        self._publish(CONF_HOLD_COMMAND_TOPIC,
            hold_command(hold_mode or "off"))

        if self._topic[CONF_HOLD_STATE_TOPIC] is not None:
            return False

        self._hold = hold_mode
        return True


import feedparser
from models.models import Feed, Entry
from app import db

def entryPolling():
    feeds = Feed.query.all()

    for feed in feeds:
        d = feedparser.parse(feed.url)
        for entry in d.entries:
            checkEntryExistence(entry, feed)

    db.session.commit()

def checkEntryExistence(entry, feed):

    new_entry = Entry.query.filter(Entry.title.ilike(entry['title'])).first()

    if new_entry == None:
        addEntry(entry, feed)
    else:
        return True

def addEntry(entry, feed):

    new_entry = Entry(entry['title'], entry['link'], entry['description'], feed)
    db.session.add(new_entry)


from modules.base_module import Module

icon = "https://raw.githubusercontent.com/AvaCity/avacity-2.0/master/" \
       "readme/star.png"
class_name = "Support"

class Support(Module):
    prefix = "spt"

    def __init__(self, server):
        self.server = server
        self.commands = {"init": self.init, "gscnl": self.get_social_channels}

    def init(self, msg, client):
        client.send(["spt.init", {"a": False}])

    def get_social_channels(self, msg, client):
        channels = []
        channels.append({"act": True, "prt": 1, "id": 1, "stid": "github",
                         "dsctn": "Поставьте звёздочку ;(", "ttl": "GitHub",
                         "icnurl": icon,
                         "lnk": "https://github.com/AvaCity/avacity-2.0"})
        client.send(["spt.gscnl", {"scls": channels}])


import numpy as np
from scipy.stats import kendalltau
from sklearn.metrics import recall_score

def scorePerformance(
        prMean_pred, prMean_true,
        rtMean_pred, rtMean_true,
        rrStd_pred, rrStd_true,
        ecgId_pred, ecgId_true):
    """
    Computes the combined multitask performance score.
    The 3 regression tasks are individually scored using Kendalls
    correlation coeffficient.
    The user classification task is scored according to macro averaged
    recall, with an adjustment for chance level. All performances are
    clipped at 0.0, so that zero indicates chance or worse performance,
    and 1.0 indicates perfect performance.
    The individual performances are then combined by taking
    the geometric mean.
    :param prMean_pred: 1D float32 numpy array.
        The predicted average P-R interval duration over the window.
    :param prMean_true: 1D float32 numpy array.
        The true average P-R interval duration over the window.
    :param rtMean_pred: 1D float32 numpy array.
        The predicted average R-T interval duration over the window.
    :param rtMean_true: 1D float32 numpy array.
        The true average R-T interval duration over the window.
    :param rrStd_pred: 1D float32 numpy array.
        The predicted R-R interval duration standard deviation over the window.
    :param rrStd_true: 1D float32 numpy array.
        The true R-R interval duration standard deviation over the window.
    :param ecgId_pred: 1D int32 numpy array.
        The predicted user ID label for each window.
    :param ecgId_true: 1D int32 numpy array.
        The true user ID label for each window.
    :return:
        - The combined performance score on all tasks;
                0.0 means at least one task has chance level performance
                or worse while 1.0 means all tasks are solved perfectly.
        - The individual task performance scores are also returned
    """

    # Input checking
    assert isinstance(ecgId_pred, np.ndarray)
    assert len(ecgId_pred.shape) == 1
    assert ecgId_pred.dtype == np.int32

    assert isinstance(ecgId_true, np.ndarray)
    assert len(ecgId_true.shape) == 1
    assert ecgId_true.dtype == np.int32

    assert isinstance(rrStd_pred, np.ndarray)
    assert len(rrStd_pred.shape) == 1
    assert rrStd_pred.dtype == np.float32

    assert isinstance(rrStd_true, np.ndarray)
    assert len(rrStd_true.shape) == 1
    assert rrStd_true.dtype == np.float32

    assert isinstance(prMean_pred, np.ndarray)
    assert len(prMean_pred.shape) == 1
    assert prMean_pred.dtype == np.float32

    assert isinstance(prMean_true, np.ndarray)
    assert len(prMean_true.shape) == 1
    assert prMean_true.dtype == np.float32

    assert isinstance(rtMean_pred, np.ndarray)
    assert len(rtMean_pred.shape) == 1
    assert rtMean_pred.dtype == np.float32

    assert isinstance(rtMean_true, np.ndarray)
    assert len(rtMean_true.shape) == 1
    assert rtMean_true.dtype == np.float32

    assert (len(ecgId_pred) == len(ecgId_true)) \
        and (len(ecgId_pred) == len(prMean_pred)) \
        and (len(ecgId_pred) == len(prMean_true)) \
        and (len(ecgId_pred) == len(rtMean_pred)) \
        and (len(ecgId_pred) == len(rtMean_true)) \
        and (len(ecgId_pred) == len(rrStd_pred)) \
        and (len(ecgId_pred) == len(rrStd_true))

    # Accuracy is computed with macro averaged recall so that accuracy
    # is computed as though the classes were balanced, even if they are not.
    # Note that provided training, validation and testing sets are balanced.
    # Unbalanced classes would only be and issue if a new train/validation
    # split is created.
    # Any accuracy value worse than random chance will be clipped at zero.

    ecgIdAccuracy = recall_score(ecgId_true, ecgId_pred, average='macro')
    adjustementTerm = 1.0 / len(np.unique(ecgId_true))
    ecgIdAccuracy = (ecgIdAccuracy - adjustementTerm) / (1 - adjustementTerm)
    if ecgIdAccuracy < 0:
        ecgIdAccuracy = 0.0

    # Compute Kendall correlation coefficients for regression tasks.
    # Any coefficients worse than chance will be clipped to zero.
    rrStdTau, _ = kendalltau(rrStd_pred, rrStd_true)
    if rrStdTau < 0:
        rrStdTau = 0.0

    prMeanTau, _ = kendalltau(prMean_pred, prMean_true)
    if prMeanTau < 0:
        prMeanTau = 0.0

    rtMeanTau, _ = kendalltau(rtMean_pred, rtMean_true)
    if rtMeanTau < 0:
        rtMeanTau = 0.0

    # Compute the final performance score as the geometric mean of the
    # individual task performances. A high geometric mean ensures that
    # there are no tasks with very poor performance that are masked by good
    # performance on the other tasks. If any task has chance performance
    # or worse, the overall performance will be zero. If all tasks are
    # perfectly solved, the overall performance will be 1.

    combinedPerformanceScore = np.power(
        rrStdTau * prMeanTau * rtMeanTau * ecgIdAccuracy, 0.25)

    return (
        combinedPerformanceScore,
        prMeanTau, rtMeanTau, rrStdTau, ecgIdAccuracy
    )

def example():
    prMean_pred = np.random.randn(480).astype(np.float32)
    prMean_true = (np.random.randn(480).astype(
        np.float32) / 10.0) + prMean_pred

    rtMean_pred = np.random.randn(480).astype(np.float32)
    rtMean_true = (np.random.randn(480).astype(
        np.float32) / 10.0) + rtMean_pred

    rrStd_pred = np.random.randn(480).astype(np.float32)
    rrStd_true = (np.random.randn(480).astype(np.float32) / 10.0) + rrStd_pred

    ecgId_pred = np.random.randint(low=0, high=32, size=(480,), dtype=np.int32)
    ecgId_true = np.random.randint(low=0, high=32, size=(480,), dtype=np.int32)

    print(
        scorePerformance(
            prMean_true, prMean_pred,
            rtMean_true, rtMean_pred,
            rrStd_true, rrStd_pred,
            ecgId_true, ecgId_pred
        )
    )

if __name__ == '__main__':

    example()


from app.models import User, Rating
import plotly
import numpy as np
import pandas as pd
import plotly.graph_objs as go

def datetime(x):
    return np.array(x, dtype=np.datetime64)

def get_ratings(shortname, rating_type='elo'):
    user = User.query.filter_by(shortname=shortname).first()
    ratings = Rating.query \
        .filter_by(rating_type=rating_type) \
        .filter_by(user_id=user.id) \
        .order_by(Rating.timestamp).all()
    values = [r.rating_value for r in ratings]
    dates = datetime([r.timestamp for r in ratings])
    match_ids = [r.match_id for r in ratings]
    source = pd.DataFrame(dict(
        date=dates,
        rating=values,
        match_id=match_ids
    ))

    return source

def plot_ratings(shortname, rating_type):
    source = get_ratings(shortname, rating_type=rating_type)
    hover_text = []
    for index, row in source.iterrows():
        match_id = row['match_id']
        if pd.isnull(match_id):
            # To allow formatting of ids laters
            match_id = 0
        hover_text.append((
            "Match ID: {match_id}
"+
            "Time: {date}
"+
            "Rating: {rating}
").format(
                match_id = format(match_id, "1.0f") ,
                date = row["date"].ctime(),
                rating = format(row["rating"], "1.0f")
            ))

    data = go.Scatter(
            x = source.loc[:,"date"],
            y = source.loc[:,"rating"],
            text = hover_text,
            hoverinfo = 'text',
            marker = dict(
                color = 'green'
            ),
            showlegend = False
        )
    data_comp = [data]

    layout_comp = go.Layout(
        title='Performance over time',
        hovermode='closest',
        xaxis=dict(
            title='Time',
        ),
        yaxis=dict(
            title='Rating',
    ))

    fig_comp = go.Figure(data=data_comp, layout=layout_comp)
    div = plotly.offline.plot(fig_comp, show_link=False, output_type="div", include_plotlyjs=False)

    return div


# Copyright 2018,2019,2020,2021 Sony Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
import nnabla as nn
import nnabla.parametric_functions as PF
import numpy as np
from nnabla.testing import assert_allclose

def ref_bcast(x_data_list, src):
    results = []
    for x_data in x_data_list:
        results.append(x_data * (src + 1))
    return results

@pytest.mark.parametrize("seed", [313])
@pytest.mark.parametrize("src", [0, 1])
@pytest.mark.parametrize("inplace", [True, False])
def test_bcast(seed, src, inplace, comm_nccl_opts):
    if comm_nccl_opts is None:
        pytest.skip(
            "Communicator test is disabled. You can turn it on by an option `--test-communicator`.")
    if len(comm_nccl_opts.devices) < 2:
        pytest.skip(
            "Communicator test is disabled. Use more than 1 gpus.")

    comm = comm_nccl_opts.comm
    device_id = int(comm_nccl_opts.device_id)
    n_devices = len(comm_nccl_opts.devices)

    # Variables
    x_list = []
    x_data_list = []
    num_layers = 20
    rng = np.random.RandomState(seed)
    for l in range(num_layers):
        x_data = rng.rand(3, 4)
        x_data_list.append(x_data)
        x = nn.Variable(x_data.shape)
        x.d = x_data * (device_id + 1)
        x_list.append(x)

    # Bcast
    comm.bcast([x.data for x in x_list], src, inplace=inplace)

    # Ref
    refs = ref_bcast(x_data_list, src)

    # Check
    for x, ref in zip(x_list, refs):
        assert_allclose(x.d, ref, rtol=1e-3, atol=1e-6)


from rewriter import *
from os.path import basename
import copy
from lan_parser import *
from cgen import *
from transformation import *
from analysis import *
fileprefix = "../test/C/"

SetNoReadBack = False
DoOptimizations = True

def LexAndParse(name, createTemp):
    import ply.yacc as yacc
    cparser = yacc.yacc()
    lex.lex()

    run = 1
    while run:
        filename = fileprefix + name + '/' + name + 'For.cpp'
        funcname = basename(os.path.splitext(filename)[0])
        try:
            f = open(filename, 'r')
            s = f.read()
            f.close()
            ## print s
        except EOFError:
            break

        lex.input(s)
        while 1:
            tok = lex.token()
            if not tok: break
            ## print tok

        ast = cparser.parse(s)
        ## ast.show()
        ## print ast
        ## print slist
        cprint = CGenerator()
        ## printres = cprint.visit(ast)
        ## print printres
        rw = Rewriter()
        rw.initOriginal(ast)
        tempfilename = fileprefix + name + '/'+'temp' +name.lower() + '.cpp'
        if createTemp:
            rw.rewrite(ast, funcname, changeAST = True)
            cprint.createTemp(ast, filename = tempfilename)

        run = 0
        filename = tempfilename
        ## funcname = basename(os.path.splitext(filename)[0])
        try:
            f = open(filename, 'r')
            s = f.read()
            f.close()
        except EOFError:
            break

        ast = cparser.parse(s)
        ## ## ast.show()
        tempast = copy.deepcopy(ast)
        tempast2 = copy.deepcopy(ast)
        return (rw, ast, tempast, tempast2, funcname)

def CGen(name, funcname, an, tempast2, ast, kernelstringname = ''):
        cprint = CGenerator()
        rw = an.rw
        an.generate_kernels(tempast2, name, fileprefix)
        ## rw.InSourceKernel(tempast2, filename = fileprefix + name + '/'+funcname + '.cl', kernelstringname = kernelstringname)
        boilerast = rw.generateBoilerplateCode(ast)
        cprint.createTemp(boilerast, filename = fileprefix + name + '/'+'boilerplate.cpp')

def matmul():
    name = 'MatMul'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)
    rw.initNewRepr(tempast)
    tf = Transformation(rw)

    an = Analysis(rw, tf)
    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()
        an.PlaceInLocalMemory()
    if SetNoReadBack:
        tf.SetNoReadBack()

    ## rw.DataStructures()
    CGen(name, funcname, an, tempast2, ast)

def jacobi():
    name = 'Jacobi'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)
    rw.initNewRepr(tempast)
    tf = Transformation(rw)

    an = Analysis(rw, tf)

    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()

        tf.localMemory(['X1'], west = 1, north = 1, east = 1, south = 1, middle = 0)
        an.PlaceInLocalMemory()
    if SetNoReadBack:
        tf.SetNoReadBack()

    CGen(name, funcname, an, tempast2, ast)

def knearest():
    name = 'KNearest'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)
    tf = Transformation(rw)
    tf.SetParDim(1)
    rw.initNewRepr(tempast)

    an = Analysis(rw, tf)
    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()
        an.PlaceInLocalMemory()
    if SetNoReadBack:
        tf.SetNoReadBack()
    ## rw.DataStructures()
    ## rw.Unroll2({'k' : 0})

    CGen(name, funcname, an, tempast2, ast)

def nbody():
    name = 'NBody'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)

    rw.initNewRepr(tempast)
    tf = Transformation(rw)
    an = Analysis(rw, tf)
    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()
        an.PlaceInLocalMemory()

    if SetNoReadBack:
        tf.SetNoReadBack()
    ## rw.Unroll2({'j': 32})
    CGen(name, funcname, an, tempast2, ast)

def laplace():
    name = 'Laplace'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)
    tf = Transformation(rw)
    tf.SetParDim(1)
    rw.initNewRepr(tempast)
    an = Analysis(rw, tf)
    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()
        an.PlaceInLocalMemory()

    if SetNoReadBack:
        tf.SetNoReadBack()

    ## rw.DataStructures()

    ## tf.Unroll2({'d' : 0, 'd_outer' : 0, 'd_inner' : 0})
    CGen(name, funcname, an, tempast2, ast)

def gaussian():
    name = 'GaussianDerivates'
    (rw, ast, tempast, tempast2, funcname) = LexAndParse(name, True)
    ## rw.SetParDim(1)
    rw.initNewRepr(tempast, dev = 'CPU')
    tf = Transformation(rw)

    an = Analysis(rw, tf)
    if DoOptimizations:
        an.Transpose()
        an.DefineArguments()
        an.PlaceInReg()
        an.PlaceInLocalMemory()

        ## tf.Unroll2({'k' : 0, 'd' : 0, 'g' : 0, 'b' : 0})
    ## rw.DataStructures()
    if SetNoReadBack:
        tf.SetNoReadBack()
    CGen(name, funcname, an, tempast2, ast)

if __name__ == "__main__":
    matmul()
    jacobi()
    knearest()
    nbody()
    laplace()
    gaussian()


# Data Engineering Trial Task
# Created by: Kovács Balázs

import datetime
import json
from google.cloud import pubsub_v1
import os.path
from os import path

# We are saving the aggregates to an internal text file.
# If it already exists, just delete it.

if path.exists('hourly.txt'):
    os.remove('hourly.txt')

# Creating an infinite loop, so the program runs continuously
while True:

    # Variables where we store the number of rides
    total = 0
    pickups = 0
    dropoffs = 0

    # We put every ID in this variable
    msgt = []

    try:

        # Creating subscription
        subscriber = pubsub_v1.SubscriberClient()
        subscription_path = subscriber.subscription_path('ems-dp-candidates', 'taxirides-kb-321783763627')

        # Callback method
        def callback(message):
            global pickups, dropoffs, total, msgt

            if message.attributes:

                msg = bytes.decode(message.data)
                jsonmsg = json.loads(msg)

                # if the ID exists in 'msgt', that means we already have this data, so we go the next iteration
                # and we won't have duplicates of the rows.
                if jsonmsg['ride_id'] in msgt:
                    message.nack()
                    return

                msgt.append(jsonmsg['ride_id'])

                # Counting total rides, pickups and dropoffs
                total += 1

                if jsonmsg['ride_status'] == 'pickup':
                    pickups += 1
                if jsonmsg['ride_status'] == 'dropoff':
                    dropoffs += 1

            message.ack()

        streaming_pull_future = subscriber.subscribe(
            subscription_path, callback=callback
        )

        # Pulling data for an hour
        streaming_pull_future.result(timeout=3600)

    except:
        # Saving aggregates to file
        f = open('hourly.txt', "a")
        f.write('Total: ' + str(total))
        f.write(' Dropoffs: ' + str(dropoffs))
        f.write(' Pickups: ' + str(pickups) + '\n')
        f.close()


import pickle

from starlette.endpoints import HTTPEndpoint
from starlette.routing import Route, Router
from starlette.responses import JSONResponse
import uvicorn

class Homepage(HTTPEndpoint):
    async def get(self, request):
        print(request)
        body = await request.json()
        fn = body['fn']
        resp = {'response': fn}

        return JSONResponse(resp)

app = Router([
    Route('/', endpoint=Homepage, methods=['POST']),
])

if __name__ == '__main__':
    uvicorn.run(app, host='0.0.0.0', port=9999, debug=True)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime as dt

class shoe_data():

	def __init__(self):

		temp_data = pd.read_excel("2019 Winter Data Science Intern Challenge Data Set.xlsx")
		temp_data[["order_id", "shop_id", "user_id", "order_amount", "total_items"]] = temp_data[["order_id", "shop_id", "user_id", "order_amount", "total_items"]].apply(pd.to_numeric)
		temp_data["created_at"] = temp_data["created_at"].apply(pd.to_datetime)

		self.data = temp_data

	def get_data(self):
		return self.data

	def poor_aov(self):

		order_sum = self.data["order_amount"].sum()
		order_len = self.data.shape[0]

		aov = order_sum / order_len
		return round(aov, 2)

	def good_aov(self):

		order_sum = self.data["order_amount"].sum()
		item_sum = self.data["total_items"].sum()

		aov = order_sum / item_sum
		return round(aov, 2)

	def average_daily_items_sold(self):

		item_sum = self.data["total_items"].sum()
		start_date = self.data["created_at"].min()
		end_date = self.data["created_at"].max()
		date_range = (end_date - start_date).round("1d").days

		adt = item_sum / date_range
		return round(adt, 2)

	def average_daily_revenue(self):

		aov = self.good_aov()
		adt = self.average_daily_items_sold()

		adr = aov * adt
		return round(adr, 2)

	def plot_daily_items_sold(self):

		temp_data = self.data.copy()
		temp_data["created_at"] = temp_data["created_at"].dt.day

		daily_data = temp_data.groupby(["created_at"]).agg({"total_items": "sum"})
		adt = self.average_daily_items_sold()

		plt.plot(daily_data)
		plt.axhline(y=adt, color="r")
		plt.xlabel("Days during March, 2017")
		plt.ylabel("Number of Items Sold")
		plt.title("Daily Items Sold during March, 2017")
		plt.show()

	def plot_daily_revenue(self):

		temp_data = self.data.copy()
		temp_data["created_at"] = temp_data["created_at"].dt.day

		daily_data = temp_data.groupby(["created_at"]).agg({"order_amount": "sum"})
		adr = self.average_daily_revenue()

		plt.plot(daily_data)
		plt.axhline(y=adr, color="r")
		plt.xlabel("Days during March, 2017")
		plt.ylabel("Revenue ($)")
		plt.title("Revenue per Day during March, 2017")
		plt.show()

	def pivot_data(self):

		start_date = self.data["created_at"].min()
		end_date = self.data["created_at"].max()
		date_range = (end_date - start_date).round("1d").days

		new_data = self.data.groupby(["shop_id"], as_index=False).agg({"order_amount": "sum", "total_items": "sum", "user_id": "nunique"})
		new_data.rename(columns={"user_id": "user_count"}, inplace=True)

		new_data["AOV"] = new_data.apply(lambda row: round(row["order_amount"] / row["total_items"], 2), axis=1)
		new_data["ADT"] = new_data.apply(lambda row: round(row["total_items"] / date_range, 2), axis=1)
		new_data["ADR"] = new_data.apply(lambda row: round(row["AOV"] * row["ADT"], 2), axis=1)

		new_data.to_excel("2019 Winter Data Science Intern Challenge Data Set (Per Shop).xlsx", header=True, index=False)

def main():
	data = shoe_data()
	print(data.poor_aov())
	print(data.good_aov())
	print(data.average_daily_items_sold())
	print(data.average_daily_revenue())
	data.pivot_data()
	data.plot_daily_items_sold()
	data.plot_daily_revenue()

if __name__ == "__main__":
	main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import params
import random
from utils import *
from sklearn.svm import SVC
from models import trainer, predictor

def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def main():

	params.print_params()
	parser = argparse.ArgumentParser()
	parser.add_argument('--local_test', type=str2bool, default=False, help='local test verbose')
	parser.add_argument('--src_id', type=int, default=0, help='source domain id')
	parser.add_argument('--tar_id', type=int, default=2, help='target domain id')
	args = parser.parse_args()

	print ('source domain:', params.domains[args.src_id])
	print ('target domain:', params.domains[args.tar_id])
	print ('===> ' + 'Start Running')

	avg_acc_svm = 0.0
	avg_acc_tr = 0.0

	train_doc, train_labels = Doc(params.domains[args.src_id], args)
	test_doc, test_labels = Doc(params.domains[args.tar_id], args)
	# print ('train data:', len(train_doc), len(train_labels)) # _, _,
	# print ('test data:', len(test_doc), len(test_labels)) # _, _,

	total_train_num = params.train_num * 2
	data_mat = Format(train_doc[:total_train_num] + test_doc[total_train_num:total_train_num+params.unlabel[args.tar_id]])

	data_train = data_mat[:total_train_num]
	label_train = train_labels[:total_train_num]

	data_test = data_mat[total_train_num:total_train_num+params.unlabel[args.tar_id]]
	label_test = test_labels[total_train_num:total_train_num+params.unlabel[args.tar_id]]

	print ('===> ' + 'Data Loaded')

	for k in range(params.rep):

		### pick small set of labeled data in target domain
		# random_list = random.sample(range(params.unlabel[args.tar_id]), params.num_data_target)

		# load pre-selected samples for books -> electronics
		# np.save('./data/b2e.npy',random_list)
		random_list = np.load('./data/b2e.npy')

		Ts = []
		l = []
		zero_cnt = 0
		one_cnt = 0
		for i in random_list:
			Ts.append(data_test[i])
			l.append(label_test[i])
			# print (label_test[i])
			if label_test[i] == -1:
				zero_cnt += 1
			else:
				one_cnt += 1

		print ()
		print (zero_cnt, one_cnt)

		Ts = np.array(Ts)
		l = np.array(l)
		data_train = np.concatenate((data_train, Ts))
		label_train = np.concatenate((label_train, l))
		data_test = np.array(data_test)
		label_test = np.array(label_test)

		if args.local_test:
			print ('train data:', data_train.shape, label_train.shape)
			print ('test data:', data_test.shape, label_test.shape)

		print ('repetition {:}:'.format(k+1))
		### our methods ###
		weakClassArr = trainer(data_train, label_train, args, params.numIt)
		acc = predictor(data_test, label_test, weakClassArr)
		print ('ours: ', acc)
		avg_acc_tr += acc

		### svm classifier (baseline) ###
		clf = SVC(gamma='auto', kernel='rbf')
		clf.fit(data_train, label_train)
		pred = clf.predict(data_test)
		acc = evaluate(pred, label_test)
		print ('svm: ', acc)
		avg_acc_svm += acc

	print ('\naverage results:')
	print ('avg acc ours:', avg_acc_tr / params.rep)
	print ('avg acc svm:', avg_acc_svm / params.rep)
	print ()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from datetime import date, timedelta
from openerp.osv import fields,osv
from openerp.tools.translate import _

# Overloaded sale_order to manage carriers :
class stock_picking(osv.osv):
    _inherit = 'stock.picking'
    _columns = {
        'wf_run':fields.integer("Run"),
        'wf_run_case':fields.integer("Run Case"),
        'wf_dhl_vers':fields.float("DHL Version"),
        'wf_nn_betrag':fields.float("NN Costs"),
        'wf_sperrgut':fields.integer("Sperrgut"),
        'wf_dhl_express_zeit':fields.integer("Expresstime"),
        'wf_dhl_samstagszustellung':fields.integer("DHL Saturday"),
        'wf_invoice_id':fields.many2one('account.invoice','Invoice'),
        'wf_dhl_zusatzp':fields.char('DHL Extra P', size=240),
        'wf_dhl_warenliste':fields.char('DHL Warenliste', size=400),
        'wf_return_label_number':fields.char('Return Label No', size=50),
        'wf_parcel_costs':fields.float("Parcel Costs"),
        'wf_parcel_lenght':fields.float("Parcel Lenght"),
        'wf_parcel_hight':fields.float("Parcel height"),
        'wf_parcel_width':fields.float("Parcel width"),
        'wf_parcel_insurance':fields.float("Parcel insurance"),
        'wf_dhl_sperrgut':fields.float("DHL Bulky"),
        'wf_date_to_stock':fields.date('Date to Stock'),
        'wf_date_shipped':fields.date('Date Shipped'),
        'wf_internal_note':fields.char('Internal Note', size=64),
        'wf_packtool_fehler':fields.char("Packtool Error", size=150),
        'state': fields.selection(
            [('draft', 'Draft'),
            ('auto', 'Waiting Another Operation'),
            ('confirmed', 'Waiting Availability'),
            ('assigned', 'Ready to Deliver'),
            ('picking', 'in Progress'),
            ('Precash_Waiting', 'Precash Waiting'),
            ('wsz_done', 'WSZ Done'),
            ('done', 'Delivered'),
            ('cancel', 'Cancelled'),],
            'Status', readonly=True, select=True, track_visibility='onchange',
            help="""* Draft: not confirmed yet and will not be scheduled until confirmed\n
                 * Waiting Another Operation: waiting for another move to proceed before it becomes automatically available (e.g. in Make-To-Order flows)\n
                 * Waiting Availability: still waiting for the availability of products\n
                 * Ready to Deliver: products reserved, simply waiting for confirmation.\n
                 * Delivered: has been processed, can't be modified or cancelled anymore\n
                 * Cancelled: has been cancelled, can't be confirmed anymore"""),
    }
    _defaults = {
        'wf_run': 0,
        'wf_dhl_vers': 0.0,
        'wf_nn_betrag': 0.0,
        'wf_sperrgut': 0,
        'wf_dhl_express_zeit': 0,
        'wf_dhl_samstagszustellung': 0,

     }

    def reset_picking(self, cr, uid, context=None):
        d=date.today()-timedelta(days=7)
        picking_obj = self.pool.get('stock.picking')
        move_obj = self.pool.get('stock.move')
        picking_ids = picking_obj.search(cr,uid, [('state','=', 'picking'), ('wf_date_to_stock','<',d), ('wf_packtool_fehler','=',False)], context=context)
        if picking_ids:
            for picking in picking_obj.browse(cr,uid, picking_ids,  context=context):
                if picking.move_lines:
                    for move in picking.move_lines:
                        if move.state not in ['done', 'cancel']:
                            move_obj.write(cr,uid, move.id, {'state': 'confirmed'})
                    picking_obj.write(cr,uid, picking.id, {'state': 'confirmed','wf_date_to_stock': False,'wf_run': False,'wf_run_case': False,'wf_packtool_fehler': 'auto_reset'})
                else:
                    picking_obj.write(cr,uid, picking.id, {'state': 'done','wf_packtool_fehler': 'keine Positionen'})
                cr.commit()

        return True

stock_picking()

class stock_picking_out(osv.osv):
    _inherit = 'stock.picking.out'
    _columns = {
        'wf_return_label_number':fields.char('Return Label No', size=50),
        'wf_run':fields.integer("Run"),
        'wf_run_case':fields.integer("Run Case"),
        'wf_packtool_fehler':fields.char("Packtool Error", size=150),
        'wf_date_to_stock':fields.date('Date to Stock'),
        'wf_date_shipped':fields.date('Date Shipped'),
        'wf_internal_note':fields.char('Internal Note', size=64),
        'state': fields.selection(
            [('draft', 'Draft'),
            ('auto', 'Waiting Another Operation'),
            ('confirmed', 'Waiting Availability'),
            ('assigned', 'Ready to Deliver'),
            ('picking', 'in Progress'),
            ('Precash_Waiting', 'Precash Waiting'),
            ('wsz_done', 'WSZ Done'),
            ('done', 'Delivered'),
            ('cancel', 'Cancelled'),],
            'Status', readonly=True, select=True, track_visibility='onchange',
            help="""* Draft: not confirmed yet and will not be scheduled until confirmed\n
                 * Waiting Another Operation: waiting for another move to proceed before it becomes automatically available (e.g. in Make-To-Order flows)\n
                 * Waiting Availability: still waiting for the availability of products\n
                 * Ready to Deliver: products reserved, simply waiting for confirmation.\n
                 * Delivered: has been processed, can't be modified or cancelled anymore\n
                 * Cancelled: has been cancelled, can't be confirmed anymore"""),
    }
    _defaults = {
        'wf_run': 0,
     }
stock_picking_out()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: