"""
A module for calculating pharmacies' current respond time.
"""

import datetime
import numpy as np
import pandas as pd
from scipy.interpolate import interp1d
from scipy.signal import medfilt
from scipy.signal import savgol_filter
import ext_connections as ext_con
import requests
import os
import json

class RespondTime:
    """
    A class for calculating pharmacies' current respond time.

    The class is designed for realtime calculating and previous calculation
    of pharmacy's respond time. After input data is received
    it's possible to create calculation math. models.

    There are 3 math. models represented:
    1. Linear interpolation with a median smoothing (a large sample of data)
    2. Linear interpolation with a Savitzky-Golay smoothing (a medium sample of data)
    3. Canonical averaging (a small sample of data)
    """

    def __init__(self):
        self.settings = RespondSettings()
        self.connection = ext_con.API()

    @property
    def connection(self):
        return self._connection

    @connection.setter
    def connection(self, connection):
        self._connection = connection

    def get_pharmacy_data_table(self, sn):
        """Creates an input table from server"""

        query = self.settings.get_setting('reaction_api')
        if not query:
            return None

        query += '/?sn=' + str(sn)

        auth = self.settings.get_setting('auth')
        if not auth:
            return None

        method = 'GET'
        headers = {
            "Authorization": ' '.join(["Basic", auth]),
            "Content-type": "application/json",
            "Accept": "application/json"
        }
        file_type = 'json'
        table = self.connection.execute(query, method, headers, file_type)

        if table is None or table.empty:
            return None

        table.FixedDT = table.FixedDT.apply(pd.to_datetime)
        table.TotalTimeSec = table.TotalTimeSec.astype(int)
        table['Hour'] = table['FixedDT'].dt.hour
        table['Minute'] = table['FixedDT'].dt.minute
        table['Weekday'] = table['FixedDT'].dt.dayofweek + 1

        return table

    def get_pharmacies_schedules(self):
        query = self.settings.get_setting('schedule_api')
        if not query:
            return None

        auth = self.settings.get_setting('auth')
        if not auth:
            return None

        method = 'GET'
        headers = {
            "Authorization": ' '.join(["Basic", auth]),
            "Content-type": "application/json",
            "Accept": "application/json"
        }
        file_type = 'json'
        table = self.connection.execute(query, method, headers, file_type)

        return table

    def get_respond_time_table(self, return_full_table=False):
        """Returns a table of calculated respond time"""

        interval_mins = int(self.settings.get_setting('interval_minutes'))
        min_res = int(self.settings.get_setting('min_res'))
        max_res = int(self.settings.get_setting('max_res'))

        current_sn = 0
        sn_str = self.settings.get_setting('pharmacy_sn')
        if sn_str:
            current_sn = int(sn_str)

        columns = ['ID_Branch', 'Weekday', 'StartTime', 'EndTime', 'RespondTime']
        result_table = pd.DataFrame([], columns=columns)
        full_table = pd.DataFrame([], columns=columns)

        is_sent = False
        sent_count = 0
        schedule_table = self.get_pharmacies_schedules()
        count = len(schedule_table)
        print(count, 'pharmacies:')
        for index, row in schedule_table.iterrows():
            is_sent = False

            num = index + 1
            pharmacy_sn = int(row.SerialNumber)
            if current_sn and pharmacy_sn != current_sn:
                continue

            print(str(pharmacy_sn), '[', str(num), ']')

            pharmacy_table = self.get_pharmacy_data_table(pharmacy_sn)
            if pharmacy_table is None or pharmacy_table.empty:
                continue
            elif len(pharmacy_table) <= min_res:
                schedule = row.Schedule.split(';')
                respond_table = self._calculate_small_sample(pharmacy_table, interval_mins, schedule)
            elif len(pharmacy_table) <= max_res:
                respond_table = self._calculate_medium_sample(pharmacy_table, interval_mins)
            else:
                respond_table = self._calculate_large_sample(pharmacy_table, interval_mins)

            if respond_table is None:
                continue

            pharmacy_id = row.ID
            respond_table['ID_Branch'] = pharmacy_id

            result_table = result_table.append(respond_table, sort=False)
            if return_full_table:
                full_table = full_table.append(respond_table, sort=False)
            sent_count += 1

            if sent_count == 5 or (num == count and not result_table.empty):
                self.send_data(result_table)
                sent_count = 0
                result_table = pd.DataFrame([], columns=columns)
                is_sent = True

        if not is_sent and not result_table.empty:
            self.send_data(result_table)

        print('Calculated respond time and sent to the server... ' + datetime.datetime.now().strftime('%H:%M:%S'))

        return full_table

    def send_data(self, table: pd.DataFrame):
        """Sends data of calculated respond time to the SQL using API"""

        if table is None or table.empty:
            return False

        query = self.settings.get_setting('send_api')
        if not query:
            return False

        auth = self.settings.get_setting('auth')
        if not auth:
            return False

        table.Weekday = table.Weekday.astype(str)
        table.RespondTime = table.RespondTime.astype(str)
        table.StartTime.map(lambda x: x.strftime('%H:%M:%S'))
        table.EndTime.map(lambda x: x.strftime('%H:%M:%S'))

        items = table.to_json(orient='records')
        items_data = json.loads(items)
        json_data = {'Items': items_data}

        headers = {
            "Authorization": ' '.join(["Basic", auth]),
            "Content-type": "application/json",
            "Accept": "application/json"
        }

        try:
            respond = requests.post(url=query, headers=headers, json=json_data)
            if respond.status_code != 200:
                return False

            json_response = respond.json()
            if json_response['Status'] == 'Error':
                print('Error:', json_response['Description'])
                return False
        except ConnectionResetError as e:
            print('Error:', e)
            return False
        except requests.exceptions.ConnectionError as e:
            print('Error:', e)
            return False

        return True

    def finish(self):
        self.connection.disconnect()

    @staticmethod
    def save_table(table: pd.DataFrame, path):
        """Saves a .csv file of calculated respond time"""

        if not path:
            return ''

        if path[-1] != '\\':
            path += '\\'

        file_date = datetime.datetime.now().strftime('%Y%m%d')
        file_name = 'respond_time_' + file_date
        file_ext = '.csv'
        full_path = path + file_name + file_ext

        table.to_csv(full_path, header=True, index=False)

        return path

    def _calculate_large_sample(self, table: pd.DataFrame, interval_minutes: int):
        """1st math. model"""

        pharmacy_table = table.copy()
        pharmacy_table.sort_values(
            ['Weekday', 'Hour', 'Minute', 'FixedDT'],
            ascending=[True, True, True, True],
            inplace=True
        )

        intervals, responds = self._get_x_y(pharmacy_table)
        median_responds = Smoothing.median_smoothing(responds, 3)

        itp_f = self._get_itp_f(intervals, median_responds)

        new_intervals = np.linspace(intervals.min(), intervals.max(), 10000)
        new_responds = itp_f(new_intervals)
        itp_func = self._get_itp_f(new_intervals, new_responds)

        respond_table = self._get_output_table(itp_func, interval_minutes, by_weekday=True)

        return respond_table

    def _calculate_medium_sample(self, table: pd.DataFrame, interval_minutes: int):
        """2nd math. model"""

        pharmacy_table = table.copy()
        pharmacy_table.sort_values(
            ['Hour', 'Minute', 'FixedDT'],
            ascending=[True, True, True],
            inplace=True
        )

        intervals, responds = self._get_x_y(pharmacy_table, by_weekday=False)

        lin_intervals = np.linspace(intervals.min(), intervals.max(), 1000)
        sg_responds = Smoothing.savgol_smoothing(intervals, responds, lin_intervals)

        itp_func = self._get_itp_f(lin_intervals, sg_responds)

        new_intervals = np.linspace(lin_intervals.min(), lin_intervals.max(), 1000)
        new_responds = itp_func(new_intervals)
        new_itp_func = self._get_itp_f(new_intervals, new_responds)

        respond_table = self._get_output_table(new_itp_func, interval_minutes, by_weekday=False)

        return respond_table

    def _calculate_small_sample(self, table: pd.DataFrame, interval_minutes: int, schedule: list):
        """3rd math. model"""

        if not schedule or not schedule[0]:
            return None

        pharmacy_table = table.copy()
        pharmacy_table.sort_values(
            ['Hour', 'Minute', 'FixedDT'],
            ascending=[True, True, True],
            inplace=True
        )

        # Calculating average respond time
        intervals, responds = self._get_x_y(pharmacy_table, by_weekday=True)

        respond_sum = responds.sum()
        respond_count = len(responds)
        avg_respond = int(respond_sum // respond_count)

        # Constants
        days_per_week = 7
        hours_per_day = 24
        minutes_per_hour = 60
        minutes_per_day = hours_per_day * minutes_per_hour

        time_format = '%H:%M'

        day_end_time_str = str(23) + ':' + str(59)
        day_end_dt = datetime.datetime.strptime(day_end_time_str, time_format)

        day_start_time_str = str(0) + ':' + str(0)
        day_start_dt = datetime.datetime.strptime(day_start_time_str, time_format)

        # Filling in minutes and respond time due to schedule
        minutes_list = []
        respond_list = []
        for weekday in range(days_per_week):
            day_schedule = ''
            if weekday < len(schedule):
                day_schedule = schedule[weekday]

            start_dt, end_dt = self._start_end_time(day_schedule)

            # Find a count of next closed days
            next_start_dt_work = start_dt
            next_closed_days = 0
            for ind in range(days_per_week):
                next_day = weekday + ind + 1
                if next_day >= days_per_week:
                    next_day -= days_per_week

                next_day_schedule = ''
                if next_day < len(schedule):
                    next_day_schedule = schedule[next_day]

                next_start_dt, next_end_dt = self._start_end_time(next_day_schedule)
                if next_start_dt == next_end_dt:
                    next_closed_days += 1
                else:
                    # Nearest work day start in a schedule
                    next_start_dt_work = next_start_dt
                    break

            for hour in range(hours_per_day):
                for minute in np.arange(0, minutes_per_hour, 1):
                    # Current calculating datetime with average respond time included
                    cur_time_str = str(hour) + ':' + str(minute)
                    cur_dt = datetime.datetime.strptime(cur_time_str, time_format)
                    cur_dt_delta = cur_dt + datetime.timedelta(seconds=avg_respond)

                    respond_time = 0
                    if cur_dt_delta >= end_dt:
                        # After schedule day's end
                        diff_from_day_start = (next_start_dt_work - day_start_dt).seconds
                        diff_till_day_end = (day_end_dt - end_dt).seconds
                        diff_schedule = diff_till_day_end + diff_from_day_start
                        diff_cur_to_end = (cur_dt - end_dt).seconds
                        diff_schedule_wo_cur = diff_schedule - diff_cur_to_end
                        if cur_dt < end_dt:
                            diff_cur_to_end = (end_dt - cur_dt).seconds
                            diff_schedule_wo_cur = diff_schedule + diff_cur_to_end
                        diff_total = diff_schedule_wo_cur + avg_respond

                        respond_time = 24 * 60 * 60 * next_closed_days + diff_total
                    elif start_dt < cur_dt_delta < end_dt:
                        # In work day schedule
                        diff_after = (start_dt - cur_dt).seconds
                        if cur_dt > start_dt:
                            diff_after = 0
                        respond_time = diff_after + avg_respond
                    elif cur_dt_delta <= start_dt:
                        # Before schedule day's start
                        diff_before = (start_dt - cur_dt).seconds
                        respond_time = diff_before + avg_respond

                    respond_list.append(respond_time)

                    cur_minute = weekday * minutes_per_day + hour * minutes_per_hour + minute
                    minutes_list.append(cur_minute)

        intervals = np.array(minutes_list)
        responds = np.array(respond_list)

        itp_func = self._get_itp_f(intervals, responds)

        respond_table = self._get_output_table(itp_func, interval_minutes, by_weekday=False)

        return respond_table

    @staticmethod
    def _get_output_table(func: interp1d, interval_minutes: int, by_weekday: bool = True):
        """Gets an output table of calculated respond time"""

        columns = ['Weekday', 'StartTime', 'EndTime', 'RespondTime']
        strings = []

        days_per_week = 7
        hours_per_day = 24
        minutes_per_hour = 60
        minutes_per_day = hours_per_day * minutes_per_hour

        for weekday in range(days_per_week):
            for hour in range(hours_per_day):
                for minute in np.arange(0, minutes_per_hour, interval_minutes):
                    new_weekday = weekday
                    if not by_weekday:
                        new_weekday = 0

                    today_minutes = new_weekday * minutes_per_day + hour * minutes_per_hour + minute
                    start_time = datetime.time(hour, minute, 0)
                    if minute + interval_minutes == minutes_per_hour:
                        if hour + 1 == hours_per_day:
                            end_time = datetime.time(0, 0, 0)
                        else:
                            end_time = datetime.time(hour + 1, 0, 0)
                    else:
                        end_time = datetime.time(hour, minute + interval_minutes, 0)
                    respond_time = int(abs(func(today_minutes)))
                    strings.append([weekday + 1, start_time, end_time, respond_time])

        respond_table = pd.DataFrame(strings, columns=columns)

        return respond_table

    @staticmethod
    def _get_x_y(table: pd.DataFrame, by_weekday: bool = True):
        """Gets x as minute of datetime and y as respond time"""

        minutes_list = []
        respond_list = []

        for index, row in table.iterrows():
            total_minute = (row['Weekday'] - 1 if by_weekday else 0) * 24 * 60 + row['Hour'] * 60 + row['Minute']
            if total_minute in minutes_list:
                continue
            minutes_list.append(total_minute)
            respond_list.append(row['TotalTimeSec'])

        if minutes_list:
            min_minute = 0
            max_minute = (7 if by_weekday else 1) * 24 * 60 - 1

            if min_minute not in minutes_list:
                first_respond = respond_list[0]
                minutes_list.insert(0, min_minute)
                respond_list.insert(0, first_respond)

            if max_minute not in minutes_list:
                last_respond = respond_list[-1]
                minutes_list.append(max_minute)
                respond_list.append(last_respond)

        arr_x = np.array(minutes_list)
        arr_y = np.array(respond_list)

        return arr_x, arr_y

    @staticmethod
    def _get_itp_f(minutes: np.ndarray, respond_time: np.ndarray):
        """Gets an interpolating function respond_time(minutes)"""

        return interp1d(minutes, respond_time, kind='linear')

    @staticmethod
    def _start_end_time(schedule):
        time_format = '%H:%M'
        default_time = datetime.datetime.strptime('0:0', time_format)
        if not schedule:
            return default_time, default_time

        start_end_list = schedule.split('-')

        # Schedule start/end time of weekday
        if start_end_list:
            start_time_str = start_end_list[0].strip()
            try:
                start_dt = datetime.datetime.strptime(start_time_str, time_format)
            except ValueError as e:
                start_dt = default_time
        else:
            start_dt = default_time

        if len(start_end_list) > 1:
            end_time_str = start_end_list[1].strip()
            try:
                end_dt = datetime.datetime.strptime(end_time_str, time_format)
            except ValueError as e:
                end_dt = default_time
        else:
            end_dt = default_time

        return start_dt, end_dt

class RespondSettings:
    _settings = {}

    def __init__(self):
        curr_dir = os.getcwd()
        if curr_dir[-1] != '\\':
            curr_dir += '\\'

        file_name = curr_dir + 'settings.ini'
        with open(file_name, 'r') as file:
            for line in file.readlines():
                set_arr = line.split('=')
                key = set_arr[0].strip()
                value = line.replace(key + '=', '').strip()
                self._settings[key] = value

    def get_setting(self, name):
        return self._settings.get(name, '')

class Smoothing:
    """A class of smoothing methods"""

    @staticmethod
    def savgol_smoothing(
            x: np.ndarray,
            y: np.ndarray,
            new_x: np.ndarray,
            window: int = 101,
            order: int = 3):
        """Savitzky-Golay smoothing"""

        itp_func = interp1d(x, y, kind='linear')
        y_smooth = savgol_filter(itp_func(new_x), window, order)

        return y_smooth

    @staticmethod
    def mab_smoothing(y: np.ndarray, box_pts: int = 3):
        """Moving average boxes smoothing"""

        box = np.ones(box_pts) / box_pts
        y_smooth = np.convolve(y, box, mode='same')

        return y_smooth

    @staticmethod
    def triangle_smoothing(
            y: np.ndarray,
            degree: int = 3,
            drop_values: bool = False):
        """Triangle smoothing"""

        triangle = np.array(list(range(degree)) + [degree] + list(range(degree)[::-1])) + 1
        y_smooth = []

        for i in range(degree, len(y) - degree * 2):
            point = y[i:i + len(triangle)] * triangle
            y_smooth.append(sum(point) / sum(triangle))

        if drop_values:
            return y_smooth

        y_smooth = [y_smooth[0]] * int(degree + degree / 2) + y_smooth
        while len(y_smooth) < len(y):
            y_smooth.append(y_smooth[-1])

        return y_smooth

    @staticmethod
    def median_smoothing(y: np.ndarray, kernel_size: int = 3):
        """Median smoothing"""

        y_smooth = medfilt(y, kernel_size)

        return y_smooth


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('group_discussion', '0003_comment_parent'),
    ]

    operations = [
        migrations.CreateModel(
            name='TopicUser',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('topic', models.ForeignKey(to='group_discussion.Topic')),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


#!/usr/bin/env python3
import os
import json
import backoff
import requests

import pandas as pd
from io import StringIO
from datetime import datetime, timedelta

import singer
from singer import utils, metadata
from singer.catalog import Catalog, CatalogEntry
from singer.schema import Schema
from singer.transform import transform

REQUIRED_CONFIG_KEYS = ["start_date", "client_id", "client_secret"]
DEFAULT_CONVERSION_WINDOW = 5
LOGGER = singer.get_logger()
URLS = {
    "get_token": "https://apis.indeed.com/oauth/v2/tokens",
    "list_employers": "https://secure.indeed.com/v2/api/appinfo",
    "ads": "https://apis.indeed.com/ads/v1/stats",
    "list_campaigns": "https://apis.indeed.com/ads/v1/campaigns",
    "campaigns": "https://apis.indeed.com/ads/v1/campaigns/{campaign_id}/stats"
}

class IndeedError(Exception):
    def __init__(self, msg):
        self.msg = msg
        super().__init__(self.msg)

def get_token(config, employer_id):
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded',
        'Accept': 'application/json'
    }
    data = {
        'grant_type': 'client_credentials',
        'client_id': config['client_id'],
        'client_secret': config['client_secret']
    }

    if employer_id:
        data["scope"] = 'employer.advertising.campaign_report.read employer.advertising.campaign.read'
        data["employer"] = employer_id
    else:
        data["scope"] = 'employer_access'

    url = URLS["get_token"]
    response = requests.post(url, headers=headers, data=data)
    return response.json()

def refresh_access_token_if_expired(config, employer_id):
    # if [expires_at not exist] or if [exist and less than current time] then it will update the token
    if config["employers"][employer_id].get('expires_at') is None or \
            config["employers"][employer_id].get('expires_at') < datetime.utcnow():
        res = get_token(config, employer_id)
        config["employers"][employer_id]['access_token'] = res["access_token"]
        config["employers"][employer_id]["expires_at"] = datetime.utcnow() + timedelta(seconds=res["expires_in"])
        return True
    return False

def get_key_properties(stream_id):
    keys = {
        "ads": ["Ad_ID", "Date", "Job_Reference_Number", "Platform"],
        "campaigns": ["Id", "Date"],
    }
    return keys[stream_id]

def get_bookmark(stream_id):
    bookmarks = {
        "ads": "Date",
        "campaigns": "Date"
    }
    return bookmarks[stream_id]

def get_abs_path(path):
    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)

def load_schemas():
    """ Load schemas from schemas folder """
    schemas = {}
    for filename in os.listdir(get_abs_path('schemas')):
        path = get_abs_path('schemas') + '/' + filename
        file_raw = filename.replace('.json', '')
        with open(path) as file:
            schemas[file_raw] = Schema.from_dict(json.load(file))
    return schemas

def create_metadata_for_report(stream_id, schema, key_properties):
    replication_key = get_bookmark(stream_id)
    mdata = [{"breadcrumb": [], "metadata": {"inclusion": "available",
                                             "forced-replication-method": "INCREMENTAL",
                                             "valid-replication-keys": [replication_key]
                                             }}]

    if key_properties:
        mdata[0]["metadata"]["table-key-properties"] = key_properties

    for key in schema.properties:
        # hence, when property is object, we will only consider properties of that object without taking object itself.
        if "object" in schema.properties.get(key).type and schema.properties.get(key).properties:
            inclusion = "available"
            mdata.extend(
                [{"breadcrumb": ["properties", key, "properties", prop], "metadata": {"inclusion": inclusion}} for prop
                 in schema.properties.get(key).properties])
        else:
            inclusion = "automatic" if key in key_properties else "available"
            mdata.append({"breadcrumb": ["properties", key], "metadata": {"inclusion": inclusion}})

    return mdata

def discover():
    raw_schemas = load_schemas()
    streams = []
    for stream_id, schema in raw_schemas.items():
        stream_metadata = create_metadata_for_report(stream_id, schema, get_key_properties(stream_id))
        key_properties = get_key_properties(stream_id)
        streams.append(
            CatalogEntry(
                tap_stream_id=stream_id,
                stream=stream_id,
                schema=schema,
                key_properties=key_properties,
                metadata=stream_metadata
            )
        )
    return Catalog(streams)

@backoff.on_exception(backoff.expo, IndeedError, max_tries=5, factor=2)
@utils.ratelimit(1, 1)
def request_data(url, request_type, headers, payload, config, employer_id=None):
    if request_type == 'POST':
        response = requests.post(url, headers=headers, data=payload)
    else:
        response = requests.get(url, headers=headers, params=payload)

    if response.status_code == 429:
        raise IndeedError(response.text)
    elif response.status_code == 401:
        # If token expired, it'll fetch new token and retry again.
        refresh_access_token_if_expired(config, employer_id)
        headers.update({'Authorization': f'Bearer {config["employers"][employer_id]["access_token"]}'})
        raise IndeedError("Token Expired, Refreshing...")
    elif response.status_code not in [200, 202]:
        raise Exception(response.text)
    return response

def get_paginated_data(url, data_key, headers, params, config, employer_id):
    """
    Retrieve data in pagination if next page url is available in response.
    """

    all_items = []
    next_page_url = url
    while True:
        response = request_data(
            next_page_url,
            "GET",
            headers,
            payload=params,
            config=config,
            employer_id=employer_id
        ).json()

        all_items += response["data"][data_key]
        next_page_location = [link["href"].split("?")[1] for link in response["meta"]["links"] if link["rel"] == "next"]

        if not next_page_location:
            break
        next_page_url = url + "?" + next_page_location[0]
    return all_items

def get_list_campaign_ids(config, employer_id):
    """
    List all campaigns under given employer account
    """

    headers = {'Authorization': 'Bearer ' + config["employers"][employer_id]['access_token']}
    params = {
        'perPage': 1000
    }

    all_campaigns = get_paginated_data(
        url=URLS["list_campaigns"],
        data_key="Campaigns",
        headers=headers,
        params=params,
        config=config,
        employer_id=employer_id
    )

    return [c["Id"] for c in all_campaigns]

def datetime_to_str(dt):
    return dt.strftime('%Y-%m-%d')

def get_next_date(_date: str):
    return datetime_to_str(datetime.strptime(_date, '%Y-%m-%d') + timedelta(days=1))

def get_valid_start_date(date_to_poll, conversion_window):
    """
    fix for data freshness
    e.g. Sunday's data is available at 3 AM UTC on Monday
    If integration is set to sync at 1AM then a problem occurs
    """

    utcnow = datetime.utcnow()
    date_to_poll = datetime.strptime(date_to_poll, "%Y-%m-%d")

    if date_to_poll >= utcnow - timedelta(days=conversion_window):
        date_to_poll = utcnow - timedelta(days=conversion_window)

    return date_to_poll.strftime("%Y-%m-%d")

def sync_ads(config, state, stream, employer_id):
    mdata = metadata.to_map(stream.metadata)
    schema = stream.schema.to_dict()

    singer.write_schema(
        stream_name=stream.tap_stream_id,
        schema=schema,
        key_properties=stream.key_properties,
    )

    # {bookmarks: {stream_id: {<employer_id>: <date>}}}
    start_date = singer.get_bookmark(state, stream.tap_stream_id, employer_id, config["start_date"])
    conversion_window = config.get("conversion_window", DEFAULT_CONVERSION_WINDOW)
    start_date = get_valid_start_date(start_date, conversion_window)

    bookmark = start_date
    stats = pd.DataFrame()
    today = datetime_to_str(datetime.utcnow().date())

    headers = {'Authorization': 'Bearer ' + config["employers"][employer_id]['access_token']}
    while bookmark <= today:
        next_date = get_next_date(bookmark)
        params = {
            'startDate': bookmark,
            'endDate': next_date
        }
        LOGGER.info("Querying Ads States, employer_id: %s,  Date: %s", employer_id, bookmark)

        # Pagination is not supported for the ads states
        tmp = request_data(URLS[stream.tap_stream_id], "GET", headers, payload=params, config=config, employer_id=employer_id).json()
        data_url = tmp['meta']['rootLocation'] + tmp['data']['location']

        tap_data = request_data(data_url, "GET", headers, payload=params, config=config, employer_id=employer_id)
        tap_data = pd.read_csv(StringIO(tap_data.text))

        if len(tap_data) != 0:
            tap_data = pd.merge(
                tap_data,
                tap_data['Spend (currency)'].fillna("0.00 -").str.split(' ', expand=True).rename(columns={0: 'Spend', 1: 'Currency'}),
                left_index=True,
                right_index=True
            )

            tap_data['Spend'] = pd.to_numeric(tap_data['Spend'])
            tap_data['Date'] = bookmark
            tap_data['EmployerName'] = config["employers"][employer_id]['name']

            stats = pd.concat([
                stats,
                tap_data]
            )
            records = stats.reset_index(drop=True)
            # Replace 'nan' with 0.0 for "Spend" column.
            records["Spend"] = records["Spend"].fillna(0).astype('float')
            records['EmployerId'] = employer_id
            records.columns = records.columns.str.replace(" ", "_")
            records = records.to_dict("records")

            with singer.metrics.record_counter(stream.tap_stream_id) as counter:
                for row in records:
                    transformed_data = transform(row, schema, metadata=mdata)

                    # write one or more rows to the stream:
                    singer.write_records(stream.tap_stream_id, [transformed_data])
                    counter.increment()

            # if there is data, then only we will print state
            if records:
                state = singer.write_bookmark(state, stream.tap_stream_id, employer_id, bookmark)
                singer.write_state(state)

        bookmark = next_date

def sync_campaigns(config, state, stream, employer_id):
    mdata = metadata.to_map(stream.metadata)
    schema = stream.schema.to_dict()

    singer.write_schema(
        stream_name=stream.tap_stream_id,
        schema=schema,
        key_properties=stream.key_properties,
    )

    today = datetime_to_str(datetime.utcnow().date())

    # {bookmarks: {stream_id: {<employer_id>: <date>}}}
    start_date = singer.get_bookmark(state, stream.tap_stream_id, employer_id, config["start_date"])
    conversion_window = config.get("conversion_window", DEFAULT_CONVERSION_WINDOW)
    start_date = get_valid_start_date(start_date, conversion_window)

    headers = {'Authorization': 'Bearer ' + config["employers"][employer_id]['access_token']}

    bookmark = start_date
    while bookmark <= today:
        next_date = get_next_date(bookmark)
        for cid in get_list_campaign_ids(config, employer_id):
            params = {
                'startDate': bookmark,
                'endDate': next_date,
                'perPage': 1000
            }
            LOGGER.info("Querying Campaign States, employer_id: %s, Campaign_Id: %s, Date: %s", employer_id, cid, bookmark)
            all_states = get_paginated_data(
                url=URLS[stream.tap_stream_id].format(campaign_id=cid),
                data_key="entries",
                headers=headers,
                params=params,
                config=config,
                employer_id=employer_id
            )

            records = pd.DataFrame(all_states)

            records['EmployerId'] = employer_id
            records['EmployerName'] = config["employers"][employer_id]['name']
            records.columns = records.columns.str.replace(" ", "_")
            records = records.to_dict("records")
            with singer.metrics.record_counter(stream.tap_stream_id) as counter:
                for row in records:
                    transformed_data = transform(row, schema, metadata=mdata)

                    # write one or more rows to the stream:
                    singer.write_records(stream.tap_stream_id, [transformed_data])
                    counter.increment()

        state = singer.write_bookmark(state, stream.tap_stream_id, employer_id, bookmark)
        singer.write_state(state)
        bookmark = next_date

def sync(config, state, catalog):
    # Get access token to list all employer_ids(employers)
    res = get_token(config, employer_id=None)
    headers = {'Authorization': f'Bearer {res["access_token"]}'}
    employers = request_data(URLS["list_employers"], "POST", headers=headers, payload={}, config=config)

    # get access token for individual employer_ids
    config["employers"] = {}
    for emp in employers.json()['employers']:
        res = get_token(config, employer_id=emp['id'])
        config["employers"][emp['id']] = {
            'name': emp['name'],
            'access_token': res['access_token'],
            "expires_at": datetime.utcnow() + timedelta(seconds=res["expires_in"])
        }

    # Loop over all employer accounts
    for emp in employers.json()['employers']:
        # Loop over selected streams in catalog
        for stream in catalog.get_selected_streams(state):
            LOGGER.info("Syncing stream:" + stream.tap_stream_id)

            if stream.tap_stream_id == "ads":
                sync_ads(config, state, stream, employer_id=emp["id"])
            elif stream.tap_stream_id == "campaigns":
                sync_campaigns(config, state, stream, employer_id=emp["id"])
    return

@utils.handle_top_exception(LOGGER)
def main():
    # Parse command line arguments
    args = utils.parse_args(REQUIRED_CONFIG_KEYS)

    if args.discover:
        catalog = discover()
        catalog.dump()
    else:
        if args.catalog:
            catalog = args.catalog
        else:
            catalog = discover()
        sync(args.config, args.state, catalog)

if __name__ == "__main__":
    main()


from threading import *
import time
import cv2
import socket
import io
import pickle
import struct

global connection
global address
cam=cv2.VideoCapture(0)
#stream = io.BytesIO()
client_socket=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client_socket.connect(('192.168.1.5', 9000))#192.168.43.205

cam.set(3, 640);
cam.set(4, 480);

img_counter = 0
print('sending')

while True:
	ret, frame = cam.read()
	data = pickle.dumps(frame, 0)
	size = len(data)

	print("frames:","{}: {}".format(img_counter, size))
	len1=str(size)

	#sending frame size
	client_socket.send(len1.encode())

	#receiving acknowledgement
	ack=client_socket.recv(2)

	#sending frame
	client_socket.send(data)

	buy=client_socket.recv(2)
	print("ack for vinnu: ",buy)

	img_counter += 1
	if cv2.waitKey(1) & 0xFF == ord('q'):
		break

	time.sleep(0.03)

cam.release()


import torch
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer, logging as tf_logging

from model_utils import predict
from utils import (
    HitsMetric,
    adjust_top_k,
    get_args,
    get_filename,
    load_data,
    prepare_input,
    update_history,
    update_metric,
    write_results,
)

tf_logging.set_verbosity_error()

if __name__ == "__main__":
    args = get_args()

    test_data, head_search_space, tail_search_space = load_data(args)

    adjust_top_k(test_data, args)

    tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.tokenizer_revision)
    tokenizer.pad_token_id = tokenizer.eos_token_id
    model = AutoModelForCausalLM.from_pretrained(
        args.model,
        torch_dtype=torch.float16 if args.fp16 else torch.float32,
        device_map="auto",
    )
    model.eval()
    print(f"model is loaded on device {model.device.type}")

    metric = HitsMetric()
    filename = get_filename(args)
    with torch.no_grad(), open(filename, "w", encoding="utf-8") as writer, tqdm(test_data) as pbar:
        for i, (x, direction) in enumerate(pbar):
            if i % args.world_size != args.rank:
                continue

            if direction == "tail":
                search_space = head_search_space
            elif direction == "head":
                search_space = tail_search_space
            else:
                raise ValueError

            model_input, candidates = prepare_input(x, search_space, args, return_prompt=True)

            predictions = predict(tokenizer, model, model_input, args)

            update_history(x, search_space, predictions, candidates, args)

            example = write_results(x, predictions, candidates, direction, writer, args)

            update_metric(example, metric, args)
            pbar.set_postfix(metric.dump())


import pandas as pd
from sklearn.model_selection import train_test_split, cross_val_predict
from sklearn.metrics import classification_report, accuracy_score
from sklearn.utils.validation import column_or_1d
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import matplotlib.pyplot as plt

datasets = ["a_affirmative", "a_conditional", "a_doubt_question", "a_emphasis", "a_negative", "a_relative", "a_topics", "a_wh_question", "a_yn_question",
             "b_affirmative", "b_conditional", "b_doubt_question", "b_emphasis", "b_negative", "b_relative", "b_topics", "b_wh_question", "b_yn_question"]

bestsParams = {}

maxd= [20,10,30,30,10,50,10,None,20,20,20,None,None,20,50,20,30,30]
maxf= [4,18,'log2','log2','auto','auto','auto','auto','auto','log2','sqrt',8,4,'log2',None,'log2',14,8]
mins= [12,12,20,20,12,20,20,12,20,20,20,20,12,20,20,20,20,20]
c= ["gini", "entropy","entropy","entropy","gini","gini","gini","entropy","gini","entropy","gini","entropy","gini","entropy","entropy","entropy","gini","gini"]
estimators=[10,20,50,100,200]

# writer= open("ForestPREPROCESSADO/forestparams.txt", "w+")

scorelist = []
combinations = range(0, 5)  # N de combinacoes

plt.title("ROC AUC Score Combinations for Random Forest")
for cnt, d in enumerate(datasets):

    X_train = pd.read_csv("../SplitPreprocessedData/{}_X_train.csv".format(d))
    X_test = pd.read_csv("../SplitPreprocessedData/{}_X_test.csv".format(d))
    y_train = pd.read_csv("../SplitPreprocessedData/{}_y_train.csv".format(d))
    y_train = column_or_1d(y_train, warn=True)
    y_test = pd.read_csv("../SplitPreprocessedData/{}_y_test.csv".format(d))
    y_test = column_or_1d(y_test, warn=True)
    # max_score=0
    # scorelist.clear()
    roc = []
    for e in estimators:

        clf = RandomForestClassifier(n_estimators=e, criterion=c[cnt],min_samples_split=mins[cnt],max_features=maxf[cnt],max_depth=maxd[cnt])
        y_train_proba = cross_val_predict(clf, X_train, y_train, cv=10, method="predict_proba")
        y_scores = y_train_proba[:, 1]
        score = roc_auc_score(y_train, y_scores)
        roc.append(score)
        # if (score > max_score):
        #     bestsParams[d]={
        #             "n_estimators":e,
        #             "ROC":score
        #             }
        #     max_score = score
    plt.plot(combinations, roc, label=d)
    # with open("ForestPREPROCESSADO/{}.txt".format(d), "w+") as wp:
    #     for x in scorelist:
    #         wp.write(str(x))
    #         wp.write('\n')

    # print(d)
    # print('\n')
    # print(str(bestsParams[d]))
    # print('\n')

# for i in bestsParams:
#     writer.write(str(i))
#     writer.write('\n')
#     writer.write(str(bestsParams[i]))
#     writer.write('\n')

# writer.close()

plt.legend()
plt.ylabel("ROC AUC")
plt.xlabel("Combination")
plt.xticks(list(combinations))
plt.legend()

plt.show()


import logging
from six import add_metaclass
from abc import ABCMeta, abstractmethod
from agentml.parser import Element
from agentml.common import attribute
from agentml.parser.trigger.response import Response

@add_metaclass(ABCMeta)
class BaseCondition(object):
    """
    AgentML Base Condition class
    """
    def __init__(self, agentml, element, **kwargs):
        """
        Initialize a new Base Condition instance
        :param agentml: The parent AgentML instance
        :type  agentml: AgentML

        :param element: The XML Element object
        :type  element: etree._Element

        :param kwargs: Default attributes
        """
        self.agentml = agentml
        self._element = element

        # Containers and default attributes
        self.statements = []
        self.else_statement = None

        self.type = kwargs['type'] if 'type' in kwargs else attribute(self._element, 'type', 'user_var')
        self._log = logging.getLogger('agentml.parser.trigger.condition')

    def evaluate(self, user):
        """
        Evaluate the conditional statement and return its contents if a successful evaluation takes place
        :param user: The active user object
        :type  user: agentml.User or None

        :return: True if the condition evaluates successfully, otherwise False
        :rtype : bool
        """
        for statement in self.statements:
            evaluated = statement.evaluate(self.agentml, user)
            if evaluated:
                return evaluated

        return self.else_statement or False

    @abstractmethod
    def get_contents(self, element):
        """
        Retrieve the contents of an element
        :param element: The XML Element object
        :type  element: etree._Element

        :return: A list of text and/or XML elements
        :rtype : list of etree._Element or str
        """
        pass

    def _parse(self):
        """
        Loop through all child elements and execute any available parse methods for them
        """
        self.type = attribute(self._element, 'type') or self.type

        for child in self._element:
            method_name = '_parse_{0}'.format(str(child.tag))  # TODO: This is a hack, skip comment objects here

            if hasattr(self, method_name):
                parse = getattr(self, method_name)
                parse(child)

    def _parse_if(self, element):
        """
        Parse the if statement
        :param element: The XML Element object
        :type  element: etree._Element
        """
        # Get the key
        name = attribute(element, 'name')
        cond_type = attribute(element, 'type', self.type)

        # Get the comparison operator and its value (if implemented)
        operator = None
        value = None
        for o in ConditionStatement.operators:
            if o in element.attrib:
                operator = o
                value = element.attrib[operator]
                break

        # Get the contents of the element in tuple form and append our if statement
        contents = tuple(self.get_contents(element))
        self.statements.append(ConditionStatement(cond_type, operator, contents, value, name))

    def _parse_elif(self, element):
        """
        Parse an elif statement
        :param element: The XML Element object
        :type  element: etree._Element
        """
        self._parse_if(element)

    def _parse_else(self, element):
        """
        Parse the else statement
        :param element: The XML Element object
        :type  element: etree._Element
        """
        self.else_statement = self.get_contents(element)

class Condition(Element, BaseCondition):
    """
    AgentML Condition object
    """
    def __init__(self, trigger, element, file_path):
        """
        Initialize a new Condition instance
        :param trigger: The Trigger instance
        :type  trigger: Trigger

        :param element: The XML Element object
        :type  element: etree._Element

        :param file_path: The absolute path to the AgentML file
        :type  file_path: str
        """
        self.trigger = trigger
        BaseCondition.__init__(self, trigger.agentml, element)
        Element.__init__(self, trigger.agentml, element, file_path)
        self._log = logging.getLogger('agentml.parser.trigger.condition')

    def get_contents(self, element):
        """
        Retrieve the contents of an element
        :param element: The XML Element object
        :type  element: etree._Element

        :return: A list of responses
        :rtype : list of Response
        """
        return [Response(self.trigger, child, self.file_path)
                for child in element if child.tag in ['response', 'template']]

class ConditionStatement:
    """
    Condition Statement object
    """
    # Condition operators
    IS = 'is'
    IS_NOT = 'is_not'
    GREATER_THAN = 'gt'
    GREATER_THAN_OR_EQUAL = 'gte'
    LESS_THAN = 'lt'
    LESS_THAN_OR_EQUAL = 'lte'

    operators = [IS, IS_NOT, GREATER_THAN, GREATER_THAN_OR_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL]

    def __init__(self, cond_type, operator, contents, value=None, name=None):
        """
        Initialize a new Condition Statement object
        :param cond_type: The type of the condition statement
        :type  cond_type: str

        :param operator: The operator of the condition statement
        :type  operator: str

        :param contents: The contents of the condition statement
        :type  contents: tuple

        :param value: The value of the condition statement
        :type  value: str, int, float or None

        :param name: The name of the variable if the condition type is USER_VAR or GLOBAL_VAR
        :type  name: str
        """
        self.type = cond_type
        self.operator = operator
        self.contents = contents
        self.value = value
        self.name = name
        self._log = logging.getLogger('agentml.parser.trigger.condition.statement')

    def evaluate(self, agentml, user=None):
        """
        Evaluate the conditional statement and return its contents if a successful evaluation takes place
        :param user: The active user object
        :type  user: agentml.User or None

        :param agentml: The active AgentML instance
        :type  agentml: AgentML

        :return: Condition contents if the condition evaluates successfully, otherwise False
        :rtype : tuple or bool
        """
        self._log.debug('Evaluating conditional statement: {statement}'
                        .format(statement=' '.join(filter(None, [self.type, self.name, self.operator, self.value]))))

        # Get the value of our key type
        if self.type not in agentml.conditions:
            self._log.error('Unknown condition type, "{type}", unable to evaluate condition statement'
                            .format(type=self.type))
            return

        key_value = agentml.conditions[self.type].get(agentml, user, self.name)

        # Atomic comparisons
        if self.operator is None and key_value:
            return self.contents

        if (self.operator == self.IS) and (key_value == self.value):
            return self.contents

        if (self.operator == self.IS_NOT) and (key_value != self.value):
            return self.contents

        # All remaining self.operators are numeric based, so key_value must contain a valid integer or float
        try:
            key_value = float(key_value)
            value = float(self.value)
        except (ValueError, TypeError):
            return False

        # Numeric comparisons
        if (self.operator == self.GREATER_THAN) and (key_value > value):
            return self.contents

        if (self.operator == self.GREATER_THAN_OR_EQUAL) and (key_value >= value):
            return self.contents

        if (self.operator == self.LESS_THAN) and (key_value < value):
            return self.contents

        if (self.operator == self.LESS_THAN_OR_EQUAL) and (key_value <= value):
            return self.contents

        return False


import boto3
import numpy as np
from tempfile import  NamedTemporaryFile
from librosa.feature import melspectrogram
import librosa

boto_session = boto3.session.Session(region_name='eu-west-3')
s3 = boto_session.resource('s3')
sukikana_bucket = 'sukikana'

def get_logmels(song_id):
    with NamedTemporaryFile('wb+') as download_file:
        s3.Object(sukikana_bucket, song_id).download_fileobj(download_file)
        download_file.seek(0)
        offset = 0  # np.random.randint(26 * 1000) / 1000.0
        duration = 3.0
        y, sr = librosa.load(download_file.name, offset=offset, duration=duration)
        return np.log10(melspectrogram(y=y, sr=sr, n_fft=1024, hop_length=512, power=2))

if __name__ == '__main__':
    song_id = 'SOAAAKE12A8C1397E9'
    feature_sequence = get_logmels(song_id)


from fastapi import APIRouter, HTTPException
from models.post import Post
from services.post_service import *

posts = APIRouter()

@posts.get("/posts/")
def get_all_post():
    try:
        return get_all()
    except Exception as e:
        return HTTPException(status_code= 400, detail= str(e))

@posts.get("/posts/{id}")
def get_post_by_id(id):
    try:
        return get_by_id(id)
    except Exception as e:
        return HTTPException(status_code= 400, detail= str(e))

@posts.post("/posts/")
def create(post : Post):
    try:
        create_post(post)
    except Exception as e:
        return HTTPException(status_code= 400, detail= str(e))

@posts.put("/posts/{id}")
def update(id, post: PostUpdateDto):
    try:
        post = update_post(id,post)
        return post
    except Exception as e:
        return HTTPException(status_code= 400, detail= str(e))

@posts.delete("/posts/{id}")
def delete(id):
    try:
        delete_post(id)
    except Exception as e:
        return HTTPException(status_code=400, detail=str(e))


import json
import logging
import random
import time
import copy
from argparse import ArgumentParser
from itertools import chain
from pprint import pformat
import copy
import numpy as np

import torch
import torch.nn.functional as F

from transformers import *
from VideoGPT2 import *
from train import SPECIAL_TOKENS, SPECIAL_TOKENS_DICT
from dataset import get_dataset, build_input_from_segments

def top_filtering(logits, top_k=0, top_p=0.0, threshold=-float('Inf'), filter_value=-float('Inf')):
    """ Filter a distribution of logits using top-k, top-p (nucleus) and/or threshold filtering
        Args:
            logits: logits distribution shape (vocabulary size)
            top_k: <=0: no filtering, >0: keep only top k tokens with highest probability.
            top_p: <=0.0: no filtering, >0.0: keep only a subset S of candidates, where S is the smallest subset
                whose total probability mass is greater than or equal to the threshold top_p.
                In practice, we select the highest probability tokens whose cumulative probability mass exceeds
                the threshold top_p.
            threshold: a minimal threshold to keep logits
    """
    assert logits.dim() == 1  # Only work for batch size 1 for now - could update but it would obfuscate a bit the code
    top_k = min(top_k, logits.size(-1))
    if top_k > 0:
        # Remove all tokens with a probability less than the last token in the top-k tokens
        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
        logits[indices_to_remove] = filter_value

    if top_p > 0.0:
        # Compute cumulative probabilities of sorted tokens
        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
        cumulative_probabilities = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)

        # Remove tokens with cumulative probability above the threshold
        sorted_indices_to_remove = cumulative_probabilities > top_p
        # Shift the indices to the right to keep also the first token above the threshold
        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
        sorted_indices_to_remove[..., 0] = 0

        # Back to unsorted indices and set them to -infinity
        indices_to_remove = sorted_indices[sorted_indices_to_remove]
        logits[indices_to_remove] = filter_value

    indices_to_remove = logits < threshold
    logits[indices_to_remove] = filter_value

    return logits

def sample_sequence(caption, history, tokenizer, model, args, current_output=None, video=None):
    special_tokens_ids = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS)
    if current_output is None:
        current_output = []
    for i in range(args.max_length):
        instance, sequence = build_input_from_segments(caption, history, current_output, tokenizer, with_eos=False, drop_caption=False)

        input_ids = torch.tensor(instance["input_ids"], device=args.device).unsqueeze(0)
        token_type_ids = torch.tensor(instance["token_type_ids"], device=args.device).unsqueeze(0)
        input_embs = model.transformer.wte(input_ids)
        if video is not None:
            input_embs = torch.cat([model.video_ff(video), input_embs], dim=1)
            token_type_ids = torch.cat([torch.ones((1, video.size(1))).long().cuda() * tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[-2]), token_type_ids], dim=1)

        logits = model(input_embs, token_type_ids=token_type_ids)
        if "gpt2" == args.model:
            logits = logits[0]
        logits = logits[0, -1, :] / args.temperature
        logits = top_filtering(logits, top_k=args.top_k, top_p=args.top_p)
        probs = F.softmax(logits, dim=-1)

        prev = torch.topk(probs, 1)[1] if args.no_sample else torch.multinomial(probs, 1)
        if i < args.min_length and prev.item() in special_tokens_ids:
            while prev.item() in special_tokens_ids:
                prev = torch.multinomial(probs, num_samples=1)

        if prev.item() in special_tokens_ids:
            break
        current_output.append(prev.item())

    return current_output

def beam_search(caption, history, tokenizer, model, args, current_output=None, video=None):
    special_tokens_ids = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS)
    if current_output is None:
        current_output = []
    hyplist = [([], 0., current_output)]
    best_state = None
    comp_hyplist = []

    for i in range(args.max_length):
        new_hyplist = []
        argmin = 0
        for out, lp, st in hyplist:
            instance, sequence = build_input_from_segments(caption, history, st, tokenizer, with_eos=False, drop_caption=False)

            input_ids = torch.tensor(instance["input_ids"], device=args.device).unsqueeze(0)
            token_type_ids = torch.tensor(instance["token_type_ids"], device=args.device).unsqueeze(0)
            input_embs = model.transformer.wte(input_ids)
            if video is not None:
                input_embs = torch.cat([model.video_ff(video), input_embs], dim=1)
                token_type_ids = torch.cat([torch.ones((1, video.size(1))).long().cuda() * tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[-2]), token_type_ids], dim=1)

            logits = model(input_embs, token_type_ids=token_type_ids)
            if "gpt2" == args.model:
                logits = logits[0]
            logp = F.log_softmax(logits, dim=-1)[:, -1, :]
            lp_vec = logp.cpu().data.numpy() + lp
            lp_vec = np.squeeze(lp_vec)
            if i >= args.min_length:
                new_lp = lp_vec[tokenizer.eos_token_id] + args.penalty * (len(out) + 1)
                comp_hyplist.append((out, new_lp))
                if best_state is None or best_state < new_lp:
                    best_state = new_lp
            count = 1
            for o in np.argsort(lp_vec)[::-1]:
                if o == tokenizer.unk_token_id or o == tokenizer.eos_token_id:
                    continue
                new_lp = lp_vec[o]
                if len(new_hyplist) == args.beam_size:
                    if new_hyplist[argmin][1] < new_lp:
                        new_st = copy.deepcopy(st)
                        new_st.append(int(o))
                        new_hyplist[argmin] = (out + [o], new_lp, new_st)
                        argmin = min(enumerate(new_hyplist), key=lambda h: h[1][1])[0]
                    else:
                        break
                else:
                    new_st = copy.deepcopy(st)
                    new_st.append(int(o))
                    new_hyplist.append((out + [o], new_lp, new_st))
                    if len(new_hyplist) == args.beam_size:
                        argmin = min(enumerate(new_hyplist), key=lambda h: h[1][1])[0]
                count += 1
        hyplist = new_hyplist
    if len(comp_hyplist) > 0:
        maxhyps = sorted(comp_hyplist, key=lambda h: -h[1])[:1]
        return maxhyps
    else:
        return [([], 0)]

def greedy_decode(caption, history, tokenizer, model, args, current_output=None, video=None):
    special_tokens_ids = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS)
    ys = []

    for i in range(args.max_length):
        instance, sequence = build_input_from_segments(caption, history, ys, tokenizer, with_eos=False, drop_caption=False)

        input_ids = torch.tensor(instance["input_ids"], device=args.device).unsqueeze(0)
        token_type_ids = torch.tensor(instance["token_type_ids"], device=args.device).unsqueeze(0)
        input_embs = model.transformer.wte(input_ids)
        if video is not None:
            input_embs = torch.cat([model.video_ff(video), input_embs], dim=1)
            token_type_ids = torch.cat([torch.ones((1, video.size(1))).long().cuda() * tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[-2]), token_type_ids], dim=1)

        logits = model(input_embs, token_type_ids=token_type_ids)
        if "gpt2" == args.model:
            logits = logits[0][0]
        logits = logits.cpu().data.numpy()
        next_word = np.argsort(logits[-1])[-1]
        if next_word == special_tokens_ids[1]:
            break
        ys.append(next_word)
    return ys

# Evaluation routine
def generate_response(model, data, dataset, args, ref_data=None):
    result_dialogs = []
    model.eval()
    with torch.no_grad():
        qa_id = 0
        for idx, dialog in enumerate(data['dialogs']):
            vid = dialog['image_id']
            out_dialog = dialog['dialog'][-1:]
            pred_dialog = {'image_id': vid,
                           'dialog': copy.deepcopy(out_dialog)}
            result_dialogs.append(pred_dialog)

            vgg = np.load("data/vggish_testset/"+vid+".npy")
            i3d_flow = np.load("data/i3d_flow_testset/"+vid+".npy")
            i3d_rgb = np.load("data/i3d_rgb_testset/"+vid+".npy")

            sample_i3d_flow = i3d_flow[range(1, i3d_flow.shape[0], 1)]
            sample_i3d_rgb = i3d_rgb[range(1, i3d_rgb.shape[0], 1)]

            vgg = torch.from_numpy(vgg).float().cuda()
            i3d_flow = torch.from_numpy(sample_i3d_flow).float().cuda()
            i3d_rgb = torch.from_numpy(sample_i3d_rgb).float().cuda()
            min_length = min([i3d_flow.size(0), i3d_rgb.size(0), vgg.size(0)])
            i3d = torch.cat([i3d_flow[:min_length], i3d_rgb[:min_length], vgg[:min_length]], dim=1).unsqueeze(0)

            for t, qa in enumerate(out_dialog):
                logging.info('%d %s_%d' % (qa_id, vid, t))
                logging.info('QS: ' + qa['question'])
                # prepare input data
                start_time = time.time()
                qa_id += 1

                if args.beam_search:
                   #hypstr = greedy_decode(dataset[idx]["caption"], dataset[idx]["history"], tokenizer, model, args, video=i3d)
                   hypstr = beam_search(dataset[idx]["caption"], dataset[idx]["history"], tokenizer, model, args, video=i3d)
                   hypstr = hypstr[0][0]
                else:
                   hypstr=sample_sequence(dataset[idx]["caption"], dataset[idx]["history"], tokenizer, model, args, video=i3d)
                hypstr=tokenizer.decode(hypstr, skip_special_tokens=True)
                logging.info('HYP: ' + hypstr)
                pred_dialog['dialog'][t]['answer'] = hypstr
                logging.info('ElapsedTime: %f' % (time.time() - start_time))
                logging.info('-----------------------')

    return {'dialogs': result_dialogs}

##################################
# main
if __name__ =="__main__":
    parser = ArgumentParser()
    parser.add_argument("--model", type=str, default="gpt2", help="Model type (gpt or gpt2)")
    parser.add_argument("--model_checkpoint", type=str, default="log_without_caption_with_valid/", help="Path, url or short name of the model")
    parser.add_argument("--max_history", type=int, default=3, help="Number of previous utterances to keep in history")
    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device (cuda or cpu)")

    parser.add_argument("--no_sample", action='store_true', help="Set to use greedy decoding instead of sampling")
    parser.add_argument("--beam_search", action='store_true', help="Set to use beam search instead of sampling")
    parser.add_argument("--beam_size", type=int, default=5, help="Beam size")
    parser.add_argument("--max_length", type=int, default=20, help="Maximum length of the output utterances")
    parser.add_argument("--min_length", type=int, default=1, help="Minimum length of the output utterances")
    parser.add_argument("--penalty", type=float, default=0.3, help="elngth penalty")
    parser.add_argument("--seed", type=int, default=42, help="Seed")
    parser.add_argument("--temperature", type=int, default=0.7, help="Sampling softmax temperature")
    parser.add_argument("--top_k", type=int, default=0, help="Filter top-k tokens before sampling (<=0: no filtering)")
    parser.add_argument("--top_p", type=float, default=0.9, help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)")
    parser.add_argument("--test_set", type=str, default="data/test_set4DSTC8-AVSD.json")
    parser.add_argument("--lbl_test_set", type=str, default="data/lbl_undisclosedonly_test_set4DSTC7-AVSD.json")
    parser.add_argument("--output", type=str, default="result.json")

    args = parser.parse_args()
    for arg in vars(args):
        print("{}={}".format(arg, getattr(args, arg)))

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s %(levelname)s: %(message)s')

    logging.info('Loading model params from ' + args.model_checkpoint)

    tokenizer_class = GPT2Tokenizer if "gpt2" == args.model else OpenAIGPTTokenizer
    tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
    tokenizer.add_special_tokens(SPECIAL_TOKENS_DICT)
    model_class = VideoGPT2LMHeadModel if "gpt2" == args.model else OpenAIGPTLMHeadModel
    model_config = GPT2Config.from_pretrained(args.model_checkpoint)
    model = model_class.from_pretrained(args.model_checkpoint+"checkpoint_mymodel_4.pth", config=model_config)
    model.to(args.device)
    model.eval()

    logging.info('Loading test data from ' + args.test_set)
    test_data = json.load(open(args.test_set,'r'))
    test_dataset = get_dataset(tokenizer, args.test_set, undisclosed_only=True, n_history=args.max_history)
    # generate sentences
    logging.info('-----------------------generate--------------------------')
    start_time = time.time()
    result = generate_response(model, test_data, test_dataset, args)
    logging.info('----------------')
    logging.info('wall time = %f' % (time.time() - start_time))
    if args.output:
        logging.info('writing results to ' + args.output)
        json.dump(result, open(args.output, 'w'), indent=4)
    logging.info('done')


from rest_framework import viewsets
from rest_framework.generics import get_object_or_404
from rest_framework.permissions import IsAuthenticated

from api.permissions import OnlyAuthorChangeContent
from api.serializers import PostSerializer, GroupSerializer, CommentSerializer
from posts.models import Post, Group

class PostViewSet(viewsets.ModelViewSet):
    queryset = Post.objects.all()
    serializer_class = PostSerializer
    permission_classes = [OnlyAuthorChangeContent, IsAuthenticated]

    def perform_create(self, serializer):
        serializer.save(author=self.request.user)

class GroupReadOnlyViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Group.objects.all()
    serializer_class = GroupSerializer

class CommentViewSet(viewsets.ModelViewSet):
    serializer_class = CommentSerializer
    permission_classes = [OnlyAuthorChangeContent, IsAuthenticated]

    def post_obj(self):
        queryset = get_object_or_404(Post, id=self.kwargs.get('post_id'))
        return queryset

    def get_queryset(self):
        queryset = self.post_obj().comments.all()
        return queryset

    def perform_create(self, serializer):
        serializer.save(author=self.request.user, post=self.post_obj())


import os
import logging
import ycm_core
from collections import defaultdict
from ycmd.completers.general_completer import GeneralCompleter
from ycmd import identifier_utils
from ycmd import utils
from ycmd.utils import ToUtf8IfNeeded
from ycmd import responses

SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX'

class IdentifierCompleter( GeneralCompleter ):
  def __init__( self, user_options ):
    super( IdentifierCompleter, self ).__init__( user_options )
    self._completer = ycm_core.IdentifierCompleter()
    self._tags_file_last_mtime = defaultdict( int )
    self._logger = logging.getLogger( __name__ )
    self._max_candidates = user_options[ 'max_num_identifier_candidates' ]

  def ShouldUseNow( self, request_data ):
    return self.QueryLengthAboveMinThreshold( request_data )

  def ComputeCandidates( self, request_data ):
    if not self.ShouldUseNow( request_data ):
      return []

    completions = self._completer.CandidatesForQueryAndType(
      ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ),
      ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) )

    completions = completions[ : self._max_candidates ]
    completions = _RemoveSmallCandidates(
      completions, self.user_options[ 'min_num_identifier_candidate_chars' ] )

    def ConvertCompletionData( x ):
        return responses.BuildCompletionData(
                insertion_text = x,
                extra_menu_info='[ID]' )

    return [ ConvertCompletionData( x ) for x in completions ]

  def AddIdentifier( self, identifier, request_data ):
    try:
      filetype = request_data[ 'filetypes' ][ 0 ]
    except KeyError:
      filetype = None
    filepath = request_data[ 'filepath' ]

    if not filetype or not filepath or not identifier:
      return

    vector = ycm_core.StringVector()
    vector.append( ToUtf8IfNeeded( identifier ) )
    self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath )
    self._completer.AddIdentifiersToDatabase( vector,
                                              ToUtf8IfNeeded( filetype ),
                                              ToUtf8IfNeeded( filepath ) )

  def AddPreviousIdentifier( self, request_data ):
    self.AddIdentifier(
      _PreviousIdentifier(
        self.user_options[ 'min_num_of_chars_for_completion' ],
        request_data ),
      request_data )

  def AddIdentifierUnderCursor( self, request_data ):
    cursor_identifier = _GetCursorIdentifier( request_data )
    if not cursor_identifier:
      return

    self.AddIdentifier( cursor_identifier, request_data )

  def AddBufferIdentifiers( self, request_data ):
    try:
      filetype = request_data[ 'filetypes' ][ 0 ]
    except KeyError:
      filetype = None
    filepath = request_data[ 'filepath' ]
    collect_from_comments_and_strings = bool( self.user_options[
      'collect_identifiers_from_comments_and_strings' ] )

    if not filetype or not filepath:
      return

    text = request_data[ 'file_data' ][ filepath ][ 'contents' ]
    self._logger.info( 'Adding buffer identifiers for file: %s', filepath )
    self._completer.ClearForFileAndAddIdentifiersToDatabase(
        _IdentifiersFromBuffer( text,
                                filetype,
                                collect_from_comments_and_strings ),
        ToUtf8IfNeeded( filetype ),
        ToUtf8IfNeeded( filepath ) )

  def AddIdentifiersFromTagFiles( self, tag_files ):
    absolute_paths_to_tag_files = ycm_core.StringVector()
    for tag_file in tag_files:
      try:
        current_mtime = os.path.getmtime( tag_file )
      except:
        continue
      last_mtime = self._tags_file_last_mtime[ tag_file ]

      # We don't want to repeatedly process the same file over and over; we only
      # process if it's changed since the last time we looked at it
      if current_mtime <= last_mtime:
        continue

      self._tags_file_last_mtime[ tag_file ] = current_mtime
      absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) )

    if not absolute_paths_to_tag_files:
      return

    self._completer.AddIdentifiersToDatabaseFromTagFiles(
      absolute_paths_to_tag_files )

  def AddIdentifiersFromSyntax( self, keyword_list, filetypes ):
    keyword_vector = ycm_core.StringVector()
    for keyword in keyword_list:
      keyword_vector.append( ToUtf8IfNeeded( keyword ) )

    filepath = SYNTAX_FILENAME + filetypes[ 0 ]
    self._completer.AddIdentifiersToDatabase( keyword_vector,
                                              ToUtf8IfNeeded( filetypes[ 0 ] ),
                                              ToUtf8IfNeeded( filepath ) )

  def OnFileReadyToParse( self, request_data ):
    self.AddBufferIdentifiers( request_data )
    if 'tag_files' in request_data:
      self.AddIdentifiersFromTagFiles( request_data[ 'tag_files' ] )
    if 'syntax_keywords' in request_data:
      self.AddIdentifiersFromSyntax( request_data[ 'syntax_keywords' ],
                                     request_data[ 'filetypes' ] )

  def OnInsertLeave( self, request_data ):
    self.AddIdentifierUnderCursor( request_data )

  def OnCurrentIdentifierFinished( self, request_data ):
    self.AddPreviousIdentifier( request_data )

# This looks for the previous identifier and returns it; this might mean looking
# at last identifier on the previous line if a new line has just been created.
def _PreviousIdentifier( min_num_candidate_size_chars, request_data ):
  def PreviousIdentifierOnLine( line, column ):
    nearest_ident = ''
    for match in identifier_utils.IdentifierRegexForFiletype(
        filetype ).finditer( line ):
      if match.end() <= column:
        nearest_ident = match.group()
    return nearest_ident

  line_num = request_data[ 'line_num' ] - 1
  column_num = request_data[ 'column_num' ] - 1
  filepath = request_data[ 'filepath' ]
  try:
    filetype = request_data[ 'filetypes' ][ 0 ]
  except KeyError:
    filetype = None

  contents_per_line = (
    request_data[ 'file_data' ][ filepath ][ 'contents' ].split( '\n' ) )

  ident = PreviousIdentifierOnLine( contents_per_line[ line_num ], column_num )
  if ident:
    if len( ident ) < min_num_candidate_size_chars:
      return ''
    return ident

  prev_line = contents_per_line[ line_num - 1 ]
  ident = PreviousIdentifierOnLine( prev_line, len( prev_line ) )
  if len( ident ) < min_num_candidate_size_chars:
    return ''
  return ident

def _RemoveSmallCandidates( candidates, min_num_candidate_size_chars ):
  if min_num_candidate_size_chars == 0:
    return candidates

  return [ x for x in candidates if len( x ) >= min_num_candidate_size_chars ]

def _GetCursorIdentifier( request_data ):
  try:
    filetype = request_data[ 'filetypes' ][ 0 ]
  except KeyError:
    filetype = None
  return identifier_utils.IdentifierAtIndex( request_data[ 'line_value' ],
                                             request_data[ 'column_num' ] - 1,
                                             filetype )

def _IdentifiersFromBuffer( text,
                            filetype,
                            collect_from_comments_and_strings ):
  if not collect_from_comments_and_strings:
    text = identifier_utils.RemoveIdentifierFreeText( text )
  idents = identifier_utils.ExtractIdentifiersFromText( text, filetype )
  vector = ycm_core.StringVector()
  for ident in idents:
    vector.append( ToUtf8IfNeeded( ident ) )
  return vector


import nltk
from nltk.corpus import wordnet

s=input('enter a negtive sentence : ')
#I am not happy
s=s.lower()

tmp=''
words=[]
for w in nltk.word_tokenize(s):
    word=w
    if tmp=='not':
        antonyms=[]
        for syn in wordnet.synsets(word):
            for s in syn.lemmas():
                for a in s.antonyms():
                    antonyms.append(a.name())
                    #print(a.name())
        if len(antonyms)>0:
            word=antonyms[0]

        else:
            word=tmp+'_'+w
        tmp=''
    if w=='not':
        tmp=w
    if w!='not':
        words.append(word)

s=' '.join(words)
print(s)


from __future__ import annotations

import logging
from pathlib import Path
from typing import TYPE_CHECKING

import pytest

from pyk.kcfg.show import KCFGShow
from pyk.proof import APRProof, APRProver, ProofStatus
from pyk.proof.show import APRProofNodePrinter
from pyk.testing import KCFGExploreTest, KProveTest
from pyk.utils import single

from ..utils import K_FILES

if TYPE_CHECKING:
    from collections.abc import Iterable
    from typing import Final, Union

    from pytest import TempPathFactory

    from pyk.kcfg import KCFGExplore
    from pyk.ktool.kprove import KProve

    STATE = Union[tuple[str, str], tuple[str, str, str]]

_LOGGER: Final = logging.getLogger(__name__)

APR_PROVE_TEST_DATA: Iterable[tuple[str, Path, str, str, int | None, int | None, Iterable[str], ProofStatus, int]] = (
    (
        'ensures-constraint-simplification',
        K_FILES / 'mini-kevm-spec.k',
        'MINI-KEVM-SPEC',
        'ensures-constraint',
        2,
        1,
        [],
        ProofStatus.PASSED,
        1,
    ),
)

def leaf_number(proof: APRProof) -> int:
    non_target_leaves = [nd for nd in proof.kcfg.leaves if not proof.is_target(nd.id)]
    return len(non_target_leaves) + len(proof.kcfg.predecessors(proof.target))

class TestMiniKEVM(KCFGExploreTest, KProveTest):
    KOMPILE_MAIN_FILE = K_FILES / 'mini-kevm.k'

    @pytest.mark.parametrize(
        'test_id,spec_file,spec_module,claim_id,max_iterations,max_depth,cut_rules,proof_status,expected_leaf_number',
        APR_PROVE_TEST_DATA,
        ids=[test_id for test_id, *_ in APR_PROVE_TEST_DATA],
    )
    def test_all_path_reachability_prove(
        self,
        kprove: KProve,
        kcfg_explore: KCFGExplore,
        test_id: str,
        spec_file: str,
        spec_module: str,
        claim_id: str,
        max_iterations: int | None,
        max_depth: int | None,
        cut_rules: Iterable[str],
        proof_status: ProofStatus,
        expected_leaf_number: int,
        tmp_path_factory: TempPathFactory,
    ) -> None:
        with tmp_path_factory.mktemp('apr_tmp_proofs') as proof_dir:
            claim = single(
                kprove.get_claims(
                    Path(spec_file), spec_module_name=spec_module, claim_labels=[f'{spec_module}.{claim_id}']
                )
            )
            proof = APRProof.from_claim(
                kprove.definition,
                claim,
                subproof_ids=[],
                logs={},
                proof_dir=proof_dir,
            )

            new_init_cterm = kcfg_explore.cterm_assume_defined(proof.kcfg.node(proof.init).cterm)
            proof.kcfg.replace_node(proof.init, new_init_cterm)
            kcfg_explore.simplify(proof.kcfg, {})

            prover = APRProver(
                proof,
                kcfg_explore=kcfg_explore,
            )
            prover.advance_proof(max_iterations=max_iterations, execute_depth=max_depth, cut_point_rules=cut_rules)

            kcfg_show = KCFGShow(
                kcfg_explore.kprint,
                node_printer=APRProofNodePrinter(proof, kcfg_explore.kprint, full_printer=True, minimize=False),
            )
            cfg_lines = kcfg_show.show(proof.kcfg)
            _LOGGER.info('\n'.join(cfg_lines))

            assert proof.status == proof_status
            assert leaf_number(proof) == expected_leaf_number


import xml.etree.ElementTree as ET
from subprocess import check_output

import logging
logger = logging.getLogger(__name__)

class AEMetric(object):
    # command to run official eval.jar to get f1 score
    commands = {
        "rest": "./java -cp reviewlab/eval/A.jar absa16.Do Eval -prd data/ft/ae/16/rest/rest_pred.xml -gld data/ft/ae/16/rest/EN_REST_SB1_TEST.xml.gold -evs 2 -phs A -sbt SB1",
        "laptop": "./java -cp reviewlab/eval/eval.jar Main.Aspects data/ft/ae/14/laptop/laptop_pred.xml data/ft/ae/14/laptop/Laptops_Test_Gold.xml"

    }

    templates = {
        "rest": "data/ft/ae/16/rest/EN_REST_SB1_TEST.xml.A",
        "laptop": "data/ft/ae/14/laptop/Laptops_Test_Data_PhaseA.xml"
    }

    @classmethod
    def metric_name(cls, index):
        if index == 0:
            return "f1"
        else:
            raise Exception("unknown index")

    @classmethod
    def append_helper(cls, ix, tag_on, domain, sent, start, opins):
        end = ix
        tag_on = False
        if domain == 'rest':
            opin = ET.Element("Opinion")
            opin.attrib['target'] = sent.find('text').text[start:end]
        elif domain == 'laptop':
            opin = ET.Element("aspectTerm")
            opin.attrib['term'] = sent.find('text').text[start:end]
        opin.attrib['from'] = str(start)
        opin.attrib['to'] = str(end)
        opins.append(opin)
        return end, tag_on, opins

    @classmethod
    def label_xml(cls, fn, output_fn, corpus, label, domain):
        dom = ET.parse(fn)
        root = dom.getroot()
        pred_y = []
        for zx, sent in enumerate(root.iter("sentence")):
            tokens = corpus[zx]
            lb = label[zx]
            if domain == 'rest':
                opins = ET.Element("Opinions")
            elif domain == 'laptop':
                opins = ET.Element("aspectTerms")
            token_idx, pt, tag_on = 0, 0, False
            start, end = -1, -1
            for ix, c in enumerate(sent.find('text').text):
                if token_idx < len(tokens) and pt >= len(tokens[token_idx]):
                    pt = 0
                    token_idx += 1

                case_1 = token_idx < len(tokens) and lb[token_idx] == 1 and pt == 0 and c != ' '
                case_2 = token_idx < len(tokens) and lb[token_idx] == 2 and pt == 0 and c != ' ' and not tag_on
                case_3 = token_idx < len(tokens) and (lb[token_idx] == 0 or lb[token_idx] == 1) and tag_on and pt == 0
                case_4 = token_idx >= len(tokens) and tag_on

                if (case_1 and tag_on) or case_3 or case_4:
                    end, tag_on, opins = AEMetric.append_helper(ix, tag_on, domain, sent, start, opins)

                if case_1 or case_2:
                    start = ix
                    tag_on = True

                quote_marks = ['``', "''"]
                if c == ' ':
                    pass
                elif tokens[token_idx][pt:pt + 2] in quote_marks:
                    pt += 2
                else:
                    pt += 1
            if tag_on:
                end, tag_on, opins = AEMetric.append_helper(len(sent.find('text').text), tag_on, domain, sent, start, opins)
            sent.append(opins)
        dom.write(output_fn)

    @classmethod
    def evaluate(cls, config, pred_json):
        y_pred = [[pred_json["labels"].index(pred) for pred in preds] for preds in pred_json["preds_list"]]

        command = AEMetric.commands[config.domain].split()
        if config.domain == "rest":
            AEMetric.label_xml(AEMetric.templates[config.domain], command[6], pred_json["sentence"], y_pred, config.domain)
            result = check_output(command).split()
            logger.info("**** java output ****")
            logger.info("%s", result)
            return [float(result[9][10:])]
        elif config.domain == "laptop":
            AEMetric.label_xml(AEMetric.templates[config.domain], command[4], pred_json["sentence"], y_pred, config.domain)
            result = check_output(command).split()
            logger.info("**** java output ****")
            logger.info("%s", result)
            return [float(result[15])]
        else:
            raise ValueError("unknown domain %s.", config.domain)


from . import DataProduct
class CTDAS(DataProduct):
  """
  CTDAS CH4 data
  """

  # List of all possible fields we expect from the data
  # (original_name, standard_name, units)
  field_list = (
    ('bio_flux_opt',  'CH4_natural_flux', 'Tg(CH4) year-1'),
    ('anth_flux_opt', 'CH4_fossil_flux',  'Tg(CH4) year-1'),
    ('fire_flux_imp', 'CH4_bioburn_flux', 'Tg(CH4) year-1'),
    ('term_flux_imp', 'CH4_agwaste_flux', 'Tg(CH4) year-1'),
    ('ocn_flux_imp',  'CH4_ocean_flux',   'Tg(CH4) year-1'),
    ('total_flux_opt','CH4_flux',         'Tg(CH4) year-1'),
    ('tempm', 'air_temperature', 'K'),
    ('presm', 'surface_pressure', 'Pa'),
    ('ch4', 'CH4', 'mol mol(semidry_air)-1'),
  )
  # Helper methods

  # Method to open a single file
  @staticmethod
  def open_file (filename):
    from pygeode.formats import netcdf as nc, hdf4
    from os.path import basename, splitext
    from pygeode.timeaxis import StandardTime
    from pygeode.axis import Lat, Lon, ZAxis
    from pygeode.var import Var
    import numpy as np
    # Case 1: netCDF flux file
    if filename.endswith('.nc'):
      f = nc.open(filename)
      # Use the date from the filename.
      yyyymm = splitext(basename(filename))[0].split('_')[-2]
      year = int(yyyymm[:4])
      month = int(yyyymm[4:])
      taxis = StandardTime(year=[year], month=[month], units='days', startdate=dict(year=2000,month=1))
      # Create the time axis.
      f = f.extend(0,taxis)
    # Case 2: HDF4 concentration file
    elif filename.endswith('.hdf'):
      f = hdf4.open(filename)
      # Use the date from the filename.
      yyyymmddhh = splitext(basename(filename))[0].split('_')[-2]
      year = int(yyyymmddhh[:4])
      month = int(yyyymmddhh[4:6])
      day = int(yyyymmddhh[6:8])
      hour = int(yyyymmddhh[8:])
      taxis = StandardTime(year=[year], month=[month], day=[day], hour=[hour], units='days', startdate=dict(year=2000,month=1))
      # Create the time axis.
      f = f.extend(0,taxis)
      # Store the hybrid a/b information for later use.
      at = f.atts['at']
      a = (at[:-1] + at[1:]) / 2
      a = Var(axes=[f.HYBRID], name='a', values=a)
      bt = f.atts['bt']
      b = (bt[:-1] + bt[1:]) / 2
      b = Var(axes=[f.HYBRID], name='b', values=b)
      f = f + a + b
      # Partial decoding of zaxis.
      # Can't create a full Hybrid axis here, because the diagnostic machinery
      # gets confused about the extra A/B auxiliary array.
      f = f.replace_axes(HYBRID=ZAxis)
      # Attach lat/lon  coordinates (for concentration files.)
      # Have to do it here in the opener, because the data_scanner can't handle
      # the default "dummy" lat/lon axes in the file.
      latbounds = np.linspace(-90,90,46)
      lat = (latbounds[:-1] + latbounds[1:]) / 2
      lat = Lat(lat)
      lonbounds = np.linspace(-180,180,61)
      lon = (lonbounds[:-1] + lonbounds[1:]) / 2
      lon = Lon(lon)
      f = f.replace_axes(LATglb600x400=lat, LONglb600x400=lon)

    # Remove idate attribute (causes problems with the data_scanner).
    for var in f:
      var.atts.pop('idate',None)
    return f

  # Method to decode an opened dataset (standardize variable names, and add any
  # extra info needed (pressure values, cell area, etc.)
  @classmethod
  def decode (cls, data):

    from ..common import ndays_in_year, compute_pressure
    from pygeode.axis import Hybrid

    # Apply fieldname conversions
    data = DataProduct.decode.__func__(cls,data)

    # Convert to a dictionary (for referencing by variable name)
    data = dict((var.name,var) for var in data)

    # Add species name for all products (to assist in things like unit conversion)
    for varname in data:
      if varname.startswith('CH4'):
        data[varname].atts['specie'] = 'CH4'
      # Convert units from mass/year to mass/day
      if 'units' in data[varname].atts:
        if data[varname].atts['units'] == 'Tg(CH4) year-1':
          data[varname] = data[varname] / ndays_in_year(data[varname].time)
          data[varname].atts['units'] = 'Tg(CH4) day-1'
          data[varname].atts['specie'] = 'CH4'
      # Add default units to unprocessed data
      else:
        data[varname].atts['units'] = ''

    # Finish defining level coordinates (for concentration files.)
    hybrid = None
    for varname in data:
      if data[varname].hasaxis('HYBRID'):
        if hybrid is None:
          hybrid = Hybrid(data[varname].HYBRID.values, name='hybrid', A=data['a'].get(), B=data['b'].get())
        data[varname] = data[varname].replace_axes(HYBRID=hybrid)
    # Remove a,b variables.
    data.pop('a',None)
    data.pop('b',None)

    # Add pressure field (if not explicitly provided).
    if 'surface_pressure' in data and hybrid is not None:
      #try:
        air_pressure = compute_pressure(hybrid, data['surface_pressure'])
        air_pressure.name = 'air_pressure'
        data['air_pressure'] = air_pressure
      #except (TypeError, ValueError, AttributeError): pass

    # Make sure the variables have the appropriate names
    for name, var in data.iteritems():  var.name = name

    # Add extra fields that will be useful for the diagnostics.
    data = cls._add_extra_fields(data)

    return data

  # Method to find all files in the given directory, which can be accessed
  # through this interface.
  @staticmethod
  def find_files (dirname):
    from glob import glob
    return glob(dirname+"/monthly_CH4_??????_1x1.nc") + glob(dirname+"mmix_??????????_??????????_glb600x400.hdf")

  # Method to find a unique identifying string for this dataset, from the
  # given directory name.
  @staticmethod
  def get_dataname (dirname):
    # No versioning information in directory name, just use interface name.
    return 'CTDAS-CH4'

# Add this interface to the table.
from . import table
table['ctdas-ch4'] = CTDAS


from sqlalchemy import Column, Integer, String, TIMESTAMP, LargeBinary
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()

class Image(Base):
    __tablename__ = 'image'
    id = Column(Integer, primary_key=True)
    filename = Column(String(255))
    content_type = Column(String(50))
    data = Column(LargeBinary)
    accessed_time = Column(TIMESTAMP)
    url = Column(String(3000))

    def __repr__(self):
        return '<Page {0} {1}: {2}>'.format(self.filename,
                                            self.url,
                                            self.data)


# -*- coding: utf-8 -*-
from django.conf.urls import *
from django.conf import settings

urlpatterns = patterns('gibbon',
                       # url(r'^admin/', include(admin.site.urls)),
                       # url(r'^$', 'views.home'),
                       url(r'^$', 'views.hello'),
                       url(r'^create_event', 'views.create_event'),
                       url(r'^list_event', 'views.list_event'),
)

if settings.DEBUG is True:
    urlpatterns += patterns('django.contrib.staticfiles.views',
            url(r'^static/(?P<path>.*)$', 'serve'),
        )


import re
import docutils.core
import docutils.io

import markdown
import textwrap
from rst2html5 import HTML5Writer

import wiki

#===============================================================================
# MARKUP BASE
#===============================================================================

class Markup(object):
    """ Base markup class."""
    NAME = 'Text'
    META_LINE = '%s: %s\n'
    EXTENSION = '.txt'
    HOWTO = """ """

    def __init__(self, raw_content):
        self.raw_content = raw_content

    @classmethod
    def render_meta(cls, key, value):
        return cls.META_LINE % (key, value)

    def process(self):
        """
        return (html, body, meta) where HTML is the rendered output
        body is the the editable content (text), and meta is
        a dictionary with at least ['title', 'tags'] keys
        """
        raise NotImplementedError("override in a subclass")

    @classmethod
    def howto(cls):
        return cls(textwrap.dedent(cls.HOWTO)).process()[0]

#===============================================================================
# MARKDOWN
#===============================================================================

class Markdown(Markup):
    NAME = 'markdown'
    META_LINE = '%s: %s\n'
    EXTENSION = '.md'
    HOWTO = """
        This editor is [markdown][] featured.

            * I am
            * a
            * list

        Turns into:

        * I am
        * a
        * list

        `**bold** and *italics*` turn into **bold** and *italics*. Very easy!

        Create links with `[Wiki](http://github.com/alexex/wiki)`.
        They turn into [Wiki][http://github.com/alexex/wiki].

        Headers are as follows:

            # Level 1
            ## Level 2
            ### Level 3

        [markdown]: http://daringfireball.net/projects/markdown/
        """

    def process(self):
        # Processes Markdown text to HTML, returns original markdown text,
        # and adds meta
        md = markdown.Markdown(['codehilite', 'fenced_code', 'meta'])
        html = md.convert(self.raw_content)
        meta_lines, body = self.raw_content.split('\n\n', 1)
        meta = md.Meta
        return html, body, meta

#===============================================================================
# RESTRUCTURED TEXT
#===============================================================================

class RestructuredText(Markup):
    NAME = 'restructuredtext'
    META_LINE = '.. %s: %s\n'
    IMAGE_LINE = '.. image:: %(url)s'
    LINK_LINE = '`%(filename)s <%(url)s>`_'

    EXTENSION = '.rst'
    HOWTO = """
        This editor is `reStructuredText`_ featured::

            * I am
            * a
            * list

        Turns into:

        *  I am
        *  a
        *  list

        ``**bold** and *italics*`` turn into **bold** and *italics*. Very easy!

        Create links with ```Wiki <http://github.com/alexex/wiki>`_``.
        They turn into `Wiki <https://github.com/alexex/wiki>`_.

        Headers are just any underline (and, optionally, overline).
        For example::

            Level 1
            *******

            Level 2
            -------

            Level 3
            +++++++

        .. _reStructuredText: http://docutils.sourceforge.net/rst.html
        """

    def process(self):
        settings = {'initial_header_level': 2,
                    'record_dependencies': True,
                    'stylesheet_path': None,
                    'link_stylesheet': True,
                    'syntax_highlight': 'short',
                    }

        html = self._rst2html(self.raw_content,
                              settings_overrides=settings)

        # Convert unknow links to internal wiki links.
        # Examples:
        #   Something_ will link to '/something'
        #  `something great`_  to '/something_great'
        #  `another thing <thing>`_  '/thing'
        refs = re.findall(r'Unknown target name: "(.*)"', html)
        if refs:
            content = self.raw_content + self.get_autolinks(refs)
            html = self._rst2html(content, settings_overrides=settings)
        meta_lines, body = self.raw_content.split('\n\n', 1)
        meta = self._parse_meta(meta_lines.split('\n'))
        return html, body, meta

    def get_autolinks(self, refs):
        autolinks = '\n'.join(['.. _%s: /%s' % (ref, wiki.urlify(ref, False))
                               for ref in refs])
        return '\n\n' + autolinks

    def _rst2html(self, source, source_path=None,
                  source_class=docutils.io.StringInput,
                  destination_path=None, reader=None, reader_name='standalone',
                  parser=None, parser_name='restructuredtext', writer=None,
                  writer_name=None, settings=None, settings_spec=None,
                  settings_overrides=None, config_section=None,
                  enable_exit_status=None):

        if not writer:
            writer = HTML5Writer()

        # Taken from Nikola
        # http://bit.ly/14CmQyh
        output, pub = docutils.core.publish_programmatically(
            source=source, source_path=source_path, source_class=source_class,
            destination_class=docutils.io.StringOutput,
            destination=None, destination_path=destination_path,
            reader=reader, reader_name=reader_name,
            parser=parser, parser_name=parser_name,
            writer=writer, writer_name=writer_name,
            settings=settings, settings_spec=settings_spec,
            settings_overrides=settings_overrides,
            config_section=config_section,
            enable_exit_status=enable_exit_status)
        return pub.writer.parts['body']

    def _parse_meta(self, lines):
        """ Parse Meta-Data. Taken from Python-Markdown"""
        META_RE = re.compile(r'^\.\.\s(?P<key>.*?): (?P<value>.*)')
        meta = {}
        key = None
        for line in lines:
            if line.strip() == '':
                continue
            m1 = META_RE.match(line)
            if m1:
                key = m1.group('key').lower().strip()
                value = m1.group('value').strip()
                try:
                    meta[key].append(value)
                except KeyError:
                    meta[key] = [value]
        return meta

#===============================================================================
# MAIN
#===============================================================================

if __name__ == "__main__":
    print(__doc__)


# -*- coding: utf-8 -*-
"""
cron: 30 7 * * *
new Env('每日新闻');
"""

import requests, time, re, json, sys, traceback
from io import StringIO
from KDconfig import getYmlConfig, send

class News:
    def __init__(self, cookie):
        self.sio = StringIO()
        self.Cookies = cookie

    def SignIn(self):
        print("【每日新闻 日志】")
        for cookie in self.Cookies:
            cookie = cookie.get("user")
            print(f"每日新闻 开始爬取中...")
            # self.sio.write(f"{cookie.get('name')}: ")
            try:
                res = requests.get(url = f'https://news.topurl.cn/api?ip={cookie.get("cookie")}').json()
                if res.get('code') == 200:
                    print(res)
                    data = res.get('data', {})
                    if data.get('newsList') != []:
                        self.sio.write("【每日新闻】\n")
                        no = 1
                        for news in data.get('newsList', []):
                            self.sio.write(f'{str(no).zfill(2)}: {news.get("title", "")}\n')
                            no += 1
                        self.sio.write('\n')
                    if data.get('historyList') != []:
                        self.sio.write('【历史上的今天】\n')
                        for history in data.get('historyList', []):
                            self.sio.write(f'{history.get("event", "")}\n')
            except:
                self.sio.write(f"每日新闻: 异常 {traceback.format_exc()}")
                if '存在异常, 请自行查看日志' not in self.sio.getvalue():
                    self.sio.write('存在异常, 请自行查看日志\n')
            break
        return self.sio

if __name__ == '__main__':
    config = getYmlConfig('Cookie.yml')
    Cookies = config.get('News')
    if Cookies != None:
        if Cookies.get('cookies') != None:
            cloud = News(Cookies['cookies'])
            sio = cloud.SignIn()
            print(f'\n{sio.getvalue()}')
            if Cookies.get('send') != None and Cookies['send'] == 1:
                send('每日新闻', sio.getvalue())
            else:
                print('推送失败: 关闭了推送 or send配置问题')
        else:
            print('配置文件 每日新闻 没有 "cookies"')
            sys.exit()
    else:
        print('配置文件没有 每日新闻')


import discord
import logging
import asyncio

import settings as stg
import db_handler
from module_registry import get_default_registry
from unz_modules import UnzBaseModule, EventHandlerStatus

logging.basicConfig(
    level=stg.LOG_LEVEL,
    format=stg.LOG_FORMAT,
    datefmt=stg.LOG_DATEFMT,
    handlers=[
        logging.FileHandler(
            stg.LOG_FILENAME,
            mode=stg.LOG_FILEMODE
        ),
        logging.StreamHandler()
    ]
)

client = discord.Client()
with open("SECRET") as f:
    SECRET = f.readline().strip()

registry = get_default_registry()
database = db_handler.load_or_create()
UnzBaseModule.global_data = database.get("unz_global", {})
loaded_modules = []
for modname in stg.LOADED_MODULES:
    config = stg.MODULE_CONFIG.get(modname, {})
    loaded_modules.append(registry.get_module(modname)(database, config))

@client.event
async def on_ready():
    app_info = await client.application_info()
    logging.info(f"Client logged in as {app_info.name}")

    for module in loaded_modules:
        status = await module.event_on_ready(client)
        if status is not None and status == EventHandlerStatus.BLOCK:
            break

@client.event
async def on_message(message):
    if message.author == client.user or not message.content:
        return
    if type(message.channel) == discord.TextChannel:
        log_msg = f"Message :: {message.channel.name} ({message.channel.id}) :: {message.author} ({message.author.id}) :: {message.content}"
    else:
        log_msg = f"Message :: {message.channel.id} :: {message.author} ({message.author.id}) :: {message.content}"
    logging.info(log_msg)
    for module in loaded_modules:
        status = await module.event_on_message(message, client)
        if status is not None and status == EventHandlerStatus.BLOCK:
            break

@client.event
async def on_disconnect():
    db_handler.save(database, loaded_modules)

async def db_saver():
    await client.wait_until_ready()
    while not client.is_closed():
        db_handler.save(database, loaded_modules)
        await asyncio.sleep(stg.DB_AUTOSAVE_INTERVAL)

async def secondary_loop():
    await client.wait_until_ready()
    time = 0
    seconds_to_wait = 1000
    while not client.is_closed():
        for module in loaded_modules:
            interval = module.secondary_interval
            if interval == 0:
                continue

            if time % interval == 0:
                await module.loop_function(client)

            next_rep = (int(time / interval) + 1) * interval
            seconds_to_wait = min((seconds_to_wait, next_rep-time))

        await asyncio.sleep(seconds_to_wait)
        seconds_to_wait = 1000

if __name__ == "__main__":
    client.loop.create_task(db_saver())
    client.loop.create_task(secondary_loop())
    client.run(SECRET)