"Abstract syntax tree types."

from __future__ import annotations

import typing
from dataclasses import field, dataclass

from .description import Description
from . import validate

class Introspect:
    "Helper class to query node type."

    @property
    def is_constant(self):
        ":return: True, if it is a `Constant` instance."
        return isinstance(self, Constant)

    @property
    def is_builtin(self):
        ":return: True, if it is a `TypeBuiltin` type"
        return isinstance(self, TypeBuiltin)

    @property
    def is_anonimous(self):
        ":return: True, if it is an `TypeAnonimous` type."
        return isinstance(self, TypeAnonimous)

    @property
    def is_array(self):
        ":return: True, if it is an `TypeArray` type."
        return isinstance(self, TypeArray)

    @property
    def is_pointer64(self):
        ":return: True, if it is an `TypePointer64` type."
        return isinstance(self, TypePointer64)

    @property
    def is_offset_pointer(self):
        ":return: True, if it is an `TypeOffsetPointer` type."
        return isinstance(self, TypeOffsetPointer)

    @property
    def is_enum(self):
        ":return: True, if it is a `TypeEnum` type."
        return isinstance(self, TypeEnum)

    @property
    def is_bitmask(self):
        ":return: True, if it is a `TypeBitmask` type."
        return isinstance(self, TypeBitmask)

    @property
    def is_enum_or_bitmask(self):
        ":return: True, if it is a `TypeEnum` or a `TypeBitmask` type."
        return self.is_enum or self.is_bitmask

    @property
    def is_struct(self):
        ":return: True, if it is a `TypeStruct` type."
        return isinstance(self, TypeStruct)

    @property
    def is_union(self):
        ":return: True, if it is a `TypeUnion` type."
        return isinstance(self, TypeUnion)

    @property
    def is_compound(self):
        ":return: True, if it is a `TypeCompound` type."
        return isinstance(self, TypeCompound)

class Node(Introspect):
    "Base class for all nodes."

@dataclass
class Named(Node):
    """
    A node with a name and a description.

    :param name: Node name. Must be a valid identifier.
    :param description: Node description.
    """

    # Parent node of this node. None for the root node.
    parent: typing.Optional["Named"] = field(
        default=None, init=False, compare=False, repr=False
    )
    name: str
    description: Description

    def __post_init__(self):
        validate.identifier(self.name)
        validate.not_known_buitin_type(self.name)

    def _set_parent_and_validate(self, collection: typing.Iterable[Named]) -> None:
        validate.uniq_names(collection)
        validate.name_not_in(self.name, collection)

        for item in collection:
            item.parent = self

    @property
    def full_name_cxx(self) -> str:
        ":return: Full name of this item for C++ language."
        return "::".join(self.__parents())

    @property
    def full_name_c(self) -> str:
        ":return: Full name of this item for C language."
        return "_".join(self.__parents())

    def __parents(self) -> typing.Iterable[str]:
        "return: All parents of this node up to the root node."
        if self.parent and self.parent.parent:
            # pylint: disable=protected-access
            yield from self.parent.__parents()

        yield self.name

_Type = typing.Union[
    "TypeArray",
    "TypeBitmask",
    "TypeBuiltin",
    "TypeCompound",
    "TypeEnum",
    "TypeOffsetPointer",
    "TypePointer64",
]

@dataclass
class TypeBuiltin(Node):
    """
    Built-in type node.

    :param name: Type name. Must be a valid built-in type.
    """

    name: str

    def __post_init__(self):
        validate.known_builtin_type(self.name)

    @property
    def full_name_cxx(self) -> str:
        ":return: Full name of this item for C++ language."
        return self.name

    @property
    def full_name_c(self) -> str:
        ":return: Full name of this item for C language."
        return self.name

    @property
    def print_as(self) -> typing.Optional[str]:
        """
        Type to cast to before printing.

        C++ ostream prints uint8_t and int8_t as chars. To avoid that,
        we cast them conditionally to a larger type, so they appear as
        numbers.

        :return: Type to cast to before printing. None if such cast is not needed.
        """

        print_as_dict = {
            "uint8_t": "uint32_t",
            "int8_t": "int32_t",
        }
        return print_as_dict.get(self.name, None)

_ScopeItemUnion = typing.Union[
    "Constant", "TypeEnum", "TypeBitmask", "TypeStruct", "TypeUnion"
]

@dataclass
class EnumValue(Named):
    """
    Enum value node.

    :param value: Enum's value.
    :param alias: If True, this value is an alias for some other value.
    """

    value: typing.Optional[str] = None
    alias: bool = False

@dataclass
class TypeEnum(Named):
    """
    Enum type node.

    :param underlying_type: Underlying type of the enum.
    :param values: Pared enum values.
    """

    underlying_type: TypeBuiltin
    values: typing.Tuple[EnumValue, ...]
    keyword: typing.ClassVar[str] = "enum"

    def __post_init__(self):
        super().__post_init__()

        if len(self.values) < 1:
            raise ValueError("A enum should contain at least one value.")

        self._set_parent_and_validate(self.values)

@dataclass
class BitmaskValue(Named):
    """
    Bitmask value node.

    :param bit_number: The bit number.
    """

    bit_number: int

@dataclass
class TypeBitmask(Named):
    """
    Bitmask type node.

    :param underlying_type: Underlying type of the bitmask.
    :param values: Pared bitmask values.
    """

    underlying_type: TypeBuiltin
    values: typing.Tuple[BitmaskValue, ...]
    keyword: typing.ClassVar[str] = "enum"

    def __post_init__(self):
        super().__post_init__()

        if len(self.values) < 1:
            raise ValueError("A bitmask should contain at least one value.")

        self._set_parent_and_validate(self.values)

@dataclass
class Constant(Named):
    """
    Constant AST node.

    :param type: Type of the constant.
    :param value: Constant value.
    """

    type: TypeBuiltin
    value: str

class TypeAnonimous(Node):
    "Type defined inline with no name and no description."

@dataclass
class TypeArray(TypeAnonimous):
    """
    Array type AST node.

    :param element_type: Array's element type.
    :param array_size: Size of the array.
    """

    element_type: TypeBuiltin
    array_size: str

@dataclass
class FieldMeta(Node):
    """
    Field metadata.

    :param padding: True if it is a padding field.
    """

    padding: bool = False

@dataclass
class Field(Named):
    """
    Compound type field AST node.

    :param type: Parsed type of the field.
    :param type: Field metadata.
    """

    type: _Type
    meta: typing.Optional[FieldMeta] = None

    def __post_init__(self):
        super().__post_init__()

        if isinstance(self.type, (TypeArray, TypePointer64, TypeOffsetPointer)):
            return

        if self.name == self.type.name:
            raise ValueError(f"Field name {self.name} is the same as type name.")

@dataclass
class TypeCompound(Named):
    """
    Compound type node.

    :param items: Collection of inner types and constants.
    :param fields: Compound type fields.
    """

    items: typing.Tuple[_ScopeItemUnion, ...] = field(default_factory=tuple)
    fields: typing.Tuple[Field, ...] = field(default_factory=tuple)

    def __post_init__(self):
        super().__post_init__()

        if not self.fields:
            raise ValueError(f"{self.name} must have at least one field.")

        self._set_parent_and_validate(self.items)
        self._set_parent_and_validate(self.fields)

    @property
    def has_union_field(self) -> bool:
        ":return: True if a type has union field, False otherwise."
        for type_field in self.fields:
            if isinstance(type_field.type, TypeUnion):
                return True
        return False

@dataclass
class TypeStruct(TypeCompound):
    "Struct type node."

    keyword: typing.ClassVar[str] = "struct"

@dataclass
class TypeUnion(TypeCompound):
    "Union type node."

    keyword: typing.ClassVar[str] = "union"

@dataclass
class TypeFwdDcl(Named):
    """
    Forward type declaration node.

    :param keyword: 'struct' or 'union'.
    """

    keyword: str

    def __post_init__(self):
        super().__post_init__()

        validate.known_keyword(self.keyword)

@dataclass
class TypePointer64(TypeAnonimous):
    """
    Pointer64 type node.

    :param value_type: Value type of the pointer.
    :param const: True if it is a const pointer, False otherwise.
    """

    value_type: typing.Union[TypeFwdDcl, _Type]
    const: bool = False

@dataclass
class TypeOffsetPointer(TypeAnonimous):
    """
    Offset pointer type node.

    :param value_type: Value type of the pointer.
    :param representation_type: Type used to store the offset value.
    :param const: True if it is a const pointer, False otherwise.
    """

    value_type: typing.Union[TypeFwdDcl, _Type]
    representation_type: TypeBuiltin
    const: bool = False

@dataclass
class Namespace(Named):
    """
    Namespace node.

    External namespace node is a collection of forward type declarations.
    :param items: Forward declared types.
    """

    forward_declarations: typing.Tuple[TypeFwdDcl, ...]

    def __post_init__(self):
        super().__post_init__()

        self._set_parent_and_validate(self.forward_declarations)

@dataclass
class Number:
    """
    Interface or command number.

    :param number: Number field.
    """

    number: int

    def __str__(self):
        return str(self.number)

    @property
    def hex(self) -> str:
        ":return: number formatted as hex."

        return hex(self.number)

@dataclass
class IoctlCommand(Named):
    """
    ioctl command AST node.

    :param number: Command number.
    :param arg_type: Argument type of the command.
    """

    number: Number
    command_type: str
    arg_type: _Type

@dataclass
class IoctlIface(Named):
    """
    ioctl interface AST node.

    :param number: Interface number.
    :param comands: Interface command nodes.
    """

    number: Number
    items: typing.Tuple[_ScopeItemUnion, ...]
    commands: typing.Tuple[IoctlCommand, ...]
    namespaces: typing.Tuple[Namespace, ...] = field(default_factory=tuple)
    code: str = ""

    def __post_init__(self):
        super().__post_init__()

        self._set_parent_and_validate(self.items)
        self._set_parent_and_validate(self.commands)
        self._set_parent_and_validate(self.namespaces)


import random
import numpy as np
import torch
import torch.nn.functional as F
import torch.optim as optim
from model import ActorNetwork, CriticNetwork

def hard_update(target, source):
    """
    Copy network parameters from source to target
    Inputs:
        target (torch.nn.Module): Net to copy parameters to
        source (torch.nn.Module): Net whose parameters to copy
    """
    for target_param, param in zip(target.parameters(), source.parameters()):
        target_param.data.copy_(param.data)

class Actor() :

    def __init__(self, state_size, action_size, random_seed, learning_rate, noise, device):
        self.state_size = state_size
        self.action_size = action_size
        self.seed = random.seed(random_seed)
        self.learning_rate = learning_rate

        self.actor_local = ActorNetwork(state_size, action_size, random_seed).to(device)
        self.actor_target = ActorNetwork(state_size, action_size, random_seed).to(device)
        hard_update(self.actor_target, self.actor_local)
        self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=self.learning_rate)

        self.noise = noise
        self.device = device

    def act(self, state, noise_factor, add_noise):
        """ Returns actions for given state as per given policy"""
        state = torch.from_numpy(state).float().to(self.device)
        self.actor_local.eval()
        with torch.no_grad() :
            action = self.actor_local(state).cpu().data.numpy()
        self.actor_local.train()
        if add_noise :
            action += (noise_factor * self.noise.sample())
        return np.clip(action, -1, 1)

    def reset(self):
        self.noise.reset()

class Critic() :

    def __init__(self, state_size, action_size, random_seed, learning_rate, weight_decay, device):

        self.state_size = state_size
        self.action_size = action_size
        self.seed= random.seed(random_seed)
        self.learning_rate = learning_rate

        self.critic_local = CriticNetwork(state_size, action_size, random_seed).to(device)
        self.critic_target = CriticNetwork(state_size, action_size, random_seed).to(device)
        hard_update(self.critic_target, self.critic_local)
        self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=self.learning_rate, weight_decay=weight_decay)


import socket
import time
import os
import subprocess
from pathlib import Path

vm_hostname = "127.0.0.1"
port = 2222

def wait():
    start = time.time()
    while time.time() - start < 5 * 60:
        try:
            connection = socket.create_connection((vm_hostname, port), timeout=10)
            data = connection.recv(3)
            connection.close()
            if data == b"SSH":
                return
        except (ConnectionRefusedError, socket.timeout):
            pass
        time.sleep(0.1)
    print("Timeout expired")

def shutdown():
    subprocess.check_call(
        [
            "ssh",
            "-o",
            "StrictHostKeyChecking=no",
            "-p",
            str(port),
            f"hacker@{vm_hostname}",
            "--",
            "Set-Service -Name sshd -StartupType Manual; "
            + "Set-Service -Name tvnserver -StartupType Manual; "
            + "Stop-Computer -Force",
        ]
    )

def qemu_running():
    procfs = Path("/proc")
    pid_dirs = [d for d in procfs.iterdir() if d.name.isdigit()]
    for pid in pid_dirs:
        try:
            exe_link = Path("/proc") / str(pid) / "exe"
            if "qemu" in os.readlink(str(exe_link)):
                return True
        except FileNotFoundError:
            pass
    return False

def wait_qemu_exit():
    start = time.time()
    while time.time() - start < 2 * 60 and qemu_running():
        time.sleep(1)

if __name__ == "__main__":
    wait()
    time.sleep(30)
    shutdown()
    wait_qemu_exit()


from telegram.ext import CommandHandler, Filters, MessageHandler, Updater
import logging
import os
import xmltodict
import requests
from datetime import datetime, timedelta, timezone
from dateutil import parser

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)

telegram_token = os.environ['TELEGRAM_TOKEN']
updater = Updater(token=telegram_token, use_context=True)
dispatcher = updater.dispatcher

telegram_id = os.environ['TELEGRAM_ID']

keywords = os.environ['KEYWORDS'].split(',')

def rule(news_string):
    for keyword in keywords:
        if keyword.lower() in news_string.lower():
            return True
    return False

def check_news(rss_url, news_filter, check_interval=10):
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
    response = requests.get(rss_url, headers=headers)
    logger.debug(response.content)
    dict_data = xmltodict.parse(response.content)
    news_items = dict_data['rss']['channel']['item']
    filtered_items =  []
    for item in news_items:
        newsdate = parser.parse(item['pubDate'])
        hour_ago = (datetime.now(timezone.utc).astimezone() - timedelta(hours=1))
        word_filter = news_filter(item['title'])
        if word_filter and (newsdate > hour_ago):
            filtered_items.append(f"{item['title']} {item['link']}")
    return filtered_items

period = int(os.environ['PERIOD'])
def send_message(event, context):
    news_sources = [
        'https://ria.ru/export/rss2/archive/index.xml',
        'http://feeds.bbci.co.uk/russian/rss.xml'
    ]

    news_items = []

    for news_source in news_sources:
        news_items.extend(check_news(news_source, rule, period))

    for news_item in news_items:
        dispatcher.bot.send_message(
            chat_id=telegram_id, text=news_item)


from anki.exporting import Exporter, AnkiExporter
from anki.utils import ids2str
from aqt import mw
import os
from anki.storage import Collection

oldInit = Exporter.__init__
def __init__(self,*args, **kwargs):
    oldInit(self, *args, **kwargs)
    self.cids = None
Exporter.__init__ = __init__

def needSiblings(self):
    userOption = mw.addonManager.getConfig(__name__)
    siblings = userOption["siblings"]
    r=siblings.get(self.key)
    if r is None:
        r= siblings.get("default")
    if r is None:
        r = True
    return r
Exporter.needSiblings = needSiblings

def siblings(cids):
    siblings = mw.col.db.list(f"select id from cards where nid in (select nid from cards where id in {ids2str(cids)})")
    return siblings

oldCardIds= Exporter.cardIds
def cardIds(self):
    if self.cids is not None:
        cids= self.cids
        self.count = len(cids)
    else:
        cids=oldCardIds(self)
    if self.needSiblings():
        cids = siblings(cids)
    return cids
Exporter.cardIds = cardIds

def exportInto(self, path: str) -> None:
    # sched info+v2 scheduler not compatible w/ older clients
    self._v2sched = self.col.schedVer() != 1 and self.includeSched

    # create a new collection at the target
    try:
        os.unlink(path)
    except (IOError, OSError):
        pass
    self.dst = Collection(path)
    self.src = self.col
    # find cards
    cids = self.cardIds()
    # copy cards, noting used nids
    nids = {}
    data = []
    for row in self.src.db.execute(
        "select * from cards where id in " + ids2str(cids)
    ):
        nids[row[1]] = True
        data.append(row)
        # clear flags
        row = list(row)
        row[-2] = 0
    self.dst.db.executemany(
        "insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", data
    )
    # notes
    strnids = ids2str(list(nids.keys()))
    notedata = []
    for row in self.src.db.all("select * from notes where id in " + strnids):
        # remove system tags if not exporting scheduling info
        if not self.includeSched:
            row = list(row)
            row[5] = self.removeSystemTags(row[5])
        notedata.append(row)
    self.dst.db.executemany(
        "insert into notes values (?,?,?,?,?,?,?,?,?,?,?)", notedata
    )
    # models used by the notes
    mids = self.dst.db.list("select distinct mid from notes where id in " + strnids)
    # card history and revlog
    if self.includeSched:
        data = self.src.db.all("select * from revlog where cid in " + ids2str(cids))
        self.dst.db.executemany(
            "insert into revlog values (?,?,?,?,?,?,?,?,?)", data
        )
    else:
        # need to reset card state
        self.dst.sched.resetCards(cids)
    # models - start with zero
    self.dst.models.models = {}
    for m in self.src.models.all():
        if int(m["id"]) in mids:
            self.dst.models.update(m)
    # decks
    dids = self.deckIds()
    print(f"dids is {dids}")
    dconfs = {}
    for d in self.src.decks.all():
        if str(d["id"]) == "1":
            continue
        if dids and d["id"] not in dids:
            continue
        if not d["dyn"] and d["conf"] != 1:
            if self.includeSched:
                dconfs[d["conf"]] = True
        if not self.includeSched:
            # scheduling not included, so reset deck settings to default
            d = dict(d)
            d["conf"] = 1
        self.dst.decks.update(d)
    # copy used deck confs
    for dc in self.src.decks.allConf():
        if dc["id"] in dconfs:
            self.dst.decks.updateConf(dc)
    # find used media
    media = {}
    self.mediaDir = self.src.media.dir()
    if self.includeMedia:
        for row in notedata:
            flds = row[6]
            mid = row[2]
            for file in self.src.media.filesInStr(mid, flds):
                # skip files in subdirs
                if file != os.path.basename(file):
                    continue
                media[file] = True
        if self.mediaDir:
            for fname in os.listdir(self.mediaDir):
                path = os.path.join(self.mediaDir, fname)
                if os.path.isdir(path):
                    continue
                if fname.startswith("_"):
                    # Scan all models in mids for reference to fname
                    for m in self.src.models.all():
                        if int(m["id"]) in mids:
                            if self._modelHasMedia(m, fname):
                                media[fname] = True
                                break
    self.mediaFiles = list(media.keys())
    self.dst.crt = self.src.crt
    # todo: tags?
    self.count = self.dst.cardCount()
    self.dst.setMod()
    self.postExport()
    self.dst.close()

def deckIds(self):
    dids: List[int]
    if self.did:
        return [self.did] + [x[1] for x in self.src.decks.children(self.did)]
    elif self.cids:
        return self.col.db.list(f"SELECT did FROM cards where id in {ids2str(self.cids)}")
    else:
        return []

AnkiExporter.deckIds = deckIds
AnkiExporter.exportInto = exportInto


from data_structures_and_algorithms.challenges.tree_intersection.tree_intersection import Node,tree_intersection,BinaryTree
import pytest

@pytest.fixture
def fir_tree():
    Bt1 = BinaryTree()
    Bt1.root = Node(6)
    Bt1.root.right = Node(9)
    Bt1.root.left = Node(2)
    Bt1.root.left.left = Node(0)
    Bt1.root.right.left = Node(10)
    return Bt1

@pytest.fixture
def sec_tree():
    Bt2 = BinaryTree()
    Bt2.root = Node(6)
    Bt2.root.right = Node(2)
    Bt2.root.left = Node(10)
    Bt2.root.left.left = Node(18)
    return Bt2

def test_intersection(fir_tree, sec_tree):

    actual=tree_intersection(fir_tree, sec_tree)

    assert actual == {2, 10, 6}


#!/usr/bin/env python3
# -*-coding:UTF-8 -*

"""
The WebStats Module
======================

This module makes stats on URL recolted from the web module.
It consider the TLD, Domain and protocol.

"""

import time
import datetime
import redis
import os
from packages import lib_words
from packages.Date import Date
from pubsublogger import publisher
from Helper import Process
from pyfaup.faup import Faup

# Config Var
threshold_total_sum = 200 # Above this value, a keyword is eligible for a progression
threshold_increase = 1.0  # The percentage representing the keyword occurence since num_day_to_look
max_set_cardinality = 10  # The cardinality of the progression set
num_day_to_look = 5       # the detection of the progression start num_day_to_look in the past

def analyse(server, field_name, date, url_parsed):
    field = url_parsed[field_name]
    if field is not None:
        field = field.decode('utf8')
        server.hincrby(field, date, 1)
        if field_name == "domain": #save domain in a set for the monthly plot
            domain_set_name = "domain_set_" + date[0:6]
            server.sadd(domain_set_name, field)
            print("added in " + domain_set_name +": "+ field)

def get_date_range(num_day):
    curr_date = datetime.date.today()
    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
    date_list = []

    for i in range(0, num_day+1):
        date_list.append(date.substract_day(i))
    return date_list

# Compute the progression for one keyword
def compute_progression_word(server, num_day, keyword):
    date_range = get_date_range(num_day)
    # check if this keyword is eligible for progression
    keyword_total_sum = 0
    value_list = []
    for date in date_range: # get value up to date_range
        curr_value = server.hget(keyword, date)
        value_list.append(int(curr_value if curr_value is not None else 0))
        keyword_total_sum += int(curr_value) if curr_value is not None else 0
    oldest_value = value_list[-1] if value_list[-1] != 0 else 1 #Avoid zero division

    # The progression is based on the ratio: value[i] / value[i-1]
    keyword_increase = 0
    value_list_reversed = value_list[:]
    value_list_reversed.reverse()
    for i in range(1, len(value_list_reversed)):
        divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
        keyword_increase += value_list_reversed[i] / divisor

    return (keyword_increase, keyword_total_sum)

'''
    recompute the set top_progression zset
        - Compute the current field progression
        - re-compute the current progression for each first 2*max_set_cardinality fields in the top_progression_zset
'''
def compute_progression(server, field_name, num_day, url_parsed):
    redis_progression_name_set = "z_top_progression_"+field_name

    keyword = url_parsed[field_name]
    if keyword is not None:

        #compute the progression of the current word
        keyword_increase, keyword_total_sum = compute_progression_word(server, num_day, keyword)

        #re-compute the progression of 2*max_set_cardinality
        current_top = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True, start=0, num=2*max_set_cardinality)
        for word, value in current_top:
            word_inc, word_tot_sum = compute_progression_word(server, num_day, word)
            server.zrem(redis_progression_name_set, word)
            if (word_tot_sum > threshold_total_sum) and (word_inc > threshold_increase):
                server.zadd(redis_progression_name_set, float(word_inc), word)

        # filter before adding
        if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):
            server.zadd(redis_progression_name_set, float(keyword_increase), keyword)

if __name__ == '__main__':
    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
    # Port of the redis instance used by pubsublogger
    publisher.port = 6380
    # Script is the default channel used for the modules.
    publisher.channel = 'Script'

    # Section name in bin/packages/modules.cfg
    config_section = 'WebStats'

    # Setup the I/O queues
    p = Process(config_section)

    # Sent to the logging a description of the module
    publisher.info("Makes statistics about valid URL")

    # REDIS #
    r_serv_trend = redis.StrictRedis(
        host=p.config.get("ARDB_Trending", "host"),
        port=p.config.get("ARDB_Trending", "port"),
        db=p.config.get("ARDB_Trending", "db"),
        decode_responses=True)

    # FILE CURVE SECTION #
    csv_path_proto = os.path.join(os.environ['AIL_HOME'],
                                  p.config.get("Directories", "protocolstrending_csv"))
    protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
                                 p.config.get("Directories", "protocolsfile"))

    csv_path_tld = os.path.join(os.environ['AIL_HOME'],
                                p.config.get("Directories", "tldstrending_csv"))
    tldsfile_path = os.path.join(os.environ['AIL_HOME'],
                                 p.config.get("Directories", "tldsfile"))

    csv_path_domain = os.path.join(os.environ['AIL_HOME'],
                                   p.config.get("Directories", "domainstrending_csv"))

    faup = Faup()
    generate_new_graph = False
    # Endless loop getting messages from the input queue
    while True:
        # Get one message from the input queue
        message = p.get_from_set()

        if message is None:
            if generate_new_graph:
                generate_new_graph = False
                today = datetime.date.today()
                year = today.year
                month = today.month

                print('Building protocol graph')
                lib_words.create_curve_with_word_file(r_serv_trend, csv_path_proto,
                                                      protocolsfile_path, year,
                                                      month)

                print('Building tld graph')
                lib_words.create_curve_with_word_file(r_serv_trend, csv_path_tld,
                                                      tldsfile_path, year,
                                                      month)

                print('Building domain graph')
                lib_words.create_curve_from_redis_set(r_serv_trend, csv_path_domain,
                                                      "domain", year,
                                                      month)
                print('end building')

            publisher.debug("{} queue is empty, waiting".format(config_section))
            print('sleeping')
            time.sleep(5*60)
            continue

        else:
            generate_new_graph = True
            # Do something with the message from the queue
            url, date, path = message.split()
            faup.decode(url)
            url_parsed = faup.get()

            # Scheme analysis
            analyse(r_serv_trend, 'scheme', date, url_parsed)
            # Tld analysis
            analyse(r_serv_trend, 'tld', date, url_parsed)
            # Domain analysis
            analyse(r_serv_trend, 'domain', date, url_parsed)

            compute_progression(r_serv_trend, 'scheme', num_day_to_look, url_parsed)
            compute_progression(r_serv_trend, 'tld', num_day_to_look, url_parsed)
            compute_progression(r_serv_trend, 'domain', num_day_to_look, url_parsed)


'''
Constants for working with Beiwe time formats.
'''
from pytz import timezone
from collections import OrderedDict

# seconds
min_s  = 60
hour_s = 60*min_s
day_s  = 24*hour_s

# milliseconds
min_ms  = 1000*min_s
hour_ms = 1000*hour_s
day_ms  = 1000*day_s
week_ms =    7*day_ms
month_ms =  30*day_ms
year_ms =  365*day_ms
time_unit_ms = OrderedDict(zip(['minutes', 'hours', 'days', 'weeks', 'months', 'years'],
                               [min_ms, hour_ms, day_ms, week_ms, month_ms, year_ms]))

# Beiwe day order
day_order = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']

# time formats
data_time_format     = '%Y-%m-%dT%H:%M:%S.%f' # used in raw Beiwe data
filename_time_format = '%Y-%m-%d %H_%M_%S'    # used in raw Beiwe file names
date_only = '%Y-%m-%d'
time_only = '%H:%M:%S'
timezone_only = '%Z'
date_time_format  = ' '.join([date_only, time_only])
local_time_format = ' '.join([date_only, time_only, timezone_only])

# commonly used time zones
UTC = timezone('UTC')


"""
Created on 2022/01/08
@author Sangwoo Han
"""
from typing import Any, Callable, Dict, List, Optional, Union

import optuna
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import Callback
from pytorch_lightning.callbacks.base import Callback
from pytorch_lightning.trainer.optimizers import _get_default_scheduler_config
from pytorch_lightning.utilities import rank_zero_info, rank_zero_warn
from pytorch_lightning.utilities.exceptions import MisconfigurationException
from torch import nn
from torch.optim.swa_utils import SWALR

from .utils.train import swa_init, swa_step, swap_swa_params

_AVG_FN = Callable[[torch.Tensor, torch.Tensor, torch.LongTensor], torch.FloatTensor]

class MLFlowExceptionCallback(Callback):
    def on_exception(
        self,
        trainer: pl.Trainer,
        pl_module: pl.LightningModule,
        exception: BaseException,
    ) -> None:
        logger = pl_module.logger
        if logger.experiment.get_run(logger.run_id):
            logger.experiment.set_terminated(logger.run_id, status="FAILED")

    def on_test_start(
        self,
        trainer: "pl.Trainer",
        pl_module: "pl.LightningModule",
    ) -> None:
        logger = pl_module.logger
        if logger.experiment.get_run(logger.run_id):
            logger.experiment.set_terminated(logger.run_id, status="RUNNING")

class StochasticWeightAveraging(Callback):
    def __init__(
        self,
        swa_epoch_start: Union[int, float] = 0.8,
        swa_lrs: Optional[Union[float, List[float]]] = None,
        annealing_epochs: int = 10,
        annealing_strategy: str = "cos",
        avg_fn: Optional[_AVG_FN] = None,
        device: Optional[torch.device] = torch.device("cpu"),
    ):
        err_msg = "swa_epoch_start should be a >0 integer or a float between 0 and 1."
        if isinstance(swa_epoch_start, int) and swa_epoch_start < 1:
            raise MisconfigurationException(err_msg)
        if isinstance(swa_epoch_start, float) and not (0 <= swa_epoch_start <= 1):
            raise MisconfigurationException(err_msg)

        wrong_type = not isinstance(swa_lrs, (float, list))
        wrong_float = isinstance(swa_lrs, float) and swa_lrs <= 0
        wrong_list = isinstance(swa_lrs, list) and not all(
            lr > 0 and isinstance(lr, float) for lr in swa_lrs
        )
        if swa_lrs is not None and (wrong_type or wrong_float or wrong_list):
            raise MisconfigurationException(
                "The `swa_lrs` should be `None`, a positive float, or a list of positive floats"
            )

        if avg_fn is not None and not isinstance(avg_fn, Callable):
            raise MisconfigurationException("The `avg_fn` should be callable.")

        self._swa_epoch_start = swa_epoch_start
        self._swa_lrs = swa_lrs
        self._annealing_epochs = annealing_epochs
        self._annealing_strategy = annealing_strategy
        self._avg_fn = avg_fn or self.avg_fn
        self._device = device
        self._model_contains_batch_norm = None
        self._average_model = None
        self._should_stop = False
        self._is_transfer_device = True

    @property
    def state_key(self) -> str:
        return self._generate_state_key(
            swa_epoch_start=self._swa_epoch_start,
            annealing_strategy=self._annealing_strategy,
        )

    @property
    def swa_start(self) -> int:
        return max(self._swa_epoch_start - 1, 0)  # 0-based

    @property
    def swa_end(self) -> int:
        return self._max_epochs - 1  # 0-based

    @staticmethod
    def pl_module_contains_batch_norm(pl_module: "pl.LightningModule"):
        return any(
            isinstance(module, nn.modules.batchnorm._BatchNorm)
            for module in pl_module.modules()
        )

    def transfer_device(self, pl_module: "pl.LightningModule") -> None:
        if not self._average_model or self._is_transfer_device:
            return

        for n, p in pl_module.named_parameters():
            self._average_model[n] = self._average_model[n].to(p.device)

        self._is_transfer_device = True

    # def on_before_accelerator_backend_setup(
    #     self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    # ):
    #     # copy the model before moving it to accelerator device.
    #     with pl_module._prevent_trainer_and_dataloaders_deepcopy():
    #         self._average_model = deepcopy(pl_module)

    def on_fit_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"):
        optimizers = trainer.optimizers
        lr_schedulers = trainer.lr_schedulers

        # print('train_dataloader', trainer.train_dataloader)

        if len(optimizers) != 1:
            raise MisconfigurationException("SWA currently works with 1 `optimizer`.")

        if len(lr_schedulers) > 1:
            raise MisconfigurationException(
                "SWA currently not supported for more than 1 `lr_scheduler`."
            )

        if isinstance(self._swa_epoch_start, float):
            self._swa_epoch_start = int(trainer.max_epochs * self._swa_epoch_start)

        # self._model_contains_batch_norm = self.pl_module_contains_batch_norm(pl_module)

        self._max_epochs = trainer.max_epochs
        # if self._model_contains_batch_norm:
        #     # virtually increase max_epochs to perform batch norm update on latest epoch.
        #     trainer.fit_loop.max_epochs += 1
        #     print("+1")

    def on_train_epoch_start(
        self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    ):
        if trainer.current_epoch == self.swa_start:
            self._average_model = {}
            swa_init(pl_module, self._average_model, verbose=False)
            # with pl_module._prevent_trainer_and_dataloaders_deepcopy():
            #     self._average_model = deepcopy(pl_module)
            # move average model to request device.
            # self._average_model = self._average_model.to(
            #     self._device or pl_module.device
            # )

            optimizer = trainer.optimizers[0]
            if self._swa_lrs is None:
                self._swa_lrs = [
                    param_group["lr"] for param_group in optimizer.param_groups
                ]
            if isinstance(self._swa_lrs, float):
                self._swa_lrs = [self._swa_lrs] * len(optimizer.param_groups)

            for lr, group in zip(self._swa_lrs, optimizer.param_groups):
                group["initial_lr"] = lr

            self._swa_scheduler = SWALR(
                optimizer,
                swa_lr=self._swa_lrs,
                anneal_epochs=self._annealing_epochs,
                anneal_strategy=self._annealing_strategy,
                last_epoch=trainer.max_epochs
                if self._annealing_strategy == "cos"
                else -1,
            )
            default_scheduler_cfg = _get_default_scheduler_config()
            assert (
                default_scheduler_cfg["interval"] == "epoch"
                and default_scheduler_cfg["frequency"] == 1
            )
            default_scheduler_cfg["scheduler"] = self._swa_scheduler

            if trainer.lr_schedulers:
                scheduler_cfg = trainer.lr_schedulers[0]
                if (
                    scheduler_cfg["interval"] != "epoch"
                    or scheduler_cfg["frequency"] != 1
                ):
                    rank_zero_warn(
                        f"SWA is currently only supported every epoch. Found {scheduler_cfg}"
                    )
                rank_zero_info(
                    f"Swapping scheduler `{scheduler_cfg['scheduler'].__class__.__name__}`"
                    f" for `{self._swa_scheduler.__class__.__name__}`"
                )
                trainer.lr_schedulers[0] = default_scheduler_cfg
            else:
                trainer.lr_schedulers.append(default_scheduler_cfg)

            # self.n_averaged = torch.tensor(0, dtype=torch.long, device=pl_module.device)

        # if self.swa_start <= trainer.current_epoch <= self.swa_end:
        #     self.update_parameters(
        #         self._average_model, pl_module, self.n_averaged, self.avg_fn
        #     )

        # Note: No > here in case the callback is saved with the model and training continues
        # if self._should_stop or trainer.current_epoch == self.swa_end + 1:
        #     # Transfer weights from average model to pl_module
        #     self.transfer_weights(self._average_model, pl_module)

        #     # Reset BatchNorm for update
        #     self.reset_batch_norm_and_save_state(pl_module)

        #     # There is no need to perform either backward or optimizer.step as we are
        #     # performing only one pass over the train data-loader to compute activation statistics
        #     # Therefore, we will virtually increase `num_training_batches` by 1 and skip backward.
        #     trainer.num_training_batches += 1
        #     trainer.fit_loop._skip_backward = True
        #     self._accumulate_grad_batches = trainer.accumulate_grad_batches

        #     trainer.accumulate_grad_batches = trainer.num_training_batches

    # def on_train_epoch_end(self, trainer: "pl.Trainer", *args):
    #     trainer.fit_loop._skip_backward = False

    # def on_train_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"):
    #     # print("swa: on_train_end")
    #     # print("should_stop", trainer.should_stop)
    #     if (
    #         self._model_contains_batch_norm
    #         and trainer.current_epoch == self.swa_end + 1
    #     ):
    #         # BatchNorm epoch update. Reset state
    #         trainer.accumulate_grad_batches = self._accumulate_grad_batches
    #         trainer.num_training_batches -= 1
    #         trainer.fit_loop.max_epochs -= 1
    #         self.reset_momenta()
    #     elif trainer.current_epoch == self.swa_end:
    #         # Last SWA epoch. Transfer weights from average model to pl_module
    #         self.transfer_weights(self._average_model, pl_module)

    def on_validation_start(
        self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    ) -> None:
        if self._average_model:
            # self.update_parameters(
            #     self._average_model, pl_module, self.n_averaged, self.avg_fn
            # )

            # self.swap_weights(self._average_model, pl_module)
            self.transfer_device(pl_module)
            swa_step(pl_module, self._average_model, verbose=False)
            swap_swa_params(pl_module, self._average_model, verbose=False)

    def on_validation_end(
        self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    ) -> None:
        # self._should_stop = trainer.should_stop
        # print("should_stop:", self._should_stop)
        if self._average_model:
            # self.swap_weights(self._average_model, pl_module)
            swap_swa_params(pl_module, self._average_model, verbose=False)

    def on_test_start(
        self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    ) -> None:
        if self._average_model:
            self.transfer_device(pl_module)
            # self.swap_weights(self._average_model, pl_module)
            swap_swa_params(pl_module, self._average_model, verbose=False)

    def on_test_end(
        self, trainer: "pl.Trainer", pl_module: "pl.LightningModule"
    ) -> None:
        if self._average_model:
            # self.swap_weights(self._average_model, pl_module)
            swap_swa_params(pl_module, self._average_model, verbose=False)

    def on_save_checkpoint(
        self,
        trainer: "pl.Trainer",
        pl_module: "pl.LightningModule",
        checkpoint: Dict[str, Any],
    ) -> Dict[str, Any]:
        if self._average_model is not None:
            return {
                "average_model": self._average_model,
                # "n_averaged": self.n_averaged,
            }
        return {}

    def on_load_checkpoint(
        self,
        trainer: "pl.Trainer",
        pl_module: "pl.LightningModule",
        callback_state: Dict[str, Any],
    ) -> None:
        if "average_model" in callback_state:
            self._is_transfer_device = False
            self._average_model = callback_state["average_model"]
            # self.n_averaged = callback_state["n_averaged"]

    @staticmethod
    def transfer_weights(
        src_pl_module: "pl.LightningModule", dst_pl_module: "pl.LightningModule"
    ):
        for src_param, dst_param in zip(
            src_pl_module.parameters(), dst_pl_module.parameters()
        ):
            dst_param.detach().copy_(src_param.to(dst_param.device))

    @staticmethod
    def swap_weights(
        src_pl_module: "pl.LightningModule", dst_pl_module: "pl.LightningModule"
    ):
        for src_param, dst_param in zip(
            src_pl_module.parameters(), dst_pl_module.parameters()
        ):
            src_param.data, dst_param.data = dst_param.data, src_param.data

    def reset_batch_norm_and_save_state(self, pl_module: "pl.LightningModule"):
        """Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L140-L154."""
        self.momenta = {}
        for module in pl_module.modules():
            if not isinstance(module, nn.modules.batchnorm._BatchNorm):
                continue
            module.running_mean = torch.zeros_like(
                module.running_mean,
                device=pl_module.device,
                dtype=module.running_mean.dtype,
            )
            module.running_var = torch.ones_like(
                module.running_var,
                device=pl_module.device,
                dtype=module.running_var.dtype,
            )
            self.momenta[module] = module.momentum
            module.momentum = None
            module.num_batches_tracked *= 0

    def reset_momenta(self):
        """Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L164-L165."""
        for bn_module in self.momenta:
            bn_module.momentum = self.momenta[bn_module]

    @staticmethod
    def update_parameters(
        average_model: "pl.LightningModule",
        model: "pl.LightningModule",
        n_averaged: torch.LongTensor,
        avg_fn: _AVG_FN,
    ):
        """Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L104-L112."""
        for p_swa, p_model in zip(average_model.parameters(), model.parameters()):
            device = p_swa.device
            p_swa_ = p_swa.detach()
            p_model_ = p_model.detach().to(device)
            src = (
                p_model_
                if n_averaged == 0
                else avg_fn(p_swa_, p_model_, n_averaged.to(device))
            )
            p_swa_.copy_(src)

        n_averaged += 1

    @staticmethod
    def avg_fn(
        averaged_model_parameter: torch.Tensor,
        model_parameter: torch.Tensor,
        num_averaged: torch.LongTensor,
    ) -> torch.FloatTensor:
        """Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L95-L97."""
        return averaged_model_parameter + (
            model_parameter - averaged_model_parameter
        ) / (num_averaged + 1)


# -*- coding: UTF-8 -*-

__author__ = 'rain'
'''
参考文档：https://anikikun.gitbooks.io/appium-girls-tutorial/content/Up_uiautomator.html
'''

from appium import webdriver
import time
from appium.webdriver.common.touch_action import TouchAction

global driver
# 设置配置信息
desired_caps = {}
desired_caps['device']='android'
desired_caps['platformName']='Android'
desired_caps['version']='4.4'
desired_caps['deviceName']='GWY7N17704003267'
# 获取PackageName, launchActivity （Android SDK tool中aapt dump badging xxx.apk）
desired_caps['appPackage']='cn.thecover.www.covermedia'
desired_caps['appActivity']='.ui.activity.FlashActivity'

# 启动回话连接appium server
driver = webdriver.Remote('http://127.0.0.1/wd/hub', desired_caps)

time.sleep(1)

'''
元素定位
    1.id定位->resouceid  # id是唯一的，多个相同resourceid可以通过index
    2.classname定位      # classname不一定唯一
    3.xpath定位          # xpath定位效率慢
    4.name定位           # appium1.6版本已去掉
    4.name定位           # appium1.6版本已去掉
'''

# id 定位控件
driver.find_element_by_id(" ")          # 控件唯一
driver.find_element_by_id(" ")[index]   # 多个相同resourceid可以通过index

# text 定位控件
driver.find_element_by_name("text")     # 已弃用
driver.find_element_by_xpath("//*[@text= '%s']" % (text))  # 可替代 find_element_by_name

# Accessibility ID在Android上面就等同于contentDescription
driver.find_element_by_accessibility_id("contentDescription")

# 层级定位 通过父节点找到子节点 list列表里面
ele = driver.find_element_by_id("parent_id")
eles = driver.find_element_by_id("child_id")[index]

# touchaction可以用来实现点击坐标，滑动等操作
# 点击操作
TouchAction(driver).press(None, x, y).release().perform()
# 滑动操作
TouchAction(driver).press(None, x, y).wait(20).move_to(None, dx, dy).release().perform()
# 长按控件
action = TouchAction(driver).long_press(el, 20000).perform()
time.sleep(20)
action.release().perform()

# Scroll 对于listview 滑动控件
driver.scroll(ele1, ele2)
# 物理按键keyenvent
driver.keyevent(3)          # Home键
# 隐藏键盘
driver.hide_keyboard()

# UI Automator 定位
driver.find_element_by_android_uiautomator('new Uiselector().text("text")')

# text 模糊定位
driver.find_element_by_android_uiautomator('new Uiselector().textContains("text")')

# text 正则匹配
driver.find_element_by_android_uiautomator('new Uiselector().texMathces("^text.*")')

# resourceID
driver.find_element_by_android_uiautomator('new Uiselector().resourceId("resourceID")')

driver.find_element_by_android_uiautomator('new Uiselector().resourceIdMatches(".+text")')

# className
driver.find_element_by_android_uiautomator('new Uiselector().className("className")')
driver.find_element_by_android_uiautomator('new Uiselector().classNameMatchs(".*Editext")')
# 滑动查找listview包含text的控件
driver.find_element_by_android_uiautomator("new UiScrollable(new UiSelector().scrollable(true)).scrollIntoView(new UiSelector().textContains(\"上课中\"))")

# 封装曲线滑动 滑动速度不能太快
def swipe(points):
    last_x = 0
    last_y = 0
    swipe_action = TouchAction(driver)
    for i in range(0, len(points)):
        x = points[i][0]
        y = points[i][1]
        if i == 0:
            swipe_action = swipe_action .press(None, x, y).wait(20)
        elif i == (len(points) - 1):
            swipe_action = swipe_action.move_to(None, x - last_x, y - last_y).release()
            swipe_action.perform()
        else:
            swipe_action = swipe_action.move_to(None, x - last_x, y - last_y).wait(20)
        last_x = x
        last_y = y

# 封装多方式定位控件函数
def fin_for_element(xpath=None, id=None, text=None, index=None, timeout=3):
    start_time = time.time()
    now_time = time.time()
    while now_time - start_time < timeout:
        try:
            if xpath is not None:
                el = driver.find_element_by_xpath(xpath)
                return el
        except:
            pass
        try:
            if id is not None:
                if index is not None:
                    return driver.find_element_by_id(id)[index]
                else:
                    return driver.find_element_by_id(id)
        except:
            pass
        try:
            if text is not None:
                return driver.find_element_by_name(text)
        except:
            pass

        now_time = time.time()

# 退出会话
driver.quit()


from plotly.graph_objs import Scattergeo, Layout
from plotly import offline
import requests

data =  requests.get('https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson')
eq_data = data.json()

#filter out dictonarys for each earthqule and the magnitudes
all_eq_dicts = eq_data['features']
mags, lats, longs, hover_texts = [], [], [], []

#sort through the earthquake dictonaries and pull the magnitues, lats and longs
for eq in all_eq_dicts:
    mags.append(int(eq['properties']['mag']))
    lats.append(eq['geometry']['coordinates'][1])
    longs.append(eq['geometry']['coordinates'][0])
    hover_texts.append(eq['properties']['title'])

#defines data sets, scatter map, the lats and longs, hover text.
#Marker makes the plots bigger for each mag, and sets a color gradiant
data = [{
    'type': 'scattergeo',
    'lon': longs,
    'lat': lats,
    'text': hover_texts,
    'marker' : {
        'size' : [5*mag for mag in mags],
        'color' : mags,
        'colorscale': 'Jet',
        'reversescale': False,
        'colorbar' : {'title': 'Magnitude'},
    },
}]

#creates a layout
my_layout = Layout(title='Global Earthquakes')

#creates a figure with the layout and data parameters. then plots
fig = {'data': data, 'layout': my_layout}
offline.plot(fig, filename = 'global_earthquakes.html')


import datetime

# Parameters not defined in here:
# In RP_RL_agent: Loss function
# In RP_RL_main: What training/test data to use for RL
# In main: Model architecture
# In RP_supervised_learning: SL loss function

run_SL = 0
run_RL = 1

# If 1, ignores model and uses LP heuristic from paper for q vals
test_with_LP = 0

# If 1, ignores model and selects actions randomly
test_with_random = 0

# If 1, runs 10x tests on test data at start
test_10x = 0

# PUT_agent learns how to fully search PUT-RP search space
f_use_PUT_agent = 0

############## Reinforcement Learning Parameters

# After how many profiles trained to test the model
# Runs test on validation data
test_every = 5000

# Whether or not to test before any RL training
test_at_start = 0

# Whether to shuffle the training data
shuffle_training_data = 0

# Number of iterations to use when testing
# Doesn't matter if using test_till_find_all_winners or testing_v2
num_test_iterations = 5

# Whether to use experience replay
f_experience_replay = 0

f_train_till_find_all_winners = 0

f_use_winners_distribution = 0

# Uses PUT_RP_using_model
f_test_using_PUT_RP = 0

# V2 has model return values over all potential edges, instead of just q value of single action
# reduces computation time, but harder to learn
f_use_v2 = 0

# Testing v2 tests number of samples to find all winners
f_use_testing_v2 = 1

learning_rate = 0.00001
# 0 = no decay
# 1 = decay over all profiles
# 2 = decay per profile (doesn't work)
# Note: If using Adam optimizer it doesn't matter
f_learning_rate_decay = 0
learning_rate_start = 0.9
learning_rate_end = 0.00001
learning_rate_decay = 2000000

exploration_rate = 0.4
f_exploration_rate_decay = 1
exploration_rate_start = 0.9
exploration_rate_end = 0.1
exploration_rate_decay = 600000

# used in boltzmann
tau_start = 1.0
tau_end = 0.05
tau_decay = 1000000

discount_factor = 0.95

# after how many iterations to update the target network to the agent's learned network
update_target_network_every = 1

# num iterations per profile
num_training_iterations = 1

# number of epochs, where one epoch = num_training_iterations per profile
num_epochs = 2

# 1 = eps greedy
# 2 = boltzmann
exploration_algo = 2

# Num candidates
m = 10.0
# Num voters
n = 10.0

################## Model Parameters

# Whether to initialize model from default values (for comparison purposes)
f_start_from_default = 0

# Path to default model (used only if f_start_from_default)
default_model_path = "C:\\Users\shepht2\Documents\School\Masters\STV Ranked Pairs\\RL\\results\\10-3\\results_RP_RL_main161490381_model.pth.tar"

# What features to include
num_polynomial = 1
use_in_out = True # out/in of u,v
use_in_out_relative = False
use_total_degree = False
use_in_out_binary = False # binary out/in of u,v
use_in_out_matrix = False # in/out of every node
use_in_out_relative_matrix = False
use_total_degree_matrix = False # total degree of every node
use_in_out_binary_matrix = False # in/out binary of every node
use_K = True # u,v in K
use_K_big = False
use_voting_rules = False
use_voting_rules_matrix = False
use_edge_weight = False
use_visited = False # I don't think visited makes sense as a feature when using SL
use_cycles = False
use_vectorized_wmg = False
use_posmat = False
use_tier_adjacency_matrix = False # adjacency matrix of just the legal actions
use_connectivity = False # takes forever to compute, don't use it
use_connectivity_matrix = False # takes forever to compute, don't use it

use_adjacency_matrix = False
use_K_representation = False
use_betweenness_centrality = False

# Compute D_in from features used
D_in = 0
if use_in_out:
    D_in += 4
if use_in_out_relative:
    D_in += 4 * num_polynomial
if use_total_degree:
    D_in += num_polynomial * 2
if use_in_out_binary:
    D_in += 4
if use_in_out_matrix:
    D_in += 2 * m * num_polynomial
if use_in_out_relative_matrix:
    D_in += 2 * m * num_polynomial
if use_total_degree_matrix:
    D_in += m * num_polynomial
if use_in_out_binary_matrix:
    D_in += 2 * m
if use_K:
    D_in += 2
if use_voting_rules:
    D_in += num_polynomial * 8
if use_voting_rules_matrix:
    D_in += m * 4 * num_polynomial
if use_edge_weight:
    D_in += num_polynomial
if use_visited:
    D_in += num_polynomial
if use_cycles:
    D_in += num_polynomial
if use_vectorized_wmg:
    D_in += m * (m-1)
if use_posmat:
    D_in += m * m
if use_adjacency_matrix:
    D_in += m * m
if use_tier_adjacency_matrix:
    D_in += m * m
if use_connectivity:
    D_in += num_polynomial * 4
if use_connectivity_matrix:
    D_in += m * (m - 1) * 2 * num_polynomial
if use_K_representation:
    D_in += m
if use_betweenness_centrality:
    D_in += 2 * num_polynomial
if f_use_PUT_agent:
    D_in += 1 # depth

D_in = int(D_in)

H1 = 10  # first hidden dimension
H2 = 32  # second hidden dimension

if f_use_v2:
    D_out = int(m * (m - 1)) # output dimension, values over all actions
else:
    D_out = 1 # just want q value

# After how many nodes to output summed loss
print_loss_every = 10000

# 1 = gradient descent
# 2 = adam
# 3 = stochastic gradient descent (pytorch optimizer)
optimizer_algo = 1

# Whether to shape reward by num times winner found
f_shape_reward = 0

# used in testing v2
tau_for_testing = 0.05
cutoff_testing_iterations = 1000

# if train_till_find_all_winners, stops after this many iterations
cutoff_training_iterations = 25000

# if f_use_PUT_agent, cutoff testing nodes in validation testing
cutoff_testing_nodes = 100000

# Experience Replay Parameters
train_from_experiences_every_iterations = 100

buffer_size = 5000
unusual_sample_factor = 0.9
batch_size = 500

# Supervised Learning Parameters

SL_test_every = 100
SL_test_at_start = 1

SL_optimal_action_learning_rate = 0.0001
SL_bad_action_learning_rate = 0.0001

SL_num_epochs = 200
SL_num_training_data = 1000
SL_num_test_data = 100

# debug_mode
# = 0: no output
# = 1: outputs minimal debug
# = 2: outputs general RL steps
# = 3: outputs everything
debug_mode = 0

# Print parameters
def print_params(parameters_file):
    parameters_file.write("run_SL\t" + str(run_SL) + '\n')
    parameters_file.write("run_RL\t" + str(run_RL) + '\n')

    parameters_file.write("test_with_LP\t" + str(test_with_LP) + '\n')
    parameters_file.write("test_with_random\t" + str(test_with_random) + '\n')
    parameters_file.write("test_10x\t" + str(test_10x) + '\n')

    parameters_file.write("f_use_PUT_agent\t" + str(f_use_PUT_agent) + '\n')

    parameters_file.write("Training Data Shuffled\t" + str(shuffle_training_data) + '\n')
    parameters_file.write("Learning Rate Decay\t" + str(f_learning_rate_decay) + '\n')
    if f_learning_rate_decay == 0:
        parameters_file.write("Learning Rate\t" + str(learning_rate) + '\n')
    elif f_learning_rate_decay == 1:
        parameters_file.write("Learning Rate Start\t" + str(learning_rate_start) + '\n')
        parameters_file.write("Learning Rate End\t" + str(learning_rate_end) + '\n')
        parameters_file.write("Learning Rate Decay Rate\t" + str(learning_rate_decay) + '\n')
    parameters_file.write("Discount Factor\t" + str(discount_factor) + '\n')
    parameters_file.write("Exploration Rate Decay\t" + str(f_exploration_rate_decay) + '\n')
    if not f_exploration_rate_decay:
        parameters_file.write("Exploration Rate\t" + str(exploration_rate) + '\n')
    else:
        parameters_file.write("Exploration Rate Decay Start\t" + str(exploration_rate_start) + '\n')
        parameters_file.write("Exploration Rate Decay End\t" + str(exploration_rate_end) + '\n')
        parameters_file.write("Exploration Rate Decay Rate\t" + str(exploration_rate_decay) + '\n')
    parameters_file.write("Exploration Algo\t" + str(exploration_algo) + '\n')
    if exploration_algo == 2:
        parameters_file.write("Tau Start\t" + str(tau_start) + '\n')
        parameters_file.write("Tau End\t" + str(tau_end) + '\n')
        parameters_file.write("Tau Decay\t" + str(tau_decay) + '\n')
    parameters_file.write("Num Training Iterations per Profile\t" + str(num_training_iterations) + '\n')
    parameters_file.write("num_epochs\t" + str(num_epochs) + '\n')
    parameters_file.write("Agent D_in\t" + str(D_in) + '\n')
    parameters_file.write("Agent H1\t" + str(H1) + '\n')
    parameters_file.write("Agent H2\t" + str(H2) + '\n')
    parameters_file.write("Agent D_out\t" + str(D_out) + '\n')
    parameters_file.write("Num Polynomial Features\t" + str(num_polynomial) + '\n')
    parameters_file.write("test_every\t" + str(test_every) + '\n')
    parameters_file.write("f_use_winners_distribution\t" + str(f_use_winners_distribution) + '\n')

    parameters_file.write("use_in_out\t" + str(use_in_out) + '\n')
    parameters_file.write("use_in_out_relative\t" + str(use_in_out_relative) + '\n')
    parameters_file.write("use_total_degree\t" + str(use_total_degree) + '\n')
    parameters_file.write("use_in_out_binary\t" + str(use_in_out_binary) + '\n')
    parameters_file.write("use_in_out_matrix\t" + str(use_in_out_matrix) + '\n')
    parameters_file.write("use_in_out_relative_matrix\t" + str(use_in_out_relative_matrix) + '\n')
    parameters_file.write("use_total_degree_matrix\t" + str(use_total_degree_matrix) + '\n')
    parameters_file.write("use_in_out_binary_matrix\t" + str(use_in_out_binary_matrix) + '\n')
    parameters_file.write("use_K\t" + str(use_K) + '\n')
    parameters_file.write("use_K_big\t" + str(use_K_big) + '\n')
    parameters_file.write("use_voting_rules\t" + str(use_voting_rules) + '\n')
    parameters_file.write("use_voting_rules_matrix\t" + str(use_voting_rules_matrix) + '\n')
    parameters_file.write("use_edge_weight\t" + str(use_edge_weight) + '\n')
    parameters_file.write("use_visited\t" + str(use_visited) + '\n')
    parameters_file.write("use_cycles\t" + str(use_cycles) + '\n')
    parameters_file.write("use_vectorized_wmg\t" + str(use_vectorized_wmg) + '\n')
    parameters_file.write("use_posmat\t" + str(use_posmat) + '\n')
    parameters_file.write("use_tier_adjacency_matrix\t" + str(use_tier_adjacency_matrix) + '\n')
    parameters_file.write("use_connectivity\t" + str(use_connectivity) + '\n')
    parameters_file.write("use_connectivity_matrix\t" + str(use_connectivity_matrix) + '\n')
    parameters_file.write("use_adjacency_matrix\t" + str(use_adjacency_matrix) + '\n')
    parameters_file.write("use_K_representation\t" + str(use_K_representation) + '\n')
    parameters_file.write("use_betweenness_centrality\t" + str(use_betweenness_centrality) + '\n')

    parameters_file.write("Agent Optimizer Algo\t" + str(optimizer_algo) + '\n')
    if f_start_from_default:
        parameters_file.write("Default Model File\t" + default_model_path + '\n')
    else:
        parameters_file.write("Default Model File\tN/A\n")
    parameters_file.write("Experience Replay\t" + str(f_experience_replay) + '\n')
    if f_experience_replay:
        parameters_file.write("Experience Replay Buffer Size\t" + str(buffer_size) + '\n')
        parameters_file.write("Experience Replay Sample Factor\t" + str(unusual_sample_factor) + '\n')
        parameters_file.write("Experience Replay Batch Size\t" + str(batch_size) + '\n')
        parameters_file.write("Experience Replay Train Every\t" + str(train_from_experiences_every_iterations) + '\n')
    else:
        parameters_file.write("print_loss_every\t" + str(print_loss_every) + '\n')
    parameters_file.write("Update Target Network Every\t" + str(update_target_network_every) + '\n')
    if not f_test_using_PUT_RP and not f_use_testing_v2:
        parameters_file.write("Num Test Iterations\t" + str(num_test_iterations) + '\n')
    parameters_file.write("Train Till Find All Winners\t" + str(f_train_till_find_all_winners) + '\n')
    if f_train_till_find_all_winners:
        parameters_file.write("cutoff_training_iterations\t" + str(cutoff_training_iterations) + '\n')
    parameters_file.write("Test Using PUT_RP\t" + str(f_test_using_PUT_RP) + '\n')
    parameters_file.write("Use V2\t" + str(f_use_v2) + '\n')
    parameters_file.write("Shape Reward\t" + str(f_shape_reward) + '\n')
    parameters_file.write("Use Testing V2\t" + str(f_use_testing_v2) + '\n')
    parameters_file.write("Tau for Testing\t" + str(tau_for_testing) + '\n')
    if f_use_testing_v2:
        parameters_file.write("cutoff_testing_iterations\t" + str(cutoff_testing_iterations) + '\n')

    parameters_file.write("SL test every\t" + str(SL_test_every) + '\n')
    parameters_file.write("SL_optimal_action_learning_rate\t" + str(SL_optimal_action_learning_rate) + '\n')
    parameters_file.write("SL_bad_action_learning_rate\t" + str(SL_bad_action_learning_rate) + '\n')
    parameters_file.write("SL_num_epochs\t" + str(SL_num_epochs) + '\n')
    parameters_file.write("SL_num_training_data\t" + str(SL_num_training_data) + '\n')
    parameters_file.write("SL_num_test_data\t" + str(SL_num_test_data) + '\n')

    parameters_file.write("Date\t" + str(datetime.datetime.now()) + '\n')

    parameters_file.flush()


#!/usr/bin/python3

import os
import sys
import pystache
import shutil
import json
import subprocess

def transform_file(template='', outfile='', data=''):
    mst_file = open(template, 'r')
    mst_content = mst_file.read()
    mst_file.close()
    rendered = pystache.render(mst_content, data)
    output_file = open(outfile, 'w')
    output_file.write(rendered)
    output_file.close()

def read_config(working_dir=None):
    config_file_path = working_dir + '/' + 'make.json'
    config_file = open(config_file_path, 'r')
    config_content = config_file.read()
    config_file.close()
    output = json.loads(config_content)
    return output

def build(kitchen_home, working_dir):
    target_dir = working_dir + '/' + 'target'

    print("Kitchen home: " + kitchen_home)
    print("Working dir: " + working_dir)
    print("Target dir: " + target_dir)

    data = read_config(working_dir)

    dockerfile_mst = kitchen_home + '/' + 'Dockerfile.mst'
    entry_mst = kitchen_home + '/' + 'entry.sh.mst'
    script_mst = working_dir + '/' + 'script.sh.mst'
    start_mst = kitchen_home + '/' + 'start.py.mst'
    ssh_mst = kitchen_home + '/' + 'access_ssh.sh.mst'

    dockerfile_output = target_dir + '/' + 'Dockerfile'
    entry_output = target_dir + '/' + 'entry.sh'
    script_output = target_dir + '/' + 'script.sh'
    start_output = target_dir + '/' + 'start_' + data['image_name'] + '.py'
    ssh_output = target_dir + '/' + 'ssh_' + data['image_name'] + '.sh'

    if os.path.exists(target_dir):
        shutil.rmtree(target_dir)

    os.mkdir(target_dir)

    transform_file(dockerfile_mst, dockerfile_output, data)
    transform_file(entry_mst, entry_output, data)
    transform_file(script_mst, script_output, data)
    transform_file(start_mst, start_output, data)
    transform_file(ssh_mst, ssh_output, data)

    subprocess.call(['chmod', '755', start_output])
    subprocess.call(['chmod', '755', ssh_output])

    subprocess.call(['cp', data['ssh_public_key'], target_dir + '/' + 'key.pub'])

    for f in data['files']:
        subprocess.call(['cp', f, target_dir + '/' + f])

    subprocess.call(['docker', 'build', '-t', data['image_name'], target_dir])

def pack(kitchen_home, working_dir):
    config = read_config(working_dir)
    return

def main():
    if len(sys.argv) < 2:
        print("Usage: cook_docker.py action")
        return

    action = sys.argv[1]

    kitchen_home = os.environ.get('DOCKER_KITCHEN_HOME')
    if kitchen_home is None:
        print('Please define DOCKER_KITCHEN_HOME variable')
        exit(1)

    working_dir = os.getcwd()

    if action == 'build':
        build(kitchen_home, working_dir)
    elif action == 'pack':
        pack(kitchen_home, working_dir)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# coding=utf-8
"""
实现日志格式和级别设置。
"""

import os
import logging
import logging.handlers

def init_log(log_path):
    """
    log_path: 日志路径
    """
    format="%(levelname)s: %(asctime)s: %(filename)s:%(lineno)d * %(thread)d %(message)s"
    datefmt="%m-%d %H:%M:%S"

    level=logging.INFO
    formatter = logging.Formatter(format, datefmt)
    logger = logging.getLogger()
    logger.setLevel(level)

    dir = os.path.dirname(log_path)
    if not os.path.isdir(dir):
        os.makedirs(dir)

    handler_0 = logging.handlers.TimedRotatingFileHandler(log_path + ".log", when='H', backupCount=10)
    handler_0.setLevel(logging.INFO)
    handler_0.setFormatter(formatter)
    logger.addHandler(handler_0)

    handler_1 = logging.handlers.TimedRotatingFileHandler(log_path + ".log.wf", when='H', backupCount=10)
    handler_1.setLevel(logging.ERROR)
    handler_1.setFormatter(formatter)
    logger.addHandler(handler_1)

if __name__ == "__main__":
    init_log("./log/mimi_spider")
    logging.info("测试 info")
    logging.warn("测试 warn")
    logging.error("测试 error")


from flask import Flask, render_template, request

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/result', methods=['POST'])
def result():
    if request.method == 'POST':
        word = request.form['word']
        return render_template('result.html', word=word)

if __name__ == '__main__':
    app.run(debug=True)


from textblob import TextBlob
import nltk

file = open("D:\instruction.txt").read()

blob = TextBlob(file)
for nouns in blob.noun_phrases:
    print(nouns)

text = nltk.word_tokenize(file)
pos_tagged = nltk.pos_tag(text)
#print(pos_tagged)
nouns = list(filter(lambda x:x[1]=='NN',pos_tagged))
print(nouns)

for word, tag in pos_tagged:
    if tag in ('VB'):
        print (word, tag)

from textblob import TextBlob
text = "Tap Save copy in the upper right corner to save your video."
text_blob_object = TextBlob(text)
for ngram in text_blob_object.ngrams(2):
    print(ngram)


from pyspark import SparkContext, SparkConf
from pyspark.sql import HiveContext, SparkSession
from pyspark.sql import SQLContext, HiveContext
import pyspark.sql.functions as func
import pandas as pd
import numpy as np
import re
from pyspark.sql.functions import UserDefinedFunction
from pyspark.sql.types import *
from pyspark.sql import Row
from pyspark.sql.types import IntegerType
from pyspark.sql.types import DoubleType
from pyspark.sql.functions import when
import logging
from pyspark.sql.functions import count, avg, sum
from pyspark.mllib.linalg.distributed import *
from pyspark.mllib.linalg import Vectors
from pyspark.sql.functions import monotonically_increasing_id
from pyspark.sql.window import Window as W
from pyspark.sql.functions import col
from pyspark.sql.functions import *
def multiply_all(t_df):
    for field in t_df.schema.fields:
        name = field.name
        if name == "total_transaction":
            print("pass")
        elif name == "analytic_id":
            print("pass")
        else:
            t_df = t_df.withColumn(name, col(name)*col('total_transaction'))
    return t_df
def divide_all(t_df):
    for field in t_df.schema.fields:
        name = field.name
        if name == "ontop_purchased":
            print("pass")
        elif name == "analytic_id":
            print("pass")
        else:
            t_df = t_df.withColumn(name, col(name)/col('ontop_purchased'))
    t_df = t_df.drop('ontop_purchased')
    return t_df

def onehotenc(t_df, column):
    categories = t_df.select(column).distinct().rdd.flatMap(lambda x : x).collect()
    categories.sort()
    for category in categories:
        function = udf(lambda item: 1 if item == category else 0, DoubleType())
        new_column_name = column+'_'+str(category)
        t_df = t_df.withColumn(new_column_name, function(col(column)))
    t_df = t_df.drop(column)
    return t_df
if __name__ == "__main__":
    logging.getLogger("py4j").setLevel(logging.ERROR)
    conf = SparkConf().setAppName("Preprocess Ontop Transaction")
    sc = SparkContext(conf=conf)
    sqlContext = SQLContext(sc)
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Preprocessing Ontop Transaction")
    print("###################################################################################################")
    df_ontop1802 = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").option("inferSchema", "true").option("delimiter", '|').load('wasb://ds-cvm-hd-rs-devprod-02-2017-09-25t08-15-40-207z@natds201708cvm1sa01.blob.core.windows.net/data_cvm/ASSOCIATION/CVM_PREPAID_RECOMMENDATION_201804.txt')
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Reading Ontop Transaction Data")
    print("###################################################################################################")
    master_jean = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").option("inferSchema", "true").option("delimiter", ',').load('/data_cvm/master_tariff_jean.csv')
    master_jean3 = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").option("inferSchema", "true").option("delimiter", ',').load('/data_cvm/master_tariff_jean5.csv')
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Reading Ontop Master Data")
    print("###################################################################################################")
    masterjean3map = master_jean3.select(col("PromotionCode").alias('Promotion Code'),col("PackageName").alias("New_Package_Name"))
    master_mapping = master_jean.select(col("Promotion Code"),col("Package Name")).join(masterjean3map, ["Promotion Code"], "left_outer")
    master_mapping = master_mapping.drop_duplicates(subset=['Package Name'])
    master_mapping = master_mapping.drop("Promotion Code")
    package_preferences = master_jean.drop_duplicates(subset=['Package Name'])
    parsed_PP = package_preferences.select(col('Package Name').alias('Package_Name'),col('Price Inc VAT').alias("Price"),col('MM Data Speed').alias("Package_Size"))
    parsed_PP_duration = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").option("inferSchema", "true").option("delimiter", ',').load('/data_cvm/parsed_pp_with_duration_datasize.csv')
    parsed_PP_duration1 = parsed_PP_duration.withColumn('Package_Duration_New2',when(parsed_PP_duration.Package_Duration <= 1,'XS').otherwise(parsed_PP_duration.Package_Duration)).drop(parsed_PP_duration.Package_Duration)
    parsed_PP_duration1 = parsed_PP_duration1.withColumn('Package_Duration',when(parsed_PP_duration1.Package_Duration_New2 < 7,'S').otherwise(parsed_PP_duration1.Package_Duration_New2)).drop(parsed_PP_duration1.Package_Duration_New2)
    parsed_PP_duration1 = parsed_PP_duration1.withColumn('Package_Duration_New2',when(parsed_PP_duration1.Package_Duration == 7,'M').otherwise(parsed_PP_duration1.Package_Duration)).drop(parsed_PP_duration1.Package_Duration)
    parsed_PP_duration1 = parsed_PP_duration1.withColumn('Package_Duration',when(parsed_PP_duration1.Package_Duration_New2 <= 15,'L').otherwise(parsed_PP_duration1.Package_Duration_New2)).drop(parsed_PP_duration1.Package_Duration_New2)
    parsed_PP_duration1 = parsed_PP_duration1.withColumn('Package_Duration_New2',when(parsed_PP_duration1.Package_Duration >= 30,'XL').otherwise(parsed_PP_duration1.Package_Duration)).drop(parsed_PP_duration1.Package_Duration)
    parsed_duration_t = parsed_PP_duration1.select(col('Package_Name'),col("Package_Duration_New2").alias("Package_Duration"),col('Price'),col('Package_Size'))
    parsed_duration_t2 = parsed_duration_t.withColumn('Price_New2',when(parsed_duration_t.Price <= 6.44,'XS').otherwise(parsed_duration_t.Price)).drop(parsed_duration_t.Price)
    parsed_duration_t2 = parsed_duration_t2.withColumn('Price',when(parsed_duration_t2.Price_New2 <= 27.62,'S').otherwise(parsed_duration_t2.Price_New2)).drop(parsed_duration_t2.Price_New2)
    parsed_duration_t2 = parsed_duration_t2.withColumn('Price_New2',when(parsed_duration_t2.Price <= 36,'M').otherwise(parsed_duration_t2.Price)).drop(parsed_duration_t2.Price)
    parsed_duration_t2 = parsed_duration_t2.withColumn('Price',when(parsed_duration_t2.Price_New2 <= 111,'L').otherwise(parsed_duration_t2.Price_New2)).drop(parsed_duration_t2.Price_New2)
    parsed_duration_t2 = parsed_duration_t2.withColumn('Price_New2',when(parsed_duration_t2.Price > 111,'XL').otherwise(parsed_duration_t2.Price)).drop(parsed_duration_t2.Price)
    parsed_master = parsed_duration_t2.select(col('Package_Name'),col("Package_Duration"),col('Price_New2').alias('Price'),col('Package_Size'))
    parsed_master_final = onehotenc(parsed_master, "Package_Duration")
    parsed_master_final = onehotenc(parsed_master_final, "Price")
    parsed_master_final = parsed_master_final.na.drop(subset=["Package_Size"])
    parsed_master_final = onehotenc(parsed_master_final, "Package_Size")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished One-hot Encoding on Ontop Master")
    print("###################################################################################################")
    parsed_master_final = parsed_master_final.join(master_mapping.select(col("Package Name").alias("Package_Name"),col("New_Package_Name")), ["Package_Name"], "left_outer")
    parsed_master_final=parsed_master_final.withColumn("Package_Duration_XS", parsed_master_final["Package_Duration_XS"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Duration_S", parsed_master_final["Package_Duration_S"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Duration_M", parsed_master_final["Package_Duration_M"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Duration_L", parsed_master_final["Package_Duration_L"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Duration_XL", parsed_master_final["Package_Duration_XL"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Price_XS", parsed_master_final["Price_XS"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Price_S", parsed_master_final["Price_S"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Price_M", parsed_master_final["Price_M"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Price_L", parsed_master_final["Price_L"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Price_XL", parsed_master_final["Price_XL"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_64Kbps", parsed_master_final["Package_Size_64Kbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_256Kbps", parsed_master_final["Package_Size_256Kbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_384Kbps", parsed_master_final["Package_Size_384Kbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_512Kbps", parsed_master_final["Package_Size_512Kbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_1Mbps", parsed_master_final["Package_Size_1Mbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_2Mbps", parsed_master_final["Package_Size_2Mbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_4Mbps", parsed_master_final["Package_Size_4Mbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_6Mbps", parsed_master_final["Package_Size_6Mbps"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Unlimit_Timed", parsed_master_final["Package_Size_Unlimit_Timed"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Unlimit", parsed_master_final["Package_Size_Unlimit"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Trotting_S", parsed_master_final["Package_Size_Trotting_S"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Trotting_M", parsed_master_final["Package_Size_Trotting_M"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Trotting_L", parsed_master_final["Package_Size_Trotting_L"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Qouta_S", parsed_master_final["Package_Size_Qouta_S"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Qouta_M", parsed_master_final["Package_Size_Qouta_M"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Qouta_L", parsed_master_final["Package_Size_Qouta_L"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Social", parsed_master_final["Package_Size_Social"].cast(IntegerType()))
    parsed_master_final=parsed_master_final.withColumn("Package_Size_Entertain", parsed_master_final["Package_Size_Entertain"].cast(IntegerType()))
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Joining Ontop Transaction with Ontop Master")
    print("###################################################################################################")
    joinedtarif = df_ontop1802.join(master_jean3.select(col("PromotionCode").alias("package_id"),col("PackageName").alias("Unique_Name")), ["package_id"], "inner")
    customer_trans = joinedtarif.na.drop(subset=["Unique_Name"])
    customer_trans_simp = customer_trans.select(col("analytic_id"),col("total_transaction"),col("Unique_Name").alias("Package_Name")).na.drop(subset=["Package_Name"])
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Joining Ontop Transaction with ontop Master")
    print("###################################################################################################")

    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Performing One-hot Encoding on Customer Ontop Transaction")
    print("###################################################################################################")
    customer_trans_final = onehotenc(customer_trans_simp, "Package_Name")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished One-hot Encoding on Customer Ontop Transaction")
    print("###################################################################################################")

    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Multiplying total_transaction to all package features")
    print("###################################################################################################")
    customer_trans_final = multiply_all(customer_trans_final)
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finish Multiplying to all package features")
    print("###################################################################################################")
    customer_trans_final = customer_trans_final.drop('total_transaction')

    #groupby and get sum of all purchased ontop and divide all onehot value to get probability of puchasing an ontop
    customer_trans_simp.registerTempTable("cust_trans_simp")
    ontop_purchased = sqlContext.sql("SELECT analytic_id, SUM(total_transaction) as ontop_purchased FROM cust_trans_simp GROUP BY analytic_id")

    customer_trans_final.registerTempTable("cust_trans_final")
    #get customer transaction ontop list to filter out dimension
    column_seq = customer_trans_simp.select('Package_Name').distinct().rdd.map(lambda r: r[0]).collect()
    query = ''
    for c in column_seq:
        query =  query + ", SUM(Package_Name_" + c + ") AS " + c

    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Group By analytic_id and get SUM of all features")
    print("###################################################################################################")
    final = sqlContext.sql("SELECT analytic_id" + query + " FROM cust_trans_final GROUP BY analytic_id ")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Group by analytic_id and get SUM of all features")
    print("###################################################################################################")

    final = final.join(ontop_purchased,["analytic_id"],"inner")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Multiplying group by result with number of transaction")
    print("###################################################################################################")
    final = divide_all(final)
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Multiplying group by result with number of transaction")
    print("###################################################################################################")
    final.registerTempTable("final")

    #filter package according to existing in customer transaction and create index list of packages
    df_index = parsed_master_final.where(parsed_master_final.New_Package_Name.isin(column_seq)).drop_duplicates(subset=['New_Package_Name']).withColumn("id", monotonically_increasing_id())
    index_pp = df_index.select("id","Package_Duration_XS","Package_Duration_S","Package_Duration_M","Package_Duration_L","Package_Duration_XL","Price_XS","Price_S",\
               "Price_M","Price_L","Price_XL","Package_Size_64Kbps","Package_Size_256Kbps","Package_Size_384Kbps","Package_Size_512Kbps",\
               "Package_Size_1Mbps","Package_Size_2Mbps","Package_Size_4Mbps","Package_Size_6Mbps","Package_Size_Unlimit_Timed",\
               "Package_Size_Unlimit","Package_Size_Trotting_S","Package_Size_Trotting_M","Package_Size_Trotting_L","Package_Size_Qouta_S",\
               "Package_Size_Qouta_M","Package_Size_Qouta_L","Package_Size_Social","Package_Size_Entertain")

    windowSpec = W.orderBy("id")
    ontop_rowid = df_index.withColumn("id", row_number().over(windowSpec))
    #df_index.select("New_Package_Name","id").withColumn("id", row_number().over(windowSpec)).show()
    ontop_rowid2 = ontop_rowid.withColumnRenamed("Package_Size_Full speed", "Package_Size_Full_speed")
    ontop_rowid2.registerTempTable("ontop_rowid2")
    index_pp2 = sqlContext.sql("SELECT id-1 as id,Package_Duration_XS,Package_Duration_S,Package_Duration_M,Package_Duration_L,Package_Duration_XL,Price_XS,Price_S,Price_M,Price_L,Price_XL,Package_Size_64Kbps,Package_Size_256Kbps,Package_Size_384Kbps,Package_Size_512Kbps,Package_Size_1Mbps,Package_Size_2Mbps,Package_Size_4Mbps,Package_Size_6Mbps,Package_Size_Unlimit_Timed,Package_Size_Unlimit,Package_Size_Trotting_S,Package_Size_Trotting_M,Package_Size_Trotting_L,Package_Size_Qouta_S,Package_Size_Qouta_M,Package_Size_Qouta_L,Package_Size_Social,Package_Size_Entertain FROM ontop_rowid2")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Writing Package Perferences to file")
    print("###################################################################################################")
    index_pp2.repartition(1).write.option("sep","|").option("header","true").csv("/preprocessed_cvm/package_preferences_rowId")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Writing Package Perferences to file")
    print("###################################################################################################")
    package_sort = ontop_rowid.select('New_Package_Name').rdd.map(lambda r: r[0]).collect()
    query = ''
    for c in package_sort:
        query =  query + ", " + c
    final_selected = sqlContext.sql("SELECT analytic_id" + query + " FROM final")
    final_index = final_selected.withColumn("id", monotonically_increasing_id())
    windowSpec = W.orderBy("id")
    final_rowindex = final_index.withColumn("id", row_number().over(windowSpec))
    final_index.select("analytic_id","id").withColumn("id", row_number().over(windowSpec)).show()
    final_rowindex.registerTempTable("final_rowindex")
    index_ct = sqlContext.sql("SELECT id-1 as id,analytic_id" + query + " FROM final_rowindex")
    index_ct.registerTempTable("index_ct ")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Start Writing Customer On-top Persona to file")
    print("###################################################################################################")
    index_ct.write.option("sep","|").option("header","true").csv("/preprocessed_cvm/customer_persona_rowId")
    print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n###################################################################################################")
    print("Finished Writing Customer On-top Persona to file")
    print("###################################################################################################")
    sc.stop()


from __future__ import absolute_import, division, print_function

if __name__ == "__main__":
  import argparse

  parser = argparse.ArgumentParser(
    description = "Server process handling far end of remote execution",
    usage = "%(prog)s job-factory queue-factory (do not run directly!)",
    )
  parser.add_argument(
    "job",
    help = "Job factory pickle string",
    )
  parser.add_argument(
    "queue",
    help = "Job factory pickle string",
    )
  parser.add_argument( "--folder", default = ".", help = "Process workdir" )
  parser.add_argument(
    "--polltime",
    type = int,
    default = 0.01,
    help = "Poll intervall",
    )

  params = parser.parse_args()

  import os
  import sys
  os.chdir( params.folder )
  sys.path.append( os.path.abspath( params.folder ) )

  from libtbx.queuing_system_utils import remote

  jfact = remote.argument_to_object( arg = params.job )
  qfact = remote.argument_to_object( arg = params.queue )

  from libtbx.queuing_system_utils import scheduling

  manager = scheduling.Scheduler(
    handler = scheduling.Unlimited(
      factory = lambda:
        scheduling.ExecutionUnit(
          factory = jfact,
          processor = scheduling.RetrieveProcessor( queue = qfact(), timeout = 1 ),
          ),
      ),
    polling_interval = params.polltime,
    )

  server = remote.SchedulerServer(
    instream = sys.stdin,
    outstream = sys.stdout,
    manager = manager,
    )

  # Redirect standard filehandles, so that the communication stream is intact
  sys.stdout = sys.stderr

  server.serve()


"""added is_closed column in PAFunctionalCompetencyRound table

Revision ID: f7b5b69d4b81
Revises: 41ef5682f91b
Create Date: 2023-11-21 16:51:22.177488

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'f7b5b69d4b81'
down_revision = '41ef5682f91b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('pa_functional_competency_round', schema=None) as batch_op:
        batch_op.add_column(sa.Column('is_closed', sa.Boolean(), nullable=True))

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('pa_functional_competency_round', schema=None) as batch_op:
        batch_op.drop_column('is_closed')

    # ### end Alembic commands ###


from sqlalchemy import Table, Column, MetaData, String
from sqlalchemy.dialects import postgresql
from sqlalchemy import create_engine
from sqlalchemy import insert, delete, select
import os

engine = create_engine(
    os.environ["POSTGRES_URL"],
    echo = True,
)

connection = engine.connect()

metadata = MetaData()

authorised = Table(
    "authorised",
    metadata,
    Column('id',String,primary_key=True)
)

table = Table(
    "stickers",
    metadata,
    Column('file_id',String,primary_key=True),
    Column('file_unique_id',String),
    Column('data', postgresql.ARRAY(String)),
    Column('type',String)
)

metadata.create_all(engine)

def add_authorised(id:str) -> None:
    stmt = (
        insert(authorised).
        values(id=id)
    )
    connection.execute(stmt)

def remove_authorised(id:str) -> None:
    stmt = (
        delete(authorised).
        where(authorised.c.id==id)
    )
    connection.execute(stmt)

def is_authorised(id:str) -> bool:
    stmt = (
        select(authorised).
        where(authorised.c.id==id)
    )
    result = connection.execute(stmt)

    try:
        return result.fetchall()[0][0] == id
    except IndexError:
        return False

def add_sticker(file_id:str,file_unique_id:str,tags:list,type:str) -> None:
    stmt = (
        insert(table).
        values(file_id=file_id,file_unique_id=file_unique_id, data=tags, type=type)
    )
    connection.execute(stmt)

def delete_sticker(file_unique_id:str) -> None:
    stmt = (
        delete(table).
        where(table.c.file_unique_id==file_unique_id)
    )
    connection.execute(stmt)

def get_tags(file_unique_id:str)->list:
    stmt = (
        select(table.c.data).
        where(table.c.file_unique_id==file_unique_id)
    )
    return connection.execute(stmt).fetchall()

def add_tag(file_id:str,file_unique_id:str,new_tags:list,type:str) -> None:
    # not using postgres array_append because it only
    # supports single argument and not multiple

    old_tags = get_tags(file_unique_id)

    if old_tags != []:
        delete_sticker(file_unique_id)
        union = list(set(old_tags[0][0]).union(set(new_tags)))
    else:
        union = new_tags
    add_sticker(file_id,file_unique_id,union,type)

def remove_tag(file_id:str,file_unique_id,new_tags:list,type:str) -> None:
    # not using postgres array_remove because it only
    # supports single argument and not multiple

    old_tags = get_tags(file_unique_id)

    if old_tags != []:
        delete_sticker(file_unique_id)
        diff_list = list(set(old_tags[0][0]).difference(set(new_tags)))
    else:
        diff_list = new_tags
    add_sticker(file_id,file_unique_id,diff_list,type)

def fetch_sticker(tags:list)->list:
    stmt = (
        select(table.c.file_id,table.c.type).
        where(table.c.data.contains(tags))
    )
    result = connection.execute(stmt)

    temp_list = []
    for i in result.fetchall():
        temp_list.append(i)

    return temp_list