from __future__ import absolute_import

from xml.sax.saxutils import XMLGenerator
from copy import copy
from . import _node_refs, OrderedDict, StringIO, _unicode
from . import _XMLCDATAPlaceholder, _XMLDictPlaceholder, _XMLListPlaceholder

__all__ = ['XMLNodeBase']

class _NoArg(object): # pylint: disable=too-few-public-methods
    """Internal Use Only"""
    def __init__(self):
        pass

_common_docstring = lambda x: """Initialize an %s object.

The optional first parameter can be the value to which the
object should be initialized. All other parameters must be
given as keywords.

Normally, the user can simply run this as::

    >>> node = %s(initializer)

In fact, the best way to use this is::

    >>> root = XMLDictNode({'root': {'branch': { 'leaf': 'a'}}})

That will set all the tags, keys, etc. correctly. However,
if you really want to customize a node, there are other
parameters available. Note that these parameters only
impact *this* node and descendants. They don't actually
add the node to a tree. Therefore, their use is
discouraged. Instead, you can probably use the :py:meth:`add_node`
method to build your tree correctly.

The one exception to this general rule is when adding a
hunk of a tree. For example, assume you currently have this XML
structure::

        <node1>a</node1>

And, assume you want to add another node ``b`` to create this
XML structure::

        <node1>a</node1>

        <node2>b</node1>

In that case, you might do something like this::

    >>> root.prettyprint()
    {u'a': {u'b': {u'node1': u'a'}}}
    >>> new_b = {'node2': 'b'}
    >>> new_b = XMLDictNode(new_b, tag="b")
    >>> _ = root['a'].add_node(tag="b", new_node=new_b)
    >>> root.prettyprint()
    {u'a': {u'b': [{u'node1': u'a'}, {'node2': u'b'}]}}

And, you can print the XML to prove it is formatted correctly::

    >>> print root.emit_xml()
    <?xml version="1.0" encoding="utf-8"?>

            <node1>a</node1>

            <node2>b</node2>

Args:
    initializer (as appropriate for node): The initial value for
        the node.
    tag (string): The XML tag for this node.
    key (string or tuple): The dictionary key used for this node.
    xml_attrs (`dict`): The XML attributes for the node.
    text (string): The node's initial CDATA value. (Note
        that this is ignored for :py:class:`XMLCDATANode` objects.)
    parent (Instance of a sub-class of :py:class:`XMLNodeBase`): A
        reference to the object's parent node in the data structure.
    convert (bool): If True, the :py:meth:`convert` method is run on
        the object's children during object initialization.
    deep (bool): If True (and the :py:obj:`convert` parameter is
        True), the :py:meth:`convert` method is run recursively
        on the object's children during object initialization.
""" % (x, x)

def _docstring_fixup(cls):
    """Fixup docstrings for a subclass of XMLNodeBase.

       This method will find any class method with an unset docstring
       and try to copy it from the corresponding method in
       XMLNodeBase.
    """
    if hasattr(cls, '__init__') and not cls.__init__.__doc__:
        docstring = """Initialize an %s object.

        See the class documentation for initializer arguments.
        """ % (cls.__name__)
        try:
            cls.__init__.__doc__ = docstring
        except AttributeError:
            cls.__init__.__func__.__doc__ = docstring
    for k in dir(cls):
        if hasattr(getattr(cls, k), '__call__') and not getattr(cls, k).__doc__:
            if hasattr(XMLNodeBase, k):
                docstring = getattr(XMLNodeBase, k).__doc__
                try:
                    getattr(cls, k).__doc__ = docstring
                except AttributeError:
                    try:
                        getattr(cls, k).__func__.__doc__ = docstring
                    except AttributeError:
                        # You just can't update some methods...
                        pass

XMLCDATANode = _XMLCDATAPlaceholder
XMLDictNode = _XMLDictPlaceholder
XMLListNode = _XMLListPlaceholder
def _resolve_references_once():
    """Internal function to resolve late references.

    There are circular references between the three node types. If we try
    to import all of the references into each module at parse time, the
    parser (rightly) complains about an infinite loop. This function "solves"
    that by doing a one-time load of the symbols the first time an
    instance of the class is changed. The function then replaces its own
    name in the module symbol table with a lambda function to turn this
    into a no-op.
    """
    # pylint: disable=global-statement
    # pylint: disable=invalid-name
    global XMLCDATANode
    global XMLDictNode
    global XMLListNode
    global _resolve_references
    XMLCDATANode = _node_refs['XMLCDATANode']
    XMLDictNode = _node_refs['XMLDictNode']
    XMLListNode = _node_refs['XMLListNode']
    _resolve_references = lambda: None

_resolve_references = _resolve_references_once

class XMLNodeBase(object):
    """This module provides methods common to the XML node classes.

    This modules is not intended for standalone use.
    """

    def __new__(cls, *args, **kwargs):
        # Resolve delayed references, if necessary
        _resolve_references()

        # Make a copy of the kwargs dictonary. (We don't want to modify
        # the original; we only want to modify the values we pass on
        # to super classes.)
        kwargs = dict(kwargs)

        # Determine if we were called with an initial value. If so,
        # make sure there was only *one* initial value.
        initializer = kwargs.pop("initializer", _NoArg())
        if not isinstance(initializer, _NoArg):
            if len(args) > 0:
                raise TypeError("got multiple values for keyword "
                                "argument 'initializer'")
            args = (initializer,)
        elif len(args) == 1:
            initializer = args[0]

        # Pop off arguments that have a special meaning at our layer.
        for k in ("tag", "key", "parent", "xml_attrs", "text",
                  "convert", "deep"):
            _ = kwargs.pop(k, None)

        # Create the object.
        return super(XMLNodeBase, cls).__new__(cls, *args, **kwargs)

    def __init__(self, *args, **kwargs):
        # Make a copy of the kwargs dictonary. (We don't want to modify
        # the original; we only want to modify the values we pass on
        # to super classes.)
        kwargs = dict(kwargs)

        # Determine if we were called with an initial value. If so,
        # make sure there was only *one* initial value.
        initializer = kwargs.pop("initializer", _NoArg())
        if not isinstance(initializer, _NoArg):
            if len(args) > 0:
                raise TypeError("got multiple values for keyword "
                                "argument 'initializer'")
            args = (initializer,)
        elif len(args) == 1:
            initializer = args[0]

        # Process arguments

        # Do not inherit tag, key, or parent in copies.
        tag = kwargs.pop("tag", None)
        key = kwargs.pop("key", tag)
        parent_node = kwargs.pop("parent", None)

        # If the initial value was an XMLNodeBase child, copy
        # node_text and xml_attrs.

        # Set some overall defaults
        xml_attrs = OrderedDict()
        node_text = _unicode()

        # Modify the defaults if the initial value was an XMLNodeBase object
        if isinstance(initializer, XMLNodeBase):
            xml_attrs = getattr(initializer, "xml_attrs", xml_attrs)
            node_text = getattr(initializer, "text", node_text)

        # Process the arguments to override the defaults
        xml_attrs = kwargs.pop("xml_attrs", xml_attrs)
        node_text = kwargs.pop("text", node_text)

        # Process other arguments
        convert = kwargs.pop('convert', True)
        deep = kwargs.pop('deep', True)

        # Initialize the object.
        super(XMLNodeBase, self).__init__(*args, **kwargs)

        # Add attributes to the object.
        self.tag = tag
        self.key = copy(key)
        self.xml_attrs = OrderedDict()
        for k in xml_attrs:
            self.xml_attrs[k] = _unicode(xml_attrs[k])
        if not hasattr(self, "text"):
            self.text = node_text
        self.parent = parent_node
        self._replacement_node = None
        if convert:
            self.standardize(deep=deep)

    def has_xml_attrs(self):
        """Determine if the node has XML attributes.

        Returns:
            A bool that is True if the node has XML attributes, and
                False otherwise.
        """
        if len(self.xml_attrs) > 0:
            return True
        return False

    def _check_replacement(self):
        # Ensure the object is current.
        if self._replacement_node:
            raise AttributeError(
                "Attempt to modify an out-of-date node. " +
                "Use get_current_node() to update the reference."
                )

    def set_xml_attr(self, attr, val):
        """Set an XML attribute.

        This method sets the XML attribute to the given value.  If the
        XML attribute already existed, its value is overridden by the
        new value.  If the XML attribute did not already exist, it is
        created.

        Args:
            attr (string): The name of the XML attribute.
            val (string): The value of the XML attribute.

        Returns:
            None

        Raises:
            :py:exc:`AttributeError`: If the node is out of date.
                (See :py:meth:`get_current_node`.)
        """
        self._check_replacement()
        self.xml_attrs[attr] = _unicode(val)

    def get_xml_attr(self, attr, defval=_NoArg()):
        """Get an XML attribute.

        This method returns the value of an XML attribute. If the XML
        attribute does not exist, it will return a user-supplied default value.
        If the user did not supply a default value, it raises a KeyError.

        Args:
            attr (string): The name of the XML attribute.
            defval (string): The default value. (Default: Raise a KeyError.)

        Returns:
            The string value of the XML attribute, or :py:obj:`defval`.

        Raises:
            :py:exc:`KeyError`: If the :py:obj:`attr` is not found and
                :py:obj:`defval` is not supplied.
        """
        try:
            return self.xml_attrs[attr]
        except KeyError:
            if not isinstance(defval, _NoArg):
                return defval
            raise

    def get_xml_attrs(self):
        """Return the XML attribute dictionary.

        This method returns the value of the XML attribute
        dictonary.  Note that it returns the actual XML attribute
        dictionary, rather than a copy.  Please take caution in
        modifying it.

        Returns:
            :py:class:`OrderedDict`: The XML attribute dictionary.
        """
        return self.xml_attrs

    def delete_xml_attr(self, attr):
        """Delete an XML attribute.

        This method deletes an XML attribute from the node.  If the
        attribute does not exist, it raises a KeyError.

        Args:
            attr (string): The name of the XML attribute.

        Returns:
            None

        Raises:
            :py:exc:`KeyError`: If the :py:obj:`attr` is not found.
            :py:exc:`AttributeError`: If the node is out of date. (See
                :py:meth:`get_current_node`.)
        """
        self._check_replacement()
        del self.xml_attrs[attr]

    def set_cdata(self, cdata, return_node=False):
        """Set a node's CDATA.

        This method sets a node's CDATA.  Note that any node can
        contain CDATA in what is called "semi-structured"
        XML. However, nodes that only contain CDATA are represented as
        :py:class:`XMLCDATANode` objects. Regardless of the node, you can use
        this same method to set the CDATA.

        **Note**: When running this on an XMLCDATANode, the actual node
        will be replaced with a new node in the tree. (This is a
        result of Python's string immutability.)  The function will
        update the XML tree, if necessary; however, any local
        references you have saved for the node will become stale.  You
        can obtain the updated node by setting the return_node
        parameter to True or by running the :py:meth:`get_current_node` method
        on the old node. For this reason, if you plan to keep a local
        reference to XML node in question, it is a good idea to run
        the method like this::

            >>> node = root['a']['b'][0]
            >>> node = node.set_cdata("foo", True)

        Args:
            cdata (string): The text value that should be used for the
                node's CDATA.
            return_node (bool): Whether the method should return the
                updated node.

        Returns:
            None or the updated node object if :py:obj:`return_node` is True.

        Raises:
            :py:exc:`AttributeError`: If the node is out of date. (See
                :py:meth:`get_current_node`.)
        """
        self._check_replacement()
        self.text = _unicode(cdata)
        if return_node:
            return self

    def append_cdata(self, cdata, return_node=False):
        """Append text to a node's CDATA.

        This method appends text to a node's CDATA.  Note that any
        node can contain CDATA in what is called "semi-structured"
        XML. However, nodes that only contain CDATA are represented as
        :py:class:`XMLCDATANode` objects. Regardless of the node, you can use
        this same method to append CDATA.

        **Note**: When running this on an :py:class:`XMLCDATANode`, the actual
        node will be replaced with a new node in the tree. (This is a
        result of Python's string immutability.)  The function will
        update the XML tree, if necessary; however, any local
        references you have saved for the node will become stale.  You
        can obtain the updated node by setting the :py:obj:`return_node`
        parameter to True or by running the :py:meth:`get_current_node` method
        on the old node. For this reason, if you plan to keep a local
        reference to XML node in question, it is a good idea to run
        the method like this::

            >>> node = root['a']['b'][0]
            >>> node = node.append_cdata("foo", True)

        Args:
            cdata (string): The text value that should be used for the
                node's CDATA.
            return_node (bool): Whether the method should return the
                updated node.

        Returns:
            None, if :py:obj:`return_node` is False, otherwise, the updated
                node object.

        Raises:
            :py:exc:`AttributeError`: If the node is out of date.
                (See :py:meth:`get_current_node`.)
        """
        self._check_replacement()
        self.text = self.text + cdata
        if return_node:
            return self

    def get_cdata(self):
        """Get a node's CDATA.

        Returns:
            A string containing the node's CDATA.
        """
        return self.text

    def strip_cdata(self, chars=None, return_node=False):
        """Strip leading/trailing characters from a node's CDATA.

        This method runs the string class' :py:meth:`strip` method on a node's
        CDATA and updates the node's CDATA with the result. (This is
        the functional equivalent to
        ``node.set_cdata(node.get_cdata().strip())``.)

        Note that any node can contain CDATA in what is called
        "semi-structured" XML. However, nodes that only contain CDATA
        are represented as :py:class:`XMLCDATANode` objects. Regardless of the
        node, you can use this same method to append CDATA.

        **Note**: When running this on an :py:class:`XMLCDATANode`, the actual
        node will be replaced with a new node in the tree. (This is a
        result of Python's string immutability.)  The function will
        update the XML tree, if necessary; however, any local
        references you have saved for the node will become stale.  You
        can obtain the updated node by setting the return_node
        parameter to True or by running the :py:meth:`get_current_node` method
        on the old node. For this reason, if you plan to keep a local
        reference to XML node in question, it is a good idea to run
        the method like this::

            >>> node = root['a']['b'][0]
            >>> node = node.strip_cdata(return_node=True)

        Args:
            chars (string): Contains the characters to strip. This is passed to
                the string class' :py:meth:`strip` method.
            return_node (bool): Whether the method should return the
                updated node.

        Returns:
            None if :py:obj:`return_node` is False; otherwise, the updated
                node object.

        Raises:
            :py:class:`AttributeError`: If the node is out of date.
                (See :py:meth:`get_current_node`.)
        """
        self._check_replacement()
        newtext = _unicode.strip(self.get_cdata(), chars)
        return self.set_cdata(newtext, return_node)

    def add_node(self, tag, key=None, text=_unicode(), new_node=None,
                 update=True, **kwargs):
        """Add an XML node to an XML tree.

        This method adds a new XML node as a child of the current
        node. If the current node is an :py:class:`XMLCDATANode`, it will be
        converted to an :py:class:`XMLDictNode` so that it can hold children.
        If the current node is an :py:class:`XMLDictNode` and you attempt to
        add a node with a duplicate key, the code will create a list to hold
        the existing node and add the new node to the list.

        By default, all new nodes are created as :py:class:`XMLCDATANode`
        objects. You can include any keyword parameters that you could
        provide when creating an :py:class:`XMLCDATANode` object. If supplied,
        these additional keyword parameters are passed to the
        :py:func:`XMLCDATANode.__init__` function.

        Args:
            tag (string): The XML tag of the node.
            key (string or tuple): The dictionary key that the method should
                use for the node. If None (the default), the :py:obj:`tag` is
                is used as the key.
            text (string): The CDATA for the new node. The default value is
                an empty string.
            new_node (instance of a subclass of :py:class:`XMLNodeBase`): If
                supplied, this will be used for the new node instead of a
                new instance of the :py:class:`XMLCDATANode`. If supplied, the
                :py:obj:`text` parameter and additional keyword arguments
                are ignored.
            update (bool): If True (the default), update the reverse linkages
                in the new node to point to the parent. If False, only create
                the one-way linkages from the parent to the child. (**Note**:
                This should always be True unless you are creating a temporary
                tree for some reason. Setting this to False may create
                inconsistent data that causes problems later.)

        Raises:
            :py:exc:`AttributeError`: If the node is out of date
                (See :py:meth:`get_current_node`) or the method encounters
                irrecoverable data inconsistency while making changes to the
                XML tree.
            :py:exc:`TypeError`: If :py:obj:`new_node` is not None and not
                an instance of :py:class:`XMLNodeBase`
        """
        raise NotImplementedError()

    def list(self, in_place=False):
        """Return a node as a list.

        This method returns a node as a list. This is useful when you
        are not sure whether a node will contain a single entry or a
        list. If the node contains a list, the node itself is
        returned. If the node does not already contain a list, the
        method creates a list, adds the node to it, and returns the
        list.

        If the :py:obj:`in_place` parameter is True, then the change is
        made in the XML tree. Otherwise, the XML tree is left unchanged and
        the method creates and returns a temporary list.

        Args:
            in_place (bool): Whether the change should be made in the
                XML tree.

        Returns:
            list or :py:class:`XMLListNode`
            If the current node is a list, the current node;
            otherwise, a list containing the current node as its sole
            member.

        Raises:
            :py:exc:`AttributeError`: If the node is out of date and
                :py:obj:`in_place` is True. (See :py:meth:`get_current_node`.)
        """
        if not in_place:
            return [self]
        new_node = XMLListNode(
            [self], tag=self.tag, key=self.key, parent=self.parent,
            convert=False
        )
        self._replace_node(new_node)

        # We weren't "replaced" in the destructive sense, so clear
        # the replacement node indicator.
        self._replacement_node = None

        # self._replace_node() may have updated our key if it was out
        # of date, so grab it.
        self.key = copy(new_node.key)

        # Reparent ourselves.
        self.parent = new_node

        # Return the list.
        return new_node

    def get_current_node(self):
        """Return the current node.

        There are times that the current node must be replaced in the
        XML tree for some reason. For example, due to the immutability
        of Python strings, a new XMLCDATANode (which masquerades as a
        string) is required anytime its CDATA value changes.

        When this occurs, you can retrieve the latest node using the
        get_current_node() method. This will attempt to find the node
        that succeeded the node in question. If the node is still
        current, it simply returns itself.

        Therefore, it should always be safe to run::

            >>> node = node.get_current_node()

        Returns:
            Subclass of :py:class:`XMLNodeBase` containing the current
            successor to the node (if any). If the node is still "current",
            the method returns the node itself.
        """
        if self._replacement_node is not None:
            # Recurse as deep as we can looking for replacement nodes.
            # If we hit an error, return the last replacement node we found.
            try:
                while self._replacement_node._replacement_node is not None:
                    self._replacement_node = (
                        self._replacement_node._replacement_node
                    )
            except: # pylint: disable=bare-except
                pass
            return self._replacement_node
        else:
            return self

    def _replace_node(self, newnode):
        # Replace a node with a new node. If the replacement node is
        # None, then the node is deleted.

        self._check_replacement()
        # We need to replace ourselves with a new node.
        if self.parent is None:
            raise AttributeError("Attempt to modify root document")
        # Record the replacement for anyone who still holds
        # references to this node.
        if newnode is not None:
            self._replacement_node = newnode
        # Case 1: Parent is dictionary
        try:
            # Plan A: Do the lookup by expected key.
            if self.key in self.parent:
                if self.parent[self.key] == self:
                    if newnode is not None:
                        self.parent[self.key] = newnode
                    else:
                        del self.parent[self.key]
                    return
            # Plan B: Brute force check, in case of some sort of
            # mismatch. We do our best, within reason.
            myiter = None
            try:
                # Python 2
                myiter = self.parent.iteritems()
            except AttributeError:
                # Python 3
                myiter = self.parent.items()
            for key, val in myiter:
                if val == self:
                    if newnode is not None:
                        # Update the new node's key
                        newnode.key = copy(key)
                        # Replace us
                        self.parent[key] = newnode
                    else:
                        del self.parent[key]
                    return
                elif isinstance(val, XMLListNode):
                    # Check the list
                    for i in range(0, len(val)):
                        if val[i] == self:
                            if newnode is not None:
                                # Update the new node's key and parent
                                newnode.key = copy(key)
                                newnode.parent = val
                                # Replace us
                                val[i] = newnode
                            else:
                                del val[i]
                                # Make sure we don't need to delete the
                                # enclosing list, too.
                                if len(val) == 0:
                                    val._replace_node(None)
                            return
        except: # pylint: disable=bare-except
            pass
        # Case 2: Parent is list
        try:
            made_change = False
            for i in range(0, len(self.parent)):
                if self.parent[i] == self and self.parent[i].xml_attrs == self.xml_attrs:
                    if newnode is not None:
                        self.parent[i] = newnode
                        return
                    else:
                        del self.parent[i]
                        made_change = True
                        break
            if made_change:
                # Make sure we don't need to delete the enclosing list,
                # too.
                if len(self.parent) == 0:
                    self.parent._replace_node(None)
                return
        except: # pylint: disable=bare-except
            pass
        raise AttributeError("Unable to find existing node in parent")

    def dict(self, attrs=None, tags=None, func=None, in_place=False,
             promote=False):
        """Return a dictionary keyed as indicated by the parameters.

        This method lets you re-key your data with some
        flexibility. It takes the current node (whether a single node
        or a list) and turns it into a dictionary. If the current node
        is a list, all the list members are added to the
        dictionary. If the current node is not a list, just the
        current node is added to the dictonary.

        The key for each node is determined by the :py:obj:`attrs`,
        :py:obj:`tags`, and :py:obj:`func` parameters, in that order of
        precedence. For *each node*, the method looks for child nodes that
        have an XML attribute that exactly matches one of the attributes
        in the :py:obj:`attrs` argument. If it finds a match, it uses the
        *node's* (not the attribute's) CDATA as the key.

        If the method does not find a matching attribute, it looks for
        child nodes that have a tag that exactly matches one of the
        tags in the :py:obj:`tags` argument. If it finds a match, it uses
        the node's CDATA as the key.

        If the method does not find a matching tag, it passes the node
        to the user-suppled function (supplied by the :py:obj:`func` parameter)
        and uses the return value as the key.

        If the :py:obj:`func` is not provided or returns a value that
        evaluates to False (e.g. None or ""), the method uses the node's
        XML tag as the key.

        If there are multiple matches, the order of precedence is like
        this (again, this is applied for *each node* independent of
        the other nodes):

        1. The attributes in the attrs parameter, in the order they
           appear in the attrs parameter.
        2. The tags in the tags parameter, in the order they appear
           in the attrs parameter.
        3. The return value of the user-supplied function.
        4. The node's XML tag.

        If the :py:obj:`in_place` parameter is True, then the method will
        replace the current node in the hierarchy with the
        dictionary. Otherwise, it will create a new dictionary and
        return it.

        If both the :py:obj:`in_place` and :py:obj:`promote` parameters are
        True, then the method will make the changes as described above;
        however, it will add the nodes to the first dictionary it finds
        enclosing the curent node.

        Some examples should help with this. Here is an example of the
        simple functionality. Note how the original nodes are turned
        into a dictionary with the appropriate keys, but the original
        root is left untouched::

            >>> root.prettyprint()
            {'a': {'b': [{'name': u'foo', 'value': u'1'},
                         {'name': u'bar', 'value': u'2'}]}}
            >>> root['a']['b'].dict(tags=['name']).prettyprint()
            {u'bar': {'name': u'bar', 'value': u'2'},
             u'foo': {'name': u'foo', 'value': u'1'}}
            >>> root.prettyprint()
            {'a': {'b': [{'name': u'foo', 'value': u'1'},
                         {'name': u'bar', 'value': u'2'}]}}

        Here is an example of a dictionary changed in place. Note how
        the original nodes are turned into a dictionary with the
        appropriate keys and this dictionary replaces the current node
        in the hierarchy::

            >>> root.prettyprint()
            {'a': {'b': [{'name': u'foo', 'value': u'1'},
                         {'name': u'bar', 'value': u'2'}]}}
            >>> root['a']['b'].dict(tags=['name'], in_place=True).prettyprint()
            {u'bar': {'name': u'bar', 'value': u'2'},
             u'foo': {'name': u'foo', 'value': u'1'}}
            >>> root.prettyprint()
            {'a': {'b': {u'bar': {'name': u'bar', 'value': u'2'},
                         u'foo': {'name': u'foo', 'value': u'1'}}}}

        Here is an example of the "promotion" functionality. Note how
        the original nodes are added directly to the ``root['a']``
        enclosing dictionary::

            >>> root.prettyprint()
            {'a': {'b': [{'name': u'foo', 'value': u'1'},
                         {'name': u'bar', 'value': u'2'}]}}
            >>> root['a']['b'].dict(tags=['name'], in_place=True, promote=True).prettyprint()
            {u'bar': {'name': u'bar', 'value': u'2'},
             u'foo': {'name': u'foo', 'value': u'1'}}
            >>> root.prettyprint()
            {'a': {u'bar': {'name': u'bar', 'value': u'2'},
                   u'foo': {'name': u'foo', 'value': u'1'}}}

        Quirks:

        1. If the current node is the only member of a list in
           the XML tree, the operation will occur on that single-node
           list instead of the node itself.
        2. If the method encounters an exception while trying to
           modify the XML tree (``in_place == True``), it will attempt
           to undo its changes; however, this logic is not
           completely reliable.

        Args:
            attrs (`list`): The list of XML attributes that signal a node
                should be used as a key.
            tags (`list`): The list of XML tags that signal a node should be
                used as a key.
            func (function): A function that will accept a node as a parameter
                and return a key.
            in_place (bool): Whether the change should be made in the XML tree.
            promote (bool): Whether the new nodes should be added to a
                dictonary placed at the current node, or they should be
                "promoted" to the first enclosing dictionary.

        Returns:
            An :py:class:`XMLDictNode`. If :py:obj:`in_place` is False, the
            dictionary formulated from the current node. If :py:obj:`in_place`
            is True, the dictionary to which the nodes were added.
            (Note: If :py:obj:`promote` is True, this dictionary may contain
            additional entries that already existed in the enclosing
            dictionary.)

        Raises:
            :py:class:`AttributeError`: If the node is out of date and
                :py:obj:`in_place` is True. (See :py:meth:`get_current_node`.)
            :py:class:`AttributeError`: If :py:obj:`in_place`
                is True and the method encounters irrecoverable data
                inconsistency while making changes to the XML tree.
        """
        if attrs is None:
            attrs = []
        if tags is None:
            tags = []
        newlist = None
        if in_place:
            # If editing in place, check that we are current.
            self._check_replacement()

            # Save some information to help restore things, if needed.
            orig = dict(parent=self.parent, tag=self.tag, key=self.key)

            # If we were editing in place, we need to figure out how to
            # properly replace ourselves.
            #
            # If our parent is a single-member list, then we can just
            # convert it. If our parent is a dictionary, then we can
            # convert ourselves in place. If our parent is anything
            # else, we just convert ourselves in place and do our best
            # to replace ourselves with a dictionary -- even if that
            # doesn't make complete sense.
            parent = self.parent
            if (isinstance(parent, XMLListNode) and len(parent) == 1
                    and parent[0] == self):
                # Easy. Just convert our parent.
                newlist = parent
            else:
                # Turn us into a list.
                newlist = self.list(in_place=True)
        if newlist is None:
            # Make a list and add us to it.
            newlist = XMLListNode(
                [self], tag=self.tag, key=self.key, convert=False
            )

        # Now, let's turn the list node into a dict.
        try:
            # pylint: disable=no-member
            return newlist.dict(
                attrs=attrs, tags=tags, func=func, in_place=in_place,
                promote=promote
            )
        except:
            if in_place:
                # Try to restore us.
                self.parent = orig['parent']
                self.tag = orig['tag']
                self.key = orig['key']
                self._replacement_node = None
                newlist._replace_node(self) # pylint: disable=no-member
            raise

    def jdict(self, in_place=False, promote=False):
        """Return a dictionary keyed appropriately for Junos output.

        This method is a shortcut to call the :py:meth:`dict`
        method with these parameters::

            attrs=[('junos:key', 'junos:key', 'junos:key'),
                   ('junos:key', 'junos:key'), 'junos:key']
            tags=['name']

        This will attempt to produce the correct key for each
        node. Some nodes have a multi-field key. If that occurs, the
        dictionary key will be a tuple. In cases where there is a
        single key, the dictionary key will be a string. If there is
        no matching node, the key will simply be the XML tag name.

        Some Junos nodes use a different tag for the key. And, in some
        cases, the ``junos:key`` attribute is not available. In those
        circumstances, you should directly call the :py:meth:`dict`
        method with the correct attributes or tags.

        Please see the documentation for the :py:meth:`dict` method for
        further information.

        Args:
            in_place (bool): Whether the change should be made in the XML tree.
            promote (bool): Whether the new nodes should be added to a
                dictonary placed at the current node, or they should be
                "promoted" to the first enclosing dictionary.

        Returns:
            An :py:class:`XMLDictNode`. If :py:obj:`in_place` is False, the
            dictionary formulated from the current node. If :py:obj:`in_place`
            is True, the dictionary to which the nodes were added.
            (Note: If :py:obj:`promote` is True, this dictionary may contain
            additional entries that already existed in the enclosing
            dictionary.)

        Raises:
            :py:class:`AttributeError`: If the node is out of date and
                :py:obj:`in_place` is True. (See :py:meth:`get_current_node`.)
            :py:class:`AttributeError`: If :py:obj:`in_place`
                is True and the method encounters irrecoverable data
                inconsistency while making changes to the XML tree.
        """
        return self.dict(attrs=[('junos:key', 'junos:key', 'junos:key'),
                                ('junos:key', 'junos:key'), 'junos:key'],
                         tags=['name'], in_place=in_place, promote=promote)

    def standardize(self, deep=True):
        """Convert all child nodes to instances of an XMLNodeBase sub-class.

        This method is useful when you have added a child node
        directly to a dictionary or list and now want to convert it to
        the appropriate :py:class:`XMLNodeBase` sub-class.

        Args:
            deep (bool): If True (the default), recursively descend
                through all children, converting all nodes, as needed. If
                False, only convert direct children of the node.
        Returns:
            None
        """
        raise NotImplementedError()

    def emit_handler(self, content_handler, pretty=True, newl='\n',
                     indent='    ', full_document=None):
        """Pass the contents of the XML tree to a ContentHandler object.

        This method will pass the contents of the XML tree to a
        :py:obj:`ContentHandler` object.

        Args:
            content_handler (:py:obj:`ContentHandler`): The
                :py:obj:`ContentHandler` object to which the XML tree wll
                be passed.
            pretty (bool): If True, this method will call the
                :py:meth:`content_handler.ignorableWhitespace` method to add
            whitespace to the output document.
            newl (string): The string which the method should use for new
                lines when adding white space (see the :py:obj:`pretty`
                parameter).
            indent (text): The string which the method should use for each
                level of indentation when adding white space (see the
                :py:obj:`pretty` parameter).
            full_document (bool): If True, the method will call the
                :py:meth:`content_handler.startDocument` and
                :py:meth:`content_handler.endDocument` methods at the start
                and end of the document, respectively. If False, it will not
                call these methods. If the parameter is not set, the method
                will attempt to determine whether the current node is the root
                of an XML tree with a single root tag. If so, it will set
                the full_document parameter to True; otherwise, it will
                set it to False.

        Returns:
            None
        """
        full_document_ok = None
        curnode = self
        # See if it is OK to be a full document. If we will have
        # multiple root nodes, it is not OK. Otherwise, we assume
        # it is.
        while full_document_ok is None:
            if isinstance(curnode, XMLCDATANode):
                # Always OK
                full_document_ok = True
            elif not isinstance(curnode, XMLNodeBase):
                # Never OK -- will probably produce an error later
                full_document_ok = False
            elif isinstance(curnode, XMLListNode) and len(curnode) > 1:
                # We have multiple tags. We cannot be a full document.
                full_document_ok = False
            elif len(curnode) == 0:
                # Empty nodes are always OK.
                full_document_ok = True
            elif ((curnode.tag is None and isinstance(curnode, XMLDictNode)) or
                  isinstance(curnode, XMLListNode)):
                # If curnode.tag is None (or this is a list), this
                # amounts to a request to output the child(ren) of
                # this node. If there are multiple children, we know
                # that it is not OK to make this a full document (as
                # there will be multiple top-level tags). If there is
                # just one child, we need to look at the child to
                # determine whether it is OK to treat it as a full
                # document.
                if len(curnode) > 1:
                    full_document_ok = False
                else:
                    if isinstance(curnode, XMLListNode):
                        curnode = curnode[0]
                    elif isinstance(curnode, XMLDictNode):
                        curnode = curnode[[i for i in curnode][0]]
            elif isinstance(curnode.tag, (_unicode, str)):
                # We have a tag. That will produce a top-level tag.
                full_document_ok = True
            else:
                # We generally shouldn't get here, but let the user
                # do what they want.
                full_document_ok = True

        if full_document is not None:
            # We were given a full_document argument. Make sure it is
            # OK to treat this as a "full document".
            if full_document and not full_document_ok:
                raise ValueError("Document will have more than one root node. "
                                 "The full_document argument must be False.")
        else:
            # Default to full_document=True, if it is OK to treat this
            # as a full document and it looks like we are at the root.
            full_document = full_document_ok
            if full_document:
                # Guess that we are at the root if:
                # - self.tag is None, or
                # - self.tag is not None, but we have no parent, or
                # - self.tag is not None, but our parent looks like a
                #   single-member, tagless root
                if (self.tag is not None and self.parent is not None and
                        (self.parent.tag is not None or len(self.parent) > 1)):
                    full_document = False

        if full_document:
            content_handler.startDocument()
        curnode._emit_handler(content_handler, depth=0, pretty=pretty, newl=newl,
                              indent=indent)
        if full_document:
            content_handler.endDocument()

    def emit_xml(self, output=None, encoding='utf-8', handler=XMLGenerator,
                 **kwargs):
        """Return the contents of the XML tree as an XML document.

        This method will create a :py:obj:`ContentHandler` by calling the
        method provided by the handler parameter.  It will call
        :py:meth:`emit_handler` with this :py:obj:`ContentHandler` object.
        In addition, this method will accept any parameter that the
        :py:meth:`emit_handler` method accepts (except the
        :py:obj:`content_handler` parameter).  It will pass
        them to the :py:meth:`emit_handler` method when it calls it.

        Args:
            output (A file-like IO object, or None): The file-like IO object
                in which output should be placed. If None, the method will
                return the XML output as a string.
            encoding (string): The encoding that should be used for the output.
            handler (function): A method that will return a
                :py:obj:`ContentHandler` object. This method will be called
                with two positional parameters: the output parameter
                (or, if None, a file-like IO object) and the encoding parameter.
        Returns:
            If :py:obj:`output` was None, the method will return the XML
            output as a string. Otherwise, None.
        """
        if output is None:
            output = StringIO()
            return_text = True
        else:
            return_text = False

        content_handler = handler(output, encoding)

        self.emit_handler(content_handler, **kwargs)

        if return_text:
            value = output.getvalue()
            try:  # pragma no cover
                value = value.decode(encoding)
            except AttributeError:  # pragma no cover
                pass
            return value

    def _emit_handler(self, content_handler, depth, pretty, newl, indent):
        raise NotImplementedError()

    def prettyprint(self, *args, **kwargs):
        """Print a "pretty" representation of the data structure.

        This uses the :py:meth:`pprint` method from the :py:mod:`pprint`
        module to print a "pretty" representation of the data structure.
        The parameters are passed unchanged to the :py:meth:`pprint` method.

        The output from this method shows only the main data and not the meta
        data (such as XML attributes).

        When using :py:meth:`pprint`, it is necessary to use this method to
        get a reasonable representation of the data; otherwise,
        :py:meth:`pprint` will not know how to represent the object in a
        "pretty" way.
        """
        raise NotImplementedError()

    def find_nodes_with_tag(self, tag, recursive=True):
        """Iterates over nodes that have a matching tag.

        This method searches the current node and its children for
        nodes that have a matching tag. The :py:obj:`tag` parameter
        accepts either a string value or a tuple, allowing you to
        search for one or more tags with a single
        operation. Optionally (by providing a False value to the
        :py:obj:`recursive` parameter), you can limit the search to
        the current node and direct children of the current node.

        The method will return a generator, which you can use to
        iterate over the matching nodes.

        For example, this will print all "name" nodes from the XML
        snippet that is shown::

            >>> root = jxmlease.parse(\"\"\"\
            ... <?xml version="1.0" encoding="utf-8"?>
            ... <root>
            ...
            ...         <name>name #1</name>
            ...
            ...             <name>name #2</name>
            ...
            ...
            ...             <c>
            ...                 <name>name #3</name>
            ...             </c>
            ...
            ...
            ... </root>\"\"\")
            >>> print root
            {u'root': {u'a': {u'b': [{u'name': u'name #2'},
                                     {u'c': {u'name': u'name #3'}}],
                              u'name': u'name #1'}}}
            >>> for node in root.find_nodes_with_tag('name'):
            ...   print node
            ...
            name #1
            name #2
            name #3

        However, if we turn off recursion, you will see that this
        returns only the direct children (if any) of the node we
        select::

            >>> for node in root.find_nodes_with_tag('name', recursive=False):
            ...   print node
            ...
            >>> for node in root['root']['a'].find_nodes_with_tag('name', recursive=False):
            ...   print node
            ...
            name #1

        If you run this against an :py:class:`XMLDictNode` without a
        tag (for example, the tagless root node), then the command is
        run on each member of the dictionary. The impact of this is
        that a non-recursive search will search for tags in the
        grandchildren of the tagless :py:class:`XMLDictNode`, rather
        than searching the children of the tagless
        :py:class:`XMLDictNode`::

            >>> root = jxmlease.parse(\"\"\"\
            ...
            ...   <name>second-level tag</name>
            ...
            ...     <name>third-level tag</name>
            ...
            ... \"\"\")
            >>> for i in root.find_nodes_with_tag('name', recursive=False):
            ...   print i
            ...
            second-level tag
            >>> for i in root['a'].find_nodes_with_tag('name', recursive=False):
            ...   print i
            ...
            second-level tag

        This method never returns a list. Instead, lists pass the
        command through to their child nodes, which may be
        returned. This ensures you get back each node you requested.

        For example, here is a root node with two top-level "name"
        elements. Searching non-recursively for the "name" tag returns
        the two "name" elements, even though they are enclosed within
        a dictionary and list::

            >>> root = XMLDictNode()
            >>> _ = root.add_node(tag='name', text='tag #1')
            >>> _ = root.add_node(tag='name', text='tag #2')
            >>> print root
            {'name': [u'tag #1', u'tag #2']}
            >>> for i in root.find_nodes_with_tag('name', recursive=False):
            ...   print i
            ...
            tag #1
            tag #2
            >>>

        Even though our examples up to this point have
        demonstrated text, it is worth noting that this method returns
        the actual node, whatever that may be::

            >>> root = jxmlease.parse(\"\"\"\
            ...
            ...
            ...     <c>
            ...       <foo>bar</foo>
            ...       <status>ok</status>
            ...     </c>
            ...
            ... \"\"\")
            >>> for i in root.find_nodes_with_tag('b'):
            ...   print i
            ...
            {u'c': {u'foo': u'bar', u'status': u'ok'}}

        If the :py:obj:`recursive` parameter is False, the code will
        check the current node. If the current node does not match,
        the code will check the current node's direct
        children. However, if the current node has a matching tag and
        the :py:obj:`recursive` parameter is False, the code will stop
        its search and not check the children of the current node.

        If the :py:obj:`recursive` parameter is True (the default),
        the code will search the current node and all of its children,
        even the children of other matching nodes. Therefore, the
        method may even return children of other matches, if you
        specify a :py:obj:`recursive` search::

            >>> root = jxmlease.parse(\"\"\"
            ...
            ...
            ...     foo
            ...     bar
            ...
            ... \"\"\")
            >>> count = 0
            >>> for i in root.find_nodes_with_tag("a"):
            ...     count += 1
            ...     print("%d: %s" % (count, i))
            ...
            1: {u'a': {u'a': [u'foo', u'bar']}}
            2: {u'a': [u'foo', u'bar']}
            3: foo
            4: bar
            >>> count = 0
            >>> for i in root.find_nodes_with_tag("a", recursive=False):
            ...     count += 1
            ...     print("%d: %s" % (count, i))
            ...
            1: {u'a': {u'a': [u'foo', u'bar']}}

        You can also use a tuple as the tag parameter, in which case
        the method will return nodes with a tag that matches any of
        the given tag values.

        You can use this function to create somewhat complicated logic
        that mimics the functionality from XPath "//tag" matches. For
        example, here we check for <xnm:warning> and <xnm:error> nodes
        and return their value::

            >>> root = jxmlease.parse(\"\"\"\
            ... <foo>
            ...   <xnm:warning>
            ...     <message>This is bad.</message>
            ...   </xnm:warning>
            ...   <bar>
            ...     <xnm:error>
            ...       <message>This is very bad.</message>
            ...     </xnm:error>
            ...   </bar>
            ... </foot>\"\"\")
            >>> if root.has_node_with_tag(('xnm:warning', 'xnm:error')):
            ...   print "Something bad happened."
            ...
            Something bad happened.
            >>> for node in root.find_nodes_with_tag(('xnm:warning', 'xnm:error')):
            ...   if node.tag == 'xnm:error':
            ...     level = "Error:"
            ...   elif node.tag == 'xnm:warning':
            ...     level = "Warning:"
            ...   else:
            ...     level = "Unknown:"
            ...   print(level + " " + node.get("message", "(unknown)"))
            ...
            Warning: This is bad.
            Error: This is very bad.

        Args:
            tag (string or tuple): The XML tag (or tags) for which to search.
            recursive (bool): If True (the default), search recursively through
                all children. If False, only search direct children.

        Returns:
            A generator which iterates over all matching nodes.

        """
        if isinstance(tag, str):
            tag = (tag,)
        return self._find_nodes_with_tag(tuple(tag), recursive=recursive,
                                         top_level=True)

    def _find_nodes_with_tag(self, tag, recursive, top_level):
        raise NotImplementedError()

    def has_node_with_tag(self, tag, recursive=True):
        """Determine whether a node with a matching tag exists.

        This method uses the :py:meth:`find_nodes_with_tag` method to
        search the current node and its children for a node that has a
        matching tag.  The method returns a boolean value to indicate
        whether at least one matching node is found.

        Because this function uses the :py:meth:`find_nodes_with_tag` method,
        the parameters and algorithm are the same as the
        :py:meth:`find_nodes_with_tag` method.

        Args:
            tag (string or tuple): The XML tag (or tags) for which to search.
            recursive (bool): If True (the default), search recursively through
                all children. If False, only search direct children.

        Returns:
            True if at least one matching node is found; otherwise, False.

        """
        for _ in self.find_nodes_with_tag(tag, recursive=recursive):
            return True
        return False

    def __repr__(self):
        return "%s(xml_attrs=%r, value=%s)" % (
            getattr(self, "__const_class_name__", self.__class__.__name__),
            self.xml_attrs, super(XMLNodeBase, self).__repr__()
        )

    def __str__(self):
        io_obj = StringIO()
        self.prettyprint(io_obj)
        return io_obj.getvalue().strip()


# Import the dependencies.
import datetime as dt

from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

from flask import Flask, jsonify

#################################################
# Database Setup
#################################################

engine = create_engine("sqlite:///Resources/hawaii.sqlite",
                       connect_args={'check_same_thread': False})

# reflect an existing database into a new model
Base = automap_base()

# reflect the tables
Base.prepare(engine, reflect=True)

# Save references to each table
measurement = Base.classes.measurement
station = Base.classes.station

# Create our session (link) from Python to the DB
session = Session(engine)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################

@app.route("/")
def homepage():
    """List all available api routes."""
    return (
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f"/api/v1.0/stations
"
        f"/api/v1.0/tobs
"
        f"/api/v1.0/tstats/<start>
"
        f"/api/v1.0/tstats/<start>/<end>"

    )

@app.route("/api/v1.0/precipitation")
def precipitation():
    """Convert the query results from your precipitation analysis (i.e. retrieve only the last 12 months of data)
       to a dictionary using date as the key and prcp as the value"""

    # Find the most recent date in the data set.
    latest_row = session.query(measurement).order_by(
        measurement.id.desc()).first()
    latest_date_string = latest_row.date
    latest_date = dt.date.fromisoformat(latest_date_string)

    # Calculate the date one year from the last date in data set.
    start_date = latest_date - dt.timedelta(days=365)

    # query the data to get precipitation data for last 12 months
    year_data = session.query(measurement.date, measurement.prcp)\
        .filter(measurement.date >= start_date).all()

    session.close()

    # Convert list of tuples into dictionary with date as key and prcp as value
    rain = dict(year_data)

    return jsonify(rain)

@app.route("/api/v1.0/stations")
def stations():
    """Return a JSON list of stations from the dataset."""

    # Query all measurements of stations and return a list of these station names (queries from jupyter notebook)
    station_list = session.query(station.station, station.name).all()

    # convert list of tuples into dictionary
    station_list = dict(station_list)

    return jsonify(station_list)

@app.route("/api/v1.0/tobs")
def tobs():
    """Return a JSON list of temperature observations for the previous year."""

    # Find the most recent date in the data set; calculate start date by subtracting 365 days from the latest date; find most active stations
    latest_row = session.query(measurement).order_by(
        measurement.id.desc()).first()
    latest_date_string = latest_row.date
    latest_date = dt.date.fromisoformat(latest_date_string)
    start_date = latest_date - dt.timedelta(days=365)

    most_active_stations = session.query(measurement.station, func.count(measurement.station)).\
        order_by(func.count(measurement.station).desc()).\
        group_by(measurement.station).all()

    most_active_station = most_active_stations[0][0]

    # Create a query that finds the TOBS and filter by start date variable
    year_data_most_active = session.query(measurement.date, measurement.tobs).\
        filter(measurement.date >= start_date).\
        filter(measurement.station == most_active_station).all()

    session.close()

    # Return a JSON list of temperature observations for the previous year.
    year_data_most_active = dict(year_data_most_active)

    return jsonify(year_data_most_active)

@app.route("/api/v1.0/tstats/<start>")
@app.route("/api/v1.0/tstats/<start>/<end>")
def tstats(start, end=None):
    """Return a JSON list of the minimum temperature, the average temperature, and the maximum temperature for a specified start
    or start-end range For a specified start, calculate TMIN, TAVG, and TMAX for all the dates greater than or equal to the start date.
    For a specified start date and end date, calculate TMIN, TAVG, and TMAX for the dates from the start date to the end date, inclusive."""

    start = dt.date.fromisoformat(start)
    end = dt.date.fromisoformat(end) if end else dt.date.max

    temp_stats = session.query(func.min(measurement.tobs), func.max(measurement.tobs), func.avg(measurement.tobs))\
        .filter(measurement.date >= start)\
        .filter(measurement.date <= end).all()

    session.close()

    temp_stats_values = []
    for min, avg, max in temp_stats:
        temp_stats_values_dict = {}
        temp_stats_values_dict["min_temp"] = min
        temp_stats_values_dict["avg_temp"] = avg
        temp_stats_values_dict["max_temp"] = max
        temp_stats_values.append(temp_stats_values_dict)

    return jsonify(temp_stats_values)

if __name__ == '__main__':
    app.run(debug=True)


import json
import argparse
from tqdm import tqdm
import os
size = 1000

parser = argparse.ArgumentParser()
parser.add_argument("--input_json", type=str, default="data/overall_collection.jsonl")
parser.add_argument("--input_qrel", type=str, default ="data/qrels/search_based/sr_collection.qrels")
parser.add_argument("--DATA_DIR", type=str, default="run_search/single")
args = parser.parse_args()

input_qrel_file = args.input_qrel
input_json_file = args.input_json
input_eval_file = args.input_qrel
input_test_eval_file = args.input_qrel
output_queries_file = os.path.join(args.DATA_DIR, "input", "queries.tsv")
output_run_fie = os.path.join(args.DATA_DIR, "input", "run.jsonl")
output_eval_file_run = os.path.join(args.DATA_DIR, "input", "eval_id.tsv")
output_test_eval_file_run = os.path.join(args.DATA_DIR, "input", "eval_test_topic_id.tsv")

input_json = open(input_json_file, 'r')
input_qrel = open(input_qrel_file, 'r')
input_eval = open(input_eval_file, 'r')
input_test_eval = open(input_test_eval_file, 'r')
output_quries = open(output_queries_file, 'w')
#output_eval = open(output_eval_file, 'w')
output_test_eval_run = open(output_test_eval_file_run, 'w')
output_eval_for_run = open(output_eval_file_run, 'w')
output_run = open(output_run_fie, 'w')

doc_dict = {}
qid_list = []

doc_positive_dict = {}
lines = input_json.readlines()

for line in lines:
    current_dict = json.loads(line)
    id = current_dict['id']
    doc_positive_dict[id] = current_dict["included_studies"]
    qid_list.append(id)
input_json.close()

lines = input_qrel.readlines()

for line in tqdm(lines):
    items = line.split()
    if items[0] in doc_dict:
        doc_dict[items[0]].append(items[2])
    else:
        doc_dict[items[0]] = [items[2]]

input_qrel.close()
# output_quries.close()

for qid in tqdm(qid_list):
    data_positive_list = set(doc_positive_dict[qid])
    data_positive_list = list(data_positive_list)
    data_list = doc_dict[qid]
    print(qid, len(data_list), len(data_positive_list))
    for positive_id in data_positive_list:
        id = qid + '_' + positive_id
        removed_list = [x for x in data_list if x != positive_id]
        removed_list = set(removed_list)
        removed_list = list(removed_list)

        new_dict = {
            'qid': id,
            'pid': removed_list
        }

        json.dump(new_dict, output_run)
        output_run.write('\n')
output_run.close()

test_eval_set = set()
lines = input_test_eval.readlines()
for line in tqdm(lines):
    items = line.split()
    qid = items[0]
    test_eval_set.add(qid)

for item in test_eval_set:
    output_test_eval_run.write(item+'\n')

for qid in tqdm(eval_list):
    output_eval_file = os.path.join(args.DATA_DIR, "input", qid+".qrel")
    output_eval = open(output_eval_file, 'w')
    data_list = set(doc_positive_dict[qid])
    for pid in data_list:
        id = qid + '_' + pid
        removed_list = [x for x in data_list if x != pid]
        for ppid in removed_list:
            output_eval.write(id+'\t' +'Q0\t' + ppid + '\t' + '1' + '\n')

        output_eval_for_run.write(id+'\n')

output_eval_for_run.close()


import pygame
import sys
import math
import random
import numpy as np

res = (1338,760)
width,height = res

pygame.init()
screen = pygame.display.set_mode(res)
clock = pygame.time.Clock()
FPS = 60

ball_list = []
bound_list = []
text_list = []

class Text:

	def __init__(self,pos,color,fontsize):
		text_list.append(self)
		self.pos = pos
		self.color = color
		self.fontsize = fontsize
		self.font = pygame.font.SysFont(None , self.fontsize)

	def size(self):
		return pygame.font.Font.size(self.font, "Test")

	def draw(self, text ):
		self.text = self.font.render(text,True,self.color,"BLACK")
		screen.blit(self.text,self.pos)

fps = Text((0,0),"WHITE",30)
#_ , fps_height = fps.size()
cursorpos = Text((0,fps.size()[1]),"WHITE",30)

class Boundary:

	def __init__(self, startx, starty, endx, endy, orientation):

		self.orientation = orientation
		self.startx,self.starty = startx,starty
		self.endx,self.endy = endx,endy
		'''
		if self.startx > self.endx:
			self.startx,self.endx = self.endx,self.startx
		if self.starty > self.endy:
			self.starty,self.endy = self.endy,self.starty
		'''
		self.length = math.sqrt((endx-startx)**2 + (endy-starty)**2)
		self.normx = (endy-starty)/self.length
		self.normy = -(endx - startx)/self.length
		self.norm = np.array([self.normx,self.normy])
		self.tan = np.array([-self.normy,self.normx])
		bound_list.append(self)

	def draw(self):
		pygame.draw.line(screen,"WHITE",(self.startx,self.starty),(self.endx,self.endy ),3)
		#pygame.draw.line(screen,"WHITE",((self.startx+self.endx)/2,(self.starty+self.endy)/2),(10*self.normx+(self.startx+self.endx)/2,10*self.normy+(self.starty+self.endy)/2),2)

bound1 = Boundary(0.1562*width,0.2461*height,0.1562*width,0.7539*height,"v")
bound2 = Boundary(0.1831*width,0.1987*height,0.4798*width,0.1987*height,"h")
bound = Boundary(0.5*width,0.5*height,0.5*width+200,0.5*height+200,"d")
bound3 = Boundary(0.1562*width,0.2461*height,0.1263*width,0.1961*height,"d")
bound4 = Boundary(0.1831*width,0.1987*height,0.1517*width,0.1474*height,"d")
bound5 = Boundary(0.4798*width,0.1987*height,0.4843*width,0.1711*height,"d")
bound5 = Boundary(0.4843*width,0.1711*height,0.4843*width,0.1329*height,"v")

class Ball:

	def __init__(self,pos,shape,color):
		#self.x,self.y = pos
		#self.vx,self.vy = 0.0,0.0
		self.pos = np.array(pos,dtype = "float")
		self.vel = np.array([0.0,0.0])
		self.shape = shape
		self.color = color
		self.radius = width/80

	def move(self):

		drag = 1
		#self.vx *= drag
		#self.vy *= drag
		self.vel *= drag

		min_speed = 0.05

		#if self.vx**2 + self.vx**2 <= min_speed: self.vx , self.vy = 0,0
		if np.dot(self.vel,self.vel) <= min_speed: self.vel = [0,0]

		#self.x += self.vx
		#self.y += self.vy
		self.pos += self.vel

	def collision_check(self):

		energy_loss = 0.8

		if self.pos[0]  > width - self.radius:
			self.pos[0] = width - self.radius
			self.vel[0] *= -energy_loss
		if self.pos[0] < self.radius:
			self.pos[0] = self.radius
			self.vel[0] *= -energy_loss
		if self.pos[1] > height - self.radius:
			self.pos[1] = height - self.radius
			self.vel[1] *= -energy_loss
		if self.pos[1] < self.radius:
			self.pos[1] = self.radius
			self.vel[1] *= -energy_loss

		vel_line = np.cross( np.hstack( (self.pos , np.ones(1) ) )  ,  np.hstack( (self.vel,np.zeros(1)) )   )
		#print(vel_line)

		for bound in bound_list:
			s = np.vstack([(bound.startx,bound.starty),(bound.endx,bound.endy)])        # s for stacked
			h = np.hstack((s, np.ones((2, 1)))) # h for homogeneous
			l1 = np.cross(h[0], h[1])
			x, y, z = np.cross(l1, vel_line)
			if z != 0 :
				x, y = x/z , y/z
				#print(x,y)
				if bound.orientation == "v":
					if (y >= bound.starty and y <= bound.endy) or (y <= bound.starty and y >= bound.endy):
						poi = np.array([x,y])
						vec = poi - self.pos
						dist = np.dot(vec,vec)
						if dist <= self.radius**2:
							self.vel[0] *= -1

				elif bound.orientation == "h":
					if (x >= bound.startx and x <= bound.endx) or (x >= bound.startx and x <= bound.endx):
						poi = np.array([x,y])
						vec = poi - self.pos
						dist = np.dot(vec,vec)
						if dist <= self.radius**2:
							self.vel[1] *= -1
				elif bound.orientation == "d":
					if ((x >= bound.startx and x <= bound.endx) or (x <= bound.startx and x >= bound.endx))  and ((y >= bound.starty and y <= bound.endy) or (y <= bound.starty and y >= bound.endy)):
						poi = np.array([x,y])
						vec =  self.pos - poi
						vec_proj = (self.pos + self.vel) - poi
						dist = np.linalg.norm(vec)
						#dist_proj = np.linalg.norm(vec_proj)
						#vnorm = np.linalg.norm(self.vel)
						if  (dist <= self.radius or np.dot(vec ,vec_proj ) < 0) :
							self.pos = poi + vec*self.radius/dist
							#self.vel = np.array([0,0])
							vtan = np.dot(self.vel,bound.tan)
							vnorm = np.dot(self.vel,bound.norm)

							vnorm *= -1

							self.vel = vtan * bound.tan + vnorm* bound.norm
				#pygame.draw.circle(screen,"WHITE",(x,y),3)

	def ball_collision(self):

		for circle in ball_list:
			if circle == self:
				continue
			dist = np.linalg.norm(circle.pos-self.pos)
			if dist <= self.radius + circle.radius:

				overlap = self.radius + circle.radius - dist

				#Calculating the normal vector and normalizing it

				n = (circle.pos - self.pos)/dist

				#Calculating the tangential vector
				t = np.array([n[1],-1*n[0]])

				#Displacing the balls along the parallel axis in case of overlap

				self.pos -= n * overlap/2
				circle.pos += n * overlap/2

				# Here I will use the collison "coordinates" , by breaking the velocity into a tangential and parralell part

				#Calculating the tangential component of velocity , which are not affected by the collision
				vtan1 = np.dot(self.vel,t)
				vtan2 = np.dot(circle.vel,t)

				#vtan1 = self.vx*tx +self.vy*ty
				#vtan2 = circle.vx*tx + circle.vy*ty

				#Calculating the parralell velocity before the colision
				vpar1b = np.dot(self.vel,n)
				vpar2b = np.dot(circle.vel,n)

				#vpar1b = self.vx * nx + self.vy * ny
				#vpar2b = circle.vx * nx + circle.vy * ny

				#Calculating the parallell velocities after the collision , using the elastic collision formula

				vpar1a = vpar2b
				vpar2a = vpar1b

				#Setting the velocities

				self.vel = vtan1 * t + n * vpar1a
				#self.vx = (vtan1*tx + nx * vpar1a)
				#self.vy = (vtan1*ty + ny * vpar1a)

				circle.vel = vtan2 * t + n * vpar2a
				#circle.vx = (vtan2*tx + nx * vpar2a)
				#circle.vy = (vtan2*ty + ny * vpar2a)

	def draw(self):

		self.ball = pygame.draw.circle(screen,self.color,(self.pos[0],self.pos[1]),self.radius,width = shapes_to_width[self.shape] )

class Pool_cue():

	def __init__(self,color):
		self.color = color
		self.playing = True

	def draw(self):
		if self.playing:

			mouse_pos = np.array(pygame.mouse.get_pos())

			vec = mouse_pos - game.white.pos

			length = np.linalg.norm(vec)

			vec /= length

			pygame.draw.line(screen,self.color,(game.white.pos[0]+vec[0]*2*r,game.white.pos[1]+vec[1]*2*r),(game.white.pos[0]+vec[0]*150,game.white.pos[1]+vec[1]*150),3)

			minimum = 1000
			for ball in ball_list:
				if ball is game.white: continue

				u = ball.pos - game.white.pos
				r2 = np.dot(u,u)
				dist = np.dot(u,vec)

				d2 = r2 - dist**2

				if d2 <= r**2 and dist < 0:
					dist = abs(dist)
					if dist < minimum: minimum = dist

			pygame.draw.line(screen,"WHITE",(game.white.pos[0]-vec[0]*2*r,game.white.pos[1]-vec[1]*2*r),(game.white.pos[0]-vec[0]*minimum , game.white.pos[1]-vec[1]*minimum ),3)

	def click(self,mouse,mousehit):
		if self.playing:
			mouse_pos = np.asarray(mouse)
			mousehit_pos = np.asarray(mousehit)
			vec = mouse_pos - game.white.pos
			#vecx = (mousex - game.white.x)
			#vecy = (mousey - game.white.y)

			length = np.linalg.norm(vec)

			vec /= length

			accel = 0.5*abs(100-100*mousehit_pos[1]/height)

			game.white.vel = -vec*accel

r = width/80
r3 = math.sqrt(3)
colors = {1:"YELLOW",2:"BLUE",3:"PINK",4:"PURPLE",5:"ORANGE",6:"GREEN",7:"RED",9:"YELLOW",10:"BLUE",11:"PINK",12:"PURPLE",13:"ORANGE",14:"GREEN",15:"RED"}
pos = {1:(0,0),2:(r3*r,-r),3:(2*r3*r,-2*r),4:(3*r3*r,-3*r),5:(4*r3*r,4*r),6:(4*r3*r,-2*r),7:(3*r3*r,r),9:(r3*r,r),10:(2*r3*r,2*r),11:(3*r3*r,3*r),12:(4*r3*r,-4*r),13:(4*r3*r,2*r),14:(3*r3*r,-r),15:(4*r3*r,0)}
shapes_to_width = {"solid":0,"white":0,"black":0,"striped":5}
class Game:

	def __init__(self):
		self.white = Ball((width*0.3,height*0.5),shape = "white", color = "WHITE")
		self.black = Ball((width*0.6+2*r3*r,height*0.5),shape = "black", color = "BLACK")
		ball_list.append(self.white)
		ball_list.append(self.black)
		'''
		for i in range(1,16):
			if i == 8 : continue
			if i <= 7 : shape = "solid"
			elif i >= 9 : shape = "striped"
			ball_list.append(Ball( (0.6*width+pos[i][0],0.5*height+pos[i][1]),shape,colors[i]))
		'''

game = Game()

cue = Pool_cue("Blue")

def event_handeler():

	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			sys.exit()
		if event.type == pygame.KEYDOWN and event.key == pygame.K_SPACE:
			game.white.pos = np.asarray(pygame.mouse.get_pos(),dtype = "float")
			game.white.vel = np.array([0.0,0.0], dtype ="float")
		if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
			cue.playing = False
			global mouse
			mouse = pygame.mouse.get_pos()
		if event.type == pygame.MOUSEBUTTONUP and event.button == 1:
			mousehit = pygame.mouse.get_pos()
			cue.playing = True
			cue.click(mouse,mousehit)

bg = pygame.image.load("pool_table.png").convert()
bg = pygame.transform.scale(bg, (int(width*0.8), int(height*0.8)))
#bg = pygame.transform.laplacian(bg)
rect = bg.get_rect()
rect.center = (width/2,height/2)

while 1:

	event_handeler()

	clock.tick(60)
	screen.fill((0,0,0))
	screen.blit(bg, rect)

	mouse = pygame.mouse.get_pos()
	fps.draw("FPS: "+str(int(clock.get_fps())))
	cursorpos.draw(f"x:{mouse[0]*100/width:.2f} y:{mouse[1]*100/height:.2f}")

	for bound in bound_list:
		bound.draw()

	for ball in ball_list:
		ball.collision_check()
		ball.ball_collision()
		ball.move()
		ball.draw()

		cue.draw()

	pygame.display.update()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script to calculate a moving average from CROCO output data for very large
datasets
@author: noam
"""

from netCDF4 import Dataset
from scipy.ndimage import convolve1d
import numpy as np
import sys
import os
from progress.bar import Bar

##############################################################################
# Parameters #################################################################
##############################################################################

out_freq = 3    # Hours
window   = 24   # Hours

in_fh = '1993_vis.nc'
out_fh = '1993_vis_filtered.nc'

variables = ['temp_surf']

##############################################################################
# File locations #############################################################
##############################################################################

this_dir = os.path.dirname(os.path.realpath(__file__))
in_fh = this_dir + '/' + in_fh
out_fh = this_dir + '/' + out_fh

##############################################################################
# Generate output file #######################################################
##############################################################################

# Firstly load to obtain dimensions
with Dataset(in_fh, mode='r') as nc:
    print('Loading files...')
    time = np.shape(nc.variables['time_counter_bounds'][:, 0])[0]
    lon = nc.variables['lon_rho'][:]
    lat = nc.variables['lat_rho'][:]

# Now generate the output file
with Dataset(out_fh, mode='w') as nc:
    print('Generating output netcdf...')
    # Create the dimensions
    nc.createDimension('time', time)
    nc.createDimension('lon', np.shape(lon)[1])
    nc.createDimension('lat', np.shape(lon)[0])

    # Create the time axis
    nc.createVariable('time', 'f8', ('time'), zlib=True)
    nc.variables['time'].long_name = 'seconds_since_start'
    nc.variables['time'].units = 'seconds'
    nc.variables['time'].field = 'time, scalar, series'
    nc.variables['time'].standard_name = 'seconds_since_start'
    nc.variables['time'][:] = time

    nc.createVariable('longitude', 'f8', ('lat', 'lon'), zlib=True)
    nc.variables['longitude'].long_name = 'longitude'
    nc.variables['longitude'].units = 'degrees_east'
    nc.variables['longitude'].field = 'longitude, scalar, series'
    nc.variables['longitude'].standard_name = 'longitude'
    nc.variables['longitude'][:] = lon

    nc.createVariable('latitude', 'f8', ('lat', 'lon'), zlib=True)
    nc.variables['latitude'].long_name = 'latitude'
    nc.variables['latitude'].units = 'degrees_north'
    nc.variables['latitude'].field = 'latitude, scalar, series'
    nc.variables['latitude'].standard_name = 'latitude'
    nc.variables['latitude'][:] = lat

    # Create the other variables
    if 'temp_surf' in variables:
        nc.createVariable('sst', 'f8', ('time', 'lat', 'lon'))
        nc.variables['sst'].long_name = 'filtered_sea_surface_temperature'
        nc.variables['sst'].units = 'Celsius'
        nc.variables['sst'].field = 'temperature, scalar, series'
        nc.variables['sst'].standard_name = 'sst'
        nc.variables['sst'].coordinates = 'lat lon'
    if 'salt_surf' in variables:
        nc.createVariable('sss', 'f8', ('time', 'lat', 'lon'))
        nc.variables['sss'].long_name = 'filtered_sea_surface_salinity'
        nc.variables['sss'].units = 'PSU'
        nc.variables['sss'].field = 'salinity, scalar, series'
        nc.variables['sss'].standard_name = 'sss'
        nc.variables['sss'].coordinates = 'lat lon'
    if 'zeta' in variables:
        nc.createVariable('ssh', 'f8', ('time', 'lat', 'lon'))
        nc.variables['ssh'].long_name = 'filtered_sea_surface_height'
        nc.variables['ssh'].units = 'meter'
        nc.variables['ssh'].field = 'free-surface, scalar, series'
        nc.variables['ssh'].standard_name = 'ssh'
        nc.variables['ssh'].coordinates = 'lat lon'

# Set up the filter
window_len = int(window/out_freq)
croco_filter = (1/window_len)*np.ones(window_len)

# Then loop through the columns
bar = Bar('Filtering data...', max=np.shape(lon)[0])
for i in range(np.shape(lon)[0]):
    with Dataset(in_fh, mode='r') as nc:
        if 'temp_surf' in variables:
            temp = nc.variables['temp_surf'][:, i, :]
            temp = convolve1d(temp, croco_filter, axis=0)
        if 'salt_surf' in variables:
            salt = nc.variables['salt_surf'][:, i, :]
            salt = convolve1d(salt, croco_filter, axis=0)
        if 'zeta' in variables:
            zeta = nc.variables['zeta'][:, i, :]
            zeta = convolve1d(zeta, croco_filter, axis=0)

    with Dataset(out_fh, mode='r+') as nc:
        if 'temp_surf' in variables:
            nc.variables['sst'][:, i, :] = temp
        if 'salt_surf' in variables:
            nc.variables['sss'][:, i, :] = salt
        if 'zeta' in variables:
            nc.variables['ssh'][:, i, :] = zeta

    bar.next()
bar.finish()

print('Processing complete!')


# write skyfield logs

from skyfield.api import load
from skyfield.api import N, W, wgs84, load, utc
from skyfield.almanac import find_discrete, risings_and_settings
from datetime import datetime, timedelta
import csv
from dateutil.relativedelta import relativedelta

def radec_list(start_year, start_month, start_day, start_hour, start_minute, start_second, name, int_days, n_days, n_months):
# for this function, pass through a start time and date. indicate whether you want a number of days or a number of months. if months, it will go for every month. if days, you can set an interval of days.
    radec = [[0 for i in range(3)] for j in range(n_days + n_months)]
    start_date = datetime(start_year, start_month, start_day, start_hour, start_minute, start_second)
    names = ['sun', 'moon', 'mercury', 'venus', 'mars', 'jupiter', 'saturn', 'uranus', 'neptune']

    if n_days == 0:
        for i in range(n_months):

            date = start_date + relativedelta(months = i)

            t = ts.utc(date.year, date.month, date.day, date.hour, date.minute, date.second)
            astrometric = eph['earth'].at(t).observe(planets[names.index(name)])
            ra, dec, distance = astrometric.apparent().radec()

            ra = ra.hours
            dec = dec.degrees

            radec[i][0] = date.strftime("%Y/%m/%d, %H:%M:%S")
            radec[i][1] = ra
            radec[i][2] = dec

    else:
        for i in range(n_days):

            date = start_date + timedelta(days = i * int_days)

            t = ts.utc(date.year, date.month, date.day, date.hour, date.minute, date.second)
            astrometric = eph['earth'].at(t).observe(planets[names.index(name)])
            ra, dec, distance = astrometric.apparent().radec()

            ra = ra.hours
            dec = dec.degrees

            radec[i][0] = date.strftime("%Y/%m/%d, %H:%M:%S")
            radec[i][1] = ra
            radec[i][2] = dec

    return(radec)

    filename = "radec_files/" + str(name) + "_radec" + "_" + str(start_date.year) + "_" + str(start_date.month) + "_" + str(start_date.day) + "-" + str(date.year) + "_" + str(date.month) + "_" + str(date.day) + ".csv"

    with open(filename, 'w') as file:
        writer = csv.writer(file)
        writer.writerows(radec)

def rise_set_list(lat, lon, start_year, start_month, start_day, start_hour, start_minute, start_second, name, n_months):
    # generate list of daily rise and set times to CSV given a number of months
    city = wgs84.latlon(float(lat), float(lon))
    names = ['sun', 'moon', 'mercury', 'venus', 'mars', 'jupiter', 'saturn', 'uranus', 'neptune']
    f = risings_and_settings(eph, planets[names.index(name)], city)
    #This returns a function taking a Time argument returning True if the body’s altazimuth altitude angle plus radius_degrees is greater than horizon_degrees, else False

    start_date = datetime(start_year, start_month, start_day, start_hour, start_minute, start_second, tzinfo=utc)
    print(start_date)
    date = start_date + relativedelta(months = n_months)
    print(date)

    t0 = ts.utc(start_date)
    t1 = ts.utc(date)

    t, values = find_discrete(t0, t1, f)
    rise_set = [[0 for i in range(2)] for j in range(len(t))]

    for i in range(len(t)):
        rise_set[i][0] = t[i].utc_strftime('%Y/%m/%d %H:%M:%S')
        rise_set[i][1] = values[i]
        # print(t[i])
        # print(rise_set[i][0])
    # print(rise_set)

    return rise_set

    filename = "rise_set_files/" + str(name) + "_rise_set" + "_" + str(start_date.year) + "_" + str(start_date.month) + "_" + str(start_date.day) + "-" + str(date.year) + "_" + str(date.month) + "_" + str(date.day) + ".csv"

    with open(filename, 'w') as file:
        writer = csv.writer(file)
        writer.writerows(rise_set)

#initialize skyfield stuff
eph = load('de421.bsp')
sun = eph['sun']
moon = eph['moon']
mercury = eph['mercury']
venus  = eph['venus']
earth  = eph['earth']
mars = eph['mars']
jupiter = eph['JUPITER BARYCENTER']
saturn = eph['SATURN BARYCENTER']
uranus = eph['URANUS BARYCENTER']
neptune = eph['NEPTUNE BARYCENTER']
planets = [sun,
           moon,
           mercury,
           venus,
           mars,
           jupiter,
           saturn,
           uranus,
           neptune]
ts = load.timescale()

# # write a CSV file of Jupiter's RA and DEC starting on 2/1/2023 every day for 365 days
# radec_list(2023, 2, 1, 0, 0, 0, 'jupiter', 1, 365, 0)

# # write a CSV file of Jupiter's RA and DEC starting on 2/1/2023 every month for 10 years
# radec_list(2023, 2, 1, 0, 0, 0, 'jupiter', 0, 0, 120)

# # write a CSV file of Jupiter's risings (1) and settings (0) starting on 2/1/2023 every month for 1 year
# lat = 38.9072
# lon = -77.0369
# rise_set_list(lat, lon, 2023, 2, 1, 0, 0, 0, 'jupiter', 12)

# with open('RADEC.csv', 'w') as file:
#     writer = csv.writer(file)
#     writer.writerows(csv_rowlist)

# now = datetime.utcnow()
# year = now.year
# month = now.month
# day = now.day
# utc = now.hour + now.minute/60 + now.second/3600

# ra = ra.hours
# dec = dec.degrees
# print('ra hours is', ra, 'dec degrees is', dec, sep='\n')


from selenium import webdriver
# from selenium.common.exceptions import TimeoutException
# from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0
# from selenium.webdriver.support import expected_conditions as EC # available since 2.26.0
import time

bot = webdriver.Firefox()
bot.get('https://wrap.co')

login_button = bot.find_element_by_link_text('LOG IN')
login_button.click()
time.sleep(5)

signup_button = bot.find_element_by_link_text('SIGNUP FOR WRAP AUTHORING')
signup_button.click()
time.sleep(5)

smb_button = bot.find_element_by_link_text('TRY SMALL BUSINESS FOR FREE')
smb_button.click()
time.sleep(5)

email_box = bot.find_element_by_tag_name('input')
email_box.send_keys('testuser11235@gmail.com')
email_box.submit()
time.sleep(5)

inputs = bot.find_elements_by_tag_name('input')
inputs[-2].send_keys('testuser11235')
inputs[-1].send_keys('P@ssword1!')

buttons = bot.find_elements_by_tag_name('button')
for button in buttons:
    if button.text.find('CREATE ACCOUNT') != -1:
        button.click()
        time.sleep(10)
        button.click()
        break

inputs = bot.find_elements_by_tag_name('input')
inputs[0].send_keys('test')
inputs[1].send_keys('user1123')
inputs[2].send_keys('test company 1123')

buttons = bot.find_elements_by_tag_name('button')
for button in buttons:
    if button.text.find('CREATE ACCOUNT') != -1:
        button.click()
        time.sleep(10)
        button.click()
        break


#!/usr/bin/env python3
# Simple https://github.com/mcRPW/rpw projects exporter
# Ignores all setting files (for now), but still does the job
# Contributions are welcome
from zipfile import ZipFile
from pathlib import Path
from sys import argv
from os import walk
from json import dumps

if len(argv) != 3:
    print("Usage: {} [project root] [output.zip]".format(argv[0]))
    exit(2)

project_root = Path(argv[1])
project_files = project_root / "project_files"
extras_directory = project_root / "extra_files"
custom_sounds = project_root / "custom_sounds"
custom_languages = project_root / "custom_languages"

pack_description = ""
pack_version = 6

def add_tree(zipf, path_from, path_to=""):
    for (dpath, _, fnames) in walk(path_from):
        for fname in fnames:
            realpath = Path(dpath, fname)
            relpath = realpath.relative_to(path_from)
            zipf.write(realpath, Path(path_to, relpath))
            print(" + {}".format(relpath))

with ZipFile(argv[2], "w") as zipf:
    add_tree(zipf, extras_directory)
    add_tree(zipf, custom_sounds, "assets/minecraft/sounds")
    add_tree(zipf, custom_languages, "assets/minecraft/lang")
    zipf.write(project_root / "sounds.json", "assets/minecraft/sounds.json")
    zipf.writestr("pack.mcmeta", dumps({
        "pack": {
            "pack_format": pack_version,
            "description": pack_description
        }
    }, indent=2))
    add_tree(zipf, project_files)
    zipf.write(project_root / "pack.png", "pack.png")


from twisted.internet import reactor, protocol
from twisted.internet.protocol import ClientFactory, Protocol
port = 9090
class ClientChat(Protocol):
     def connectionMade(self):
         login = input('Enter your  login')
         fio = input('Enter your  FIO:')
         addres = input('Enter  your  address:')
         a1 = {}
         a1['login'] = login
         a1['fio'] = fio
         a1['addres'] = addres
         data2 = ','.join([f'{key},{value}' for key, value in a1.items()])
         a = data2.encode('utf-8')
         self.transport.write(a)
     def dataReceived(self, data: str):
         print(data)
         self.connectionMade()

class ClientChatFactory(ClientFactory):
    def startedConnecting(self, connector):
        print('connected')
    def buildProtocol(self, addr):
         return ClientChat()
    def clientConnectionFailed(self, connector, reason):
        print('ConnectionFailed, reason:', reason)
        reactor.stop()
    def clientConnectionLost(self, connector, reason):
        print('ConnectionLost, reason:', reason)
        connector.connect()
reactor.connectTCP('localhost',port, ClientChatFactory())
reactor.run()


import numpy as np
from matplotlib import pyplot as plt
#from scipy.integrate import quad

t= np.arange(0,15,0.0001)
## s = a1 t**3 + a2 t**2 + a3t + a4: 3次多項式，可以用更多次方，會得到不同的結果
## condition: s(0)=0, s(15)=415.75, v(t)=s'(t), v(0)=v(15)=0 --> 用這些邊界條件來找多項式係數
a1 = -415.75*2/(15**3)
a2 = 3*415.75/15**2
s = a1*(t**3) +a2*(t**2)
v = 3*a1*(t**2) + 2*a2*t
a = 6*a1*t +2*a2
ax1 = plt.subplot(311)
ax1.set_title("S")
ax1.set_ylim([0,415.75])
plt.plot(t,s)
ax2 = plt.subplot(312,sharex=ax1)
ax2.set_title("V")
plt.plot(t,v)
ax3 = plt.subplot(313,sharex=ax1)
ax3.set_title("a")
plt.plot(t,a)
plt.subplots_adjust(hspace=0.5)
plt.show()


import os
from datetime import datetime
import time
from fpdf import FPDF
from selenium.webdriver.common.by import By
from selenium.webdriver import Firefox
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.common.alert import Alert
vowelList = ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U', 'u']

def next_page(driver):
    buttons = driver.find_elements(By.TAG_NAME, "button")
    header = driver.find_element(By.TAG_NAME, 'h1')
    for text in buttons:
        words = text.get_attribute('innerHTML')
        if 'Continue' in words or 'appl' in words or 'Apply now' in words:
            driver.execute_script("arguments[0].click()", text)
            # text.click()
            #if the next header is the same as the previous, we haven't moved
            time.sleep(2)
            new_header = driver.find_element(By.TAG_NAME, 'h1')
            if new_header == header:
                return 0
            else:
                return 1

def indeed(options, service):
    while True:
        try:
            driver = Firefox(service=service, options=options)
            try:
                driver.get("https://myjobs.indeed.com/saved?hl=en&co=US&from=_atweb_gnav-homepage")
            except Exception:
                pass
            if driver.current_url == "https://myjobs.indeed.com/saved?hl=en&co=US&from=_atweb_gnav-homepage":
                break
        except Exception:
            driver.close()

        #find the user icon and click on saved jobs

    postings = driver.find_elements(By.CSS_SELECTOR, 'a.atw-ApplyButton')

    applications = [x.get_attribute('href') for x in postings]
    for x in applications:
        driver.get(x)
        try:
            Alert(driver).accept()
        except Exception:
            pass
        time.sleep(2)
        header = ''
        redundancy = 1
        next_page(driver)
        link = x
        while("Please Review" not in header):
            time.sleep(2)
            try:
                #get the header each time and behave accordingly
                header = driver.find_element(By.XPATH, '//h1[@class="ia-BasePage-heading fs-unmask"]')
                header = header.get_attribute('innerHTML')
                if not redundancy:
                    break
                #for preliminary questions
                elif "Questions" in header:
                    questions = driver.find_elements(By.XPATH, '//div[@class="ia-Questions-item css-e9ld6l eu4oa1w0"]')
                    questions_text = [x.get_attribute('innerHTML') for x in questions]
                    for i, text in enumerate(questions_text):
                        if "sponsor" in text:
                            try:
                                driver.execute_script("arguments[0].click()", questions[i].find_element(By.XPATH, '//input[@value="0"]'))
                            except Exception:
                                pass
                        elif "education" in text:
                            driver.execute_script("arguments[0].click()", questions[i].find_element(By.XPATH, '//input[@value="Bachelor\'s"]'))
                        elif "experience" in text:
                            #find the selection and select it
                            try:
                                answers = questions[i].find_elements(By.XPATH, '//input[@type="radio"]')
                                for selection in answers:
                                    choice = selection.find_element(By.CSS_SELECTOR, "span.css-19kaor0 eu4oa1w0")
                                    if '3' in choice.get_attribute('innerHTML'):
                                        selection.click()
                                        continue
                            except Exception:
                                if 'value=""' in text:
                                    response = questions[i].find_element(By.XPATH, '//input[@value=""]')
                                    response.send_keys(Keys.CONTROL+'a')
                                    response.send_keys('3')
                                continue
                        elif "salary" in text:
                            questions[i].find_element(By.CSS_SELECTOR, "div.css-d8iwdi e1jgz0i3").send_keys('65000 - 85000')
                        else:

                            #weird thing to click, because element obscures
                            try:
                                driver.execute_script("arguments[0].click()", questions[i].find_element(By.XPATH, '//input[@value="1"]'))
                            except Exception:
                                pass
                        #TODO: case handling for commuting and protected veteran
                    #next page
                    redundancy = next_page(driver)
                    continue
                #for the letter
                elif "Want to" in header:
                    #tricky part is the letter
                    job = driver.find_element(By.XPATH, '//div[@class="ia-JobHeader-details"]')
                    job = job.get_attribute('innerHTML')
                    #name of company
                    company = job[:job.find(" - ")-1]

                    job = driver.find_element(By.XPATH, '//div[@class="ia-JobHeader-title ia-JobHeader-title--withJobDetails"]')
                    job = job.get_attribute('innerHTML')
                    #correct grammar
                    if job[0] in vowelList:
                        job = 'an ' + job
                    else:
                        job = 'a ' + job
                    #find the location of the coverletter field and click
                    coverletter = driver.find_element(By.XPATH, '//span[@title="Write cover letter"]')
                    coverletter.click()
                    coverletter = driver.find_element(By.CSS_SELECTOR, "textarea.ia-Coverletter-textarea")
                    coverletter.click()
                    #don't forget to clear the text field of previous response
                    coverletter.send_keys(Keys.CONTROL+'a')
                    coverletter.send_keys(Keys.BACK_SPACE)
                    #open the coverletter
                    file = open('coverletter.txt', 'r')
                    #put these two entries into the coverletter
                    for i, x in enumerate(file):
                        time.sleep(0.3)
                        if i == 1:
                            index = x.find('this company')
                            first = x[0:index]
                            second = x[index+12:]
                            x = first + company + ' as ' + job + second
                        coverletter.send_keys(x)
                    #sweet this works great
                    #next page
                    redundancy = next_page(driver)
                    continue
                elif "Want to" in header:
                    redundancy = next_page(driver)
                #if the header asks for resume, confirmations, past jobs, or otherwise by default
                else:
                    #continued weirdness. Now just look for buttons with innerHTML with 'continue'
                    redundancy = next_page(driver)
                    continue
            except Exception:
                print("Something didn't work")
                f = open('unfinishedApps.txt', 'a+')
                now = datetime.now()
                current_time = now.strftime("%m-%d-%Y %H:%M:%S")
                f.write(f"{current_time} unfinished app: {link}\n{Exception.with_traceback}\n\n")
                f.close()
                break
        print("End of process")
        next_page(driver)
    driver.close()

def letterEdit(company, position):
    pdf = FPDF()

    pdf.add_page()

    pdf.set_font("Arial", size = 11)
    file = open("coverletter.txt", "r")

    for i, x in enumerate(file):

        print(x)
        pdf.cell(200, 10, txt = x, ln = 1, align = 'L')

    pdf.output("test.pdf")
# submissions = {
#     "resume": "",
#     "coverletter": "",
#     "references" : ""
# }

options = Options()
service = Service()
service.path = "F:\\Dropbox\\UNT\\TiredofApps\\geckodriver.exe"
options.add_argument("-profile F:\\Dropbox\\UNT\\TiredofApps\\ed53ej1d.Gecko")
options.set_preference("remote.prefs.recommended", False)
options.set_preference("dom.allow_scripts_to_close_windows", True)
options.set_preference("browser.tabs.closeWindowWithLastTab", True)
options.add_argument("--marionette-port 5555")
options.add_argument("--disable-gpu-shader-disk-cache ")

# cwd = os.getcwd()
# print(cwd)
indeed(options, service)