import os
import json

# third party imports
import mock

# CubicWeb specific imports
from cubicweb.devtools import PostgresApptestConfiguration
from cubicweb.devtools.testlib import CubicWebTC

# library specific imports
from cubicweb_francearchives.views import STRING_SEP
from cubicweb_frarchives_edition.alignments import get_externaluri_data
from cubicweb_frarchives_edition.alignments.align import AgentAligner, AgentRecord
from cubicweb_frarchives_edition.alignments.databnf import DataBnfDatabase
from cubicweb_frarchives_edition.alignments.wikidata import WikidataDatabase

from cubicweb_frarchives_edition.alignments import DATABNF_RE, DATABNF_ARK_RE

from utils import FrACubicConfigMixIn
from pgfixtures import setup_module, teardown_module  # noqa

class AlignementUtilsTC(CubicWebTC):
    """Agent alignements utils tests class."""

    def test_databnf_re(self):
        uri = "https://data.bnf.fr/fr/11894985/albert_camus/"
        self.assertEqual("11894985", DATABNF_RE.search(uri).group(1))
        uri = "https://data.bnf.fr/11907966/victor_hugo"
        self.assertEqual("11907966", DATABNF_RE.search(uri).group(1))
        uri = "https://data.bnf.fr/11907966"
        self.assertFalse(DATABNF_RE.search(uri))
        uri = "https://data.bnf.fr/ark:/12148/cb13005429m"
        self.assertFalse(DATABNF_RE.search(uri))

    def test_databnf_ark_re(self):
        uri = "https://data.bnf.fr/fr/11894985/albert_camus/"
        self.assertFalse(DATABNF_ARK_RE.search(uri))
        uri = "https://data.bnf.fr/11907966/victor_hugo"
        self.assertFalse(DATABNF_ARK_RE.search(uri))
        uri = "https://data.bnf.fr/11907966"
        self.assertFalse(DATABNF_ARK_RE.search(uri))
        uri = "https://data.bnf.fr/ark:/12148/cb11909252t"
        self.assertEqual("11909252", DATABNF_ARK_RE.search(uri).group(1))
        uri = "https://data.bnf.fr/fr/ark:/12148/cb13005429m"
        self.assertEqual("13005429", DATABNF_ARK_RE.search(uri).group(1))

    def test_uri_source(self):
        for source, extid, uri in (
            ("databnf", "11907966", "https://data.bnf.fr/11907966/victor_hugo"),
            ("databnf", "11907966", "https://data.bnf.fr/en/11907966/victor_hugo"),
            ("databnf", "13005429", "https://data.bnf.fr/fr/ark:/12148/cb13005429m"),
            ("databnf", "11921467", "http://data.bnf.fr/ark:/12148/cb11921467w"),
            ("databnf", "13946072", "https://data.bnf.fr/ark:/12148/cb139460728"),
            ("databnf", "13926902", "https://data.bnf.fr/ark:/12148/cb139269025"),
            ("databnf", "39269025", "https://data.bnf.fr/ark:/12148/cb39269025q"),
            ("wikidata", "Q158768", "https://www.wikidata.org/wiki/Q158768"),
            ("wikidata", "Q131412", "http://www.wikidata.org/wiki/Q131412"),
            ("fr.wikipedia.org", None, "https://fr.wikipedia.org/wiki/Edmond_Maire"),
            ("geoname", "3033123", "https://www.geonames.org/3033123"),
            ("geoname", "2986302", "http://www.geonames.org/2986302"),
        ):
            got_source, got_extid = get_externaluri_data(uri)
            self.assertEqual(source, got_source)
            self.assertEqual(extid, got_extid)

class AlignmentTC(FrACubicConfigMixIn, CubicWebTC):
    """Manual alignment test cases base class."""

    configcls = PostgresApptestConfiguration

class WikidataTC(AlignmentTC):
    """Wikidata test cases."""

    def test_agentinfo_from_wikidata(self):
        """Retrieve some data from Wikidata.

        Trying: retrieve and process mock data
        Expecting: data is processed correctly
        """
        return_value = (
            (
                "Igor Stravinsky",
                "+1882-06-18T00:00:00Z",
                "+1971-04-06T00:00:00Z",
                "11",
                "11",
                "pianiste et compositeur",
            ),
        )
        with mock.patch.object(WikidataDatabase, "agent_query", return_value=return_value):
            wikidata = WikidataDatabase()
            data_infos = wikidata.agent_infos("Q7314")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual("1882-06-18", birthdate["timestamp"])
            self.assertEqual("d", birthdate["precision"])
            self.assertTrue(birthdate["isiso"])
            deathdate = dates["deathdate"]
            self.assertEqual("1971-04-06", deathdate["timestamp"])
            self.assertEqual("d", deathdate["precision"])
            self.assertTrue(deathdate["isiso"])
            self.assertEqual("Igor Stravinsky", data_infos["label"])
            self.assertEqual("pianiste et compositeur", data_infos["description"])

    def test_agentinfo_from_wikidata_bc(self):
        """Retrieve some data from Wikidata.

        Trying: birthdate and deathdate are BC
        Expecting: timestamp is unsigned and isbc flag is True
        """
        return_value = (
            (
                "Jules César",
                "-0099-07-01T00:00:00Z",
                "-0043-03-13T00:00:00Z",
                "10",
                "11",
                "homme politique et général romain du Ier siècle avant J-C.",
            ),
        )
        with mock.patch.object(WikidataDatabase, "agent_query", return_value=return_value):
            wikidata = WikidataDatabase()
            data_infos = wikidata.agent_infos("Q1048")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual("0099-07-01", birthdate["timestamp"])
            self.assertEqual("m", birthdate["precision"])
            self.assertTrue(birthdate["isiso"])
            self.assertTrue(birthdate["isbc"])
            deathdate = dates["deathdate"]
            self.assertEqual("0043-03-13", deathdate["timestamp"])
            self.assertEqual("d", deathdate["precision"])
            self.assertTrue(deathdate["isiso"])
            self.assertTrue(deathdate["isbc"])
            self.assertEqual("Jules César", data_infos["label"])
            self.assertEqual(
                "homme politique et général romain du Ier siècle avant J-C.",
                data_infos["description"],
            )

    def test_agentinfo_from_wikidata_year(self):
        """Retrieve some data from Wikidata.

        Trying: birthdate has precision '9'
        Expecting: birthdate has precision 'y'(ear)
        """
        return_value = (
            (
                "François Rabelais",
                "+1494-01-01T00:00:00Z",
                "+1553-04-19T00:00:00Z",
                "9",
                "11",
                "auteur français du 16e siècle",
            ),
        )
        with mock.patch.object(WikidataDatabase, "agent_query", return_value=return_value):
            wikidata = WikidataDatabase()
            data_infos = wikidata.agent_infos("Q131018")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual("1494-01-01", birthdate["timestamp"])
            self.assertEqual("y", birthdate["precision"])
            self.assertTrue(birthdate["isiso"])
            deathdate = dates["deathdate"]
            self.assertEqual("1553-04-19", deathdate["timestamp"])
            self.assertEqual("d", deathdate["precision"])
            self.assertTrue(deathdate["isiso"])
            self.assertEqual("François Rabelais", data_infos["label"])
            self.assertEqual("auteur français du 16e siècle", data_infos["description"])

class DataBnfTC(AlignmentTC):
    """Data.bnf.fr test cases."""

    def test_agentinfo_from_databnf_date(self):
        """
        Retrieve some data from data.bnf.fr
        Trying: add an same_as relation to a databnf ExternalUri
        Expecting: retrive dates (full date) and notes
        """
        return_value = (
            (
                "Igor Stravinsky (1882-1971)",
                None,
                "1882-06-17",
                None,
                "1971-04-06",
                "Compositeur. - Pianiste. - Chef d'orchestre",
            ),
        )
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("12405560")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual("1882-06-17", birthdate["timestamp"])
            self.assertEqual("d", birthdate["precision"])
            self.assertFalse(birthdate["isiso"])
            deathdate = dates["deathdate"]
            self.assertEqual("1971-04-06", deathdate["timestamp"])
            self.assertEqual("d", deathdate["precision"])
            self.assertFalse(deathdate["isiso"])
            self.assertEqual("Igor Stravinsky (1882-1971)", data_infos["label"])
            self.assertEqual(
                "Compositeur. - Pianiste. - Chef d'orchestre", data_infos["description"]
            )

    def test_agentinfo_description_from_databnf_date(self):
        """
        Retrieve some data from data.bnf.fr
        Trying: add an same_as relation to a databnf ExternalUri
        Expecting: retrive dates all notes
        """
        descriptions = (
            "A utilisé le pseudonyme de Morland dans la Résistance",
            "Avocat et homme politique. - Quatrième président de la Ve République (1981-1995)",
        )
        return_value = (
            (
                "François Mitterrand (1916-1996)",
                None,
                "1916-10-26",
                None,
                "1996-01-08",
                descriptions[0],
            ),
            (
                "François Mitterrand (1916-1996)",
                None,
                "1916-10-26",
                None,
                "1996-01-08",
                descriptions[1],
            ),
        )
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("11916320")
            self.assertCountEqual(descriptions, data_infos["description"].split(STRING_SEP))

    def test_agentinfo_from_databnf_year(self):
        """
        Retrieve some data from data.bnf.fr
        Trying: add an same_as relation to a databnf ExternalUri
        Expecting: retrive year and notes
        """
        return_value = (
            (
                "Hervé (saint, 05..-0575?)",
                "05..",
                None,
                "0575",
                None,
                "Barde d'origine galloise. - Fonda un ermitage en Bretagne",
            ),
        )
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("14978856")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual(birthdate["timestamp"], "05..")
            self.assertFalse(birthdate["isdate"])
            self.assertFalse(birthdate["isiso"])
            deathdate = dates["deathdate"]
            self.assertEqual(deathdate["timestamp"], "0575-01-01")
            self.assertTrue(deathdate["isdate"])
            self.assertFalse(deathdate["isiso"])
            self.assertEqual(deathdate["precision"], "y")

    def test_agentinfo_from_databnf_bc(self):
        """Test processing BCE dates.

        Trying: retrieve BCE dates from data.bnf.fr
        Expecting: valid date as timestamp and marked as BCE
        """
        return_value = (
            (
                "Sénèque (0004 av. J.-C.-0065)",
                None,
                "- 4",
                None,
                "65",
                (
                    "Philosophe stoïcien. - Auteur de tragédies. - Fils de : "
                    "Sénèque le rhéteur (0060 av. J.-C.?-0039?)"
                ),
            ),
        )
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("11887555")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual(birthdate["timestamp"], "0004-01-01")
            self.assertTrue(birthdate["isdate"])
            self.assertTrue(birthdate["isbc"])
            self.assertFalse(birthdate["isiso"])
            self.assertEqual(birthdate["precision"], "y")
            deathdate = dates["deathdate"]
            self.assertEqual(deathdate["timestamp"], "0065-01-01")
            self.assertFalse(deathdate["isbc"])
            self.assertFalse(deathdate["isiso"])

    def test_agentinfo_one_date_from_databnf_date(self):
        """
        Retrieve some data from data.bnf.fr
        Trying: add an same_as relation to a databnf ExternalUri
        Expecting: only the start date exist
        """
        return_value = (("Alain Jupp\xe9", 1945, None, None, None, "Inspecteur des finances"),)
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("11909252")
            dates = json.loads(data_infos["dates"])
            birthdate = dates["birthdate"]
            self.assertEqual("1945-01-01", birthdate["timestamp"])
            self.assertEqual("y", birthdate["precision"])
            self.assertFalse(birthdate["isiso"])
            self.assertIsNone(dates.get("deathdate"))
            self.assertEqual("Alain Juppé", data_infos["label"])
            self.assertEqual("Inspecteur des finances", data_infos["description"])

    def test_agentinfo_cast(self):
        """Test retrieving information from data.bnf.fr.

        Trying: using int
        Expecting: no birthdate is returned and deathdate is of the proper type (str)
        despite failing date evaluation (no dates > 9999 allowed in datetime)
        """
        return_value = (("Max Mustermann", None, None, 10000, None, ""),)
        with mock.patch.object(DataBnfDatabase, "author_query", return_value=return_value):
            databnf = DataBnfDatabase()
            data_infos = databnf.agent_infos("1234567890")
            dates = json.loads(data_infos["dates"])
            self.assertIsNone(dates.get("birthdate"))
            self.assertEqual("10000", dates["deathdate"]["timestamp"])

class AgentAlignerTC(AlignmentTC):
    """Wikidata and data.bnf.fr test cases (automatic)."""

    configcls = PostgresApptestConfiguration
    # TODO remove test_isdate and test_isbc as soon as
    # manual alignment and automatic alignment behaviour is
    # the same and test compute_dates function instead of
    # process_alignment (Wikidata)

    def test_process_csv(self):
        """Test processing CSV file.

        Trying: processing CSV file
        Expecting: corresponding alignments
        """
        with self.admin_access.cnx() as cnx:
            file_name = os.path.join(self.datapath(), "agent-alignments.csv")
            aligner = AgentAligner(cnx)
            with open(file_name) as fp:
                expected = AgentRecord(
                    {
                        "eid": "19034398",
                        "date naissance": "+1600-01-01T00:00:00Z",
                        "precision date naissance": "9",
                        "date mort": "+1674-01-01T00:00:00Z",
                        "precision date mort": "9",
                        "description": "Marin",
                        "uri": "https://www.wikidata.org/wiki/Q646986",
                        "source": "wikidata",
                        "confidence": "0.9",
                        "extlabel": "Marin le Roy de Gomberville",
                    }
                )
                alignments = aligner.process_csv(fp)
                self.assertEqual(1, len(alignments))
                actual = alignments[("19034398", "https://www.wikidata.org/wiki/Q646986")]
                self.assertEqual(actual.autheid, expected.autheid)
                self.assertEqual(actual.date_birth, expected.date_birth)
                self.assertEqual(actual.date_death, expected.date_death)
                self.assertEqual(actual.description, expected.description)
                self.assertEqual(actual.exturi, expected.exturi),
                self.assertEqual(actual.source, expected.source)
                self.assertEqual(actual.confidence, expected.confidence)
                self.assertEqual(actual.extlabel, expected.extlabel)

    def test_process_alignments_existing_externaluris(self):
        """Test updating database.

        Trying: existing ExternalUris
        Expecting: ExternalUri is kept
        """
        with self.admin_access.cnx() as cnx:
            external_uri = cnx.create_entity(
                "ExternalUri",
                source="wikidata",
                uri="https://www.wikidata.org/wiki/Q646986",
                label="Marin le Roy de Gomberville",
            )
            cnx.commit()
            agent_authority = cnx.create_entity("AgentAuthority")
            cnx.commit()
            record = AgentRecord(
                {
                    "eid": str(agent_authority.eid),
                    "date naissance": "+1600-01-01T00:00:00Z",
                    "precision date naissance": "9",
                    "date mort": "+1674-01-01T00:00:00Z",
                    "precision date mort": "9",
                    "description": "Marin",
                    "uri": "https://www.wikidata.org/wiki/Q646986",
                    "source": "wikidata",
                    "confidence": "0.9",
                    "external label": "Marin le Roy de Gomberville",
                }
            )
            alignments = {(agent_authority.eid, "https://www.wikidata.org/wiki/Q646986"): record}
            dates = {
                "birthdate": {
                    "timestamp": "1600-01-01",
                    "isbc": False,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
                "deathdate": {
                    "timestamp": "1674-01-01",
                    "isbc": False,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
            }
            aligner = AgentAligner(cnx)
            aligner.process_alignments(alignments)
            agent_info = cnx.execute(
                "Any X WHERE X is AgentInfo, X agent_info_of %(eid)s", {"eid": external_uri.eid}
            ).one()
            self.assertEqual(agent_info.dates, dates)
            self.assertEqual(agent_info.description, record.description)
            self.assertEqual(external_uri.label, record.extlabel)

    def test_process_alignments_nonexisting_externaluris(self):
        """Test updating database.

        Trying: nonexisting ExternalUris
        Expecting: new ExternalUris and AgentInfos are created
        """
        with self.admin_access.cnx() as cnx:
            self.assertFalse(cnx.find("ExternalUri"))
            agent_authority = cnx.create_entity("AgentAuthority")
            cnx.commit()
            record = AgentRecord(
                {
                    "eid": str(agent_authority.eid),
                    "date naissance": "+1600-01-01T00:00:00Z",
                    "precision date naissance": "9",
                    "date mort": "+1674-01-01T00:00:00Z",
                    "precision date mort": "9",
                    "description": "Marin",
                    "uri": "https://www.wikidata.org/wiki/Q646986",
                    "source": "wikidata",
                    "confidence": "0.9",
                    "external label": "Marin le Roy de Gomberville",
                }
            )
            alignments = {(agent_authority.eid, "https://www.wikidata.org/wiki/Q646986"): record}
            dates = {
                "birthdate": {
                    "timestamp": "1600-01-01",
                    "isbc": False,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
                "deathdate": {
                    "timestamp": "1674-01-01",
                    "isbc": False,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
            }
            aligner = AgentAligner(cnx)
            aligner.process_alignments(alignments)
            agent_info = cnx.find("AgentInfo").one()
            self.assertEqual(agent_info.dates, dates)
            self.assertEqual(agent_info.description, record.description)
            external_uri = cnx.find("ExternalUri").one()
            self.assertEqual(external_uri.label, record.extlabel)

    def test_process_date_wikidata(self):
        """Test processing date.

        Trying: source is Wikidata
        Expecting: is successfully processed
        """
        with self.admin_access.cnx() as cnx:
            aligner = AgentAligner(cnx)
            self.assertEqual(
                aligner._process_date("+1600-01-01T00:00:00Z", "9", "wikidata"),
                {
                    "timestamp": "1600-01-01",
                    "isbc": False,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
            )

    def test_process_date_databnf(self):
        """Test processing date.

        Trying: source is data.bnf.fr
        Expecting: is successfully processed
        """
        with self.admin_access.cnx() as cnx:
            aligner = AgentAligner(cnx)
            self.assertEqual(
                aligner._process_date("05..", "", "databnf"),
                {
                    "timestamp": "05..",
                    "isbc": False,
                    "isdate": False,
                    "precision": "d",
                    "isiso": False,
                },
            )

    def test_isbc(self):
        """Test updating database.

        Trying: date is Before Christ
        Expecting: timestamp is formatted date and it is flagged as Before Christ
        """
        with self.admin_access.cnx() as cnx:
            agent_authority = cnx.create_entity("AgentAuthority")
            cnx.commit()
            record = AgentRecord(
                {
                    "eid": str(agent_authority.eid),
                    "date naissance": "-0400-01-01T00:00:00Z",
                    "precision date naissance": "9",
                    "date mort": "+0065-04-12T00:00:00Z",
                    "precision date mort": "11",
                    "description": "philosophe stoïcien, dramaturge et homme d'État romain",
                    "uri": "https://www.wikidata.org/wiki/Q2054",
                    "source": "wikidata",
                    "confidence": "1.0",
                }
            )
            alignments = {(agent_authority.eid, "https://www.wikidata.org/wiki/Q2054"): record}
            dates = {
                "birthdate": {
                    "timestamp": "0400-01-01",
                    "isbc": True,
                    "isdate": True,
                    "precision": "y",
                    "isiso": True,
                },
                "deathdate": {
                    "timestamp": "0065-04-12",
                    "isbc": False,
                    "isdate": True,
                    "precision": "d",
                    "isiso": True,
                },
            }
            aligner = AgentAligner(cnx)
            aligner.process_alignments(alignments)
            agent_info = cnx.find("AgentInfo").one()
            self.assertEqual(agent_info.dates, dates)

    def test_isdate(self):
        """Test updating database.

        Trying: date is not valid date format
        Expecting: date is not retrieved
        """
        with self.admin_access.cnx() as cnx:
            agent_authority = cnx.create_entity("AgentAuthority")
            cnx.commit()
            record = AgentRecord(
                {
                    "eid": str(agent_authority.eid),
                    "date naissance": "0061",
                    "precision date naissance": "11",
                    "date mort": "0114",
                    "precision date mort": "11",
                    "description": "orateur et homme politique romain",
                    "uri": "https://www.wikidata.org/wiki/Q168707",
                    "source": "wikidata",
                    "confidence": "1.0",
                }
            )
            alignments = {(agent_authority.eid, "https://www.wikidata.org/wiki/Q168707"): record}
            aligner = AgentAligner(cnx)
            aligner.process_alignments(alignments)
            agent_info = cnx.find("AgentInfo").one()
            self.assertEqual(agent_info.dates, {})


"""Decorator utils."""
# Standard Python Libraries
from functools import wraps
import hashlib
from http import HTTPStatus

# Third-Party Libraries
import cognitojwt
from flask import abort, g, request

# cisagov Libraries
from api.config import (
    AWS_REGION,
    COGNITO_CLIENT_ID,
    COGNITO_DEFAULT_ADMIN,
    COGNTIO_ENABLED,
    COGNTIO_USER_POOL_ID,
    logger,
)
from api.manager import DomainManager, UserManager
from utils.users import user_can_access_domain

domain_manager = DomainManager()
user_manager = UserManager()

class RequestAuth:
    """Authorization class for requests."""

    def __init__(self, request):
        """Initialize class with cognito settings and associated request."""
        self.request = request
        self.username = ""
        self.groups = []

    def validate(self):
        """Validate request."""
        if self.check_api_key(request):
            g.is_admin = True
            return True
        if not COGNTIO_ENABLED:
            g.is_admin = True
            return True
        if self.check_cognito_jwt(request):
            return True

        return False

    def check_api_key(self, request):
        """Check if API Key is valid."""
        if "api_key" in request.headers:
            hash_val = hashlib.sha256(
                str.encode(request.headers.get("api_key"))
            ).hexdigest()
            user = user_manager.all(params={"HashedAPI": hash_val})
            if not user:
                return False
            if len(user) > 1:
                logger.info(f"Hash Key collision - {hash_val}")
                return False

            matched_user = user[0]
            self.username = matched_user["Username"]
            return True
        return False

    def get_authorization_header(self, request):
        """Get auth header."""
        auth_header = request.headers.get("Authorization", "")
        return auth_header

    def get_jwt_token(self, request):
        """Get JWT Token."""
        auth = self.get_authorization_header(request).split()
        if not auth or str(auth[0].lower()) != "bearer":
            return None
        if len(auth) != 2:
            return None
        return auth[1]

    def check_cognito_jwt(self, request):
        """Check if valid cognito jwt."""
        jwt = self.get_jwt_token(request)
        if not jwt:
            return False
        try:
            resp = cognitojwt.decode(
                jwt,
                AWS_REGION,
                COGNTIO_USER_POOL_ID,
                app_client_id=COGNITO_CLIENT_ID,
            )
            self.username = resp["cognito:username"]
            self.groups = resp.get("cognito:groups", [])
            return self.username
        except Exception as e:
            if str(e) != "Token is expired":
                logger.exception(e)
            else:
                print("Token is expired")
            return False

    def check_admin_status(self):
        """Check if user is in the admin cognito group."""
        if not COGNTIO_ENABLED:
            return True
        if COGNITO_DEFAULT_ADMIN:
            return True
        if "admin" in self.groups:
            return True
        return False

def auth_required(view):
    """Authorize requests."""

    @wraps(view)
    def decorated(*args, **kwargs):
        """Decorate."""
        auth = RequestAuth(request)
        if auth.validate():
            g.username = auth.username
            if auth.check_admin_status():
                g.is_admin = True
            else:
                g.is_admin = False
            return view(*args, **kwargs)
        else:
            abort(HTTPStatus.UNAUTHORIZED.value)

    return decorated

def auth_admin_required(view):
    """Authorize requests."""

    @wraps(view)
    def decorated(*args, **kwargs):
        """Decorate."""
        if not g.is_admin:
            abort(HTTPStatus.FORBIDDEN.value)
        return view(*args, **kwargs)

    return decorated

def can_access_domain(view):
    """Check if user can access domain."""

    @wraps(view)
    def decorated(*args, **kwargs):
        """Decorate."""
        if kwargs.get("domain_id"):
            domain = domain_manager.get(
                document_id=kwargs.get("domain_id"), fields=["application_id"]
            )
            if user_can_access_domain(domain):
                return view(*args, **kwargs)
            else:
                return (
                    "User does not have permission to domain.",
                    HTTPStatus.BAD_REQUEST.value,
                )
        else:
            return (
                "URL Path configured improperly.",
                HTTPStatus.INTERNAL_SERVER_ERROR.value,
            )

    return decorated

def can_access_user(view):
    """Check if user can access user."""

    @wraps(view)
    def decorated(*args, **kwargs):
        """Decorate."""
        if kwargs.get("username"):
            if g.is_admin or g.username == kwargs.get("username"):
                return view(*args, **kwargs)
            else:
                return (
                    "User does not have permission to user.",
                    HTTPStatus.BAD_REQUEST.value,
                )
        else:
            return (
                "URL Path configured improperly.",
                HTTPStatus.INTERNAL_SERVER_ERROR.value,
            )

    return decorated


from pprint import PrettyPrinter
from requests import get
'''
Code gets the head coaches from all NBA teams from an API
'''
#better way to print json
printer = PrettyPrinter()

def get_links(second_key, base_url = 'https://data.nba.net/',
         all_json ='/prod/v1/today.json' , fist_key = 'links',):
    '''getting the URL for the desired info'''
    response = get(base_url + all_json).json()
    final_url = response[fist_key][second_key]
    return base_url + final_url

def get_info (final_url):
    '''getting info from the final json'''
    my_info = get(final_url).json()
    return my_info

def get_hc(my_info):
    '''print all HC'''
    my_coaches = my_info['league']['standard']
    for coach in my_coaches:
        if coach['isAssistant'] is False:
            print(f"{coach['teamSitesOnly']['displayName']} Team: {coach['teamSitesOnly']['teamCode']}")

coaches_url = get_links('leagueRosterCoaches')

coach_info = get_info(coaches_url)

get_hc(coach_info)
'''
HELPS TO VISUALIZE THE JSON DATA
printer.pprint(my_info['displayName'])
YOU CAN CALL TO SEE THE KEYS
printer.pprint(data.keys())
'''


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from datetime import datetime

from CaptainConsole.views import get_cart_items
from checkout.forms.shipping_info import ShippingForm, PaymentForm
from checkout.models import OrderInformation, OrderProduct
from products.models import Product
from user.models import Profile

@login_required
def checkout(request):
    request.session["confirmed"] = False
    profile = Profile.objects.filter(user=request.user).first()
    if request.method == "POST":
        form = ShippingForm(instance=profile, data=request.POST)
        if form.is_valid():
            profile = form.save(commit=False)
            profile.user = request.user
            profile.save()
            request.session["checkout_state"] = 1
            return redirect("payment_info")
    return render(request, 'checkout/checkout.html', {
        'form': ShippingForm(instance=profile)
    })

@login_required
def payment_info(request):
    request.session["confirmed"] = False
    if request.method == "POST":
        form = PaymentForm(data=request.POST)
        if form.is_valid():
            clean_data = form.cleaned_data
            clean_data["expiry_date"] = clean_data["expiry_date"].isoformat()
            request.session["card_info"] = clean_data

            return redirect("order_review")
    if "card_info" in request.session:
        card_data = dict(request.session["card_info"])
        card_data["expiry_date"] = datetime.fromisoformat(card_data["expiry_date"])
        form = PaymentForm(card_data)
    else:
        form = PaymentForm()
    request.session["checkout_state"] = 1
    return render(request, 'checkout/payment.html', {"form": form})

@login_required
def order_review(request):
    profile, card_info, cart = compile_order_info(request)
    request.session["confirmed"] = True
    return render(request, 'checkout/review.html', {"shipping": profile, "card_info": card_info, "cart": cart})

@login_required
def order_confirmation(request):
    profile, card_info, cart = compile_order_info(request)
    if "confirmed" not in request.session or not request.session["confirmed"]:
        return redirect("cart")
    del request.session["confirmed"]
    order = OrderInformation()
    order.postal_code = profile["postal_code"]
    order.city = profile["city"]
    order.country = profile["country"]
    order.name = profile["name"]
    order.address = profile["address"]
    order.order_total = cart["subtotal"]
    order.user = request.user
    order.save()
    cart_items = request.session["cart"]
    for product_id in cart_items:
        product_amount = cart_items[product_id]
        for i in range(product_amount):
            product = Product.objects.get(pk=product_id)
            ordered_product = OrderProduct()
            ordered_product.order = order
            ordered_product.product = product
            ordered_product.save()
    del request.session["cart"]
    del request.session["card_info"]
    return render(request, 'checkout/confirmation.html', {"shipping": profile, "card_info": card_info, "cart": cart})

def compile_order_info(request):
    profile = Profile.objects.filter(user=request.user) \
        .values("name", "address", "city", "country", "postal_code").first()
    card_info = request.session["card_info"]
    cart = get_cart_items(request.session)
    return profile, card_info, cart

def is_session_state_correct(session, state):
    if "checkout_state" not in session or session["checkout_state"] != state:
        return False
    return True


from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User, Group

# Register your models here.

from .models import UserProfile, Clinic, Note, CommunicationNote, ProcedureNote, InstructionNote, ResourceNote, SelfCareNote, Notebook, Attachment, NoteReply, Notification, AppointmentNote, ContactNote, MedicationNote, Address, Privacy

class UserProfileInline(admin.StackedInline):
	model = UserProfile
	can_delete = False
	verbose_name_plural = 'User Profile'
	fieldsets = (
		(None, {'fields': ('profile_picture', 'role', 'title', 'associates', 'phone_number',
			'medical_history', )}),
		('Address', {'fields':['address_unit', 'address_street', 'address_city', 'address_province', 'address_country', 'address_postal_code'],
		 'classes':['show']}),
	)

class UserAdmin(UserAdmin):
	inlines = (UserProfileInline, )
	fieldsets = (
		('Basic Information', {'fields': (('username', 'password'), ('first_name', 'last_name'), 'email',)}),
		('Date Information' , {'fields': ('last_login', 'date_joined',)}),
		('User Permissions', {'fields': ('is_active', 'is_staff', 'is_superuser',),})
	)

class RepliesInline(admin.TabularInline):
	model = NoteReply
	extra = 1

class AttachmentInline(admin.TabularInline):
	model = Attachment
	extra = 1

@admin.register(Note)
class NoteAdmin(admin.ModelAdmin):
	list_display = ('subject', 'author', 'date_created', 'date_accessed', 'note_type', )
	fieldsets = (
		(None, {'fields': ('subject', 'note_type', 'note_content', 'url', 'follow_up', 'done_status')}),
		('Users', {'fields':['author', 'editors', 'viewers'],'classes':['show']}),
	)
	inlines=[RepliesInline, AttachmentInline]
	list_filter = ['date_created']
	search_fields = ['subject', 'note_content']

@admin.register(Clinic)
class ClinicAdmin(admin.ModelAdmin):
	list_display = ('name', 'address', 'email', 'phone_number', )
	fieldsets = (
		(None, {'fields': ('name', 'address', 'email', 'phone_number', 'users',)}),
	)

@admin.register(Notebook)
class NotebookAdmin(admin.ModelAdmin):
	list_display = ('name', 'description', 'date_created', 'date_modified', 'date_accessed')
	fieldsets = (
		(None, {'fields': ('name', 'description', 'notes')}),
		('Users', {'fields':['editors', 'viewers'],'classes':['show']}),
	)

admin.site.unregister(User)
admin.site.unregister(Group)
admin.site.register(User, UserAdmin)
admin.site.register(CommunicationNote)
admin.site.register(ProcedureNote)
admin.site.register(InstructionNote)
admin.site.register(ResourceNote)
# admin.site.register(MedicalInformationNote)
admin.site.register(SelfCareNote)
# admin.site.register(Notebook)
admin.site.register(Attachment)
admin.site.register(NoteReply)
admin.site.register(Notification)
admin.site.register(UserProfile)
admin.site.register(AppointmentNote)
admin.site.register(ContactNote)
admin.site.register(MedicationNote)
admin.site.register(Address)
admin.site.register(Privacy)


#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

from django import forms
from mytask.models import Task , Comment , Type
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit
from crispy_forms.bootstrap import InlineCheckboxes, FormActions

class TaskForm(forms.ModelForm):
    LEVEL_CHOICES = (
        ('delay', 'delay'), ('normal', 'normal'),
        ('quick', 'quick'), ('now', 'now'),
    )

    PROGRESS_CHOICES = (
        ('0', '0'), ('10', '10'), ('20', '20'),
        ('30', '30'), ('40', '40'), ('50', '50'),
        ('60', '60'), ('70', '70'), ('80', '80'),
        ('90', '90'), ('100', '100'),
    )

    # name
    name = forms.CharField(
        label=u'任务名称',
        max_length=20 ,
        required=True ,
        error_messages={'required': u'请输入任务名称', 'error':u'任务名称输入太长，请精简'},
    )
    # 文章类型
    type = forms.ModelChoiceField(
        label=u'类型 ',
        empty_label=u'请选择文章类别' ,
        queryset=Type.objects.filter(state=True),
        to_field_name='id',
        required=True ,
        error_messages={'required': u'请选择任务类型'},
    )
    # 关键字
    level = forms.ChoiceField(
        label=u'任务级别',
        choices=LEVEL_CHOICES,
        error_messages={'required': u'请选择任务级别'},
        required=True,
    )
    progress = forms.ChoiceField(
        label=u'任务进度',
        choices=PROGRESS_CHOICES,
        error_messages={'required': u'请选择任务进度'},
        required=True,
    )
    is_fixed = forms.BooleanField(
        label=u'是否结束',
        required=False,
    )
    # 文章内容
    content = forms.Textarea()

    class Meta:
        model = Task
        fields = [ 'name' , 'type', 'level' , 'progress', 'content' , 'is_fixed' ]

    def __init__(self , *args, **kwargs):
        super(TaskForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-8'
        self.helper.layout = Layout(
            'name',
            'type' ,
            'level',
            'progress' ,
            'content',
            'is_fixed',
            FormActions(
                Submit('save', u'保存'),
            )
        )

class CommentForm(forms.ModelForm):
    content = forms.Textarea()

    class Meta:
        model = Comment
        fields = ['content' ]

    def __init__(self , *args, **kwargs):
        super(CommentForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-8'
        self.helper.layout = Layout(
            'content',
            FormActions(
                Submit('save', u'发布'),
            )
        )


import io

POPULATE_TEMPLATE = """
COPY public.\"{}\"({})
FROM '{}'
DELIMITER ','
CSV HEADER
ENCODING 'utf-8';\n\n
"""

def _write_lines_to_file(file_name_prefix, lines_per_file, line_generator, title_line=None, file_type="csv"):
    file_count = 1
    make_file_name = lambda: "/out/{}-{}.{}".format(file_name_prefix, file_count, file_type)
    file_names_written = []
    lines_to_write = []
    for line in line_generator():
        lines_to_write.append(line)
        if len(lines_to_write) == lines_per_file:
            file_name = make_file_name()
            _write_file(file_name, lines_to_write, title_line)
            file_names_written.append(file_name)
            file_count += 1
            lines_to_write = []
    if len(lines_to_write) > 0:
        file_name = make_file_name()
        _write_file(file_name, lines_to_write, title_line)
        file_names_written.append(file_name)
    return file_names_written

def _write_file(file_name, lines, title_line):
    with io.open(file_name, "w", encoding="utf-8") as f:
        if title_line is not None:
            f.write("{}\n".format(title_line))
        for line in lines:
            f.write("{}\n".format(line))
    print("wrote file {}".format(file_name))

class SQLTemplate:
    def __init__(self, table_name, column_headers):
        self.table_name = table_name
        self.column_headers = column_headers
        self._files = []

    def write_files_for_template(self, line_generator, max_lines):
        self._files = _write_lines_to_file(self.table_name, max_lines, line_generator, self.column_headers)

    def yield_sql_template(self):
        if len(self._files) == 0:
            raise Exception("Must have written files")
        for f in self._files:
            relative_file_name = "/home/postgres/wordsmith-data/{}".format(f.split("/")[-1])
            yield POPULATE_TEMPLATE.format(self.table_name, self.column_headers, relative_file_name)


'''
预测数据,多维单步，预测几步后的那一时刻输出
'''
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.metrics import *
from math import sqrt
import time
import csv
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm

from train import get_test_data,get_train_data,lstm

def read_cpu(filename):
    with open(filename, 'rb') as f:
        df = pd.read_csv(f,sep=';')  # 读入数据
        cpudata = df.values
        cpudata = cpudata[:, 1:]
        cpudata = cpudata.astype(np.float)
        x = np.sum(cpudata, axis=1)[:,np.newaxis]
        print('cpudata.shape:', cpudata.shape)
        print('x.shape:',x.shape)
        return x

#——————————————————训练模型—————————————————
def train_lstm(data,input_size,output_size,lr,train_time,rnn_unit,weights,biases,train_end,
               batch_size,time_step,kp,train_begin=0):
    X=tf.placeholder(tf.float32, shape=[None,time_step,input_size])
    Y=tf.placeholder(tf.float32, shape=[None,output_size])
    keep_prob = tf.placeholder('float')
    batch_index,train_x,train_y=get_train_data(data,batch_size,time_step,train_end,predict_step,train_begin)
    #train_y = np.array(train_y)[:, 1:input_size].tolist()  # 如果输入加上时间维度，这里就需要加上
    print(np.array(train_x).shape)
    print(np.array(train_y).shape)
    pred,_,m, mm=lstm(X,weights,biases,input_size,rnn_unit,keep_prob)
    #损失函数
    loss=tf.reduce_mean(tf.square(tf.reshape(pred, [-1, output_size]) - tf.reshape(Y, [-1, output_size])))
    train_op=tf.train.AdamOptimizer(lr).minimize(loss)

    saver = tf.train.Saver(max_to_keep=1)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        #saver.save(sess, './save/MyModel')
        ckpt = tf.train.get_checkpoint_state(save_model_path)  # checkpoint存在的目录
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)  # 自动恢复model_checkpoint_path保存模型一般是最新
            print("Model restored...")
        else:
            print('No Model')

        #重复训练
        for i in range(train_time):
            for step in range(len(batch_index)-1):
                _,loss_,M,MM=sess.run([train_op,loss,m,mm],feed_dict={X:train_x[batch_index[step]:batch_index[step+1]],Y:train_y[batch_index[step]:batch_index[step+1]],keep_prob: kp})
            print(i,loss_)
        saver.save(sess, save_model_path + save_model_name)#保存模型

        #预测
        maxvalue, mean, std, test_x, test_y = get_test_data(data,time_step,train_end - time_step-(predict_step-1),predict_step)
        #test_y = np.array(test_y)[:, 1:input_size].tolist() #如果输入加上时间维度，这里就需要加上
        test_predict = []
        for step in range(len(test_x)):
            prob = sess.run(pred, feed_dict={X: [test_x[step]],keep_prob: 1})
            predict = prob.reshape((-1))
            test_predict.extend(predict)

        print('test_predict:', np.array(test_predict).shape)
        print('truedata:', np.array(test_y).shape)
        test_predict = np.array(test_predict).reshape(-1, output_size)
        print('test_predict:', test_predict.shape)
        print('test_y:', np.array(test_y).shape)

        test_y = np.array(test_y)
        #for i in range(output_size):
            #test_y[:, i] = test_y[:, i] * std[i+1] + mean[i+1]
            #test_predict[:, i] = test_predict[:, i] * std[i+1] + mean[i+1]
        for i in range(output_size):
            test_y[:, i] = test_y[:, i] * maxvalue[i]
            test_predict[:, i] = test_predict[:, i] * maxvalue[i]

        # 预测误差
        predict_y = test_predict.reshape(-1, )
        true_y = test_y.reshape(-1, )
        mae = mean_absolute_error(test_predict, test_y)/1024
        mse = mean_squared_error(test_predict, test_y)
        rmse = sqrt(mse)
        r_square = r2_score(test_predict, test_y,multioutput="raw_values")
        r_square1 = r2_score(test_predict, test_y)
        print(('平均绝对误差(mae): %dKB' % mae),
              (mean_absolute_error(test_predict, test_y,multioutput="raw_values")/1024).astype(np.int))
        aa = test_y.sum()/(len(true_y)*1024)
        print('实际值的平均值：%dKB' % aa)
        print('误差百分比：%.2f' % (100*mae/aa)+'%')
        print('均方误差: %d' % mse)
        print('均方根误差: %d' % rmse)
        #print('R_square: %.6f' % r_square)
        print('r2_score:')#越接近1表示预测值和实际值越接近
        print(r_square)
        print(r_square1)

        # 画图表示结果
        fig = plt.figure()
        ax1 = fig.add_subplot(111)
        ax1.set_xlabel('time/min')
        ax1.set_ylabel('cpu usage')
        ax1.set_title('CPU')
        #ax2 = fig.add_subplot(122)
        #ax2.set_xlabel('time/min')
        #ax2.set_ylabel('area')
        #ax2.set_title('table2')
        x = list(range(len(data)))
        X = list(range(train_end, train_end + len(test_predict)))
        y = [[] for i in range(output_size)]
        Y = [[] for i in range(output_size)]
        for i in range(output_size):
            y[i] = data[:, i]
            Y[i] = test_predict[:, i]
        ax1.plot(x, y[0], color='b', label='real')
        ax1.plot(X, Y[0], color='r', label='predict')
        #ax2.plot(x, y[1], color='r', label='real')
        #ax2.plot(X, Y[1], color='b', label='predict')
        plt.legend()
        plt.show()

if __name__=="__main__":
    input_size = 1  # 输入维度
    output_size = 1  # 输出维度
    rnn_unit = 12  # 隐藏层节点
    lr = 0.0004  # 学习率
    batch_size = 30  # 每次训练的一个批次的大小
    time_step = 20  # 前time_step步来预测下一步
    predict_step = 5 #预测predict_step分钟后的负载
    kp = 1  # dropout保留节点的比例
    smooth = 0  # 为1则在时间维度上平滑数据
    train_time = 50  # 所有数据的训练轮次
    train_end = 1200  # 训练集截取到的位置140/190,320/386,4200/4537
    filename = 'csv_for_train/grafana_data_export_4-18_60.csv'
    save_model_path = './save/predict_cpu_4-18_60/'  # checkpoint存在的目录
    save_model_name = 'MyModel'    #saver.save(sess, './save/MyModel') 保存模型

    data = read_cpu(filename)

    if smooth == 1:  # 平滑数据
        newdata = data
        for i in range(2, len(data) - 2):
            for j in range(input_size * 2):
                newdata[i][j] = (data[i - 2][j] + data[i - 1][j] + data[i][j] + data[i + 1][j] + data[i + 2][j]) / 5
        data = newdata

    # ——————————————————定义神经网络变量——————————————————
    # 输入层、输出层权重、偏置

    # random_normal
    # random_uniform
    # truncated_normal
    init_orthogonal = tf.orthogonal_initializer(gain=1.0, seed=None, dtype=tf.float32)
    init_glorot_uniform = tf.glorot_uniform_initializer()
    init_glorot_normal = tf.glorot_normal_initializer()

    weights = {
        #'in': tf.get_variable('in', shape=[input_size, rnn_unit], initializer=init_glorot_normal),
        #'out': tf.get_variable('out', shape=[rnn_unit, output_size], initializer=init_glorot_normal)
        'in': tf.Variable(tf.random_uniform([input_size, rnn_unit])),#maxval=0.125
        'out': tf.Variable(tf.random_uniform([rnn_unit, output_size]))
    }

    biases = {
        'in': tf.Variable(tf.constant(0.1, shape=[rnn_unit, ])),
        'out': tf.Variable(tf.constant(0.1, shape=[output_size, ]))
    }
    t0 = time.time()
    train_lstm(data,input_size,output_size,lr,train_time,rnn_unit,weights,biases,train_end,batch_size,time_step,kp)
    t1 = time.time()
    print("时间:%.4fs" % (t1 - t0))


"""
The :mod:`websockets.client` module defines a simple WebSocket client API.

"""

import asyncio
import collections.abc

from .exceptions import InvalidHandshake, InvalidMessage, InvalidStatusCode
from .handshake import build_request, check_response
from .http import USER_AGENT, build_headers, read_response
from .protocol import CONNECTING, OPEN, WebSocketCommonProtocol
from .uri import parse_uri

__all__ = ['connect', 'WebSocketClientProtocol']

class WebSocketClientProtocol(WebSocketCommonProtocol):
    """
    Complete WebSocket client implementation as an :class:`asyncio.Protocol`.

    This class inherits most of its methods from
    :class:`~websockets.protocol.WebSocketCommonProtocol`.

    """
    is_client = True
    state = CONNECTING

    @asyncio.coroutine
    def write_http_request(self, path, headers):
        """
        Write request line and headers to the HTTP request.

        """
        self.path = path
        self.request_headers = build_headers(headers)
        self.raw_request_headers = headers

        # Since the path and headers only contain ASCII characters,
        # we can keep this simple.
        request = ['GET {path} HTTP/1.1'.format(path=path)]
        request.extend('{}: {}'.format(k, v) for k, v in headers)
        request.append('\r\n')
        request = '\r\n'.join(request).encode()

        self.writer.write(request)

    @asyncio.coroutine
    def read_http_response(self):
        """
        Read status line and headers from the HTTP response.

        Raise :exc:`~websockets.exceptions.InvalidMessage` if the HTTP message
        is malformed or isn't an HTTP/1.1 GET request.

        Don't attempt to read the response body because WebSocket handshake
        responses don't have one. If the response contains a body, it may be
        read from ``self.reader`` after this coroutine returns.

        """
        try:
            status_code, headers = yield from read_response(self.reader)
        except ValueError as exc:
            raise InvalidMessage("Malformed HTTP message") from exc

        self.response_headers = build_headers(headers)
        self.raw_response_headers = headers

        return status_code, self.response_headers

    def process_subprotocol(self, get_header, subprotocols=None):
        """
        Handle the Sec-WebSocket-Protocol HTTP header.

        """
        subprotocol = get_header('Sec-WebSocket-Protocol')
        if subprotocol:
            if subprotocols is None or subprotocol not in subprotocols:
                raise InvalidHandshake(
                    "Unknown subprotocol: {}".format(subprotocol))
            return subprotocol

    @asyncio.coroutine
    def handshake(self, wsuri,
                  origin=None, subprotocols=None, extra_headers=None):
        """
        Perform the client side of the opening handshake.

        If provided, ``origin`` sets the Origin HTTP header.

        If provided, ``subprotocols`` is a list of supported subprotocols in
        order of decreasing preference.

        If provided, ``extra_headers`` sets additional HTTP request headers.
        It must be a mapping or an iterable of (name, value) pairs.

        """
        headers = []
        set_header = lambda k, v: headers.append((k, v))

        if wsuri.port == (443 if wsuri.secure else 80):     # pragma: no cover
            set_header('Host', wsuri.host)
        else:
            set_header('Host', '{}:{}'.format(wsuri.host, wsuri.port))
        if origin is not None:
            set_header('Origin', origin)
        if subprotocols is not None:
            set_header('Sec-WebSocket-Protocol', ', '.join(subprotocols))
        if extra_headers is not None:
            if isinstance(extra_headers, collections.abc.Mapping):
                extra_headers = extra_headers.items()
            for name, value in extra_headers:
                set_header(name, value)
        set_header('User-Agent', USER_AGENT)

        key = build_request(set_header)

        yield from self.write_http_request(wsuri.resource_name, headers)

        status_code, headers = yield from self.read_http_response()
        get_header = lambda k: headers.get(k, '')

        if status_code != 101:
            raise InvalidStatusCode(status_code)

        check_response(get_header, key)

        self.subprotocol = self.process_subprotocol(get_header, subprotocols)

        assert self.state == CONNECTING
        self.state = OPEN
        self.opening_handshake.set_result(True)

@asyncio.coroutine
def connect(uri, *,
            create_protocol=None,
            timeout=10, max_size=2 ** 20, max_queue=2 ** 5,
            read_limit=2 ** 16, write_limit=2 ** 16,
            loop=None, legacy_recv=False, klass=None,
            origin=None, subprotocols=None, extra_headers=None,
            **kwds):
    """
    This coroutine connects to a WebSocket server at a given ``uri``.

    It yields a :class:`WebSocketClientProtocol` which can then be used to
    send and receive messages.

    :func:`connect` is a wrapper around the event loop's
    :meth:`~asyncio.BaseEventLoop.create_connection` method. Unknown keyword
    arguments are passed to :meth:`~asyncio.BaseEventLoop.create_connection`.

    For example, you can set the ``ssl`` keyword argument to a
    :class:`~ssl.SSLContext` to enforce some TLS settings. When connecting to
    a ``wss://`` URI, if this argument isn't provided explicitly, it's set to
    ``True``, which means Python's default :class:`~ssl.SSLContext` is used.

    The behavior of the ``timeout``, ``max_size``, and ``max_queue``,
    ``read_limit``, and ``write_limit`` optional arguments is described in the
    documentation of :class:`~websockets.protocol.WebSocketCommonProtocol`.

    The ``create_protocol`` parameter allows customizing the asyncio protocol
    that manages the connection. It should be a callable or class accepting
    the same arguments as :class:`WebSocketClientProtocol` and returning a
    :class:`WebSocketClientProtocol` instance. It defaults to
    :class:`WebSocketClientProtocol`.

    :func:`connect` also accepts the following optional arguments:

    * ``origin`` sets the Origin HTTP header
    * ``subprotocols`` is a list of supported subprotocols in order of
      decreasing preference
    * ``extra_headers`` sets additional HTTP request headers – it can be a
      mapping or an iterable of (name, value) pairs

    :func:`connect` raises :exc:`~websockets.uri.InvalidURI` if ``uri`` is
    invalid and :exc:`~websockets.handshake.InvalidHandshake` if the opening
    handshake fails.

    On Python 3.5, :func:`connect` can be used as a asynchronous context
    manager. In that case, the connection is closed when exiting the context.

    """
    if loop is None:
        loop = asyncio.get_event_loop()

    # Backwards-compatibility: create_protocol used to be called klass.
    # In the unlikely event that both are specified, klass is ignored.
    if create_protocol is None:
        create_protocol = klass

    if create_protocol is None:
        create_protocol = WebSocketClientProtocol

    wsuri = parse_uri(uri)
    if wsuri.secure:
        kwds.setdefault('ssl', True)
    elif kwds.get('ssl') is not None:
        raise ValueError("connect() received a SSL context for a ws:// URI. "
                         "Use a wss:// URI to enable TLS.")
    factory = lambda: create_protocol(
        host=wsuri.host, port=wsuri.port, secure=wsuri.secure,
        timeout=timeout, max_size=max_size, max_queue=max_queue,
        read_limit=read_limit, write_limit=write_limit,
        loop=loop, legacy_recv=legacy_recv,
    )

    transport, protocol = yield from loop.create_connection(
        factory, wsuri.host, wsuri.port, **kwds)

    try:
        yield from protocol.handshake(
            wsuri, origin=origin, subprotocols=subprotocols,
            extra_headers=extra_headers)
    except Exception:
        yield from protocol.close_connection(force=True)
        raise

    return protocol

try:
    from .py35.client import Connect
except (SyntaxError, ImportError):                          # pragma: no cover
    pass
else:
    Connect.__wrapped__ = connect
    # Copy over docstring to support building documentation on Python 3.5.
    Connect.__doc__ = connect.__doc__
    connect = Connect


import sys
import csv
import numpy as np
import scipy.stats as sp
import math
import os
from collections import defaultdict

def mean_confidence_interval(data, confidence):
	a = 1.0*np.array([float(data0) for data0 in data])
	n = len(a)
	mean, std = np.mean(a), np.std(a,ddof=1)
	h = sp.norm.interval(confidence,mean,std/math.sqrt(n))
	return mean, h[1]-mean

path_topology_data = sys.argv[1]
numberOfIterations = int(sys.argv[2])

#routingProtocols = ["BMRF_BROADCAST" , "BMRF_UNICAST" ]
routingProtocols = ["UNICAST"]
topologies = ['sim-3']

dirpath = os.getcwd()
foldername = os.path.basename(dirpath)

#for each property we create a defaultdict to save the data

for topology in topologies:
    csv_header = ["hops"]
    #for each topology we have to open the specfic folder and correct files to gather the data and save them in de defaultdict
    path = foldername + "/" + topology
    allData = defaultdict(list) #this defaultdict wil save all the calculated data

    allDelay = defaultdict(list)
    allPDR = defaultdict(list)

    for routing_pr in routingProtocols:
        #csv_header.append("Delay_" + routing_pr)
        #csv_header.append("CI")
        #csv_header.append("PDR_" + routing_pr)
        #csv_header.append("CI")
        #for each routing protocol we create defaultdicts
        delay_rpr = defaultdict(list)
        pdr_rpr = defaultdict(list)

        for i in range(1,numberOfIterations +1):
            #calculate the mean and ci of all the iterations per node
            delays = defaultdict(list)
            pdr = defaultdict(list)

            print("Calculating PDRs for : " +routing_pr + "-" + topology + "-iteration-" + str(i))

            with open(topology + "/" + "pdr_results_" + routing_pr + "_" + str(i) + ".csv" ,'rU') as fdel:
                reader = csv.reader(fdel , delimiter =";" , dialect= csv.excel_tab)
                #save the data at correct node
                for row in reader:
                    pdr[int(row[2])].append(int(row[1]))

                    #we calculate the confidence interval for each key pair
                    for p in pdr:
                        mean, ci = mean_confidence_interval(pdr[p],0.95)
                        print("mean = " + str(mean) + "   confidence interval = " + str(ci))
                        pdr_rpr[p].append(mean)

            print("Calculating delays for : " +routing_pr + "-" + topology + "-iteration-" + str(i))
            with open(topology + "/" + "delay_results_" + routing_pr + "_" + str(i) + ".csv" ,'rU') as fdel:
                reader = csv.reader(fdel , delimiter =";" , dialect= csv.excel_tab)
                #save the data at correct node
                for row in reader:
                    delays[int(row[3])].append(int(row[2]))

                    #we calculate the confidence interval for each key pair
                    for d in delays:
                        mean, ci = mean_confidence_interval(delays[d],0.95)
                        print("mean = " + str(mean) + "   confidence interval = " + str(ci))
                        delay_rpr[d].append(mean)

            print(delay_rpr)
            print(pdr_rpr)

        allPDR[routing_pr] = pdr_rpr
        allDelay[routing_pr] = delay_rpr

    allData['Delay'] = allDelay
    allData['PDR'] = allPDR

    #all this data needs to be saved to csv in differect files (for each mac protocol a different file)
    #see how many nodes are used in each topology

    nodes_numbers = set([])
    for properties , routing_protocols in allData.items():
        for routing_protocol , nodes in routing_protocols.items():
            for node , data in nodes.items():
                nodes_numbers.add(node)
    print(nodes_numbers)

    nodes_numbers = list(nodes_numbers)
    nodes_numbers = np.array(nodes_numbers)

    totaldata = nodes_numbers.T

    for data_name , methods in allData.items():
        #print("Data_name = " + data_name)
        data_nodes = []
        data_nodes_np = np.array(data_nodes)
        for method, nodes in methods.items():
            print("Method = "+ data_name + "_" + method)
            title = data_name + "_" + method
            csv_header.append(title)
            csv_header.append("CI")
            all_nodes = []
            for node , data in nodes.items():
                mean, ci = mean_confidence_interval(data, 0.95)
                row = [mean , ci]
                all_nodes.append(row)

            all_nodes_np = np.array(all_nodes)
            totaldata = np.column_stack((totaldata, all_nodes_np))

    filename = "results_" + topology + "-"+ ".csv"
    np.savetxt(filename, totaldata, delimiter="," , header=str(csv_header), comments='')


# !python3
# -*- coding: utf-8 -*-

"""
Create on 2018 08 29 10:29
@author: wuyu
"""

import os
import cv2
import numpy as np
from matplotlib import pyplot as plt

def test_opencv_1():
    img_path = "D:/project/py_test/test_open_cv_image/rose_1.jpg"

    # 彩色图像 cv2.IMREAD_COLOR
    # img = cv2.imread(img_path, cv2.IMREAD_COLOR)

    # 灰度图 cv2.IMREAD_GRAYSCALE
    # img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)

    # 带alpha通道  cv2.IMREAD_UNCHANGED     测试和cv2.IMREAD_COLOR效果相同
    img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)

    # print(img)

    if img is not None:
        # cv2.WINDOW_NORMAL 可调整图框尺寸   默认: cv2.WINDOW_AUTOSIZE
        cv2.namedWindow('image', cv2.WINDOW_NORMAL)
        cv2.imshow("image", img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()
    else:
        print("img is None, img_path={}, exists={}.".format(img_path, os.path.exists(img_path)))

def test_2():
    img_path = "D:/project/py_test/test_open_cv_image/rose_1.jpg"
    img = cv2.imread(img_path, 0)
    cv2.imshow('image', img)
    # k = cv2.waitKey(0)
    # 64位机器
    k = cv2.waitKey(0) & 0xff

    # wait for ESC key to exit
    if k == 27:
        cv2.destroyAllWindows()
    # wait for 's' key to save and exit
    elif k == ord('s'):
        cv2.imwrite('D:/project/py_test/test_open_cv_image/save_rose_1.jpg', img)
        cv2.destroyAllWindows()

def test_3():
    img_path = "D:/project/py_test/test_open_cv_image/rose_1.jpg"
    img = cv2.imread(img_path, 0)
    plt.imshow(img, cmap='gray', interpolation='bicubic')
    plt.xticks([]), plt.yticks([])
    plt.show()

def test_4():
    """
    使用 matplotlib 正确显示彩色图片  BGR -> RGB
    """

    img_path = "D:/project/py_test/test_open_cv_image/rose_1.jpg"
    img = cv2.imread(img_path, 1)
    # print(img)
    # b, g, r = cv2.split(img)
    # img2 = cv2.merge([r, g, b])
    img2 = img[:, :, ::-1]
    plt.subplot(121)
    plt.imshow(img)  # expects distorted color
    plt.subplot(122)
    plt.imshow(img2)  # expect true color
    plt.show()

    cv2.imshow('bgr image', img)  # expects true color
    cv2.imshow('rgb image', img2)  # expects distorted color
    cv2.waitKey(0)
    cv2.destroyAllWindows()

def test_video():
    # path = "rtsp://admin:1234qwer@192.168.16.202/h264/ch1/main"
    # path = 0
    path = "output.avi"

    cap = cv2.VideoCapture(path)
    print("cap.isOpened() = ", cap.isOpened())
    if not cap.isOpened():
        """检查摄像头是否成功初始化"""
        cap.open()

    for i in range(18):
        print("cap.get({}) = {}".format(i, cap.get(i)))

    # not use ???
    cap.set(3, 640)
    cap.set(4, 480)
    print("cap.get(3) = {}, cap.get(4) = {}".format(cap.get(3), cap.get(4)))

    while True:
        try:
            ret, frame = cap.read()
        except Exception as e:
            print(str(e))
        if ret:
            cv2.namedWindow('gray Video', cv2.WINDOW_NORMAL)
            cv2.namedWindow('Oto Video', cv2.WINDOW_NORMAL)
            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            cv2.imshow("gray Video", gray)
            cv2.imshow("Oto Video", frame)
        else:
            print("camera catch image error!")
            break
        key = cv2.waitKey(1) & 0xFF
        if key == ord("q"):
            break
    cap.release()
    cv2.destroyAllWindows()

def test_5():
    path = "rtsp://admin:1234qwer@192.168.16.202/h264/ch1/main"
    cap = cv2.VideoCapture(path)

    print(cap.get(3), cap.get(4))

    # Define the codec and create VideoWriter object
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    out = cv2.VideoWriter('output.avi', fourcc, 20.0, (1920, 1080), True)

    while cap.isOpened():
        ret, frame = cap.read()

        if ret:
            # > 0 : L/R turn    0:Top/down turn    <0:L/R and Top/down turn
            frame = cv2.flip(frame, -1)
            # write the flipped frame
            out.write(frame)
            cv2.imshow('frame', frame)

            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        else:
            break

    # Release everything if job is finished
    cap.release()
    out.release()
    cv2.destroyAllWindows()

def test_draw():
    img = np.zeros((512, 512, 3), np.uint8)
    cv2.line(img, (0, 0), (511, 511), (255, 0, 0), 5)
    cv2.rectangle(img, (384, 0), (510, 128), (0, 255, 0), 3)
    cv2.circle(img, (447, 63), 63, (0, 0, 255), -1, cv2.LINE_AA)
    cv2.ellipse(img, (256, 256), (100, 50), 0, 0, 180, (125, 125, 125), -1, cv2.LINE_AA)

    pts = np.array([[10, 5], [20, 30], [70, 20], [50, 10]], np.int32)
    pts = pts.reshape((-1, 1, 2))
    cv2.polylines(img, [pts], True, (0, 0, 255), 3, lineType=cv2.LINE_AA)

    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(img, 'OpenCV', (10, 450), font, 4, (255, 255, 255), 2, lineType=cv2.LINE_AA, bottomLeftOrigin=False)

    winname = "test_draw"
    cv2.namedWindow(winname, cv2.WINDOW_NORMAL)
    cv2.imshow(winname, img)

    k = cv2.waitKey(0) & 0xff

    if ord('q') == k:
        cv2.destroyAllWindows()

def test_mouse():
    # events = [i for i in dir(cv2) if 'EVENT' in i]
    # print(events)
    """['EVENT_FLAG_ALTKEY', 'EVENT_FLAG_CTRLKEY', 'EVENT_FLAG_LBUTTON', 'EVENT_FLAG_MBUTTON', 'EVENT_FLAG_RBUTTON',
     'EVENT_FLAG_SHIFTKEY', 'EVENT_LBUTTONDBLCLK', 'EVENT_LBUTTONDOWN', 'EVENT_LBUTTONUP', 'EVENT_MBUTTONDBLCLK',
     'EVENT_MBUTTONDOWN', 'EVENT_MBUTTONUP', 'EVENT_MOUSEHWHEEL', 'EVENT_MOUSEMOVE', 'EVENT_MOUSEWHEEL',
     'EVENT_RBUTTONDBLCLK', 'EVENT_RBUTTONDOWN', 'EVENT_RBUTTONUP']"""

    drawing = False
    mode = True
    ix, iy = -1, -1

    def draw_circle(event, x, y, flags, param):
        nonlocal drawing, mode, ix, iy

        if event == cv2.EVENT_LBUTTONDBLCLK:
            cv2.circle(img, (x, y), 20, (255, 0, 0), 3)

        if event == cv2.EVENT_LBUTTONDOWN:
            drawing = True
            ix, iy = x, y
        # 当鼠标左键按下并移动是绘制图形。 event可以查看移动， flag查看是否按下
        elif event == cv2.EVENT_MOUSEMOVE and flags == cv2.EVENT_FLAG_LBUTTON:
            if drawing is True:
                if mode is True:
                    cv2.rectangle(img, (ix, iy), (x, y), (0, 255, 0), -1)
                else:
                    # 绘制圆圈，小圆点连在一起就成了线， 3 代表了笔画的粗细
                    cv2.circle(img, (x, y), 3, (0, 0, 255), -1)
                    # 下面注释掉的代码是起始点为圆心，起点到终点为半径的
                    #  r=int(np.sqrt((x-ix)**2+(y-iy)**2))
                    #  cv2.circle(img,(x,y),r,(0,0,255),-1)
        # 当鼠标松开停止绘画。
        elif event == cv2.EVENT_LBUTTONUP:
            drawing = False
            mode = not mode
            # if mode==True:
            #     cv2.rectangle(img,(ix,iy),(x,y),(0,255,0),-1)
            # else:
            #     cv2.circle(img,(x,y),5,(0,0,255),-1)

    # 创建图像与窗口并将窗口与回调函数绑定
    img = np.zeros((512, 512, 3), np.uint8)
    cv2.namedWindow('image')
    cv2.setMouseCallback('image', draw_circle)

    while True:
        cv2.imshow('image', img)

        if cv2.waitKey(20) & 0xFF == ord('q'):
            break

    cv2.destroyAllWindows()

def test_progressbar():
    def nothing(x):
        pass

    # 创建一副黑色图像
    img = np.zeros((300, 512, 3), np.uint8)
    cv2.namedWindow('image')
    cv2.createTrackbar('R', 'image', 0, 255, nothing)
    cv2.createTrackbar('G', 'image', 0, 255, nothing)
    cv2.createTrackbar('B', 'image', 0, 255, nothing)

    switch = "0:OFF\n1:ON"
    cv2.createTrackbar(switch, 'image', 0, 1, nothing)

    while True:
        cv2.imshow('image', img)
        k = cv2.waitKey(1) & 0xFF

        if k == 27:
            break

        r = cv2.getTrackbarPos('R', 'image')
        g = cv2.getTrackbarPos('G', 'image')
        b = cv2.getTrackbarPos('B', 'image')
        s = cv2.getTrackbarPos(switch, 'image')

        if s == 0:
            img[:] = 0
        else:
            img[:] = [b, g, r]

    cv2.destroyAllWindows()

def test_6():
    img = cv2.imread("D:/project/py_test/test_open_cv_image/rose_1.jpg")
    bgr_100_100 = img[100, 100]
    b_100_100 = img[100, 100, 0]
    # print(img, type(img))
    print(bgr_100_100)     # (100, 100)像素点的bgr值
    print(b_100_100)       # (100, 100)像素点的blue值

    # 修改像素点的值
    # img[100, 100] = [255, 255, 255]
    # print(">>>", img[100, 100])
    print(img.item(100, 100, 0))
    img.itemset((100, 100, 0), 100)
    print(img.item(100, 100, 0))

    print("(高度，宽度，通道数) = ", img.shape)
    print("像素数：", img.size, 768*1366*3)
    print("数据类型: ", img.dtype)
    log = img[50:140, 1120:1300]
    cv2.imshow('log', log)

    img[300:390, 1120:1300] = log
    cv2.imshow('image', img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    # cv2.split()是一个比较耗时的操作，尽量不要用
    # b, g , r = cv2.split(img)
    b, g, r = img[:, :, 0], img[:, :, 1], img[:, :, 2]
    print(b, g, r)

def test_border():
    """给图片添加边框"""

    BLUE = [0, 0, 255]
    img = cv2.imread('../test_open_cv_image/rose_3.jpg')
    img2 = img[:, :, ::-1]

    replicate = cv2.copyMakeBorder(img2, 50, 50, 50, 50, cv2.BORDER_REPLICATE)
    reflect = cv2.copyMakeBorder(img2, 50, 50, 50, 50, cv2.BORDER_REFLECT)
    reflect101 = cv2.copyMakeBorder(img2, 50, 50, 50, 50, cv2.BORDER_REFLECT_101)
    wrap = cv2.copyMakeBorder(img2, 50, 50, 50, 50, cv2.BORDER_WRAP)
    constant = cv2.copyMakeBorder(img2, 50, 50, 50, 50, cv2.BORDER_CONSTANT, value=BLUE)
    plt.subplot(231), plt.imshow(img2, 'gray'), plt.title('ORIGINAL')
    plt.subplot(232), plt.imshow(replicate, 'gray'), plt.title('REPLICATE')
    plt.subplot(233), plt.imshow(reflect, 'gray'), plt.title('REFLECT')
    plt.subplot(234), plt.imshow(reflect101, 'gray'), plt.title('REFLECT_101')
    plt.subplot(235), plt.imshow(wrap, 'gray'), plt.title('WRAP')
    plt.subplot(236), plt.imshow(constant, 'gray'), plt.title('CONSTANT')
    plt.show()

def test_cv_add():
    """openCvd的加法是一种饱和操作，Numpy的加法是一种模操作"""
    x = np.uint8([250])       # 250+10 = 260 => 255         [[255]]
    y = np.uint8([10])        # 250+10 = 260 % 256 = 4      [4]
    print("cv2.add(x, y) = {}, x + y = {}".format(cv2.add(x, y), x + y))

    img1 = cv2.imread('../test_open_cv_image/rose_1920_1080_1.jpg')
    img2 = cv2.imread('../test_open_cv_image/rose_1920_1080_2.jpg')

    # 图片大小、类型必须一致
    dst = cv2.addWeighted(img1, 0.7, img2, 0.3, 0)
    cv2.imshow('dst', dst)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == "__main__":
    # test_opencv_1()
    # test_2()
    # test_3()
    # test_4()
    # test_video()
    # test_5()
    # test_draw()
    # test_mouse()
    # test_progressbar()
    # test_6()
    # test_border()
    test_cv_add()


import os

import pytest

from renku.cli import cli
from renku.core.management.repository import DEFAULT_DATA_DIR as DATA_DIR
from tests.utils import format_result_exception, modified_environ

@pytest.mark.parametrize("revision", [None, "HEAD", "HEAD^^", "HEAD^^..HEAD"])
@pytest.mark.parametrize("format", ["json-ld", "rdf", "nt"])
def test_graph_export_validation(runner, client, directory_tree, run, revision, format):
    """Test graph validation when exporting."""
    assert 0 == runner.invoke(cli, ["dataset", "add", "-c", "my-data", str(directory_tree)]).exit_code

    file1 = client.path / DATA_DIR / "my-data" / directory_tree.name / "file1"
    file2 = client.path / DATA_DIR / "my-data" / directory_tree.name / "dir1" / "file2"
    assert 0 == run(["run", "head", str(file1)], stdout="out1")
    assert 0 == run(["run", "tail", str(file2)], stdout="out2")

    result = runner.invoke(cli, ["graph", "export", "--format", format, "--strict", "--revision", revision])

    assert 0 == result.exit_code, format_result_exception(result)

    assert "https://localhost" in result.output
    assert "https://renkulab.io" not in result.output

    with modified_environ(RENKU_DOMAIN="https://renkulab.io"):
        result = runner.invoke(cli, ["graph", "export", "--format", format, "--strict", "--revision", revision])

        assert 0 == result.exit_code, format_result_exception(result)

        assert "https://localhost" not in result.output
        assert "https://renkulab.io" in result.output

@pytest.mark.serial
@pytest.mark.shelled
@pytest.mark.parametrize("format", ["json-ld", "nt", "rdf"])
def test_graph_export_strict_run(runner, project, run_shell, format):
    """Test graph export output of run command."""
    # Run a shell command with pipe.
    result = run_shell('renku run echo "my input string" > my_output_file')

    # Assert created output file.
    result = runner.invoke(cli, ["graph", "export", "--strict", "--format={}".format(format)])
    assert 0 == result.exit_code, format_result_exception(result)
    assert "my_output_file" in result.output
    assert "my input string" in result.output

@pytest.mark.parametrize("format", ["json-ld", "nt", "rdf"])
def test_graph_export_strict_dataset(tmpdir, runner, project, client, format, subdirectory):
    """Test output of graph export for dataset add."""
    result = runner.invoke(cli, ["dataset", "create", "my-dataset"])
    assert 0 == result.exit_code, format_result_exception(result)
    paths = []
    test_paths = []
    for i in range(3):
        new_file = tmpdir.join("file_{0}".format(i))
        new_file.write(str(i))
        paths.append(str(new_file))
        test_paths.append(os.path.relpath(str(new_file), str(project)))

    # add data
    result = runner.invoke(cli, ["dataset", "add", "my-dataset"] + paths)
    assert 0 == result.exit_code, format_result_exception(result)

    result = runner.invoke(cli, ["graph", "export", "--strict", f"--format={format}"])
    assert 0 == result.exit_code, format_result_exception(result)
    assert all(p in result.output for p in test_paths), result.output


import os
import logging
from dotenv import load_dotenv
from ingestion.logconfig import c_handler
# from ingestion.logconfig import f_handler

# Setup Logger
logger = logging.getLogger(__name__)
logger.setLevel('DEBUG')
logger.addHandler(c_handler)
# logger.addHandler(f_handler)

class BatchIngestor:
    '''
    Represents a base class for ingestor.

    Params:
        bearer_token = [str] Twitter's BearerToken.
        query_params = [dict] Query and filds to request the API, please see Twitter's API Documentation for more info.
        n_requests = [int] Number of hits at the API to pull tweets.
        api_cls = [subclass of TweetOauth2API] API Object to pull tweets.
        writers_cls = [class FileWriter or list of FileWriter] Class responsible for writing data permanently, more than one writer can be used.
        folder_name = [str] Optional. Folder name, usually API's name.
    '''

    def __init__(self, query_params, n_requests, api_cls, folder_name, writers_cls):
        if os.getenv('ENVIRONMENT') == "local":
            load_dotenv()
        self.api_cls = api_cls
        self.bearer_token = os.environ.get("BEARER_TOKEN")
        self.query_params = query_params
        self.n_requests = n_requests
        self.folder_name = folder_name
        if not isinstance(writers_cls, list):
            self.writers_cls = [writers_cls]
        else:
            self.writers_cls = writers_cls
            logger.debug("List of writers ",self.writers_cls)

    def _get_data(self):
        return self.api_cls(self.bearer_token).get_tweets(self.query_params, self.n_requests)

    def ingest(self):
        data = self._get_data()
        for writer in self.writers_cls:
            writer(self.folder_name).write_file(data)


from app.models import db, Topping, environment, SCHEMA
from sqlalchemy.sql import text

def seed_toppings(all_customizations):

    topping1 = Topping(
        customization_id=1,
        caramelCrunchTopping='light'
    )

    topping2 = Topping(
        customization_id=2,
        cookieCrumbleTopping='light'
    )

    topping3 = Topping(
        customization_id=3,
        caramelDrizzle='extra'
    )

    topping4 = Topping(
        customization_id=4,
        whippedCream='regular'
    )

    all_toppings = [topping1, topping2, topping3, topping4]
    add_toppings = [db.session.add(topping)
                      for topping in all_toppings]
    db.session.commit()

    return all_toppings

def undo_toppings():
    if environment == "production":
        db.session.execute(
            f"TRUNCATE table {SCHEMA}.toppings RESTART IDENTITY CASCADE;")
    else:
        db.session.execute(text("DELETE FROM toppings"))

    db.session.commit()


from bs4 import BeautifulSoup  # type: ignore
import requests
import urllib.parse
import pywikibot  # type: ignore
import pywikibot.pagegenerators  # type: ignore
import mwparserfromhell as mwph  # type: ignore
import json
import os
import toolforge
import argparse
import time
import re
import acnutils as utils

from mwparserfromhell.wikicode import Wikicode  # type: ignore
from bs4.element import Tag  # type: ignore
from pywikibot.page import BasePage  # type: ignore
from typing import Dict, List, Set, Any, Optional, Tuple

__version__ = "0.7"

_conf_dir = os.path.realpath(os.path.dirname(__file__) + "/..")

logger = utils.getInitLogger(
    "harvcheck" if __name__ == "__main__" else __name__,
    level="VERBOSE",
    filename="harvcheck.log",
)

config = utils.load_config("harvcheck", __file__)
simulate = config.get("simulate", True)

session = requests.Session()
session.headers.update(
    {
        "User-Agent": config.get(
            "useragent", "harvcheck" + toolforge.set_user_agent(config["tool"])
        )
    }
)
# check if on toolforge
try:
    f = open("/etc/wmcs-project")
except FileNotFoundError:
    wmcs = False
else:
    wmcs = True
    f.close()

site = pywikibot.Site("en", "wikipedia")
last_edit = float()

def get_html(title: str, revision: str = "") -> Tuple[str, str]:
    """Get Parsoid HTML for a page (and optional revision)"""
    url = "https://en.wikipedia.org/api/rest_v1/page/html/" + "/".join(
        urllib.parse.quote(i.replace(" ", "_"), safe="") for i in (title, revision) if i
    )
    resp = session.get(url)
    raw_html = resp.text
    etag = resp.headers["ETag"]
    return raw_html, etag

def parse_citeref_ids(soup: BeautifulSoup) -> Set[str]:
    """Parse Parsoid HTML and return html ids for citations"""
    ids = set()
    for element in soup.find_all(class_="citation"):
        el_id = element.get("id")
        if el_id and el_id.startswith("CITEREF"):
            ids.add(el_id)

    return ids

def parse_citeref_links(title: str, soup: BeautifulSoup) -> Dict[str, List[Tag]]:
    """Parse Parsoid HTML for same-page CITEREF links
    Returns a dict of fragments and HTML a tags
    """
    # The inline footnote boxes have "cite_ref" ids and link to "cite_note" ids.
    # The reference list items have "cite_note" ids and link to "cite_ref" ids.
    # sfn inline boxes link to cite_note-FOOTNOTE ids
    # Harvard citations link to CITEREF ids
    # ref=harv references have CITEREF ids

    links: Dict[str, List[Tag]] = {}
    for link in soup.find_all("a"):
        link_page, sep, fragment = link.get("href", "").partition("#")
        if link_page.endswith(title) and sep and fragment.startswith("CITEREF"):
            links.setdefault(fragment, []).append(link)

    return links

def parse_refs(title: str, soup: BeautifulSoup) -> Dict[str, List[Tag]]:
    """Parse Parsoid HTML for reflist citations
    Returns a dict of fragments and HTML citations
    """
    refs: Dict[str, List[Tag]] = {}
    for ref in soup.find_all(class_="mw-ref"):
        link_page, sep, fragment = ref.find("a").get("href", "").partition("#")
        if link_page.endswith(title) and sep and fragment.startswith("cite_note"):
            refs.setdefault(fragment, []).append(ref)

    return refs

def find_mismatch(ids: Set[str], links: Dict[str, Any]) -> Dict[str, Any]:
    """Returns dict of links where links are all in ids"""
    return {key: value for key, value in links.items() if key not in ids}

def find_note_id(element: Tag) -> str:
    """Finds the HTML ID for a citation"""
    note_id = element.parent.parent.get("id", "")
    return note_id

def find_ref_for_note(note: Tag, page_refs: Dict[str, Any]) -> Any:
    """Finds the corresponding [1] refs for the reflist note"""
    if "mw-reference-text" not in note.parent.get("class", [""]):
        return [note]
    note_id = find_note_id(note)
    ref = page_refs.get(note_id)
    return ref

def find_wikitext_for_ref(ref: Tag, note: Tag, title: str, etag: str) -> str:
    wikitext = html_to_wikitext(str(ref), title, etag)
    # if the ref is a template, it gets returned
    if not wikitext:
        # if the ref is not a template, it must be a tag
        raw_wikitext = html_to_wikitext(str(ref) + str(note.parent), title, etag)
        wikitext = "".join(raw_wikitext.partition("</ref>")[0:2])

    return wikitext

def html_to_wikitext(html: str, title: str, etag: str) -> str:
    """Converts html to wikitext in a page and etag context using the Parsoid API"""
    url = (
        "https://en.wikipedia.org/api/rest_v1/transform/html/to/wikitext/"
        + urllib.parse.quote(title.replace(" ", "_"), safe="")
    )
    data = {"html": str(html), "scrub_wikitext": True}
    headers = {"if-match": etag}
    resp = session.post(url, json=data, headers=headers)
    wikitext = resp.text
    return wikitext

def index_nodes(wikitext, obj):
    try:
        index = wikitext.index(obj)
    except ValueError:
        # obj is inside something
        parent = wikitext.get_parent(obj)
        if isinstance(parent, mwph.nodes.tag.Tag):
            if str(parent.tag) == "ref":
                index, nodes, ret_obj = index_nodes(wikitext, parent)
            else:
                ret_obj = obj
                index = parent.contents.index(obj)
                nodes = parent.contents.nodes
    else:
        ret_obj = obj
        nodes = wikitext.nodes

    return index, nodes, ret_obj

def append_tags(wikitext: Wikicode, target: str) -> Wikicode:
    """Appends a tag for occurances of target in wikitext"""
    tag = config["tag"]
    skip_tags = config["skip_tags"]

    def match(n):
        return n == target

    if target.startswith("<"):
        matches = wikitext.filter_tags(matches=match)
        target_type = "tag"
    elif target.startswith("{{"):
        matches = wikitext.filter_templates(matches=match)
        target_type = "template"
    else:
        matches = wikitext.filter(matches=match)
        target_type = ""

    for raw_obj in matches:
        index, nodes, obj = index_nodes(wikitext, raw_obj)
        try:
            # skip if there's already an inline maint tag
            next_obj = nodes[index + 1]
            skip = next_obj.name.matches(skip_tags)
        except (AttributeError, IndexError):
            # assume that it's the end of a section or something and tag anyway
            skip = False

        # make sure this is the right object unless we changed it already
        skip = skip or (str(obj) != target and str(obj) == str(raw_obj))

        if not skip:
            wikitext.insert_after(obj, tag)

    if not matches:
        if target_type == "tag":
            # strip tag out
            new_target = target.partition(">")[2].rpartition("</ref>")[0]
            wikitext = append_tags(wikitext, new_target)

    return wikitext

def broken_anchors(title: str, revision: str = "") -> Dict[str, Set[str]]:
    """Returns a dict of broken anchors and the refs that contain them"""
    title = title.replace(" ", "_")
    raw_html, etag = get_html(title, revision)
    soup = BeautifulSoup(raw_html, "html.parser")

    citeref_links = parse_citeref_links(title, soup)
    citeref_ids = parse_citeref_ids(soup)
    page_refs = parse_refs(title, soup)

    missing_link_notes = find_mismatch(citeref_ids, citeref_links)
    broken_harvs: Dict[str, Set[str]] = {}
    for link_id, notes in missing_link_notes.items():
        for note in notes:
            refs = find_ref_for_note(note, page_refs)
            for ref in refs:
                ref_wikitext = find_wikitext_for_ref(ref, note, title, etag)
                if (
                    ref_wikitext.endswith("/>")
                    and ref_wikitext.startswith("<ref")
                    and not ref_wikitext.startswith("<ref>")
                ) or (ref_wikitext.startswith("{{r")):
                    # skip self-closed ref tags, ref text defined elsewhere
                    continue
                broken_harvs.setdefault(link_id, set()).add(ref_wikitext)

    return broken_harvs

def save_page(page: BasePage, wikitext: str, summary: str) -> bool:
    """Saves wikitext to the on-wiki page, unless simulate is set"""
    if not wikitext:
        raise ValueError
    if wikitext == page.text:
        return False
    page.text = wikitext

    check_runpage()
    if simulate:
        print(page.title(), summary)
        print(page.text)
    else:
        throttle()
        page.save(summary=summary)

    return True

def check_runpage() -> None:
    """Raises utils.RunpageError if on-wiki runpage is not True"""
    utils.check_runpage(site, title=config["runpage"])

def check_many_nosave(pages: List[str]) -> None:
    for page in pages:
        broken_harvs = broken_anchors(page)
        print(f"* [[{page}]] - <nowiki>{broken_harvs}</nowiki>")

def main(title: Optional[str] = None, page: Optional[BasePage] = None) -> bool:
    """Checks one page, returns True if problems found"""
    assert page or title

    if page and not title:
        title = page.title()
    elif title and not page:
        page = pywikibot.Page(site, title)
    elif page and title:
        if page.title() != title:
            raise ValueError("Specified title and page do not match")

    assert page and title
    logger.info(f"Checking {title}")
    check_runpage()
    wikitext = mwph.parse(page.text)

    broken_harvs = broken_anchors(title)

    for link_id, ref_text_list in broken_harvs.items():
        for ref_wikitext in ref_text_list:
            wikitext = append_tags(wikitext, ref_wikitext)

    changes = len(broken_harvs)
    return save_page(
        page,
        str(wikitext),
        config["summary"].format(version=__version__, changes=changes),
    )

def throttle() -> None:
    """Prevents edits from happening faster than configured"""
    global last_edit
    now = time.monotonic()
    length = now - last_edit
    rate = config["rate"]
    if length < rate:
        logger.info(f"Throttling for {round(rate - length,1)} seconds")
        time.sleep(rate - length)
    last_edit = time.monotonic()

def iter_json_lines_pages(filename):
    """Iterates pages from a json lines file with page_title and page_namespace"""
    with open(filename) as f:
        for line in f:
            data = json.loads(line)
            yield pywikibot.Page(
                site, title=data["page_title"], ns=data["page_namespace"]
            )

def query_quarry(url):
    """Gets a json lines file from a Quarry url and iterates pages from it"""
    urldata = url.split("/")
    if urldata[3] == "query":
        run_url = quarry_get_run_url(url)
    else:
        run_url = url
    logger.info(f"Getting JSON lines from {run_url}")
    req = session.get(run_url)
    req.raise_for_status()
    # json file can be big, write it to disk and remove from memory
    filename = _conf_dir + str(id(req)) + ".json-lines"
    with open(filename, "x") as f:
        f.write(req.text)
    del req
    try:
        for page in iter_json_lines_pages(filename):
            yield page
    finally:
        os.remove(filename)

def quarry_get_run_url(query_url):
    """Gets the Quarry runid from a Quarry query page"""
    query = session.get(query_url)
    query.raise_for_status()
    regex = r"(?<=\"qrun_id\": )\d*"
    match = re.search(regex, query.text)
    if match:
        run_id = match.group(0)
        return f"https://quarry.wmflabs.org/run/{run_id}/output/0/json-lines"
    else:
        raise ValueError("No Quarry run id found")

def auto(method, limit: int = 0, start: str = "!", url: str = ""):
    """Checks multiple pages"""
    logger.info("Starting up")
    try:
        check_runpage()
        if method == "alpha":
            iterpages = site.allpages(start=start, filterredir=False)
        elif method == "random":
            iterpages = site.randompages(namespaces=0, redirects=False)
        elif method == "quarry":
            iterpages = query_quarry(url if url else config["quarry_url"])
        else:
            raise KeyError("Generator is invalid")

        checked, edited = 0, 0
        for page in iterpages:
            checked += 1
            if limit and edited >= limit:
                break
            result = main(page=page)
            if result:
                edited += 1
    except Exception as err:
        logger.exception(err)
        raise err
    finally:
        logger.info(f"Finished! {checked} articles scanned, {edited} articles edited.")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    runtype = parser.add_mutually_exclusive_group(required=True)
    dryrun = parser.add_mutually_exclusive_group()
    runtype.add_argument(
        "--auto",
        help="runs the bot continuously",
        action="store",
        nargs="?",
        const=config.get("auto_method"),
    )
    runtype.add_argument("--page", help="run the bot on this page only")
    parser.add_argument(
        "--limit",
        type=int,
        help="how many pages to edit",
        action="store",
        nargs="?",
        const=config.get("limit", 0),
        default=0,
    )
    parser.add_argument("--start", help="page to start iterating from", default="!")
    dryrun.add_argument(
        "--simulate", action="store_true", help="prevents bot from saving"
    )
    dryrun.add_argument(
        "--run", action="store_true", help="overrides config to force saving"
    )
    parser.add_argument("--version", action="version", version=__version__)
    parser.add_argument("--url", default="")
    args = parser.parse_args()
    if args.simulate:
        simulate = True
    elif args.run:
        simulate = False

    if args.auto:
        auto(args.auto, limit=args.limit, start=args.start, url=args.url)
    elif args.page:
        if main(title=args.page):
            print("Done")
        else:
            print("No change")


import os
import numpy as np
from PIL import Image
from torch.utils.data import Dataset
import torch

class VIMEO(Dataset):

    def __init__(self, path, train=True, transform=None, add_noise=False):
        assert os.path.exists(
            path), 'Invalid path to VIMEO data set: ' + path
        self.path = path
        self.transform = transform
        if train:
            self.video_list = os.path.join(path, 'sep_trainlist.txt')
        else:
            self.video_list = os.path.join(path, 'sep_testlist.txt')
        self.video_list = np.loadtxt(self.video_list, dtype=str)
        self.video_list = np.core.defchararray.add(f'{os.path.join(path, "sequences")}/', self.video_list)

        self.add_noise = add_noise

    def __getitem__(self, ind):
        # load the images from the ind directory to get list of PIL images
        img_names = os.listdir(str(self.video_list[ind]))
        imgs = [Image.open(os.path.join(self.video_list[ind], str(img_name)))
                for img_name in img_names]
        if self.transform is not None:
            imgs = self.transform(imgs)

        if self.add_noise:
            imgs = imgs + (torch.rand_like(imgs)-0.5) / 256.

        return imgs

    def __len__(self):
        # total number of videos
        return len(self.video_list)


import json
import psycopg2

def create_database(database_name: str, params: dict):
    """Создание базы данных и таблиц для сохранения данных."""

    conn = psycopg2.connect(dbname='postgres', **params)
    conn.autocommit = True
    cur = conn.cursor()

    cur.execute(f"DROP DATABASE IF EXISTS {database_name}")
    cur.execute(f"CREATE DATABASE {database_name}")

    cur.close()
    conn.close()

    conn = psycopg2.connect(dbname=database_name, **params)

    with conn.cursor() as cur:
        cur.execute("""CREATE TABLE employers (
                    id_employer SERIAL PRIMARY KEY,
                    name VARCHAR(255) NOT NULL,
                    url_employer TEXT,
                    open_vacancies INTEGER
                )
            """)

    with conn.cursor() as cur:
        cur.execute("""CREATE TABLE vacancies (
                    id_vacancy SERIAL PRIMARY KEY,
                    name_employer VARCHAR(255) NOT NULL,
                    name_vacancy VARCHAR(255) NOT NULL,
                    salary_from INTEGER,
                    salary_to INTEGER,
                    url_vacancy TEXT
                )
            """)

    conn.commit()
    conn.close()

def save_data_to_database_emp(database_name: str, params: dict):
    """Заполнение БД данными о работодателях из json файла """

    conn = psycopg2.connect(dbname=database_name, **params)
    with conn.cursor() as cur:
        with open('src/employer.json', 'r', encoding='utf-8') as file:
            employer = json.load(file)
            for emp in employer:
                name = emp['name']
                url_employer = emp['alternate_url']
                open_vacancies = emp['open_vacancies']
                cur.execute(
                    """
                    INSERT INTO employers (name, url_employer, open_vacancies)
                    VALUES (%s, %s, %s)
                    """,
                    (name, url_employer, open_vacancies)
                )

    conn.commit()
    conn.close()

def save_data_to_database_vac(database_name: str, params: dict):
    """Заполнение БД данными о вакансиях из json файла """

    conn = psycopg2.connect(dbname=database_name, **params)
    with conn.cursor() as cur:
        with open('src/vacancies.json', 'r', encoding='utf-8') as file:
            vacancies = json.load(file)
            for vac in vacancies:
                name_employer = vac['employer']['name']
                name_vacancy = vac['name']
                if vac['salary'] is None:
                    salary_from = 0
                    salary_to = 0
                else:
                    salary_from = vac['salary']['from']
                    if salary_from is None:
                        salary_from = 0

                    salary_to = vac['salary']['to']
                    if salary_to is None:
                        salary_to = 0

                url_vacancy = vac['alternate_url']
                cur.execute(
                    """
                    INSERT INTO vacancies (name_employer, name_vacancy, salary_from, salary_to,
                    url_vacancy)
                    VALUES (%s, %s, %s, %s, %s)
                    """,
                    (name_employer, name_vacancy, salary_from, salary_to,
                     url_vacancy)
                )

    conn.commit()
    conn.close()