from datetime import date, datetime, timedelta
from unittest.mock import Mock, call, patch

from django.contrib.auth.models import Group
from django.core import mail
from django.core.cache import caches
from django.core.exceptions import ValidationError
from django.test import TestCase, override_settings
from django_webtest import WebTest
from model_bakery import baker

from evap.evaluation.models import (
    Contribution,
    Course,
    CourseType,
    EmailTemplate,
    Evaluation,
    NotArchiveable,
    Question,
    Questionnaire,
    QuestionType,
    RatingAnswerCounter,
    Semester,
    TextAnswer,
    UserProfile,
)
from evap.evaluation.tests.tools import (
    let_user_vote_for_evaluation,
    make_contributor,
    make_editor,
    make_rating_answer_counters,
)
from evap.grades.models import GradeDocument
from evap.results.tools import cache_results, calculate_average_distribution
from evap.results.views import get_evaluation_result_template_fragment_cache_key

class TestSemester(WebTest):
    def test_can_be_deleted_by_manager(self):
        semester = baker.make(Semester)
        self.assertTrue(semester.can_be_deleted_by_manager)

        semester.is_active = True
        self.assertFalse(semester.can_be_deleted_by_manager)
        semester.is_active = False

        voter = baker.make(UserProfile)
        baker.make(Evaluation, course__semester=semester, state=Evaluation.State.PUBLISHED, voters=[voter])
        self.assertFalse(semester.can_be_deleted_by_manager)

        semester.archive()
        self.assertFalse(semester.can_be_deleted_by_manager)

        semester.delete_grade_documents()
        self.assertFalse(semester.can_be_deleted_by_manager)

        semester.archive_results()
        self.assertTrue(semester.can_be_deleted_by_manager)

class TestQuestionnaire(WebTest):
    def test_can_be_deleted_by_manager(self):
        questionnaire = baker.make(Questionnaire)
        self.assertTrue(questionnaire.can_be_deleted_by_manager)

        baker.make(Contribution, questionnaires=[questionnaire])
        self.assertFalse(questionnaire.can_be_deleted_by_manager)

@override_settings(EVALUATION_END_OFFSET_HOURS=0)
class TestEvaluations(WebTest):
    def test_approved_to_in_evaluation(self):
        evaluation = baker.make(Evaluation, state=Evaluation.State.APPROVED, vote_start_datetime=datetime.now())

        with patch("evap.evaluation.models.EmailTemplate") as mock:
            mock.EVALUATION_STARTED = EmailTemplate.EVALUATION_STARTED
            mock.Recipients.ALL_PARTICIPANTS = EmailTemplate.Recipients.ALL_PARTICIPANTS
            mock.objects.get.return_value = mock
            Evaluation.update_evaluations()

        self.assertEqual(mock.objects.get.call_args_list[0][1]["name"], EmailTemplate.EVALUATION_STARTED)
        mock.send_to_users_in_evaluations.assert_called_once_with(
            [evaluation], [EmailTemplate.Recipients.ALL_PARTICIPANTS], use_cc=False, request=None
        )

        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(evaluation.state, Evaluation.State.IN_EVALUATION)

    def test_in_evaluation_to_evaluated(self):
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.IN_EVALUATION,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today() - timedelta(days=1),
        )

        with patch("evap.evaluation.models.Evaluation.is_fully_reviewed") as mock:
            mock.__get__ = Mock(return_value=False)
            Evaluation.update_evaluations()

        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(evaluation.state, Evaluation.State.EVALUATED)

    def test_in_evaluation_to_reviewed(self):
        # Evaluation is "fully reviewed" as no open text answers are present by default.
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.IN_EVALUATION,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today() - timedelta(days=1),
        )

        Evaluation.update_evaluations()

        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(evaluation.state, Evaluation.State.REVIEWED)

    def test_in_evaluation_to_published(self):
        # Evaluation is "fully reviewed" and not graded, thus gets published immediately.
        course = baker.make(Course)
        evaluation = baker.make(
            Evaluation,
            course=course,
            state=Evaluation.State.IN_EVALUATION,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today() - timedelta(days=1),
            wait_for_grade_upload_before_publishing=False,
        )

        with patch(
            "evap.evaluation.models.EmailTemplate.send_participant_publish_notifications"
        ) as participant_mock, patch(
            "evap.evaluation.models.EmailTemplate.send_contributor_publish_notifications"
        ) as contributor_mock:
            Evaluation.update_evaluations()

        participant_mock.assert_called_once_with([evaluation])
        contributor_mock.assert_called_once_with([evaluation])

        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(evaluation.state, Evaluation.State.PUBLISHED)

    @override_settings(EVALUATION_END_WARNING_PERIOD=24)
    def test_ends_soon(self):
        evaluation = baker.make(
            Evaluation,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today() + timedelta(hours=24),
        )

        self.assertFalse(evaluation.ends_soon)

        evaluation.vote_end_date = date.today()
        self.assertTrue(evaluation.ends_soon)

        evaluation.vote_end_date = date.today() - timedelta(hours=48)
        self.assertFalse(evaluation.ends_soon)

    @override_settings(EVALUATION_END_WARNING_PERIOD=24, EVALUATION_END_OFFSET_HOURS=24)
    def test_ends_soon_with_offset(self):
        evaluation = baker.make(
            Evaluation, vote_start_datetime=datetime.now() - timedelta(days=2), vote_end_date=date.today()
        )

        self.assertFalse(evaluation.ends_soon)

        evaluation.vote_end_date = date.today() - timedelta(hours=24)
        self.assertTrue(evaluation.ends_soon)

        evaluation.vote_end_date = date.today() - timedelta(hours=72)
        self.assertFalse(evaluation.ends_soon)

    def test_evaluation_ended(self):
        # Evaluation is out of evaluation period.
        course_1 = baker.make(Course)
        course_2 = baker.make(Course)
        baker.make(
            Evaluation,
            course=course_1,
            state=Evaluation.State.IN_EVALUATION,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today() - timedelta(days=1),
            wait_for_grade_upload_before_publishing=False,
        )
        # This evaluation is not.
        baker.make(
            Evaluation,
            course=course_2,
            state=Evaluation.State.IN_EVALUATION,
            vote_start_datetime=datetime.now() - timedelta(days=2),
            vote_end_date=date.today(),
            wait_for_grade_upload_before_publishing=False,
        )

        with patch("evap.evaluation.models.Evaluation.end_evaluation") as mock:
            Evaluation.update_evaluations()

        self.assertEqual(mock.call_count, 1)

    def test_approved_to_in_evaluation_sends_emails(self):
        """Regression test for #945"""
        participant = baker.make(UserProfile, email="foo@example.com")
        evaluation = baker.make(
            Evaluation, state=Evaluation.State.APPROVED, vote_start_datetime=datetime.now(), participants=[participant]
        )

        Evaluation.update_evaluations()

        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(evaluation.state, Evaluation.State.IN_EVALUATION)

    def test_has_enough_questionnaires(self):
        # manually circumvent Evaluation's save() method to have a Evaluation without a general contribution
        # the semester must be specified because of https://github.com/vandersonmota/model_bakery/issues/258
        course = baker.make(Course, semester=baker.make(Semester), type=baker.make(CourseType))
        Evaluation.objects.bulk_create([baker.prepare(Evaluation, course=course)])
        evaluation = Evaluation.objects.get()
        self.assertEqual(evaluation.contributions.count(), 0)
        self.assertFalse(evaluation.general_contribution_has_questionnaires)
        self.assertFalse(evaluation.all_contributions_have_questionnaires)

        editor_contribution = baker.make(
            Contribution,
            evaluation=evaluation,
            contributor=baker.make(UserProfile),
            role=Contribution.Role.EDITOR,
            textanswer_visibility=Contribution.TextAnswerVisibility.GENERAL_TEXTANSWERS,
        )
        evaluation = Evaluation.objects.get()
        self.assertFalse(evaluation.general_contribution_has_questionnaires)
        self.assertFalse(evaluation.all_contributions_have_questionnaires)

        general_contribution = baker.make(Contribution, evaluation=evaluation, contributor=None)
        evaluation = Evaluation.objects.get()
        self.assertFalse(evaluation.general_contribution_has_questionnaires)
        self.assertFalse(evaluation.all_contributions_have_questionnaires)

        questionnaire = baker.make(Questionnaire)
        general_contribution.questionnaires.add(questionnaire)
        self.assertTrue(evaluation.general_contribution_has_questionnaires)
        self.assertFalse(evaluation.all_contributions_have_questionnaires)

        editor_contribution.questionnaires.add(questionnaire)
        self.assertTrue(evaluation.general_contribution_has_questionnaires)
        self.assertTrue(evaluation.all_contributions_have_questionnaires)

    def test_single_result_can_be_deleted_only_in_reviewed(self):
        responsible = baker.make(UserProfile)
        evaluation = baker.make(Evaluation, is_single_result=True)
        contribution = baker.make(
            Contribution,
            evaluation=evaluation,
            contributor=responsible,
            questionnaires=[Questionnaire.single_result_questionnaire()],
            role=Contribution.Role.EDITOR,
            textanswer_visibility=Contribution.TextAnswerVisibility.GENERAL_TEXTANSWERS,
        )
        make_rating_answer_counters(Questionnaire.single_result_questionnaire().questions.first(), contribution)
        evaluation.skip_review_single_result()
        evaluation.publish()
        evaluation.save()

        self.assertTrue(Evaluation.objects.filter(pk=evaluation.pk).exists())
        self.assertFalse(evaluation.can_be_deleted_by_manager)

        evaluation.unpublish()
        self.assertTrue(evaluation.can_be_deleted_by_manager)

        RatingAnswerCounter.objects.filter(contribution__evaluation=evaluation).delete()
        evaluation.delete()
        self.assertFalse(Evaluation.objects.filter(pk=evaluation.pk).exists())

    @staticmethod
    def test_single_result_can_be_published():
        """Regression test for #1238"""
        responsible = baker.make(UserProfile)
        single_result = baker.make(Evaluation, is_single_result=True, _participant_count=5, _voter_count=5)
        contribution = baker.make(
            Contribution,
            evaluation=single_result,
            contributor=responsible,
            questionnaires=[Questionnaire.single_result_questionnaire()],
            role=Contribution.Role.EDITOR,
            textanswer_visibility=Contribution.TextAnswerVisibility.GENERAL_TEXTANSWERS,
        )
        make_rating_answer_counters(Questionnaire.single_result_questionnaire().questions.first(), contribution)

        single_result.skip_review_single_result()
        single_result.publish()  # used to crash

    def test_second_vote_sets_can_publish_text_results_to_true(self):
        student1 = baker.make(UserProfile, email="student1@institution.example.com")
        student2 = baker.make(UserProfile, email="student2@example.com")
        evaluation = baker.make(
            Evaluation, participants=[student1, student2], voters=[student1], state=Evaluation.State.IN_EVALUATION
        )
        evaluation.save()
        top_general_questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        baker.make(Question, questionnaire=top_general_questionnaire, type=QuestionType.POSITIVE_LIKERT)
        evaluation.general_contribution.questionnaires.set([top_general_questionnaire])

        self.assertFalse(evaluation.can_publish_text_results)

        let_user_vote_for_evaluation(student2, evaluation)
        evaluation = Evaluation.objects.get(pk=evaluation.pk)

        self.assertTrue(evaluation.can_publish_text_results)

    def test_textanswers_get_deleted_if_they_cannot_be_published(self):
        student = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.REVIEWED,
            participants=[student],
            voters=[student],
            can_publish_text_results=False,
        )
        questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        question = baker.make(Question, type=QuestionType.TEXT, questionnaire=questionnaire)
        evaluation.general_contribution.questionnaires.set([questionnaire])
        baker.make(TextAnswer, question=question, contribution=evaluation.general_contribution)

        self.assertEqual(evaluation.textanswer_set.count(), 1)
        evaluation.publish()
        self.assertEqual(evaluation.textanswer_set.count(), 0)

    def test_textanswers_do_not_get_deleted_if_they_can_be_published(self):
        student = baker.make(UserProfile)
        student2 = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.REVIEWED,
            participants=[student, student2],
            voters=[student, student2],
            can_publish_text_results=True,
        )
        questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        question = baker.make(Question, type=QuestionType.TEXT, questionnaire=questionnaire)
        evaluation.general_contribution.questionnaires.set([questionnaire])
        baker.make(TextAnswer, question=question, contribution=evaluation.general_contribution)

        self.assertEqual(evaluation.textanswer_set.count(), 1)
        evaluation.publish()
        self.assertEqual(evaluation.textanswer_set.count(), 1)

    def test_textanswers_to_delete_get_deleted_on_publish(self):
        student = baker.make(UserProfile)
        student2 = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.REVIEWED,
            participants=[student, student2],
            voters=[student, student2],
            can_publish_text_results=True,
        )
        questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        question = baker.make(Question, type=QuestionType.TEXT, questionnaire=questionnaire)
        evaluation.general_contribution.questionnaires.set([questionnaire])
        baker.make(
            TextAnswer,
            question=question,
            contribution=evaluation.general_contribution,
            answer=iter(["deleted", "public", "private"]),
            review_decision=iter(
                [
                    TextAnswer.ReviewDecision.DELETED,
                    TextAnswer.ReviewDecision.PUBLIC,
                    TextAnswer.ReviewDecision.PRIVATE,
                ]
            ),
            _quantity=3,
            _bulk_create=True,
        )

        self.assertEqual(evaluation.textanswer_set.count(), 3)
        evaluation.publish()
        self.assertEqual(evaluation.textanswer_set.count(), 2)
        self.assertFalse(TextAnswer.objects.filter(answer="deleted").exists())

    def test_original_textanswers_get_deleted_on_publish(self):
        student = baker.make(UserProfile)
        student2 = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.REVIEWED,
            participants=[student, student2],
            voters=[student, student2],
            can_publish_text_results=True,
        )
        questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        question = baker.make(Question, type=QuestionType.TEXT, questionnaire=questionnaire)
        evaluation.general_contribution.questionnaires.set([questionnaire])
        baker.make(
            TextAnswer,
            question=question,
            contribution=evaluation.general_contribution,
            answer="published answer",
            original_answer="original answer",
            review_decision=TextAnswer.ReviewDecision.PUBLIC,
        )

        self.assertEqual(evaluation.textanswer_set.count(), 1)
        self.assertFalse(TextAnswer.objects.get().original_answer is None)
        evaluation.publish()
        self.assertEqual(evaluation.textanswer_set.count(), 1)
        self.assertTrue(TextAnswer.objects.get().original_answer is None)

    def test_publishing_and_unpublishing_effect_on_template_cache(self):
        student = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.REVIEWED,
            participants=[student],
            voters=[student],
            can_publish_text_results=True,
        )

        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", True))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", False))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", True))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", False))
        )

        evaluation.publish()
        evaluation.save()

        self.assertIsNotNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", True))
        )
        self.assertIsNotNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", False))
        )
        self.assertIsNotNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", True))
        )
        self.assertIsNotNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", False))
        )

        evaluation.unpublish()
        evaluation.save()

        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", True))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "en", False))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", True))
        )
        self.assertIsNone(
            caches["results"].get(get_evaluation_result_template_fragment_cache_key(evaluation.id, "de", False))
        )

    def assert_textanswer_review_state(
        self,
        evaluation,
        expected_default_value,
        expected_value_with_gets_no_grade_documents,
        expected_value_with_wait_for_grade_upload_before_publishing,
        expected_value_after_grade_upload,
    ):
        self.assertEqual(evaluation.textanswer_review_state, expected_default_value)

        evaluation.course.gets_no_grade_documents = True
        self.assertEqual(evaluation.textanswer_review_state, expected_value_with_gets_no_grade_documents)
        evaluation.course.gets_no_grade_documents = False

        evaluation.wait_for_grade_upload_before_publishing = True
        self.assertEqual(
            evaluation.textanswer_review_state, expected_value_with_wait_for_grade_upload_before_publishing
        )

        grade_document = baker.make(GradeDocument, type=GradeDocument.Type.FINAL_GRADES, course=evaluation.course)
        self.assertEqual(evaluation.textanswer_review_state, expected_value_after_grade_upload)
        grade_document.delete()

        evaluation.wait_for_grade_upload_before_publishing = False

    def test_textanswer_review_state(self):
        evaluation = baker.make(
            Evaluation,
            state=Evaluation.State.IN_EVALUATION,
            can_publish_text_results=True,
            wait_for_grade_upload_before_publishing=False,
        )

        self.assert_textanswer_review_state(
            evaluation,
            evaluation.TextAnswerReviewState.NO_TEXTANSWERS,
            evaluation.TextAnswerReviewState.NO_TEXTANSWERS,
            evaluation.TextAnswerReviewState.NO_TEXTANSWERS,
            evaluation.TextAnswerReviewState.NO_TEXTANSWERS,
        )

        textanswer = baker.make(TextAnswer, contribution=evaluation.general_contribution)
        del evaluation.num_textanswers  # reset cached_property cache

        # text_answer_review_state should be NO_REVIEW_NEEDED as long as we are still in_evaluation
        self.assert_textanswer_review_state(
            evaluation,
            evaluation.TextAnswerReviewState.NO_REVIEW_NEEDED,
            evaluation.TextAnswerReviewState.NO_REVIEW_NEEDED,
            evaluation.TextAnswerReviewState.NO_REVIEW_NEEDED,
            evaluation.TextAnswerReviewState.NO_REVIEW_NEEDED,
        )

        evaluation.end_evaluation()
        evaluation.save()

        self.assert_textanswer_review_state(
            evaluation,
            evaluation.TextAnswerReviewState.REVIEW_URGENT,
            evaluation.TextAnswerReviewState.REVIEW_URGENT,  # course has `gets_no_grade_documents`
            evaluation.TextAnswerReviewState.REVIEW_NEEDED,  # still waiting for grades
            evaluation.TextAnswerReviewState.REVIEW_URGENT,  # grades were uploaded
        )

        textanswer.review_decision = TextAnswer.ReviewDecision.PUBLIC
        textanswer.save()
        del evaluation.num_reviewed_textanswers  # reset cached_property cache

        self.assert_textanswer_review_state(
            evaluation,
            evaluation.TextAnswerReviewState.REVIEWED,
            evaluation.TextAnswerReviewState.REVIEWED,
            evaluation.TextAnswerReviewState.REVIEWED,
            evaluation.TextAnswerReviewState.REVIEWED,
        )

class TestCourse(TestCase):
    def test_can_be_deleted_by_manager(self):
        course = baker.make(Course)
        evaluation = baker.make(Evaluation, course=course)
        self.assertFalse(course.can_be_deleted_by_manager)

        evaluation.delete()
        self.assertTrue(course.can_be_deleted_by_manager)

    def test_responsibles_names(self):
        # last names required for sorting
        user1 = baker.make(UserProfile, last_name="Doe")
        user2 = baker.make(UserProfile, last_name="Meyer")
        course = baker.make(Course, responsibles=[user1, user2])
        self.assertEqual(course.responsibles_names, f"{user1.full_name}, {user2.full_name}")

class TestUserProfile(TestCase):
    def test_is_student(self):
        some_user = baker.make(UserProfile)
        self.assertFalse(some_user.is_student)

        student = baker.make(UserProfile, evaluations_participating_in=[baker.make(Evaluation)])
        self.assertTrue(student.is_student)

        contributor = baker.make(UserProfile, contributions=[baker.make(Contribution)])
        self.assertFalse(contributor.is_student)

        semester_contributed_to = baker.make(Semester, created_at=date.today())
        semester_participated_in = baker.make(Semester, created_at=date.today())
        course_contributed_to = baker.make(Course, semester=semester_contributed_to)
        course_participated_in = baker.make(Course, semester=semester_participated_in)
        evaluation_contributed_to = baker.make(Evaluation, course=course_contributed_to)
        evaluation_participated_in = baker.make(Evaluation, course=course_participated_in)
        contribution = baker.make(Contribution, evaluation=evaluation_contributed_to)
        user = baker.make(
            UserProfile, contributions=[contribution], evaluations_participating_in=[evaluation_participated_in]
        )

        self.assertTrue(user.is_student)

        semester_contributed_to.created_at = date.today() - timedelta(days=1)
        semester_contributed_to.save()

        # invalidate cached_property
        del user.is_student
        self.assertTrue(user.is_student)

        semester_participated_in.created_at = date.today() - timedelta(days=2)
        semester_participated_in.save()

        # invalidate cached_property
        del user.is_student
        self.assertFalse(user.is_student)

    def test_can_be_deleted_by_manager(self):
        user = baker.make(UserProfile)
        baker.make(Evaluation, participants=[user], state=Evaluation.State.NEW)
        self.assertFalse(user.can_be_deleted_by_manager)

        user2 = baker.make(UserProfile)
        baker.make(Evaluation, participants=[user2], state=Evaluation.State.IN_EVALUATION)
        self.assertFalse(user2.can_be_deleted_by_manager)

        contributor = baker.make(UserProfile)
        baker.make(Contribution, contributor=contributor)
        self.assertFalse(contributor.can_be_deleted_by_manager)

        proxy_user = baker.make(UserProfile, is_proxy_user=True)
        self.assertFalse(proxy_user.can_be_deleted_by_manager)

    def test_inactive_users_hidden(self):
        active_user = baker.make(UserProfile)
        baker.make(UserProfile, is_active=False)

        self.assertEqual(list(UserProfile.objects.exclude(is_active=False)), [active_user])

    def test_inactive_users_shown(self):
        active_user = baker.make(UserProfile)
        inactive_user = baker.make(UserProfile, is_active=False)

        user_list = list(UserProfile.objects.all())
        self.assertIn(active_user, user_list)
        self.assertIn(inactive_user, user_list)

    def test_can_be_marked_inactive_by_manager(self):
        user = baker.make(UserProfile)
        evaluation = baker.make(Evaluation, state=Evaluation.State.NEW)
        self.assertTrue(user.can_be_marked_inactive_by_manager)
        evaluation.participants.set([user])
        evaluation.save()
        self.assertFalse(user.can_be_marked_inactive_by_manager)

        contributor = baker.make(UserProfile)
        baker.make(Contribution, contributor=contributor)
        self.assertFalse(contributor.can_be_marked_inactive_by_manager)

        reviewer = baker.make(UserProfile, groups=[Group.objects.get(name="Reviewer")])
        self.assertFalse(reviewer.can_be_marked_inactive_by_manager)

        grade_publisher = baker.make(UserProfile, groups=[Group.objects.get(name="Grade publisher")])
        self.assertFalse(grade_publisher.can_be_marked_inactive_by_manager)

        super_user = baker.make(UserProfile, is_superuser=True)
        self.assertFalse(super_user.can_be_marked_inactive_by_manager)

        proxy_user = baker.make(UserProfile, is_proxy_user=True)
        self.assertFalse(proxy_user.can_be_marked_inactive_by_manager)

    @override_settings(INSTITUTION_EMAIL_REPLACEMENTS=[("example.com", "institution.com")])
    def test_email_domain_replacement(self):
        user = baker.make(UserProfile, email="test@example.com")
        self.assertEqual(user.email, "test@institution.com")

    def test_get_sorted_due_evaluations(self):
        student = baker.make(UserProfile, email="student@example.com")
        course = baker.make(Course)

        evaluations = baker.make(
            Evaluation,
            course=course,
            name_en=iter(["C", "B", "A"]),
            name_de=iter(["C", "B", "A"]),
            vote_end_date=iter([date.today(), date.today(), date.today() + timedelta(days=1)]),
            state=Evaluation.State.IN_EVALUATION,
            participants=[student],
            _quantity=3,
        )

        sorted_evaluations = student.get_sorted_due_evaluations()
        self.assertEqual(sorted_evaluations, [(evaluations[1], 0), (evaluations[0], 0), (evaluations[2], 1)])

    def test_correct_sorting(self):
        baker.make(
            UserProfile,
            last_name=iter(["Y", "x", "", ""]),
            first_name_given=iter(["x", "x", "a", ""]),
            email=iter(["3xy@example.com", "4xx@example.com", "2a@example.com", "1unnamed@example.com"]),
            _quantity=4,
            _bulk_create=True,
        )
        email_list = [user.email for user in UserProfile.objects.all()]
        self.assertEqual(email_list, ["4xx@example.com", "3xy@example.com", "2a@example.com", "1unnamed@example.com"])

class ParticipationArchivingTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.semester = baker.make(Semester)
        cls.evaluation = baker.make(
            Evaluation, state=Evaluation.State.PUBLISHED, course=baker.make(Course, semester=cls.semester)
        )
        cls.evaluation.general_contribution.questionnaires.set([baker.make(Questionnaire)])

        users = baker.make(UserProfile, _bulk_create=True, _quantity=3)
        cls.evaluation.participants.set(users)
        cls.evaluation.voters.set(users[:2])

    def refresh_evaluation(self):
        """refresh_from_db does not work with evaluations"""
        self.evaluation = self.semester.evaluations.first()

    def test_counts_dont_change(self):
        """
        Asserts that evaluation.num_voters evaluation.num_participants don't change after archiving.
        """
        voter_count = self.evaluation.num_voters
        participant_count = self.evaluation.num_participants

        self.semester.archive()
        self.refresh_evaluation()

        self.assertEqual(voter_count, self.evaluation.num_voters)
        self.assertEqual(participant_count, self.evaluation.num_participants)

    def test_participants_do_not_loose_evaluations(self):
        """
        Asserts that participants still participate in their evaluations after the participations get archived.
        """
        some_participant = self.evaluation.participants.first()

        self.semester.archive()

        self.assertEqual(list(some_participant.evaluations_participating_in.all()), [self.evaluation])

    def test_participations_are_archived(self):
        """
        Tests whether participations_are_archived returns True on semesters and evaluations with archived participations.
        """
        self.assertFalse(self.evaluation.participations_are_archived)

        self.semester.archive()
        self.refresh_evaluation()

        self.assertTrue(self.evaluation.participations_are_archived)

    def test_archiving_participations_does_not_change_results(self):
        cache_results(self.evaluation)
        distribution = calculate_average_distribution(self.evaluation)

        self.semester.archive()
        self.refresh_evaluation()
        caches["results"].clear()

        cache_results(self.evaluation)
        new_distribution = calculate_average_distribution(self.evaluation)
        self.assertEqual(new_distribution, distribution)

    def test_archiving_participations_twice_raises_exception(self):
        self.semester.archive()
        with self.assertRaises(NotArchiveable):
            self.semester.archive()
        with self.assertRaises(NotArchiveable):
            self.semester.courses.first().evaluations.first()._archive()

    def test_evaluation_participations_are_not_archived_if_participant_count_is_set(self):
        evaluation = baker.make(Evaluation, state=Evaluation.State.PUBLISHED, _participant_count=1, _voter_count=1)
        self.assertFalse(evaluation.participations_are_archived)
        self.assertTrue(evaluation.participations_can_be_archived)

    def test_archiving_participations_doesnt_change_single_results_participant_count(self):
        responsible = baker.make(UserProfile)
        evaluation = baker.make(
            Evaluation, state=Evaluation.State.PUBLISHED, is_single_result=True, _participant_count=5, _voter_count=5
        )
        contribution = baker.make(
            Contribution,
            evaluation=evaluation,
            contributor=responsible,
            role=Contribution.Role.EDITOR,
            textanswer_visibility=Contribution.TextAnswerVisibility.GENERAL_TEXTANSWERS,
        )
        contribution.questionnaires.add(Questionnaire.single_result_questionnaire())

        evaluation.course.semester.archive()
        evaluation = Evaluation.objects.get(pk=evaluation.pk)
        self.assertEqual(evaluation._participant_count, 5)
        self.assertEqual(evaluation._voter_count, 5)

class TestLoginUrlEmail(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.other_user = baker.make(UserProfile, email="other@extern.com")
        cls.user = baker.make(UserProfile, email="extern@extern.com")
        cls.user.ensure_valid_login_key()

        cls.evaluation = baker.make(Evaluation)
        baker.make(
            Contribution,
            evaluation=cls.evaluation,
            contributor=cls.user,
            role=Contribution.Role.EDITOR,
            textanswer_visibility=Contribution.TextAnswerVisibility.GENERAL_TEXTANSWERS,
        )

        cls.template = baker.make(EmailTemplate, plain_content="{{ login_url }}")

        EmailTemplate.objects.filter(name="Login Key Created").update(plain_content="{{ user.login_url }}")

    @override_settings(PAGE_URL="https://example.com")
    def test_no_login_url_when_delegates_in_cc(self):
        self.user.delegates.add(self.other_user)
        self.template.send_to_users_in_evaluations(
            [self.evaluation], EmailTemplate.Recipients.CONTRIBUTORS, use_cc=True, request=None
        )
        self.assertEqual(len(mail.outbox), 2)
        self.assertEqual(mail.outbox[0].body, "")  # message does not contain the login url
        self.assertEqual(mail.outbox[1].body, self.user.login_url)  # separate email with login url was sent
        self.assertEqual(len(mail.outbox[1].cc), 0)
        self.assertEqual(mail.outbox[1].to, [self.user.email])

    def test_no_login_url_when_cc_users_in_cc(self):
        self.user.cc_users.add(self.other_user)
        self.template.send_to_users_in_evaluations(
            [self.evaluation], [EmailTemplate.Recipients.CONTRIBUTORS], use_cc=True, request=None
        )
        self.assertEqual(len(mail.outbox), 2)
        self.assertEqual(mail.outbox[0].body, "")  # message does not contain the login url
        self.assertEqual(mail.outbox[1].body, self.user.login_url)  # separate email with login url was sent
        self.assertEqual(len(mail.outbox[1].cc), 0)
        self.assertEqual(mail.outbox[1].to, [self.user.email])

    def test_login_url_when_nobody_in_cc(self):
        # message is not sent to others in cc
        self.template.send_to_users_in_evaluations(
            [self.evaluation], [EmailTemplate.Recipients.CONTRIBUTORS], use_cc=True, request=None
        )
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].body, self.user.login_url)  # message does contain the login url

    def test_login_url_when_use_cc_is_false(self):
        # message is not sent to others in cc
        self.user.delegates.add(self.other_user)
        self.template.send_to_users_in_evaluations(
            [self.evaluation], [EmailTemplate.Recipients.CONTRIBUTORS], use_cc=False, request=None
        )
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].body, self.user.login_url)  # message does contain the login url

@override_settings(INSTITUTION_EMAIL_DOMAINS=["example.com"])
class TestEmailTemplate(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.user = baker.make(UserProfile, email="user@example.com")
        cls.additional_cc = baker.make(UserProfile, email="additional@example.com")
        cls.template = EmailTemplate.objects.get(name=EmailTemplate.EDITOR_REVIEW_NOTICE)

    @staticmethod
    def test_missing_email_address():
        """
        Tests send_to_user when the user has no email address.
        Regression test to https://github.com/e-valuation/EvaP/issues/825
        """
        user = baker.make(UserProfile, email=None)
        template = EmailTemplate.objects.get(name=EmailTemplate.STUDENT_REMINDER)
        template.send_to_user(user, {}, {}, False, None)

    def test_send_multi_alternatives_email(self):
        template = EmailTemplate(
            subject="Example", plain_content="Example plain content", html_content="<p>Example html content</p>"
        )
        template.send_to_user(self.user, {}, {}, False)
        self.assertEqual(len(mail.outbox), 1)
        self.assertTrue(isinstance(mail.outbox[0], mail.message.EmailMultiAlternatives))
        self.assertEqual(mail.outbox[0].body, "Example plain content")
        self.assertEqual(len(mail.outbox[0].alternatives), 1)
        self.assertEqual(mail.outbox[0].alternatives[0][1], "text/html")
        self.assertIn("<p>Example html content</p>", mail.outbox[0].alternatives[0][0])

    def test_plain_content_escaped_and_copied_on_empty_html_content(self):
        template = EmailTemplate(subject="Subject <>&", plain_content="A\nB <>& {{ some_var }}", html_content="")
        template.send_to_user(self.user, {}, {"some_var": "0 < 1"}, False)

        self.assertEqual(len(mail.outbox), 1)
        message = mail.outbox[0]
        self.assertEqual(message.alternatives[0][1], "text/html")
        html_content = message.alternatives[0][0]

        self.assertEqual("Subject <>&", message.subject)
        self.assertEqual("A\nB <>& 0 < 1", message.body)
        self.assertIn("A<br>B <>& 0 < 1\n", html_content)
        self.assertNotIn("<>&", html_content)

    def test_escaping_with_html_content(self):
        template = EmailTemplate(
            subject="Subject <>&",
            plain_content="A\nB <>& {{ some_var }}",
            html_content="Html content &<br/> {{ some_var }}",
        )
        template.send_to_user(self.user, {}, {"some_var": "0 < 1"}, False)

        self.assertEqual(len(mail.outbox), 1)
        message = mail.outbox[0]
        self.assertEqual(message.alternatives[0][1], "text/html")
        html_content = message.alternatives[0][0]

        self.assertEqual("Subject <>&", message.subject)
        self.assertEqual("A\nB <>& 0 < 1", message.body)
        self.assertIn("Html content &<br/> 0 < 1", html_content)
        self.assertNotIn("<>&", html_content)

    def test_put_delegates_in_cc(self):
        delegate_a = baker.make(UserProfile, email="delegate-a@example.com")
        delegate_b = baker.make(UserProfile, email="delegate-b@example.com")
        self.user.delegates.add(delegate_a, delegate_b)
        self.template.send_to_user(self.user, {}, {}, use_cc=True)

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(set(mail.outbox[0].cc), {delegate_a.email, delegate_b.email})

    def test_put_cc_users_in_cc(self):
        cc_a = baker.make(UserProfile, email="cc-a@example.com")
        cc_b = baker.make(UserProfile, email="cc-b@example.com")
        self.user.cc_users.add(cc_a, cc_b)
        self.template.send_to_user(self.user, {}, {}, use_cc=True)

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(set(mail.outbox[0].cc), {cc_a.email, cc_b.email})

    def test_put_additional_cc_users_in_cc(self):
        additional_cc_b = baker.make(UserProfile, email="additional-b@example.com")
        self.template.send_to_user(
            self.user, {}, {}, use_cc=True, additional_cc_users=[self.additional_cc, additional_cc_b]
        )

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(set(mail.outbox[0].cc), {self.additional_cc.email, additional_cc_b.email})

    def test_put_delegates_of_additional_cc_user_in_cc(self):
        additional_delegate_a = baker.make(UserProfile, email="additional-delegate-a@example.com")
        additional_delegate_b = baker.make(UserProfile, email="additional-delegate-b@example.com")
        self.additional_cc.delegates.add(additional_delegate_a, additional_delegate_b)
        self.template.send_to_user(self.user, {}, {}, use_cc=True, additional_cc_users=[self.additional_cc])

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(
            set(mail.outbox[0].cc), {self.additional_cc.email, additional_delegate_a.email, additional_delegate_b.email}
        )

    def test_cc_does_not_contain_duplicates(self):
        user_a = baker.make(UserProfile, email="a@example.com")
        user_b = baker.make(UserProfile, email="b@example.com")
        user_c = baker.make(UserProfile, email="c@example.com")
        self.user.delegates.add(user_a)
        self.user.cc_users.add(self.additional_cc, user_b)
        self.additional_cc.delegates.add(user_b, user_c)
        self.additional_cc.cc_users.add(user_a, user_c)
        self.template.send_to_user(self.user, {}, {}, use_cc=True, additional_cc_users=[self.additional_cc])

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(len(mail.outbox[0].cc), 4)
        self.assertEqual(set(mail.outbox[0].cc), {self.additional_cc.email, user_a.email, user_b.email, user_c.email})

    def test_disable_cc(self):
        delegate = baker.make(UserProfile, email="delegate@example.com")
        cc_user = baker.make(UserProfile, email="cc@example.com")
        self.user.delegates.add(delegate)
        self.user.cc_users.add(cc_user)
        self.additional_cc.delegates.add(delegate)
        self.additional_cc.cc_users.add(cc_user)
        self.template.send_to_user(self.user, {}, {}, use_cc=False, additional_cc_users=[self.additional_cc])

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(set(mail.outbox[0].cc), {self.additional_cc.email})

    @staticmethod
    def test_send_contributor_publish_notifications():
        responsible1 = baker.make(UserProfile)
        responsible2 = baker.make(UserProfile)
        # use is_single_result to get can_publish_average_grade to become true
        evaluation1 = baker.make(Evaluation, course__responsibles=[responsible1], is_single_result=True)
        evaluation2 = baker.make(Evaluation, course__responsibles=[responsible2])

        editor1 = baker.make(UserProfile)
        contributor1 = baker.make(UserProfile)

        contributor2 = baker.make(UserProfile)
        editor2 = baker.make(UserProfile)
        contributor_both = baker.make(UserProfile)

        # Contributions for evaluation1
        make_contributor(responsible1, evaluation1)
        make_contributor(contributor1, evaluation1)
        make_contributor(contributor_both, evaluation1)
        make_editor(editor1, evaluation1)

        # Contributions for evaluation2
        make_editor(editor2, evaluation2)
        contributor_both_contribution = make_contributor(contributor_both, evaluation2)
        contributor2_contribution = make_contributor(contributor2, evaluation2)

        baker.make(TextAnswer, contribution=contributor_both_contribution)
        baker.make(TextAnswer, contribution=contributor2_contribution)

        expected_calls = [
            # these 4 are included since they are contributors for evaluation1 which can publish the average grade
            call(responsible1, {}, {"user": responsible1, "evaluations": {evaluation1}}, use_cc=True),
            call(editor1, {}, {"user": editor1, "evaluations": {evaluation1}}, use_cc=True),
            call(contributor1, {}, {"user": contributor1, "evaluations": {evaluation1}}, use_cc=True),
            call(
                contributor_both, {}, {"user": contributor_both, "evaluations": {evaluation1, evaluation2}}, use_cc=True
            ),
            # contributor2 has textanswers, so they are notified
            call(contributor2, {}, {"user": contributor2, "evaluations": {evaluation2}}, use_cc=True),
        ]

        with patch("evap.evaluation.models.EmailTemplate.send_to_user") as send_to_user_mock:
            EmailTemplate.send_contributor_publish_notifications({evaluation1, evaluation2})
            # Assert that all expected publish notifications are sent to contributors.
            send_to_user_mock.assert_has_calls(expected_calls, any_order=True)

        # if general textanswers for an evaluation exist, all responsibles should also be notified
        baker.make(TextAnswer, contribution=evaluation2.general_contribution)
        expected_calls.append(call(responsible2, {}, {"user": responsible2, "evaluations": {evaluation2}}, use_cc=True))

        with patch("evap.evaluation.models.EmailTemplate.send_to_user") as send_to_user_mock:
            EmailTemplate.send_contributor_publish_notifications({evaluation1, evaluation2})
            send_to_user_mock.assert_has_calls(expected_calls, any_order=True)

class TestEmailRecipientList(TestCase):
    def test_recipient_list(self):
        evaluation = baker.make(Evaluation)
        responsible = baker.make(UserProfile)
        editor = baker.make(UserProfile)
        contributor = baker.make(UserProfile)
        evaluation.course.responsibles.set([responsible])
        baker.make(Contribution, evaluation=evaluation, contributor=editor, role=Contribution.Role.EDITOR)
        baker.make(Contribution, evaluation=evaluation, contributor=contributor)

        participant1 = baker.make(UserProfile, evaluations_participating_in=[evaluation])
        participant2 = baker.make(UserProfile, evaluations_participating_in=[evaluation])
        evaluation.voters.set([participant1])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(evaluation, [], filter_users_in_cc=False)
        self.assertCountEqual(recipient_list, [])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.RESPONSIBLE], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [responsible])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.EDITORS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [responsible, editor])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.CONTRIBUTORS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [responsible, editor, contributor])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.ALL_PARTICIPANTS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [participant1, participant2])

        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.DUE_PARTICIPANTS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [participant2])

    def test_recipient_list_filtering(self):
        evaluation = baker.make(Evaluation)

        contributor1 = baker.make(UserProfile)
        contributor2 = baker.make(UserProfile, delegates=[contributor1])

        baker.make(Contribution, evaluation=evaluation, contributor=contributor1)
        baker.make(Contribution, evaluation=evaluation, contributor=contributor2)

        # no-one should get filtered.
        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.CONTRIBUTORS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [contributor1, contributor2])

        # contributor1 is in cc of contributor2 and gets filtered.
        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.CONTRIBUTORS], filter_users_in_cc=True
        )
        self.assertCountEqual(recipient_list, [contributor2])

        contributor3 = baker.make(UserProfile, delegates=[contributor2])
        baker.make(Contribution, evaluation=evaluation, contributor=contributor3)

        # again, no-one should get filtered.
        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.CONTRIBUTORS], filter_users_in_cc=False
        )
        self.assertCountEqual(recipient_list, [contributor1, contributor2, contributor3])

        # contributor1 is in cc of contributor2 and gets filtered.
        # contributor2 is in cc of contributor3 but is not filtered since contributor1 wouldn't get an email at all then.
        recipient_list = EmailTemplate.recipient_list_for_evaluation(
            evaluation, [EmailTemplate.Recipients.CONTRIBUTORS], filter_users_in_cc=True
        )
        self.assertCountEqual(recipient_list, [contributor2, contributor3])

class QuestionnaireTests(TestCase):
    def test_locked_contributor_questionnaire(self):
        questionnaire = baker.prepare(Questionnaire, is_locked=True, type=Questionnaire.Type.CONTRIBUTOR)
        self.assertRaises(ValidationError, questionnaire.clean)


import json
import logging
from abc import abstractmethod

import attr
import gevent

from metadata.backend.interface import RawBackendType
from metadata.event_system.register import EventType
from metadata.middleware.rabbitmq import ExchangeType, RabbitMQProducer
from metadata.runtime import rt_context
from metadata.type_system.core import MetaData
from metadata.util.common import StrictABCMeta, camel_to_snake
from metadata_biz.types import default_registry

module_logger = logging.getLogger(__name__)
listener_registry = {}
"""
监听者注册表
"""

def listener_registrar(cls):
    """
    监听者注册装饰器

    :param cls: 监听者类
    :return: boolean True
    """
    listener_name = cls.name if cls.name else camel_to_snake(cls.__name__)
    if listener_name not in listener_registry:
        listener_registry[listener_name] = cls
    return cls

def listener_dispatcher(target_listener, message_dict):
    """
    监听者调度器

    :param target_listener: list 目标监听者列表
    :param message_dict: dict 传输内容
    :return: None
    """
    event_processed = False
    try:
        with gevent.Timeout(0.5, exception=False):
            for listener_name in target_listener:
                listener_cls = listener_registry[listener_name] if listener_name in listener_registry else None
                listener = listener_cls(message_dict)
                if listener and listener.followed_events and listener.check_event():
                    listener.publish_event()
                    break
            event_processed = True
    except Exception as le:
        # Todo: 暂时仅告警，后续补上记录和补录的步骤
        module_logger.warning('[Event_system] event dispatch failed: {}'.format(le))
    finally:
        module_logger.info('[Event_system] event process state: {}'.format(event_processed))

class ListenerPrototype(object, metaclass=StrictABCMeta):
    __abstract__ = True
    followed_events = []

    def __init__(self):
        """
        监听者根据实际情况设置关注事件列表
        """
        self.db_conf = rt_context.config_collection.db_config
        self.follow_registered_event()

    @abstractmethod
    def follow_registered_event(self):
        """
        监听者注册关注的事件列表
        :return:
        """
        return

    @abstractmethod
    def check_event(self, *args, **kwargs):
        """
        检查是否满足触发事件的条件
        :param args:
        :param kwargs:
        :return:
        """
        return

    @abstractmethod
    def publish_event(self):
        """
        触发事件发布消息
        :return:
        """
        return

@listener_registrar
class DgraphMutationListener(ListenerPrototype):
    """
    Dgraph元数据变更事件监听者
    """

    name = 'dgraph_mutation_listener'

    def __init__(self, message_dict):
        """
        对<dgraph主集群>的元数据变更进行监听

        :param message_dict: dict 监听内容
        """

        self.logger = module_logger
        self.captured_event = None
        self.message_dict = message_dict
        backend_type = message_dict.get('target', {}).get('type')
        backend_id = message_dict.get('target', {}).get('id')
        master_id = None
        try:
            system_state = getattr(rt_context, str('system_state'), {})
            master_id = system_state.cache['backends_ha_state'][backend_type]['master']['id']
        except Exception as e:
            self.logger.warning('[backends_ha_state is not existed in zk cache] {}'.format(e))
        if backend_type == RawBackendType.DGRAPH.value:
            if backend_id and backend_id == master_id:
                super(DgraphMutationListener, self).__init__()

    @staticmethod
    def get_md_cls(type_name):
        if type_name:
            return default_registry.get(type_name, None)
        return None

    @staticmethod
    def _get_unicode_value(val):
        """
        获取unicode字符返回

        :param val: 输入
        :return: unicode输出
        """
        if not val:
            return ''
        if isinstance(val, str):
            return val
        try:
            ret_val = str(val)
        except Exception as e:
            module_logger.warning('[get unicode val failed] {}'.format(e))
            ret_val = val
        return ret_val

    def standardize_data(self, mutate_record, mutate_result):
        """
        标准化变更前数据和变更后数据

        :param mutate_record: 变更记录
        :param mutate_result: 变更结果
        :return: tuple (变更前数据dict, 变更后数据dict)
        """
        origin_data_ret = mutate_result.get('data', {}).get('queries', {}).get('get_subject_info', [])
        origin_data_ret = origin_data_ret.pop(0) if origin_data_ret else dict()
        origin_data = {
            key.replace('{}.'.format(mutate_record.type_name), ''): self._get_unicode_value(val)
            for key, val in list(origin_data_ret.items())
            if key.startswith(mutate_record.type_name)
        }
        changed_data = json.loads(getattr(mutate_record, 'changed_data', '{}'))
        changed_data = {
            key: self._get_unicode_value(val)
            for key, val in list(changed_data.items())
            if key not in ('created_at', 'updated_at', 'typed')
        }
        return origin_data, changed_data

    def follow_registered_event(self):
        """
        关注元数据变更类型的已注册事件
        :return:
        """
        self.followed_events = rt_context.event_list.get(EventType.MUTATION.value, None)

    def check_event(self, *args, **kwargs):
        """
        检查事件是否触发

        :return: boolean True/False
        """

        dispatch_id = self.message_dict.get('dispatch_id', None)
        mutate_result = self.message_dict.get('result', {})
        mutate_record = self.message_dict.get('record', {})
        md_cls = self.get_md_cls(mutate_record.type_name)
        md_mapping = {attr_def.name: attr_def for attr_def in attr.fields(md_cls)}
        method = mutate_record.method
        identifier_key = md_cls.metadata['identifier'].name if md_cls else None
        identifier_value = mutate_record.identifier_value
        extra = mutate_record.extra_
        origin_data, changed_data = self.standardize_data(mutate_record, mutate_result)
        if identifier_key not in origin_data:
            origin_data[identifier_key] = identifier_value
        differ = {
            key: {'b': origin_data.get(key, None), 'a': changed_data[key]}
            for key in list(changed_data.keys())
            if changed_data[key] != origin_data.get(key, None)
        }
        if identifier_key not in differ:
            differ[identifier_key] = dict()
        event_info = dict(
            dispatch_id=dispatch_id,
            md_name=md_cls.__name__,
            identifier_key=identifier_key,
            identifier_value=identifier_value,
            triggered_by=extra.get('updated_by', ''),
            triggered_at=extra.get('updated_at', ''),
            origin_data=origin_data,
            changed_data=changed_data,
            differ=differ,
        )
        # 主键需要第一个检查
        check_list = [check_key for check_key in list(differ.keys()) if check_key != identifier_key]
        check_list.insert(0, identifier_key)
        for pred_name in check_list:
            attr_def = md_mapping.get(pred_name, None)
            if not attr_def:
                continue
            is_link = True if issubclass(attr_def.type, MetaData) else False
            is_identifier = True if identifier_key == pred_name else False
            capture_types = getattr(attr_def, 'metadata', {}).get('event', [])
            if not capture_types:
                continue
            scope_dict = dict(
                name=attr_def.name,
                method=method,
                capture_types=capture_types,
                is_identifier=is_identifier,
                is_foreignkey=True if is_link and not is_identifier else False,
            )
            # 检查捕获到的事件, 因各事件之间互斥，检测到一个即可
            for event_name, event_cls in list(self.followed_events.items()):
                if event_cls.in_scope(scope_dict):
                    self.captured_event = event_cls(event_info)
                    return True
        return False

    def publish_event(self):
        """
        发布事件

        :return: None
        """

        event_message = self.captured_event.gen_message()
        producer = RabbitMQProducer(
            conn_addr=self.db_conf.RABBITMQ_ADDR,
            exchange_name='meta_event_system',
            exchange_type=ExchangeType.TOPIC.value,
            routing_key='{}.{}'.format(self.captured_event.namespace, self.captured_event.event_type),
        )
        producer.publish(event_message)
        producer.release()

@listener_registrar
class ExceptionListener(ListenerPrototype):
    """
    错误事件监听者
    """

    name = 'exception_listener'

    def follow_registered_event(self):
        """
        关注错误类型的已注册事件
        :return:
        """
        self.followed_events = rt_context.event_list.get(EventType.EXCEPTION.value, None)

    def check_event(self, *args, **kwargs):
        pass

    def publish_event(self):
        pass


#!/usr/bin/env python
# coding: utf-8

# In[6]:

from sklearn.metrics import r2_score,mean_absolute_error,mean_squared_error

from sklearn import datasets
iris = datasets.load_iris()
# print(iris.DESCR)

# In[9]:

targ = iris.target
# print(targ)
# print(iris.target_names)

# In[20]:

data = iris.data
# print(data)

# In[22]:

from sklearn.model_selection import train_test_split

data_train,data_test,targ_train,targ_test=train_test_split(data,targ,test_size=0.2,random_state=101)

# In[25]:

from sklearn.linear_model import LinearRegression

LRclf = LinearRegression()
LRclf.fit(data_train,targ_train)

# In[26]:

targ_pred = LRclf.predict(data_test)
targ_pred

# In[27]:

# import matplotlib.pyplot as plt

#sepal length
# plt.scatter(data_test[:,0],targ_pred)
# plt.show()

# In[28]:

#sepal width
# plt.scatter(data_test[:,1],targ_pred)
# plt.show()

# In[29]:

#petal length
# plt.scatter(data_test[:,2],targ_pred)
# plt.show()

# In[30]:

#petal width
# plt.scatter(data_test[:,3],targ_pred)
# plt.show()

# In[35]:

import matplotlib.pyplot as plt

fig = plt.figure(figsize = (15, 10))
plt.subplot(2, 2, 1)

plt.title('Sepal Length')
plt.scatter(data_test[:,0],targ_pred)

plt.subplot(2, 2, 2)
plt.title('Sepal Width')
plt.scatter(data_test[:,1],targ_pred)

plt.subplot(2, 2, 3)
plt.title('Petal Width')
plt.scatter(data_test[:,2],targ_pred)

plt.subplot(2, 2, 4)
plt.title('Petal Width')
plt.scatter(data_test[:,3],targ_pred)

# In[ ]:


from typing import Optional, Dict

import pandas as pd

from kbc_pul.popularity.entity_counting.total_ent_count import EntityTripleCountController

EntityStr = str

class EntityCountInOtherRelationsBothPositions:

    def __init__(self,
                 target_relation: str,
                 entity_triple_count_controller: EntityTripleCountController,
                 verbose: bool = False
                 ):
        self.target_relation: str = target_relation
        self.verbose: bool = verbose
        self.entity_triple_count_controller: EntityTripleCountController = entity_triple_count_controller

        self.o_entity_to_count_map: Optional[
            Dict[
                EntityStr,
                int
            ]
        ] = None

        self.relations_to_include = self.entity_triple_count_controller.get_all_relations_but(
            {target_relation}
        )
        self.positions_to_include = None

    def get_count(self, entity: EntityStr) -> int:
        if self.o_entity_to_count_map is not None:
            return self.o_entity_to_count_map.get(entity, 0)
        else:
            return self.entity_triple_count_controller.get_count_of(
                entity=entity,
                relations_to_include=self.relations_to_include,
                positions_to_include=self.positions_to_include
            )

    def are_counts_materialized(self) -> bool:
        return self.o_entity_to_count_map is not None

    def materialize_entity_counts(self) -> None:
        if self.verbose:
            print("Materializing entity counts...")
        self.o_entity_to_count_map = self.entity_triple_count_controller.materialize_entity_to_count_map(
            relations_to_include=self.relations_to_include,
            positions_to_include=self.positions_to_include
        )

    def get_entity_counts_as_df(self) -> pd.DataFrame:
        if not self.are_counts_materialized():
            self.materialize_entity_counts()
        return EntityTripleCountController.entity_to_count_map_to_df(self.o_entity_to_count_map)


import numpy as np
import matplotlib.pyplot as plt

gridWorld = np.zeros((6, 6))
tmp = np.zeros((6, 6))
theta = 0.01

if_convergence = False
while not if_convergence:
    delta = 0
    for i in range(6):
        for j in range(6):
            if (i == 0 and j == 1) or (i == 5 and j == 5):
                continue
            else:
                # i--
                gridW = gridWorld[i][j-1] - 1 if j > 0 else gridWorld[i][j] - 1
                # i++
                gridE = gridWorld[i][j+1] - 1 if j < 5 else gridWorld[i][j] - 1
                # j--
                gridN = gridWorld[i-1][j] - 1 if i > 0 else gridWorld[i][j] - 1
                # j++
                gridS = gridWorld[i+1][j] - 1 if i < 5 else gridWorld[i][j] - 1
                tmp[i][j] = 0.25 * (gridS + gridN + gridE + gridW)
                delta = max(delta, abs(tmp[i][j] - gridWorld[i][j]))
                gridWorld[i][j] = tmp[i][j]
    print(delta)
    if_convergence = True if (delta < theta) else False

print(gridWorld)
plt.matshow(gridWorld)
plt.show()
mat = [[1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1],
       [1, 1, 1, 1, 1, 1]]
plt.matshow(mat)
for i in range(6):
    for j in range(6):
        west = gridWorld[i][j-1] - gridWorld[i][j] if (j > 0) else 0
        east = gridWorld[i][j+1] - gridWorld[i][j] if (j < 5) else 0
        north = gridWorld[i-1][j] - gridWorld[i][j] if (i > 0) else 0
        south = gridWorld[i+1][j] - gridWorld[i][j] if (i < 5) else 0
        go = ""
        max_n = max(west, east, south, north)
        if west == max_n:
            go += "w /"
        if east == max_n:
            go += "e /"
        if south == max_n:
            go += "s /"
        if north == max_n:
            go += "n /"
        go = go[:-1]
        plt.text(x=j, y=i, s=go)
plt.show()


from random import shuffle
import pygame as pg
from lib.config import *

def shuffler (r):

    shuff = [i for i in range(r)]
    shuffleagain = 1

    while shuffleagain:
        # re-shuffle until no two sequential cards are neighbors
        shuffle(shuff)
        idx = 0
        for j in range(r - 1):
            if shuff[j] - shuff[j+1] == 1 or shuff[j] - shuff[j+1] == -1:
                shuffleagain = 1
            else:
                idx += 1
            if idx == 5:
                shuffleagain = 0
                return shuff

def scaler (file, factor):
    file = pg.image.load(file)
    return pg.transform.scale(file, factor)

def mouse_chk (mp, pos, size):
    return (mp[0]>=pos.x) and mp[0]<=(pos.x + size) and mp[1]<=(pos.y + size) and mp[1]>=(pos.y)


from numpy import *

def loadDataSet(fileName):
    dataMat = []
    fr = open(fileName)
    for line in fr.readlines():
        curLine = line.strip().split('\t')
        fltLine = list(map(float,curLine))###map()返回值已经不再是list,而是iterators；所以需要将iterator 转换成list，即list(map())
        dataMat.append(fltLine)
    return dataMat

def distEclud(vecA, vecB):##欧几里得距离
    return sqrt(sum(power(vecA - vecB, 2)))

def randCent(dataSet, k):##生成k个中心
    n = shape(dataSet)[1]##列数
    centroids = mat(zeros((k,n)))
    for j in range(n):
        minJ = min(dataSet[:,j])#每一列最值
        #print(minJ)
        rangeJ = float(max(dataSet[:,j]) - minJ)#最大值与最小值差
        #print(rangeJ)
        centroids[:,j] = mat(minJ + rangeJ * random.rand(k,1))#生成k*1的矩阵，每个数在最小值和最大值之间。
    return centroids

def kMeans(dataSet, k, distMeas=distEclud, createCent=randCent):##得到k近邻中心和每个数据最近的中心分布。
    m = shape(dataSet)[0]#行数
    #print(m)
    clusterAssment = mat(zeros((m,2)))
    centroids = createCent(dataSet, k)
    clusterChanged = True
    while clusterChanged:
        clusterChanged = False
        for i in range(m):
            minDist = inf; minIndex = -1
            for j in range(k):
                distJI = distMeas(centroids[j,:],dataSet[i,:])#测量每个数据与每个中心的距离
                if distJI < minDist:
                    minDist = distJI; minIndex = j
            if clusterAssment[i,0] != minIndex: clusterChanged = True##只要原数据中有一个点对应的最近的距离点编号有改变就再次循环。
            ##如果所有的数据对应的最近的中心点不发生任何改变，则在下一个循环中每一个中心点所收集的原数据相同，则不会生成新的k近邻点。
            ##所有最后一次调用时下面的部分多余。可以考虑添加if判断停止循环。
            clusterAssment[i,:] = minIndex,minDist**2##存储最近的中心点及距离平方
        #print (centroids)
        if clusterChanged==False:return centroids, clusterAssment##最后一次循环可省略下面的for循环部分。
        for cent in range(k):#recalculate centroids
            ptsInClust = dataSet[nonzero(clusterAssment[:,0].A==cent)[0]]#最后的[0]是输出的符合要求的点的存储点号的原数据
            ##这里nonzero内部的值是True,False；返回的是True值的位置。
            #比如aaa=mat([[0,1],[1,1]]),nonzero(aaa[:,0].A==0)，aaa[:,0].A==0 返回的是 True, False; 所以nonzero返回的是array[0]，array[0]
            #print(ptsInClust)
            #print(shape(ptsInClust))
            centroids[cent,:] = mean(ptsInClust, axis=0) #
        #print(centroids)
        #print()
    return centroids, clusterAssment

def biKmeans(dataSet, k, distMeas=distEclud):
    m = shape(dataSet)[0]
    clusterAssment = mat(zeros((m,2)))
    centroid0 = mean(dataSet, axis=0).tolist()[0]
    #print(centroid0)#
    centList =[centroid0]
    #print(len(centList))#1
    for j in range(m):#calc initial Error
        clusterAssment[j,1] = distMeas(mat(centroid0), dataSet[j,:])**2 #第二项存储中心点与每个数据距离平方
    while (len(centList) < k):#k限制了总的循环次数
        lowestSSE = inf
        for i in range(len(centList)):
            #print(i)
            ptsInCurrCluster = dataSet[nonzero(clusterAssment[:,0].A==i)[0],:]
            #aido=dataSet[nonzero(clusterAssment[:,0].A==i)[0]]#和上面的写法结果一致。
            centroidMat, splitClustAss = kMeans(ptsInCurrCluster, 2, distMeas)#对剥离出来的部分数据采取k近邻方法，生成2个近邻点。
            #print(shape(splitClustAss))
            sseSplit = sum(splitClustAss[:,1])##
            sseNotSplit = sum(clusterAssment[nonzero(clusterAssment[:,0].A!=i)[0],1])#第一次循环到这里时所有都为0，即该行为空。因为初始设定时值没有改变。
            #第二次循环到这里时，假设此时i=0，则计算前次分组i=1时的所有距离。而sseSplit计算的是前次分组i=0时的数据中再次划分两个点之后的所有距离。
            #print(shape(clusterAssment[nonzero(clusterAssment[:,0].A!=i)[0],1]))
            #print ("sseSplit, and notSplit: ",sseSplit,sseNotSplit)
            if (sseSplit + sseNotSplit) < lowestSSE:
                bestCentToSplit = i
                bestNewCents = centroidMat
                bestClustAss = splitClustAss.copy()
                lowestSSE = sseSplit + sseNotSplit
        print(lowestSSE)
        bestClustAss[nonzero(bestClustAss[:,0].A == 1)[0],0] = len(centList)
        bestClustAss[nonzero(bestClustAss[:,0].A == 0)[0],0] = bestCentToSplit
        #print(len(centList))
        #print(shape(bestClustAss))
        #print ('the bestCentToSplit is: ',bestCentToSplit)
        #print ('the len of bestClustAss is: ', len(bestClustAss))
        centList[bestCentToSplit] = bestNewCents[0,:].tolist()[0]#replace a centroid with two best centroids
        centList.append(bestNewCents[1,:].tolist()[0])##这两行需要先转成list才能最后运行mat()
        clusterAssment[nonzero(clusterAssment[:,0].A == bestCentToSplit)[0],:]= bestClustAss
    return mat(centList), clusterAssment

import urllib.request
import json
def geoGrab(stAddress, city):
    apiStem = 'http://where.yahooapis.com/geocode?'  #create a dict and constants for the goecoder
    params = {}
    params['flags'] = 'J'#JSON return type
    params['appid'] = 'ppp68N8t'
    params['location'] = '%s %s' % (stAddress, city)
    url_params = urllib.parse.urlencode(params)
    yahooApi = apiStem + url_params      #print url_params
    print (yahooApi)
    c=urllib.request.urlopen(yahooApi)
    return json.loads(c.read())

from time import sleep
def massPlaceFind(fileName):
    fw = open('places.txt', 'w')
    for line in open(fileName).readlines():
        line = line.strip()
        lineArr = line.split('\t')
        retDict = geoGrab(lineArr[1], lineArr[2])
        if retDict['ResultSet']['Error'] == 0:
            lat = float(retDict['ResultSet']['Results'][0]['latitude'])
            lng = float(retDict['ResultSet']['Results'][0]['longitude'])
            print ("%s\t%f\t%f" % (lineArr[0], lat, lng))
            fw.write('%s\t%f\t%f\n' % (line, lat, lng))
        else: print ("error fetching")
        sleep(1)
    fw.close()


from dataclasses import dataclass
import warnings

from cropharvest.countries import BBox
from cropharvest.datasets import Task, CropHarvestLabels, CropHarvest
from cropharvest.config import TEST_REGIONS, TEST_DATASETS
from cropharvest import countries

from .config import TEST_CROP_TO_CLASSIFICATION

from typing import List, Optional, Union, Tuple

@dataclass
class TIMLTask(Task):
    classification_label: Optional[str] = None

    def __post_init__(self):
        if self.target_label is None:
            self.target_label = "crop"
            self.classification_label = "crop"

            if self.balance_negative_crops is True:
                warnings.warn(
                    "Balance negative crops not meaningful for the crop vs. non crop tasks"
                )
        else:
            assert (
                self.classification_label is not None
            ), "Higher level classsification must be provided if a target label is provided"

        if self.bounding_box is None:
            self.bounding_box = BBox(
                min_lat=-90, max_lat=90, min_lon=-180, max_lon=180, name="global"
            )

class TIMLCropHarvestLabels(CropHarvestLabels):
    def classes_in_bbox(
        self, bounding_box: BBox, return_classifications: bool = False
    ) -> Union[List[str], List[Tuple[str, str]]]:
        bbox_geojson = self.filter_geojson(self.as_geojson(), bounding_box)
        unique_labels = [x for x in bbox_geojson.label.unique() if x is not None]
        if not return_classifications:
            return unique_labels
        else:
            label_classification_pairs: List[Tuple[str, str]] = []
            for unique_label in unique_labels:
                classification = bbox_geojson[
                    bbox_geojson.label == unique_label
                ].classification_label.iloc[0]
                label_classification_pairs.append((unique_label, classification))
            return label_classification_pairs

class TIMLCropHarvest(CropHarvest):
    @classmethod
    def create_benchmark_datasets(
        cls,
        root,
        balance_negative_crops: bool = True,
        download: bool = True,
    ) -> List:
        r"""
        Create the benchmark datasets.
        :param root: The path to the data, where the training data and labels are (or will be)
            saved
        :param balance_negative_crops: Whether to ensure the crops are equally represented in
            a dataset's negative labels. This is only used for datasets where there is a
            target_label, and that target_label is a crop
        :param download: Whether to download the labels and training data if they don't
            already exist
        :returns: A list of evaluation CropHarvest datasets according to the TEST_REGIONS and
            TEST_DATASETS in the config
        """

        output_datasets: List = []

        for identifier, bbox in TEST_REGIONS.items():
            country, crop, _, _ = identifier.split("_")

            country_bboxes = countries.get_country_bbox(country)
            for country_bbox in country_bboxes:
                task = TIMLTask(
                    country_bbox,
                    crop,
                    balance_negative_crops,
                    f"{country}_{crop}",
                    classification_label=TEST_CROP_TO_CLASSIFICATION[crop].name,
                )
                if task.id not in [x.id for x in output_datasets]:
                    if country_bbox.contains_bbox(bbox):
                        output_datasets.append(cls(root, task, download=download))

        for country, test_dataset in TEST_DATASETS.items():
            # TODO; for now, the only country here is Togo, which
            # only has one bounding box. In the future, it might
            # be nice to confirm its the right index (maybe by checking against
            # some points in the test h5py file?)
            country_bbox = countries.get_country_bbox(country)[0]
            output_datasets.append(
                cls(
                    root,
                    TIMLTask(country_bbox, None, test_identifier=test_dataset),
                    download=download,
                )
            )
        return output_datasets


import copy
import imp
import numpy as np
from ...utils import common_utils
from .nuscenes_dataset import NuScenesDataset
from collections import defaultdict
import torch
from ...ops.roiaware_pool3d import roiaware_pool3d_utils
from pathlib import Path

class NuScenesDatasetMT(NuScenesDataset):
    def __init__(self, dataset_cfg, class_names, training=True, root_path=None, logger=None):
        super().__init__(
            dataset_cfg=dataset_cfg, class_names=class_names, training=training, root_path=root_path, logger=logger
        )
    def __getitem__(self, index):
        if self._merge_all_iters_to_one_epoch:
            index = index % len(self.infos)

        info = copy.deepcopy(self.infos[index])
        points = self.get_lidar_with_sweeps(index, max_sweeps=self.dataset_cfg.MAX_SWEEPS)

        if self.dataset_cfg.get('SHIFT_COOR', None):
            points[:, 0:3] += np.array(self.dataset_cfg.SHIFT_COOR, dtype=np.float32)

        input_dict = {
            'points': points,
            'frame_id': Path(info['lidar_path']).stem,
            'metadata': {'token': info['token']}
        }

        if 'gt_boxes' in info:
            if self.dataset_cfg.get('FILTER_MIN_POINTS_IN_GT', False):
                mask = (info['num_lidar_pts'] > self.dataset_cfg.FILTER_MIN_POINTS_IN_GT - 1)
            else:
                mask = None

            input_dict.update({
                'gt_names': info['gt_names'] if mask is None else info['gt_names'][mask],
                'gt_boxes': info['gt_boxes'] if mask is None else info['gt_boxes'][mask]
            })

            if self.dataset_cfg.get('SHIFT_COOR', None):
                input_dict['gt_boxes'][:, 0:3] += self.dataset_cfg.SHIFT_COOR

            if self.dataset_cfg.get('USE_PSEUDO_LABEL', None) and self.training:
                input_dict['gt_boxes'] = None

            # for debug only
            # gt_boxes_mask = np.array([n in self.class_names for n in input_dict['gt_names']], dtype=np.bool_)
            # debug_dict = {'gt_boxes': copy.deepcopy(input_dict['gt_boxes'][gt_boxes_mask])}

        if self.dataset_cfg.get('FOV_POINTS_ONLY', None):
            input_dict['points'] = self.extract_fov_data(
                input_dict['points'], self.dataset_cfg.FOV_DEGREE, self.dataset_cfg.FOV_ANGLE
            )
            if input_dict['gt_boxes'] is not None:
                fov_gt_flag = self.extract_fov_gt(
                    input_dict['gt_boxes'], self.dataset_cfg.FOV_DEGREE, self.dataset_cfg.FOV_ANGLE
                )
                input_dict.update({
                    'gt_names': input_dict['gt_names'][fov_gt_flag],
                    'gt_boxes': input_dict['gt_boxes'][fov_gt_flag],
                })

        if self.dataset_cfg.get('USE_PSEUDO_LABEL', None) and self.training:
            self.fill_pseudo_labels(input_dict)

        if self.dataset_cfg.get('SET_NAN_VELOCITY_TO_ZEROS', False) and not self.dataset_cfg.get('USE_PSEUDO_LABEL', None):
            gt_boxes = input_dict['gt_boxes']
            gt_boxes[np.isnan(gt_boxes)] = 0
            input_dict['gt_boxes'] = gt_boxes

        if not self.dataset_cfg.PRED_VELOCITY and 'gt_boxes' in input_dict and not self.dataset_cfg.get('USE_PSEUDO_LABEL', None):
            input_dict['gt_boxes'] = input_dict['gt_boxes'][:, [0, 1, 2, 3, 4, 5, 6]]

        if self.dataset_cfg.get('USE_PSEUDO_LABEL', None) and self.training:
            self.fill_pseudo_labels(input_dict)

        data_dict1 = copy.deepcopy(input_dict)
        data_dict2 = copy.deepcopy(input_dict)
        data_dict1 = self.prepare_data(data_dict=data_dict1)
        if isinstance(data_dict1, list):    # len(data_dict['gt_boxes']) == 0
            return data_dict1
        data_dict2 = self.prepare_data_teacher(data_dict=data_dict2)
        if isinstance(data_dict2, list):    # len(data_dict['gt_boxes']) == 0
            return data_dict2

        output = [data_dict1, data_dict2]
        return output

    def prepare_data_teacher(self, data_dict):
        """
        Args:
            data_dict:
                points: optional, (N, 3 + C_in)
                gt_boxes: optional, (N, 7 + C) [x, y, z, dx, dy, dz, heading, ...]
                gt_names: optional, (N), string
                ...

        Returns:
            data_dict:
                frame_id: string
                points: (N, 3 + C_in)
                gt_boxes: optional, (N, 7 + C) [x, y, z, dx, dy, dz, heading, ...]
                gt_names: optional, (N), string
                use_lead_xyz: bool
                voxels: optional (num_voxels, max_points_per_voxel, 3 + C)
                voxel_coords: optional (num_voxels, 3)
                voxel_num_points: optional (num_voxels)
                ...
        """
        if self.training:
            # filter gt_boxes without points
            num_points_in_gt = data_dict.get('num_points_in_gt', None)
            if num_points_in_gt is None:
                num_points_in_gt = roiaware_pool3d_utils.points_in_boxes_cpu(
                    torch.from_numpy(data_dict['points'][:, :3]),
                    torch.from_numpy(data_dict['gt_boxes'][:, :7])).numpy().sum(axis=1)

            mask = (num_points_in_gt >= self.dataset_cfg.get('MIN_POINTS_OF_GT', 1))
            data_dict['gt_boxes'] = data_dict['gt_boxes'][mask]
            data_dict['gt_names'] = data_dict['gt_names'][mask]
            if 'gt_classes' in data_dict:
                data_dict['gt_classes'] = data_dict['gt_classes'][mask]
                data_dict['gt_scores'] = data_dict['gt_scores'][mask]

            assert 'gt_boxes' in data_dict, 'gt_boxes should be provided for training'
            gt_boxes_mask = np.array([n in self.class_names for n in data_dict['gt_names']], dtype=np.bool_)

            ## note: teacher has no augmentation
            if self.dataset_cfg.get('TEACHER_RBRS', None):
                augmentor_configs = self.dataset_cfg.DATA_AUGMENTOR.AUG_CONFIG_LIST
                aug_config_list = augmentor_configs if isinstance(augmentor_configs, list) \
                    else augmentor_configs.AUG_CONFIG_LIST
                for cur_cfg in aug_config_list:
                    if cur_cfg.NAME == 'random_beam_downsample':
                        beam = cur_cfg.BEAM
                        beam_prob = cur_cfg.BEAM_PROB
                points = data_dict['points']
                beam_label = data_dict['beam_labels']
                beam_mask = np.random.rand(beam) < beam_prob
                points_mask = beam_mask[beam_label]
                data_dict['points'] = points[points_mask]

            data_dict['gt_boxes'] = data_dict['gt_boxes'][gt_boxes_mask]
            data_dict['gt_names'] = data_dict['gt_names'][gt_boxes_mask]

            if 'gt_boxes2d' in data_dict:
                data_dict['gt_boxes2d'] = data_dict['gt_boxes2d'][gt_boxes_mask]
            if len(data_dict['gt_boxes']) == 0:
                new_index = np.random.randint(self.__len__())
                return self.__getitem__(new_index)

        if data_dict.get('gt_boxes', None) is not None:
            selected = common_utils.keep_arrays_by_name(data_dict['gt_names'], self.class_names)
            data_dict['gt_boxes'] = data_dict['gt_boxes'][selected]
            data_dict['gt_names'] = data_dict['gt_names'][selected]
            # for pseudo label has ignore labels.
            if 'gt_classes' not in data_dict:
                gt_classes = np.array([self.class_names.index(n) + 1 for n in data_dict['gt_names']], dtype=np.int32)
            else:
                gt_classes = data_dict['gt_classes'][selected]
                data_dict['gt_scores'] = data_dict['gt_scores'][selected]
            gt_boxes = np.concatenate((data_dict['gt_boxes'], gt_classes.reshape(-1, 1).astype(np.float32)), axis=1)
            data_dict['gt_boxes'] = gt_boxes

            if data_dict.get('gt_boxes2d', None) is not None:
                data_dict['gt_boxes2d'] = data_dict['gt_boxes2d'][selected]

        if data_dict.get('points', None) is not None:
            data_dict = self.point_feature_encoder.forward(data_dict)

        data_dict = self.data_processor.forward(
            data_dict=data_dict
        )

        if self.training and len(data_dict['gt_boxes']) == 0:
            new_index = np.random.randint(self.__len__())
            return self.__getitem__(new_index)

        data_dict.pop('gt_names', None)
        data_dict.pop('gt_classes', None)

        return data_dict

    @staticmethod
    def collate_batch(batch_list, _unused=False):
        def collate_fn(batch_list):
            data_dict = defaultdict(list)
            for cur_sample in batch_list:
                for key, val in cur_sample.items():
                    data_dict[key].append(val)
            batch_size = len(batch_list)
            ret = {}

            for key, val in data_dict.items():
                try:
                    if key in ['voxels', 'voxel_num_points']:
                        ret[key] = np.concatenate(val, axis=0)
                    elif key in ['points', 'voxel_coords']:
                        coors = []
                        for i, coor in enumerate(val):
                            coor_pad = np.pad(coor, ((0, 0), (1, 0)), mode='constant', constant_values=i)
                            coors.append(coor_pad)
                        ret[key] = np.concatenate(coors, axis=0)
                    elif key in ['gt_boxes']:
                        max_gt = max([len(x) for x in val])
                        batch_gt_boxes3d = np.zeros((batch_size, max_gt, val[0].shape[-1]), dtype=np.float32)
                        for k in range(batch_size):
                            batch_gt_boxes3d[k, :val[k].__len__(), :] = val[k]
                        ret[key] = batch_gt_boxes3d
                    elif key in ['gt_scores']:
                        max_gt = max([len(x) for x in val])
                        batch_scores = np.zeros((batch_size, max_gt), dtype=np.float32)
                        for k in range(batch_size):
                            batch_scores[k, :val[k].__len__()] = val[k]
                        ret[key] = batch_scores
                    elif key in ['gt_boxes2d']:
                        max_boxes = 0
                        max_boxes = max([len(x) for x in val])
                        batch_boxes2d = np.zeros((batch_size, max_boxes, val[0].shape[-1]), dtype=np.float32)
                        for k in range(batch_size):
                            if val[k].size > 0:
                                batch_boxes2d[k, :val[k].__len__(), :] = val[k]
                        ret[key] = batch_boxes2d
                    elif key in ["images", "depth_maps"]:
                        # Get largest image size (H, W)
                        max_h = 0
                        max_w = 0
                        for image in val:
                            max_h = max(max_h, image.shape[0])
                            max_w = max(max_w, image.shape[1])

                        # Change size of images
                        images = []
                        for image in val:
                            pad_h = common_utils.get_pad_params(desired_size=max_h, cur_size=image.shape[0])
                            pad_w = common_utils.get_pad_params(desired_size=max_w, cur_size=image.shape[1])
                            pad_width = (pad_h, pad_w)
                            # Pad with nan, to be replaced later in the pipeline.
                            pad_value = np.nan

                            if key == "images":
                                pad_width = (pad_h, pad_w, (0, 0))
                            elif key == "depth_maps":
                                pad_width = (pad_h, pad_w)

                            image_pad = np.pad(image,
                                            pad_width=pad_width,
                                            mode='constant',
                                            constant_values=pad_value)

                            images.append(image_pad)
                        ret[key] = np.stack(images, axis=0)
                    elif key in ['object_scale_noise', 'object_rotate_noise']:
                        max_noise = max([len(x) for x in val])
                        batch_noise = np.zeros((batch_size, max_noise), dtype=np.float32)
                        for k in range(batch_size):
                            batch_noise[k, :val[k].__len__()] = val[k]
                        ret[key] = batch_noise
                    elif key in ['beam_labels']:
                        continue
                    else:
                        ret[key] = np.stack(val, axis=0)
                except:
                    print('Error in collate_batch: key=%s' % key)
                    raise TypeError

            ret['batch_size'] = batch_size
            return ret

        if isinstance(batch_list[0], dict):
            return collate_fn(batch_list)

        else:
            assert isinstance(batch_list[0], list)
            batch_list1 = [x[0] for x in batch_list]
            batch_list2 = [x[1] for x in batch_list]
            ret1 = collate_fn(batch_list1)
            ret2 = collate_fn(batch_list2)
            return [ret1, ret2]


__author__ = 'Xavier Basty <xbasty@gmail.com>'
__version__ = '0.1a3'

# The base package metadata to be used by both distutils and setuptools
METADATA = dict(
  name = "grapefruit",
  version = __version__,
  py_modules = ['grapefruit'],
  author = 'Xavier Basty',
  author_email = 'xbasty@gmail.com',
  description = 'A module to manipulate color information easily.',
  license = 'Apache License 2.0',
  url = 'http://code.google.com/p/grapefruit/',
  download_url = 'http://grapefruit.googlecode.com/files/grapefruit-%s.tar.gz' % \
    __version__,
  keywords ='color conversion',
)

# Extra package metadata to be used only if setuptools is installed
SETUPTOOLS_METADATA = dict(
  install_requires = ['setuptools'],
  include_package_data = True,
  zip_safe = True,
  classifiers = [
    'Development Status :: 3 - Alpha',
    'Intended Audience :: Developers',
    'License :: OSI Approved :: Apache Software License',
    'Topic :: Software Development :: Libraries :: Python Modules',
    'Topic :: Multimedia :: Graphics',
  ],
  test_suite = 'grapefruit_test.suite',
)

def Read(file):
  return open(file).read()

def BuildLongDescription():
  return '\n'.join([Read('README'), Read('CHANGES')])

def Main():
  # Build the long_description from the README and CHANGES files
  METADATA['long_description'] = BuildLongDescription()

  # Use setuptools if available, otherwise fallback and use distutils
  try:
    import setuptools
    METADATA.update(SETUPTOOLS_METADATA)
    setuptools.setup(**METADATA)
  except ImportError:
    import distutils.core
    distutils.core.setup(**METADATA)

if __name__ == '__main__':
  Main()


from neuron import h

h.load_file('stdrun.hoc')

soma = h.Section(name='soma')
seg = soma(0.5)
soma.insert('hh')

ic = h.IClamp(seg)
ic.amp = 40  # since by default a very large section
ic.delay = 1
ic.dur = 1

h.xpanel("neuron")
h.xlabel("voltage graph")
g = h.Graph()
g.addvar("v", "v(0.5)")   #hoc string example
g.addvar("m", seg.hh._ref_m)   #pointer example
#g.addvar("v", soma(0.5)._ref_v) #pointer
#g.addvar("v(0.5)")   #no label (not yet)

h.xpanel()

h.finitialize(-65)


import numpy as np
import networkx as nx
import scrubber
import model
import os.path

def neglog(c):
    return -np.log((c + 1)/2)

def heavyneglog(c):
    return (- np.log((c + 1)/2)) ** 5

def invfun(c):
    return (1 - c)/(1 + c)

def cov(c):
    return c

def cotan(c):
    return 1/np.tan(np.pi * (c + 1)/ 2)

def read_or_scrub (year_from,year_to,fun = cov):
    cache_string_cov = "cache/covariances_{}_{}.csv".format(year_from,year_to)
    cache_string_list = "cache/countrylist_{}_{}.csv".format(year_from,year_to)
    if os.path.exists(cache_string_cov) == False or\
       os.path.exists(cache_string_list) == False:
        vote_matrix, country_list = scrubber.scrub_un_docs(year_from,year_to)

        coalition_matrix = np.matmul(vote_matrix,np.transpose(vote_matrix))

        average_votes = np.sum(vote_matrix,1) / vote_matrix.shape[1]
        covariance_matrix = coalition_matrix - vote_matrix.shape[1] * np.outer(average_votes,average_votes)

        stddevs = np.zeros(covariance_matrix.shape[0])
        for i in range(0,covariance_matrix.shape[0]):
            if (covariance_matrix[i][i] <= 0):
                print(country_list[i], covariance_matrix[i][i], vote_matrix[i])
            stddevs[i] = np.sqrt(covariance_matrix[i][i])

        for i in range(0,covariance_matrix.shape[0]):
            covariance_matrix[i][i] = 0
            for j in range(i + 1,covariance_matrix.shape[1]):
                covariance_matrix[i,j] = covariance_matrix[i,j]/(stddevs[i] * stddevs[j])
                covariance_matrix[j,i] = covariance_matrix[j,i]/(stddevs[i] * stddevs[j])

        voting_matrix = np.zeros(covariance_matrix.shape)
        for i in range(0,covariance_matrix.shape[0]):
            for j in range(0,covariance_matrix.shape[1]):
                voting_matrix[i,j] = fun(covariance_matrix[i,j])

        covariance_graph = nx.from_numpy_matrix(covariance_matrix)
        voting_graph = nx.from_numpy_matrix(voting_matrix)
        nx.write_weighted_edgelist(covariance_graph,cache_string_cov,delimiter=';')
        country_list_file = open(cache_string_list,"w")
        for str in country_list:
            country_list_file.write(str)
            country_list_file.write("\n")
        country_list_file.close()
        return voting_graph, country_list, voting_matrix
    else:
        covariance_graph = nx.read_weighted_edgelist(cache_string_cov,delimiter=';')
        country_list_file = open(cache_string_list,'r')
        country_list = country_list_file.read().split("\n")
        country_list_file.close()
        covariance_matrix = nx.to_numpy_matrix(covariance_graph)
        voting_matrix = np.zeros(covariance_matrix.shape)
        for i in range(0,covariance_matrix.shape[0]):
            for j in range(0,covariance_matrix.shape[1]):
                voting_matrix[i,j] = fun(covariance_matrix[i,j])

        voting_graph = nx.from_numpy_matrix(voting_matrix)

        return voting_graph, country_list, voting_matrix

def filter_graph (G, filter_dict, verbose = False) :
    bad_edges = []
    for i in range(len(filter_dict)):
        for j in range(i + 1, len(filter_dict)):
            if filter_dict[i] == filter_dict[j]:
                bad_edges.append((i,j))
                bad_edges.append((j,i))
    if verbose:
        print ("Removing {} edges".format(len(bad_edges)))
    G.remove_edges_from(bad_edges)
    return G

def generate_filtered_record (year_from ,  year_to, blocs):
    cache_string_graph = "cache/filtered_{}_{}_{}.csv".format(year_from,year_to,blocs)
    cache_string_list = "cache/countrylist_{}_{}.csv".format(year_from,year_to)
    if os.path.exists(cache_string_graph) == False or\
       os.path.exists(cache_string_list) == False:
        voting_graph, country_list, voting_matrix = read_or_scrub(year_from,year_to)
        indset_dict = model.optimize_scoring(voting_matrix, blocs)
        filtered_graph = filter_graph(voting_graph, indset_dict)
        nx.write_weighted_edgelist(filtered_graph,cache_string_graph,delimiter=';')
        country_list_file = open(cache_string_list,"w")
        for str in country_list:
            country_list_file.write(str)
            country_list_file.write("\n")
        country_list_file.close()
        return filtered_graph, country_list
    else:
        filtered_graph = nx.read_weighted_edgelist(cache_string_graph,delimiter=';')
        country_list_file = open(cache_string_list,"r")
        country_list = country_list_file.read().split("\n")
        country_list_file.close()
        return filtered_graph, country_list

def spell_community_difference(G, Graw, res, country_list):
    c = nx.algorithms.community.greedy_modularity_communities(G, weight = 'weight', resolution = res)
    c_raw = nx.algorithms.community.greedy_modularity_communities(Graw, weight = 'weight', resolution = res)

    min, max = 0,0
    if len(c_raw) > len(c):
        min = len(c)
        max = len(c_raw)
    else:
        min = len(c_raw)
        max = len(c)
    for j in range(min):
        print("|Community {0:>30}|".format(j + 1))
        print("------------------------------------------------------------------------------------")
        for i in range(len(country_list)):
            bold_str = "*{}*".format(country_list[i])
            if i in c_raw[j] and str(i) in c[j]:
                print("|{0:>40}|{1:>41}|".format(country_list[i],country_list[i]))
            elif i in c_raw[j]:
                print("|                                        |{0:41}|".format(bold_str))
            elif str(i) in c[j]:
                print("|{0:>40}|                                         |".format(bold_str))
    for j in range(min,max):
        print("|Community {0:>30}|".format(j + 1))
        print("------------------------------------------------------------------------------------")
        for i in range(len(country_list)):
            bold_str = "*{}*".format(country_list[i])
            if min == len(c):
                if i in c_raw[j]:
                    print("|                                        |{0:>40}|".format(bold_str))
            else:
                if str(i) in c[j]:
                    print("|{0:>40}|                                         |".format(bold_str))


#!/usr/bin/env python

from setuptools import setup, find_packages
import os

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(name='franca2ros',
      version='0.0.1',
      description='Franca IDL to ROS2 translator',
      author='Mikhail Tsukerman',
      packages=find_packages(include=['franca2ros.*','franca2ros.translator.*']),
      author_email='miketsukerman@gmail.com',
      license="Apache2",
      keywords="fidl someip ros2",
      long_description=read('readme.md'),
      install_requires=[
          'jinja2',
          'pyfranca',
          'argparse'
      ],
      entry_points={
          'console_scripts': ['fidl2ros=franca2ros.scripts.fidl2ros:main']
      }
      )