#!/usr/bin/python
# coding: utf-8
__author__ = 'lhfcws'

from setuptools import setup, find_packages

setup(
      name="rdir",
      version="0.52",
      description="More powerful recursive dir. Support HTML pretty view in tree structure.",
      long_description="Documentation and bug repot: http://www.github.com/Lhfcws/rdir\n \
                       Sorry for the deploy bugs in the previous versions\n \
                       If you came across ImportError or NoScriptError, please `rm /usr/local/bin/rdir*`",
      author="Lhfcws Wu",
      author_email="lhfcws@gmail.com",
      url="http://www.github.com/Lhfcws/rdir",
      license="MIT",
      packages=["rdir", "rdir/core", "rdir/generateHTML"],
      include_package_data=True,
      package_data={"rdir": ["rdir/generateHTML/template/*.html", "rdir/generateHTML/bin/*.js"]},
      install_requires=['colorama', 'pyquery'],
      keywords=["dir", "doc", "pydoc", "html"],
)


import time

def app(environ, start_response):
    if environ['PATH_INFO'] == '/sleepy':
        time.sleep(2)
        body = b'sleepy returned'
    else:
        body = b'notsleepy returned'
    cl = str(len(body))
    start_response(
        '200 OK',
        [('Content-Length', cl), ('Content-Type', 'text/plain')]
        )
    return [body]

if __name__ == '__main__':
    from waitress.tests.support import start_server
    start_server(app)


import collections

# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def widthOfBinaryTree(self, root):
        """
        :type root: TreeNode
        :rtype: int
        """

        res = 0
        value_list = collections.deque([(root, 0)])

        if not root:

            return res

        while value_list:
            temp_value = len(value_list)
            res = max(res, value_list[-1][1] - value_list[0][1] + 1)

            for _ in range(temp_value):
                root, cost = value_list.popleft()

                if root.left:
                    value_list.append((root.left, cost * 2))

                if root.right:
                    value_list.append((root.right, cost * 2 + 1))

        return res


"""Fire up the browser to search for missing images."""

import os
import sys
import urllib.parse
import webbrowser
import unicodedata

import pyperclip

duckduckgo = 'https://duckduckgo.com/?t=canonical&q=%s&iax=images&ia=images&iaf=type%%3Aclipart'

images = {
    unicodedata.normalize('NFC', os.path.splitext(f)[0])
    for f in os.listdir('img')
}

with open(sys.argv[1]) as fin:
    for line in fin:
        try:
            syllable, word = line.strip().split(' ')
        except ValueError:
            continue

        word = unicodedata.normalize('NFC', word)
        if word not in images:
            pyperclip.copy(word)
            url = duckduckgo % urllib.parse.quote(word)
            webbrowser.open(url)
            input('searched for %r, press Enter to continue or Ctrl+C to exit' % word)


import json
from unittest.case import TestCase

import pytest
from django.urls import reverse
from rest_framework import status
from rest_framework.test import APIClient

from audit.models import AuditLog, RelatedObjectType
from environments.models import Environment
from features.models import Feature, FeatureSegment
from organisations.models import Organisation, OrganisationRole
from projects.models import Project
from segments.models import Segment
from util.tests import Helper

@pytest.mark.django_db
class FeatureSegmentViewSetTestCase(TestCase):
    def setUp(self) -> None:
        self.client = APIClient()
        user = Helper.create_ffadminuser()
        self.client.force_authenticate(user=user)

        organisation = Organisation.objects.create(name="Test Org")

        user.add_organisation(organisation, OrganisationRole.ADMIN)

        self.project = Project.objects.create(
            organisation=organisation, name="Test project"
        )
        self.environment_1 = Environment.objects.create(
            project=self.project, name="Test environment 1"
        )
        self.environment_2 = Environment.objects.create(
            project=self.project, name="Test environment 2"
        )
        self.feature = Feature.objects.create(project=self.project, name="Test feature")
        self.segment = Segment.objects.create(project=self.project, name="Test segment")

    def test_list_feature_segments(self):
        # Given
        base_url = reverse("api-v1:features:feature-segment-list")
        url = (
            f"{base_url}?environment={self.environment_1.id}&feature={self.feature.id}"
        )
        segment_2 = Segment.objects.create(project=self.project, name="Segment 2")
        segment_3 = Segment.objects.create(project=self.project, name="Segment 3")

        FeatureSegment.objects.create(
            feature=self.feature, segment=self.segment, environment=self.environment_1
        )
        FeatureSegment.objects.create(
            feature=self.feature, segment=segment_2, environment=self.environment_1
        )
        FeatureSegment.objects.create(
            feature=self.feature, segment=segment_3, environment=self.environment_1
        )
        FeatureSegment.objects.create(
            feature=self.feature, segment=self.segment, environment=self.environment_2
        )

        # When
        response = self.client.get(url)

        # Then
        assert response.status_code == status.HTTP_200_OK
        response_json = response.json()
        assert response_json["count"] == 3
        for result in response_json["results"]:
            assert result["environment"] == self.environment_1.id

    def test_create_feature_segment(self):
        # Given
        data = {
            "feature": self.feature.id,
            "segment": self.segment.id,
            "environment": self.environment_1.id,
        }
        url = reverse("api-v1:features:feature-segment-list")

        # When
        response = self.client.post(
            url, data=json.dumps(data), content_type="application/json"
        )

        # Then
        assert response.status_code == status.HTTP_201_CREATED
        response_json = response.json()
        assert response_json["id"]

    def test_delete_feature_segment(self):
        # Given
        feature_segment = FeatureSegment.objects.create(
            feature=self.feature, environment=self.environment_1, segment=self.segment
        )
        url = reverse(
            "api-v1:features:feature-segment-detail", args=[feature_segment.id]
        )

        # When
        response = self.client.delete(url)

        # Then
        assert response.status_code == status.HTTP_204_NO_CONTENT
        assert not FeatureSegment.objects.filter(id=feature_segment.id).exists()

    def test_audit_log_created_when_feature_segment_created(self):
        # Given
        url = reverse("api-v1:features:feature-segment-list")
        data = {
            "segment": self.segment.id,
            "feature": self.feature.id,
            "environment": self.environment_1.id,
        }

        # When
        response = self.client.post(url, data=data)

        # Then
        assert response.status_code == status.HTTP_201_CREATED
        assert (
            AuditLog.objects.filter(
                related_object_type=RelatedObjectType.FEATURE.name
            ).count()
            == 1
        )

    def test_priority_of_multiple_feature_segments(self):
        # Given
        url = reverse("api-v1:features:feature-segment-update-priorities")

        # another segment and 2 feature segments for the same feature / the 2 segments
        another_segment = Segment.objects.create(
            name="Another segment", project=self.project
        )
        feature_segment_default_data = {
            "environment": self.environment_1,
            "feature": self.feature,
        }
        feature_segment_1 = FeatureSegment.objects.create(
            segment=self.segment, **feature_segment_default_data
        )
        feature_segment_2 = FeatureSegment.objects.create(
            segment=another_segment, **feature_segment_default_data
        )

        # reorder the feature segments
        assert feature_segment_1.priority == 0
        assert feature_segment_2.priority == 1
        data = [
            {"id": feature_segment_1.id, "priority": 1},
            {"id": feature_segment_2.id, "priority": 0},
        ]

        # When
        response = self.client.post(
            url, data=json.dumps(data), content_type="application/json"
        )

        # Then the segments are reordered
        assert response.status_code == status.HTTP_200_OK
        json_response = response.json()
        assert json_response[0]["id"] == feature_segment_1.id
        assert json_response[1]["id"] == feature_segment_2.id


# -*- coding: utf-8 -*-
from django.db import connection
from collections import namedtuple
from operator import itemgetter
from itertools import groupby

from django.db.models import Q

from common.lib.utils import dictfetchall
from common.lib.date_utils import create_date_range
from data_sources.models import YandexDirectStat, GoogleAdwordsStat, DirectPhraseStat, AdwordsPhraseStat, \
    DirectBannerStat, AdwordsBannerStat
from data_sources.serializers import YandexDirectStatQuerySetSerializer, GoogleAdwordsStatQuerySetSerializer, \
    DirectPhraseStatQuerySetSerializer, AdwordsPhraseStatQuerySetSerializer, DirectBannerStatQuerySetSerializer, \
    AdwordsBannerStatQuerySetSerializer
from statistics.lib.xls_stats_builder import StatsXlsTotalBuilder, StatsXlsCampaignBuilder, StatsXlsPhraseBuilder, \
    StatsXlsBannerBuilder

DIRECT_NAME = 'Yandex Direct'
ADWORDS_NAME = 'Google Adwords'

class StatsGetterTotal(object):

    ADWORDS_NAME = ADWORDS_NAME
    DIRECT_NAME = DIRECT_NAME

    def __init__(self, project, start_date, end_date, **kwargs):
        self.project = project
        self.range = create_date_range(start_date, end_date)

        self.previous_range = None
        prev_range = kwargs.get('prev_range')
        if prev_range:
            if prev_range == 'previous':
                self.previous_range = self.range.create_previous()
            else:
                dates = prev_range.split('_')
                self.previous_range = create_date_range(*dates)

    def get_stats_summary(self):
        populated_record_direct = self._create_stats_record(
            YandexDirectStat, self.DIRECT_NAME, DirectRecordUpdater
        )

        populated_record_adwords = self._create_stats_record(
            GoogleAdwordsStat, self.ADWORDS_NAME, AdwordsRecordUpdater
        )

        return [populated_record_direct, populated_record_adwords]

    def _create_stats_record(self, model_cls, source_name, record_updater_cls):
        stats = self._get_serialized_stats(model_cls, self.range)
        record = self._get_initial_record(source_name)
        populated_primary_record = self._get_record_populated_by_helper(record, stats, record_updater_cls)

        if self.previous_range:
            stats = self._get_serialized_stats(model_cls, self.previous_range)
            populated_record = self._get_record_populated_by_helper(populated_primary_record, stats, record_updater_cls.PREVIOUS)
        else:
            populated_record = populated_primary_record
        return populated_record

    def get_stats_summary_as_xls(self):
        stats = self.get_stats_summary()
        xls_builder = self._get_xls_builder()
        return xls_builder.get_xls_obj(stats)

    def _get_xls_builder(self):
        return StatsXlsTotalBuilder()

    def _get_serialized_stats(self, model_cls, _range):
        select_related_strings = [
            'campaign'
        ]
        stats = model_cls.objects.filter(
            campaign__data_resource__projects=self.project,
            date__range=_range.as_tuple
        ).select_related(*select_related_strings)

        qs_serializer = model_cls.get_qs_serializer()(
            stats, select_related_strings=select_related_strings
        )

        return qs_serializer.serialize()

    def _get_record_populated_by_helper(self, record, stats, helper_cls):
        updater = helper_cls(record)
        for st in stats:
            updater.update_by(st)
        updater.after_update()
        return record

    def _get_initial_record(self, source):
        return {
            'source': source,
            'sessions': 0,
            'shows': 0,
            'clicks': 0,
            'cost': 0,
            'clicks_search': 0,
            'shows_search': 0,
            'shows_context': 0,
            'shows_search_premium': 0,
            'shows_search_other': 0,
            'clicks_positions': 0,
            'shows_positions': 0,
            'income': 0,
            'leads': 0,
            'users': 0,
            'new_users': 0,
            'page_views': 0,
            'bounces': 0,
            'session_duration': 0,
            'transaction_revenue': 0,
            'prev_sessions': 0,
            'prev_shows': 0,
            'prev_clicks': 0,
            'prev_cost': 0,
            'prev_clicks_search': 0,
            'prev_shows_context': 0,
            'prev_shows_search': 0,
            'prev_shows_search_premium': 0,
            'prev_shows_search_other': 0,
            'prev_clicks_positions': 0,
            'prev_shows_positions': 0,
            'prev_income': 0,
            'prev_leads': 0,
            'prev_users': 0,
            'prev_new_users': 0,
            'prev_page_views': 0,
            'prev_bounces': 0,
            'prev_session_duration': 0,
            'prev_transaction_revenue': 0,
        }

class StatsGetterItem(StatsGetterTotal):
    DIRECT_STAT_MODEL = None
    ADWORDS_STAT_MODEL = None

    def __init__(self, project, start_date, end_date, **kwargs):
        super(StatsGetterItem, self).__init__(project, start_date, end_date, **kwargs)
        self.source_name = kwargs.get('source_name')

    def get_stats_summary(self):
        stats = self.get_direct_stats()
        stats.extend(self.get_adwords_stats())
        return stats

    def is_adwords(self):
        return self.source_name == self.ADWORDS_NAME.replace(' ', '_').lower()

    def get_direct_stats(self):
        return self._get_source_stats(self.DIRECT_STAT_MODEL, self.DIRECT_NAME, DirectRecordUpdater)

    def get_adwords_stats(self):
        return self._get_source_stats(self.ADWORDS_STAT_MODEL, self.ADWORDS_NAME, AdwordsRecordUpdater)

    def _get_source_stats(self, stat_model, source_name, record_updater):
        group_func = self._get_group_func()
        serialized_stats = self._get_stats(stat_model, self.range)

        prev_serialized_stats = []
        if self.previous_range:
            prev_serialized_stats = self._get_stats(stat_model, self.previous_range)

        result = self._get_result(
            serialized_stats, prev_serialized_stats, group_func, source_name, record_updater
        )
        return result

    def _get_result(self, serialized_stats, prev_serialized_stats, group_func, source_name, record_updater):
        serialized_stats.sort(key=group_func)
        prev_serialized_stats.sort(key=group_func)

        result = []
        prev_dict = {}
        for item, groupped_stats in groupby(prev_serialized_stats, key=group_func):
            stats_list = list(groupped_stats)
            ids_dict = self._get_items_ids_dict(item, stats_list)
            prev_dict[ids_dict['id']] = stats_list

        for item, groupped_stats in groupby(serialized_stats, key=group_func):
            stats_list = list(groupped_stats)
            record = self._get_initial_record(source_name, item, stats_list)

            ids_dict = self._get_items_ids_dict(item, stats_list)

            populated_primary_record = self._get_record_populated_by_helper(record, stats_list, record_updater)
            populated_record = self._get_record_populated_by_helper(
                populated_primary_record,
                prev_dict.get(ids_dict['id'], []),
                record_updater.PREVIOUS
            )

            result.append(populated_record)
        return result

    def _get_items_ids_dict(self, item, stats_list):
        return {'id': item['id']}

    def _get_stats(self, model_cls, _range):
        select_related_strings = self._get_select_related_strings()
        stats = model_cls.objects.filter(
            self._get_stat_filter_q(),
            date__range=_range.as_tuple
        ).select_related(*select_related_strings)

        serialized_stats = model_cls.get_qs_serializer()(
            stats, select_all_relations=False,
            select_related_strings=select_related_strings
        ).serialize()
        return serialized_stats

    def _get_initial_record(self, source, item, stats_list):
        return super(StatsGetterItem, self)._get_initial_record(source)

    def _get_stat_filter_q(self):
        raise NotImplementedError

    def _get_group_func(self):
        raise NotImplementedError

    def _get_select_related_strings(self):
        raise NotImplementedError

class StatsGetterCampaign(StatsGetterItem):

    DIRECT_STAT_MODEL = YandexDirectStat
    ADWORDS_STAT_MODEL = GoogleAdwordsStat

    def _get_stat_filter_q(self):
        return Q(campaign__data_resource__projects=self.project)

    def _get_group_func(self):
        return itemgetter('campaign')

    def _get_select_related_strings(self):
        return ['campaign']

    def _get_xls_builder(self):
        return StatsXlsCampaignBuilder()

    def _get_initial_record(self, source_name, campaign, stats_list):
        record = super(StatsGetterCampaign, self)._get_initial_record(source_name, campaign, stats_list)
        record.update({
            'campaign_id': campaign['id'],
            'campaign': campaign['name']
        })
        return record

class StatsGetterPhrase(StatsGetterItem):

    DIRECT_STAT_MODEL = DirectPhraseStat
    ADWORDS_STAT_MODEL = AdwordsPhraseStat

    def get_direct_stats(self):
        return self._get_source_stats(self._get_direct_sql, self.DIRECT_NAME, DirectRecordUpdater)

    def get_adwords_stats(self):
        return self._get_source_stats(self._get_adwords_sql, self.ADWORDS_NAME, AdwordsRecordUpdater)

    def _get_xls_builder(self):
        return StatsXlsPhraseBuilder()

    def _get_source_stats(self, sql_getter, source_name, record_updater):
        group_func = self._get_group_func()
        serialized_stats = self._get_stats(sql_getter, self.range)

        prev_serialized_stats = []
        if self.previous_range:
            prev_serialized_stats = self._get_stats(sql_getter, self.previous_range)

        result = self._get_result(
            serialized_stats, prev_serialized_stats, group_func, source_name, record_updater
        )
        return result

    def _get_initial_record(self, source_name, item, stats_list):
        record = super(StatsGetterPhrase, self)._get_initial_record(source_name, item, stats_list)

        ids_dict = self._get_items_ids_dict(item, stats_list)

        record.update({
            'phrase': item.phrase,
            'phrase_id': ids_dict['id'],
            'system_id': ids_dict['system_id'],
            'campaign': item.campaign_name,
            'campaign_id': item.campaign_id
        })
        return record

    def _get_stats(self, sql_getter, _range):
        with connection.cursor() as cursor:
            params = [self.project.pk, _range.start.isoformat(), _range.end.isoformat()]
            select_sql = sql_getter()
            cursor.execute(select_sql, params)
            return dictfetchall(cursor)

    def _get_direct_sql(self):
        return """
            SELECT
                dps.*,
                dp.id dp_phrase_id, dp.phrase_id phrase_external_id, dp.phrase, dp.banner_id,
                ydc.id campaign_id, ydc.name campaign_name
            FROM direct_phrase_stat dps
            JOIN direct_phrase dp ON dps.phrase_id=dp.id
            JOIN yandex_direct_campaign ydc ON dp.campaign_id=ydc.id
            JOIN projects_data_resources pdr ON ydc.data_resource_id=pdr.data_resource_id
            WHERE pdr.project_id=%s AND dps.date BETWEEN %s AND %s;
        """

    def _get_adwords_sql(self):
        return """
            SELECT
                aps.*,
                ap.id dp_phrase_id, ap.phrase_id phrase_external_id, ap.phrase,
                ac.id campaign_id, ac.name campaign_name
            FROM adwords_phrase_stat aps
            JOIN adwords_phrase ap ON aps.phrase_id=ap.id
            JOIN adwords_campaign ac ON ap.campaign_id=ac.id
            JOIN projects_data_resources pdr ON ac.data_resource_id=pdr.data_resource_id
            WHERE pdr.project_id=%s AND aps.date BETWEEN %s AND %s;
        """

    def _get_items_ids_dict(self, item, stats_list):
        item_id = stats_list[0]['dp_phrase_id']
        system_id = item.phrase_external_id
        if not system_id:
            item_id = self._get_unique_joined_string(stats_list, lambda st: str(st['dp_phrase_id']))
            system_id = self._get_unique_joined_string(stats_list, lambda st: str(st['banner_id']))

        return {
            'id': item_id,
            'system_id': system_id
        }

    def _get_unique_joined_string(self, stats_list, getter_fn):
        return ','.join(set(map(getter_fn, stats_list)))

    def _get_stat_filter_q(self):
        return Q(phrase__campaign__data_resource__projects=self.project)

    def _get_select_related_strings(self):
        return ['phrase__campaign']

    def _get_group_func(self):
        nt = namedtuple('GroupItem', ['phrase_external_id', 'phrase', 'campaign_id', 'campaign_name'])
        return lambda n: nt(n['phrase_external_id'], n['phrase'], n['campaign_id'], n['campaign_name'])

class StatsGetterBanner(StatsGetterItem):

    DIRECT_STAT_MODEL = DirectBannerStat
    ADWORDS_STAT_MODEL = AdwordsBannerStat

    def _get_initial_record(self, source_name, item, stats_list):
        record = super(StatsGetterBanner, self)._get_initial_record(source_name, item, stats_list)
        record.update({
            'banner': item['title'],
            'banner_id': item['id'],
            'campaign': item['campaign']['name'],
            'campaign_id': item['campaign']['id']
        })
        return record

    def _get_stat_filter_q(self):
        return Q(banner__campaign__data_resource__projects=self.project)

    def _get_xls_builder(self):
        return StatsXlsBannerBuilder()

    def _get_select_related_strings(self):
        return ['banner__campaign']

    def _get_group_func(self):
        return lambda n: {k: v for k, v in n['banner'].iteritems() if k != 'date_to'}

class RecordUpdaterBase(object):

    def __init__(self, record):
        self.handled_stats_count = 0
        self.record = record

    def update_by(self, st):
        self.handled_stats_count += 1

    def after_update(self):
        pass

class CommonRecordUpdater(RecordUpdaterBase):

    def update_by(self, st):
        super(CommonRecordUpdater, self).update_by(st)
        self.record['sessions'] += st['sessions']
        self.record['leads'] += st['leads']
        self.record['users'] += st['users']
        self.record['new_users'] += st['new_users']
        self.record['page_views'] += st['page_views']
        self.record['bounces'] += st['bounces']
        self.record['session_duration'] += st['session_duration']
        self.record['transaction_revenue'] += st['transaction_revenue']
        self.record['clicks_positions'] += st['clicks_positions']
        self.record['shows_positions'] += st['shows_positions']

class CommonPreviousRecordUpdater(RecordUpdaterBase):

    def update_by(self, st):
        super(CommonPreviousRecordUpdater, self).update_by(st)
        self.record['prev_sessions'] += st['sessions']
        self.record['prev_leads'] += st['leads']
        self.record['prev_users'] += st['users']
        self.record['prev_new_users'] += st['new_users']
        self.record['prev_page_views'] += st['page_views']
        self.record['prev_bounces'] += st['bounces']
        self.record['prev_session_duration'] += st['session_duration']
        self.record['prev_transaction_revenue'] += st['transaction_revenue']
        self.record['prev_clicks_positions'] += st['clicks_positions']
        self.record['prev_shows_positions'] += st['shows_positions']

class DirectPreviousRecordUpdater(CommonPreviousRecordUpdater):

    def update_by(self, st):
        super(DirectPreviousRecordUpdater, self).update_by(st)
        self.record['prev_shows'] += st['shows_search'] + st['shows_context']
        self.record['prev_cost'] += st['sum_search'] + st['sum_context']
        self.record['prev_clicks'] += st['clicks_search'] + st['clicks_context']
        self.record['prev_clicks_search'] += st['clicks_search']
        self.record['prev_shows_search'] += st['shows_search']
        self.record['prev_shows_search_premium'] += st['shows_search_premium']
        self.record['prev_shows_search_other'] += st['shows_search_other']

    def after_update(self):
        super(DirectPreviousRecordUpdater, self).after_update()
        self.record['prev_avg_shows_position'] = 0
        self.record['prev_avg_clicks_position'] = 0
        if self.record['prev_shows_search']:
            self.record['prev_avg_shows_position'] = self.record['prev_shows_positions'] / self.record['prev_shows_search']
        if self.record['prev_clicks_search']:
            self.record['prev_avg_clicks_position'] = self.record['prev_clicks_positions'] / self.record['prev_clicks_search']

class AdwordsPreviousRecordUpdater(CommonPreviousRecordUpdater):

    def update_by(self, st):
        super(AdwordsPreviousRecordUpdater, self).update_by(st)
        self.record['prev_shows'] += st['shows']
        self.record['prev_cost'] += st['amount']
        self.record['prev_clicks'] += st['clicks']

class DirectRecordUpdater(CommonRecordUpdater):
    PREVIOUS = DirectPreviousRecordUpdater

    def update_by(self, st):
        super(DirectRecordUpdater, self).update_by(st)
        self.record['shows'] += st['shows_search'] + st['shows_context']
        self.record['cost'] += st['sum_search'] + st['sum_context']
        self.record['clicks'] += st['clicks_search'] + st['clicks_context']
        self.record['clicks_search'] += st['clicks_search']
        self.record['shows_search'] += st['shows_search']
        self.record['shows_search_premium'] += st['shows_search_premium']
        self.record['shows_search_other'] += st['shows_search_other']

    def after_update(self):
        super(DirectRecordUpdater, self).after_update()
        self.record['avg_shows_position'] = 0
        self.record['avg_clicks_position'] = 0
        if self.record['shows_search']:
            self.record['avg_shows_position'] = self.record['shows_positions'] / self.record['shows_search']
        if self.record['clicks_search']:
            self.record['avg_clicks_position'] = self.record['clicks_positions'] / self.record['clicks_search']

class AdwordsRecordUpdater(CommonRecordUpdater):
    PREVIOUS = AdwordsPreviousRecordUpdater

    def update_by(self, st):
        super(AdwordsRecordUpdater, self).update_by(st)
        self.record['shows'] += st['shows']
        self.record['cost'] += st['amount']
        self.record['clicks'] += st['clicks']

class DetailStatsGetter(object):
    SOURCE_NAME = None
    STAT_MODEL = None
    SERIALIZER = None

    def __init__(self, project, start_date, end_date, **kwargs):
        self.project = project
        self.start_date = start_date
        self.end_date = end_date
        self.campaign_id = kwargs.get('campaign_id')
        self.phrase_ids = kwargs.get('phrase_ids')
        self.banner_id = kwargs.get('banner_id')
        self.select_related_strings = self._get_select_related_strings()

    def get_stats_by_date(self):
        stats = self._get_stats()
        result = []
        serialized_stats = self.SERIALIZER(
            stats, select_related_strings=self.select_related_strings
        ).serialize()
        serialized_stats.sort(key=lambda n: n['date'])
        for _date, groupped_stats in groupby(serialized_stats, key=itemgetter('date')):
            stats_list = list(groupped_stats)
            record = self._get_record(_date)
            for st in stats_list:
                self._update_record(record, st)
            result.append(record)
        return result

    def _get_stats(self):
        stats = self.STAT_MODEL.objects.filter(
            campaign__data_resource__projects=self.project,
            date__range=(self.start_date, self.end_date),
        ).select_related(*self.select_related_strings)
        if self.campaign_id:
            stats = stats.filter(campaign=self.campaign_id)
        return stats

    def _get_select_related_strings(self):
        return []

    def _get_record(self, date):
        rec = self._get_initial_record()
        rec.update({
            'date': date,
            'sessions': 0,
            'shows': 0,
            'clicks': 0,
            'shows_search': 0,
            'shows_context': 0,
            'shows_search_premium': 0,
            'shows_search_other': 0,
            'clicks_search': 0,
            'shows_positions': 0,
            'clicks_positions': 0,
            'cost': 0,
            'income': 0,
            'leads': 0,
            'users': 0,
            'new_users': 0,
            'page_views': 0,
            'bounces': 0,
            'session_duration': 0,
            'transaction_revenue': 0,
        })
        return rec

    def _update_record(self, record, st):
        self.RECORD_UPDATER(record).update_by(st)

    def _get_initial_record(self):
        return {
            'source': self.SOURCE_NAME
        }

class DetailPhrasesStatsGetter(DetailStatsGetter):

    def _get_stats(self):
        return self.STAT_MODEL.objects.filter(
            self._get_item_q(),
            date__range=(self.start_date, self.end_date)
        )

    def _get_item_q(self):
        return Q(phrase__in=self.phrase_ids)

class DetailBannersStatsGetter(DetailPhrasesStatsGetter):

    def _get_item_q(self):
        return Q(banner=self.banner_id)

class DirectDetailStatsGetter(DetailStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = YandexDirectStat
    SERIALIZER = YandexDirectStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['campaign']

class AdwordsDetailStatsGetter(DetailStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = GoogleAdwordsStat
    SERIALIZER = GoogleAdwordsStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['campaign']

class DirectDetailPhrasesStatsGetter(DetailPhrasesStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = DirectPhraseStat
    SERIALIZER = DirectPhraseStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['phrase__campaign']

class AdwordsDetailPhrasesStatsGetter(DetailPhrasesStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = AdwordsPhraseStat
    SERIALIZER = AdwordsPhraseStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['phrase__campaign']

class DirectDetailBannersStatsGetter(DetailBannersStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = DirectBannerStat
    SERIALIZER = DirectBannerStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['banner__campaign']

class AdwordsDetailBannersStatsGetter(DetailBannersStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = AdwordsBannerStat
    SERIALIZER = AdwordsBannerStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['banner__campaign']


# -*- coding: utf-8 -*-
import json
import logging

from django.db.models.expressions import Window
from django.db.models.functions import RowNumber
from django.db.models import F

from django.shortcuts import render, get_object_or_404
from django.template import loader
from django.http import HttpResponse, JsonResponse, Http404
from django.views.decorators.csrf import csrf_exempt
from django.core import serializers
# from pip._vendor.requests import Response
from django.views.generic import ListView
from django.views.generic .edit import CreateView
from django.views import View
from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly
from rest_framework.response import Response

from sakila.forms import CustomerForm
from sakila.models_views import CustomerList
from sakila.permissions import UserIsOwnerTodo, IsOwnerOrReadOnly
from .models import Customer

from .models import Film

from .serializers import FilmSerializer, CustomerListSerializer
from rest_framework import generics

logger = logging.getLogger('default')
# logger = logging.getLogger(__name__)
# logging.disable(logging.NOTSET)
# logger.setLevel(logging.DEBUG)

def index(request):
    print('@@@@@@@@@@@@@@@@ sakila')
    context = {}
    template = loader.get_template('sakila/index.html')
    return HttpResponse(template.render(context, request))

def gentella_html(request):
    print('@@@@@@@@@@@@@@@@')
    context = {}
    # The template to be loaded as per gentelella.
    # All resource paths for gentelella end in .html.

    # Pick out the html file name from the url. And load that template.
    load_template = request.path.split('/')[-1]
    logger.debug(' function name gentella_html : %s ' % load_template)
    if load_template == 'tables_dynamic.html':
        list = Film.objects.all()
        list_cnt = Film.objects.count()
        logger.debug(' function name list_cnt : %s ' % list_cnt)
        context = {'list': list}
    elif load_template == 'tables_dynamic_customer_list.html':
        # list = CustomerList.objects.all()
        # sqlite3 3.25 버전이상에서 Window 함수를 사용할수 있음.
        list = CustomerList.objects.annotate(row_number=Window(
            expression=RowNumber(),
            partition_by=[F('sid')],
            order_by=F('id').desc())
        ).order_by('row_number', 'id')
        # ).order_by('row_number', 'client')

        list_cnt = CustomerList.objects.count()
        logger.debug(' function name list_cnt : %s ' % list_cnt)
        context = {'list': list}
    elif load_template == 'tables_dynamic_customer_list2.html':
        # # list = CustomerList.objects.all()
        # # sqlite3 3.25 버전이상에서 Window 함수를 사용할수 있음.
        # list = CustomerList.objects.annotate(row_number=Window(
        #     expression=RowNumber(),
        #     partition_by=[F('sid')],
        #     order_by=F('id').desc())
        # ).order_by('row_number', 'id')
        # # ).order_by('row_number', 'client')
        # list_cnt = CustomerList.objects.count()
        # logger.debug(' function name list_cnt : %s ' % list_cnt)
        # context = {'list': list}
        pass

    template = loader.get_template('sakila/' + load_template)
    return HttpResponse(template.render(context, request))

# 여기서에서 @csrf_exempt 는 csrf 보안을 사용 하기 않겠다는 의미 입니다.  참조사이트 [ django csrf 문서 ]
# @csrf_exempt
def film_modify(request):
    logger.debug(' function name : %s ' % 'film_modify')
    film_id = request.POST['film_id']
    description = request.POST['description']
    special_features = request.POST['special_features']

    film = get_object_or_404(Film, pk=film_id)
    film.description = description
    film.special_features = special_features
    film.save()
    # tmpJson = serializers.serialize("json",film)
    # tmpObj = json.loads(tmpJson)
    context = {
        "msg": "성공",
    }
    # return HttpResponse(context, content_type="application/json; charset=utf-8")
    # return HttpResponse(json.dumps(context), content_type="application/json; charset=utf-8")
    # return JsonResponse({
    #     'message': '안녕 파이썬 장고',
    #     'items': ['파이썬', '장고', 'AWS', 'Azure'],
    # }, json_dumps_params={'ensure_ascii': True})
    # 한글 잘 보임
    return HttpResponse(json.dumps(context, ensure_ascii=False), content_type="application/json")
    # return JsonResponse(json.dumps(context, ensure_ascii=False), safe=False)
    # return Response(context, content_type='application/json; charset=utf-8')

def list_staff(request):
    print('@@@@@@@@@@@@@@@@')
    load_template = request.path.split('/')[-1]
    logger.debug(' function name load_template : %s ' % load_template)
    context = {}
    # list = Staff.objects.all()
    # list_cnt = Staff.objects.count()
    # logger.debug(' function name list_cnt : %s ' % list_cnt)
    # context = {'list': list}
    template = loader.get_template('sakila/tables_dynamic.html')
    return HttpResponse(template.render(context, request))

class CustomerListView(ListView):
    """
    클래스 뷰
    """
    model = Customer
    template_name = 'sakila/tables_dynamic_customer.html'
    context_object_name = 'list'

class CustomerAjaxView(View):
    """
    Ajax 처리
    """

    form_class = CustomerForm
    initial = {'key': 'value'}

    def post(self, request):
        if request.is_ajax():
            # data = {"lat":20.586, "lon":-89.530}
            # print(request.POST.get('value'))
            logger.debug(' class name : %s ' % 'CustomerAjaxView')
            logger.debug(' function name : %s ' % 'post')
            # film_id = request.POST['film_id']
            # description = request.POST['description']
            # special_features = request.POST['special_features']

            form = self.form_class(request.POST)

            if form.is_valid():
                data = {
                    "msg": "성공",
                }
                form.save()
                return JsonResponse(data, json_dumps_params={'ensure_ascii': False})

            logger.debug(' error msg : %s ' % form.errors)
            # film = get_object_or_404(Film, pk=film_id)
            # form = CustomerForm(request.P)
            # film.description = description
            # film.special_features = special_features
            # film.save()
            # tmpJson = serializers.serialize("json",film)
            # tmpObj = json.loads(tmpJson)
            data = {
                "msg": "실패",
                "errors": str(form.errors) # str() 자동형 변환 하지 못하게 함 그러면 html 로 생성됨.
                # "errors": form.errors
            }
            return JsonResponse(data,  json_dumps_params={'ensure_ascii': False})

    def get(self, request):
        form = self.form_class(initial=self.initial)
        return render(request, 'landing.html', {'form': form})

class AjaxableResponseMixin:
    """
    Mixin to add AJAX support to a form.
    Must be used with an object-based FormView (e.g. CreateView)
    """
    def form_invalid(self, form):
        response = super().form_invalid(form)
        if self.request.is_ajax():
            return JsonResponse(form.errors, status=400)
        else:
            return response

    def form_valid(self, form):
        # We make sure to call the parent's form_valid() method because
        # it might do some processing (in the case of CreateView, it will
        # call form.save() for example).
        response = super().form_valid(form)
        if self.request.is_ajax():
            data = {
                'pk': self.object.pk,
            }
            return JsonResponse(data)
        else:
            return response

# class CustomerAjaxCreateView(AjaxableResponseMixin, CreateView):
class CustomerAjaxCreateView(CreateView):
    """
    Ajax 처리 Create
    """
    model = Customer
    # fields = ['first_name']
    # initial={'date_of_death':'05/01/2018',}
    # initial = {'key': 'value'}
    # # form_class = Customer
    # #
    def get(self, request):
        form = CustomerForm()
        # form = self.form_class()
        # form = self.form_class(initial=self.initial)
        data = {
            'form_as_table': form.as_table(),
        }
        return JsonResponse(data,  json_dumps_params={'ensure_ascii': False})

    def post(self, request):
        self.form_class = CustomerForm
        # initial = {'key': 'value'}
        if request.is_ajax():
            # data = {"lat":20.586, "lon":-89.530}
            # print(request.POST.get('value'))
            logger.debug(' class name : %s ' % 'CustomerAjaxCreateView')
            logger.debug(' function name : %s ' % 'post')
            # film_id = request.POST['film_id']
            # description = request.POST['description']
            # special_features = request.POST['special_features']

            form = self.form_class(request.POST)

            if form.is_valid():
                data = {
                    "msg": "성공",
                }
                form.save()
                return JsonResponse(data, json_dumps_params={'ensure_ascii': False})

            logger.debug(' error msg : %s ' % form.errors)
            # film = get_object_or_404(Film, pk=film_id)
            # form = CustomerForm(request.P)
            # film.description = description
            # film.special_features = special_features
            # film.save()
            # tmpJson = serializers.serialize("json",film)
            # tmpObj = json.loads(tmpJson)
            data = {
                "msg": "실패",
                "errors": str(form.errors) # str() 자동형 변환 하지 못하게 함 그러면 html 로 생성됨.
                # "errors": form.errors
            }
            return JsonResponse(data,  json_dumps_params={'ensure_ascii': False})

class FilmListApi(generics.ListCreateAPIView):
    logger.debug('@@@@@@@@@@ FilmListApi ')
    # queryset = Film.objects.all()
    serializer_class = FilmSerializer
    permission_classes = [IsAuthenticatedOrReadOnly,
                          IsOwnerOrReadOnly]
    # permission_classes = (IsAuthenticated, UserIsOwnerTodo)

    def get_queryset(self, request):
        logger.debug('@@@@@@@@@@ FilmListApi.get_queryset ')
        # OFFSET 5 / LIMIT 5
        req_draw = int(request.GET['draw'])
        req_start = int(request.GET['start'])
        req_length = int(request.GET['length'])
        req_search_value = str(request.GET['search[value]'])
        logger.debug('req_search_value : %s' % req_search_value)
        return Film.objects.filter(title__icontains=req_search_value)[req_start:req_length + req_start]

    def list(self, request):
        # Note the use of `get_queryset()` instead of `self.queryset`
        queryset = self.get_queryset(request)
        serializer = FilmSerializer(queryset, many=True)
        return Response({'data': serializer.data})

# class FilmDetailApi(generics.RetrieveAPIView):
class FilmDetailApi(generics.RetrieveUpdateDestroyAPIView):
    logger.debug('@@@@@@@@@@ FilmDetailApi ')
    queryset = Film.objects.all()
    serializer_class = FilmSerializer

    # def get_queryset(self, pk):
    #     logger.debug('@@@@@@@@@@ FilmDetailApi.get_queryset ')
    #     return Film.objects.all(pk)

class CustomerListListApi(generics.ListCreateAPIView):
    logger.debug('@@@@@@@@@@ FilmListApi ')
    # queryset = Film.objects.all()
    serializer_class = CustomerListSerializer
    permission_classes = [IsAuthenticatedOrReadOnly,
                          IsOwnerOrReadOnly]
    # permission_classes = (IsAuthenticated, UserIsOwnerTodo)

    def get_queryset(self, request):
        logger.debug('@@@@@@@@@@ FilmListApi.get_queryset ')
        # OFFSET 5 / LIMIT 5
        req_draw = int(request.GET['draw'])
        req_start = int(request.GET['start'])
        req_length = int(request.GET['length'])
        req_search_value = str(request.GET['search[value]'])
        logger.debug('req_search_value : %s' % req_search_value)
        req_order_column = str(request.GET['order[0][column]'])
        logger.debug('req_order_column : %s' % req_order_column)
        req_order_dir = str(request.GET['order[0][dir]'])

        req_order_column_nm = None
        logger.debug('req_order_dir : %s' % req_order_dir)
        if req_order_column:
            req_order_column_nm = CustomerList._meta.fields[int(req_order_column)].attname
            logger.debug('req_order_column_nm : %s' % req_order_column_nm)

        if req_order_dir == 'desc':
            return CustomerList.objects.filter(name__icontains=req_search_value).order_by('-'+req_order_column_nm)[req_start:req_length + req_start]
        elif req_order_dir == 'asc':
            return CustomerList.objects.filter(name__icontains=req_search_value).order_by(req_order_column_nm)[req_start:req_length + req_start]

        return CustomerList.objects.filter(name__icontains=req_search_value)[req_start:req_length + req_start]

    def list(self, request):
        # Note the use of `get_queryset()` instead of `self.queryset`
        queryset = self.get_queryset(request)
        serializer = CustomerListSerializer(queryset, many=True)
        return Response({'data': serializer.data})


"""
usbProtocol.py

The following module has been created to manage the bluetooth interface between the control system and the connected devices

Michael Xynidis
Fluvio L Lobo Fenoglietto
09/01/2016

"""

# Import Libraries and/or Modules
import  os, serial, time
#from    timeStamp           import  *

"""
FUNCTIONS
"""

def createUSBPort( deviceName, portPrefix, portNumber, baudrate):
    """
    CREATE USB PORT
        input;
            deviceName (string) --name of the input device (for reference)
            portPrefix (string) --prefix based on the operating system of the control system/computer
                                = /dev/ttyACM --Linux
                                = COM --Windows
            portNumber (num)    --communication port number
            baudrate   (num)    --communication baudrate (bps)

        output;
            usbObject  (object) --serial object
    """
    #print fullStamp() + " createUSBPort()"
    usbObject = serial.Serial(
        port = portPrefix + str(portNumber),
        baudrate = baudrate)
    time.sleep(1)
    #usbConnectionCheck(usbObject,deviceName,portNumber,baudrate,attempts)
    usbObject.close()
    return usbObject


"""
- Bu modül kameradan görüntü almak için kullanılır.
- Alınan görüntüyü gösterebilir veya göstermeyebilir.
- Görüntüler tekrar boyutlandırılabilir.
"""

import cv2

# Jetson Nano için gstreamer pipeline'ının oluşturulması
def gstreamer_pipeline(
        capture_width=1280,
        capture_height=720,
        display_width=1280,
        display_height=720,
        framerate=60,
        flip_method=0,
):
    return (
            "nvarguscamerasrc ! "
            "video/x-raw(memory:NVMM), "
            "width=(int)%d, height=(int)%d, "
            "format=(string)NV12, framerate=(fraction)%d/1 ! "
            "nvvidconv flip-method=%d ! "
            "video/x-raw, width=(int)%d, height=(int)%d, format=(string)BGRx ! "
            "videoconvert ! "
            "video/x-raw, format=(string)BGR ! appsink"
            % (
                capture_width,
                capture_height,
                framerate,
                flip_method,
                display_width,
                display_height,
            )
    )

# Kameranın tanımlanması
cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0))

# Görüntü almak için metot
def getImg(display=False, size=[480, 240]):
    _, img = cap.read()
    img = cv2.resize(img, (size[0], size[1]))
    # display değişkeni 1 ise görüntüleri göster
    if display:
        cv2.imshow('IMG', img)
    return img

# Kod yalnız çalıştırılırsa görüntü al
if __name__ == '__main__':
    while True:
        img = getImg(True)


# python imports
import datetime
import itertools
import os

# django imports
from django.conf import settings

# third-party imports
from whoosh import qparser
from whoosh.analysis import CharsetFilter, StandardAnalyzer
from whoosh.fields import Schema, ID, TEXT
from whoosh.index import create_in, open_dir, EmptyIndexError
from whoosh.support.charset import default_charset, charset_table_to_dict

# The last time an indexing process was run
LAST_INDEXING = None

def get_schema():
    # Analyzer to ignore accents-related differences
    # ( https://pythonhosted.org/Whoosh/stemming.html#character-folding )
    charmap = charset_table_to_dict(default_charset)
    name_analyzer = StandardAnalyzer() | CharsetFilter(charmap)

    return Schema(
        id=ID(stored=True, unique=True),
        name=TEXT(stored=True, analyzer=name_analyzer)
    )

def get_search_index():
    """
        Return the search index and a boolean value indicating if it had to be
            created now.
    """
    if not os.path.exists(settings.SEARCH_INDEX_DIR):
        os.mkdir(settings.SEARCH_INDEX_DIR)

    try:
        index = open_dir(settings.SEARCH_INDEX_DIR)
        index_created_now = False
    except EmptyIndexError:
        index = create_in(settings.SEARCH_INDEX_DIR, get_schema())
        index_created_now = True

    return index, index_created_now

def update_index():
    """
        Update the search index.
    """
    from .models import Spot
    global LAST_INDEXING

    index, index_created_now = get_search_index()

    spots_to_index = Spot.objects.only('name').available()

    if not index_created_now:
        if LAST_INDEXING:
            spots_to_index = spots_to_index.filter(
                last_modified__gt=LAST_INDEXING
            )

    writer = index.writer()

    for spot in spots_to_index:
        writer.update_document(
            id=unicode(spot.id),
            name=unicode(spot.name)
        )

    writer.commit()

    LAST_INDEXING = datetime.datetime.now()

def delete_spot_from_index(spot):
    from .models import Spot
    assert isinstance(spot, Spot)
    index, index_created_now = get_search_index()
    if not index_created_now:
        writer = index.writer()
        writer.delete_by_term('id', unicode(spot.id))
        writer.commit()

def search_spots(search_term, limit=250):
    from .models import Spot
    assert isinstance(limit, int) and limit > 0
    index = get_search_index()[0]
    parser = qparser.QueryParser(
        'name',
        get_schema(),
        group=qparser.OrGroup.factory(0.9)
    )
    words = search_term.split()
    query_string = ' '.join(['*%s*' % w for w in words])
    query = parser.parse(query_string)
    with index.searcher() as searcher:
        spots = [
            Spot(id=result['id'], name=result['name'])
            for result in searcher.search(query, limit=limit)
        ]
        corrector = searcher.corrector('name')
        suggestions = [corrector.suggest(w, maxdist=4) for w in words]
    suggestions = itertools.chain(*suggestions)
    suggestions = list(set(suggestions))  # Remove duplicates
    suggestions = [w for w in suggestions if len(w) >= 3]  # Min length
    suggestions.sort(key=len, reverse=True)  # Biggest words first
    return {'spots': spots, 'suggestions': suggestions}


import chainer
import chainer.functions as F
import chainer.links.caffe as C
import numpy as np
from PIL import Image

from model import ImageCaptionNN

TEST_IMAGES = ('test-{}.jpg'.format(i) for i in range(1, 6))

index2word = {0: '', 1:''}
with open('words.txt', 'r') as f:
    for line in f:
        line = line.strip().split(',')
        index2word[int(line[0])] = line[1]

cnn = C.CaffeFunction('bvlc_alexnet.caffemodel')
model = ImageCaptionNN(max(index2word.keys()) + 1, 512)
chainer.serializers.load_hdf5('chap09.hdf5', model)

words_max = 15
beam_w = 5

def tree_traverse():
    global max_score
    cur_word = cur_sentence[-1]
    score = np.prod(cur_score)
    depth = len(cur_sentence)
    if depth > 5 and score < max_score * 0.6:
        return
    if cur_word == 1 or depth > words_max:
        data = np.array(cur_sentence)
        sentences.append((score, data))
        if score > max_score:
            max_score = score
        return
    cur_model = model_history[-1].copy()
    x = np.array([cur_word], dtype=np.int32)
    y = cur_model.decode(x)
    z = F.softmax(y)
    result = z.data[0]
    argsort = np.argsort(result)[::-1]
    model_history.append(cur_model)
    for i in range(beam_w):
        cur_sentence.append(argsort[i])
        cur_score.append(result[argsort[i]])
        tree_traverse()
        cur_sentence.pop()
        cur_score.pop()
    model_history.pop()

for image_filename in TEST_IMAGES:
    with open(image_filename, 'r') as f:
        print(image_filename)
        image = Image.open(image_filename).resize((400, 400)).convert('RGB')

        vectors = []

        for s in ((0, 0, 227, 227),
                  (173, 0, 400, 227),
                  (0, 173, 227, 400),
                  (173, 173, 400, 400),
                  (86, 86, 313, 313)):
            crop_image = image.crop(s)
            pix = np.array(crop_image, dtype=np.float32)
            pix = pix[::-1].transpose(2, 0, 1)
            pix = np.array([pix], dtype=np.float32)
            x, = cnn(inputs={'data': pix}, outputs=['fc6'], disable=['drop6'])
            model.encode(x)

        c, h = model.get_state()
        model.set_state(c, h)

        sentences = []
        model_history = [model]
        cur_sentence = [0]
        cur_score = []
        max_score = 0

        tree_traverse()

        sentences = sorted(sentences, key=lambda x: x[0])[::-1]

        for i in range(min([5, len(sentences)])):
            score, description = sentences[i]
            print('{}  {}'.format(score, ''.join(map(index2word.get, description))))


from decimal import Decimal

from rest_framework import serializers

from tests.test_case import AppTestCase
from wip.models import JobTiming
from wip.serializers import JobTimingSerializer

class TestSerializer(AppTestCase):
    fixtures = ['wip/tests/fixtures/test.yaml']

    def test_subclass(self):
        self.assertTrue(issubclass(JobTimingSerializer, serializers.ModelSerializer))

    def test_model(self):
        self.assertEqual(JobTimingSerializer.Meta.model, JobTiming)

    def test_fields(self):
        self.assertEqual(
            JobTimingSerializer.Meta.fields,
            [
                'allocated_hours',
                'time_spent_hours',
                'is_over_allocated_hours'
            ]
        )

    def test_serialized_data(self):
        instance = JobTiming.objects.first()
        serializer = JobTimingSerializer(instance=instance)
        self.assertEqual(
            serializer.data,
            {
                'allocated_hours': str(Decimal(instance.allocated_hours).quantize(Decimal('0.00'))),
                'time_spent_hours': str(Decimal(instance.time_spent_hours).quantize(Decimal('0.00'))),
                'is_over_allocated_hours': instance.is_over_allocated_hours
            }
        )


# coding: utf-8

import collections
import csv
import decimal
import logging
import numbers
import re
from io import BytesIO

from openpyxl import load_workbook
from openpyxl.utils.cell import get_column_letter
from six import string_types

from ..codes import FileParseCodes
from ..utilities import ErrorAggregator, ParseError

logger = logging.getLogger(__name__)

def _has_value(*args):
    """
    Tests whether any of the provided arguments should be treated as having a valid value
    """

    def _valid_value(arg):
        if isinstance(arg, string_types):
            return bool(arg)  # assume it's already .strip()ed
        else:
            return arg is not None

    return any(filter(_valid_value, args))

class TableParser(object):
    """
    A parser for the tabular import files that allows supports semi-tolerant parsing of
    user-provided spreadsheets. Basic conditions for parsing are:

    1) Required columns can be provided in any order
    2) Optional columns will be identified if present
    3) Column headers are case-insensitive
    4) Any column whose header doesn't match those specified by the file format will be
        ignored (with a warning)
    5) Whitespace parsing is tolerant, e.g. different number and type of whitespace is allowed in
        column headers than what's suggested, though space is still needed in between words if
        column headers are defined with internal whitespace.
    """

    # TODO: capture common method parameters as a class to simplify interface
    def __init__(
        self,
        req_cols,
        opt_cols=None,
        value_opt_cols=None,
        numeric_cols=None,
        supported_units=None,
        aggregator=None,
    ):
        """
        Initializes the parser using client-provided lists of required and optional column headers.

        :param req_cols: an iterable of human-readable strings containing required column
            headers. The parser will take care of tolerating variations in optional leading,
            trailing, and internal whitespace.
        :param opt_cols: an iterable of human-readable strings with optional column headers.
            Subject to the same processing as req_cols.
        :param value_opt_cols: an optional iterable of column headers from either or both of
            req_cols & opt_cols. Any col header in value_opt_cols will have missing
            values tolerated in any row.  Any column headers not listed here are by default
            required to have a value in every row, assuming the column is present.
        :param numeric_cols: an  optional list of column names for columns that will be verified
            during parsing to have numeric content.
        :param aggregator: an optional main.importer2.ErrorAggregator instance to report errors
            and warnings to
        """
        super(TableParser, self).__init__()

        # TODO: add support for vector input
        # TODO: add optional column aliases?
        self.aggregator = aggregator if aggregator else ErrorAggregator()

        # keep inputs to use as keys and as human-readable for use in err messages
        self.req_cols = req_cols
        self.opt_cols = opt_cols if opt_cols is not None else set()
        self.value_opt_cols = value_opt_cols if value_opt_cols is not None else set()
        self.numeric_cols = numeric_cols if numeric_cols is not None else set()
        self.supported_units = supported_units if supported_units is not None else {}

        # maps canonical col name -> observed text (e.g. maybe including whitespace, mixed case,
        # alias)
        self.obs_col_names = {}
        self.obs_col_units = {}

        self._is_excel = None

        # build up regex patterns for each supported column header that allow us to do
        # semi-tolerant matching against user-provided values.
        # maps human-readable col name => pattern
        self.req_col_patterns = self._build_col_header_patterns(req_cols)

        if opt_cols:
            self.opt_col_patterns = self._build_col_header_patterns(self.opt_cols)
        else:
            self.opt_col_patterns = []

        self._build_unit_patterns()

    @property
    def has_all_times(self):
        """
        Tests whether the parsed file contained time values for all measurements.  Incomplete time
        values should be treated as a parse error.
        """
        return False  # children should override

    def has_all_units(self):
        """
        Tests whether the parsed file contained units for all measurements
        Partial units should be treated as a parse error.
        """
        return False  # children should override

    def _build_col_header_patterns(self, col_headers):
        """
        Builds patterns to facilitate reasonably tolerant parsing of fixed string values expected
        to match during parsing.  Patterns are constructed to match case-insensitive input and
        are tolerant of leading and trailing whitespace insertions, or of added internal
        whitespace.  Internal whitespace in the input is required, but may use different
        whitespace characters or have additional whitespace added.
        """
        reg = r"^\s*{title}\s*$"
        return [
            re.compile(
                reg.format(title=TableParser._process_label(col_header)), re.IGNORECASE
            )
            for col_header in col_headers
        ]

    def _build_unit_patterns(self):
        self._unit_patterns = {}
        for col, units in self.supported_units.items():
            logger.debug(f'Building unit patterns for column "{col}": {units}')
            # note : maintaining case-sensitivity is important! SI units use case!
            vals = "|".join([TableParser._process_label(unit) for unit in units])
            pat = re.compile(r"^\s*({vals})\s*$".format(vals=vals))
            self._unit_patterns[col] = pat

    @staticmethod
    def _process_label(s):
        tokens = re.split(r"\s+", s)  # tokenize label on whitespace
        escaped_tokens = (re.escape(part) for part in tokens)  # escape each part
        return r"\s+".join(escaped_tokens)  # gather together with whitespace regex

    def parse_excel(self, file):
        """
        Parses the input file as an Excel workbook.
        :param file: a path-like object identifying the file location
        :return: if parsing was successful, the number of valid rows read from file, otherwise None
        :raise: OSError If the file can't be opened or ParseError if the file format or content
        is bad
        """
        self._is_excel = True
        wb = load_workbook(BytesIO(file.read()), read_only=True, data_only=True)
        logger.debug("In parse(). workbook has %d sheets" % len(wb.worksheets))
        if not wb.worksheets:
            self.aggregator.add_error(FileParseCodes.EMPTY_FILE)
            raise ParseError(self.aggregator)
        elif len(wb.worksheets) > 1:
            sheet_name = wb.sheetnames[0]
            msg = (
                f'Only the first sheet in your workbook, "{sheet_name}", was processed.  '
                f"All other sheets will be ignored."
            )
            self.aggregator.add_warning(
                FileParseCodes.IGNORED_WORKSHEET, occurrence=msg
            )
        worksheet = wb.worksheets[0]
        return self._parse(worksheet.iter_rows())

    def parse_csv(self, file):
        """
        Parses the input file as CSV.
        :param file: a path-like object identifying the file location
        :return: if parsing was successful, the number of valid rows read from file, otherwise None
        :raise: OSError If the file can't be opened, or ParseError if the file format or content is
        bad
        """
        self._is_excel = False
        reader = csv.reader(file)
        return self._parse(reader)

    def _parse(self, rows_iter):
        """
        A workhorse method that performs parsing, independent of how the file content is stored
        (CSV or Excel).
        :return: the number of non-whitespace rows parsed
        :raise ParseError: if the file format or content was bad
        """
        # Clear out state from any previous use of this parser instance
        self.column_layout = None
        obs_required_cols = set()

        # loop over rows
        for row_index, cols_list in enumerate(rows_iter):
            logger.debug(f"Parsing row {row_index+1}")

            # identify columns of interest first by looking for required labels
            if not self.column_layout:
                self.column_layout = self._parse_col_layout(
                    cols_list, row_index, obs_required_cols
                )

                if self.column_layout:
                    self._verify_layout(self.aggregator, row_index)

            # if column labels have been identified, look for data
            else:
                self._parse_row(cols_list, row_index)

        # if we got all the way through the file without finding a single required column header,
        # it's badly formatted
        if not self.column_layout:
            logger.debug(f"Required columns were not read {self.req_cols}")
            self.aggregator.add_error(
                FileParseCodes.MISSING_REQ_COL_HEADER, ", ".join(self.req_cols)
            )

        if self.aggregator.errors:
            raise ParseError(self.aggregator)

        return self.measurement_count

    def cell_content_desc(self, content, row_index, col_index):
        """
        Builds a succinct, human-readable description of the cell content and location
        :param content: the cell content
        :param row_index: index into the file where this row is located
        :param col_index: index into the file where this column is located
        :return: a string description of the cell content and location
        """
        return f'"{content}" ({self.cell_coords(row_index, col_index)})'

    def _parse_col_layout(self, row, row_index, obs_required_cols):
        """
        Scans a row of the file to see if it contains required column headers that define the file
        layout.
        :param row: the row to inspect for column headers
        :param row_index: the index into the file of the row being checked for required col headers
        :param obs_required_cols: a set to populate with required col names observed so far in the
            file.  Since detection of any required column header counts as a match, this parameter
            is essentially just an optimization to prevent creating a new set each time we try
            to read a row in poorly-formatted files.
        :return: the column layout if required columns were found, or None otherwise
        """
        layout = {}  # maps col name to col index
        importer = self.aggregator
        req_cols_set = set(self.req_cols)

        ###########################################################################################
        # loop over columns in the current row
        ###########################################################################################
        obs_opt_cols = set()
        non_header_vals = []
        for col_index, cell in enumerate(row):
            cell_content = self._raw_cell_value(cell)

            # skip this cell if it has no non-whitespace content
            # (strings are stripped by _get_raw_value())
            if not cell_content:
                continue

            # ignore non-string cells since they can't be the column headers we're looking for
            if not isinstance(cell_content, string_types):
                if cell_content is not None:
                    col_desc = self.cell_content_desc(
                        cell_content, row_index, col_index
                    )
                    non_header_vals.append(col_desc)
                continue

            #######################################################################################
            # check whether column label matches one of the canonical column names specified by
            # the format
            #######################################################################################
            self._process_col_name(
                cell_content,
                row_index,
                col_index,
                non_header_vals,
                obs_required_cols,
                obs_opt_cols,
                layout,
            )

        # if at least the required columns were found, consider this a successful read
        if obs_required_cols == req_cols_set:
            for bad_val in non_header_vals:
                importer.add_warning(FileParseCodes.COLUMN_IGNORED, occurrence=bad_val)
            logger.debug(
                f"Found all {len(self.req_cols)} required column headers in row "
                f"{row_index+1}"
            )
            return layout

        # if some--but not all--required columns were found in this row, consider this a
        # poorly-formatted file
        if obs_required_cols:
            missing_cols = req_cols_set.difference(obs_required_cols)
            logger.debug(f"Required column headers missing: {missing_cols}")
            importer.raise_errors(
                FileParseCodes.MISSING_REQ_COL_HEADER, occurrences=missing_cols
            )
        else:
            importer.add_warnings(
                FileParseCodes.IGNORED_VALUE_BEFORE_HEADERS, non_header_vals
            )

        return None

    def _process_col_name(
        self,
        cell_content,
        row_index,
        col_index,
        non_header_vals,
        obs_req_cols,
        obs_opt_cols,
        layout,
    ):
        """
        Process the string content of a cell when looking for columns
        :param cell_content: the string content (non empty, non-whitespace)
        :param non_header_vals: a list of non column header values read from the file before
        any valid column header is found
        :param obs_req_cols: a set of required column headers already observed in the row
        :param obs_opt_cols: a set of optional columns headers already observed in the row
        :param layout: a dict that maps col name to the column index where it was observed
        """
        req_name = self._parse_col_header(
            self.req_col_patterns, self.req_cols, row_index, col_index, cell_content
        )
        opt_name = self._parse_col_header(
            self.opt_col_patterns, self.opt_cols, row_index, col_index, cell_content
        )
        agg = self.aggregator
        if req_name:
            logger.debug(
                f'Found required column "{req_name}"'
                f"({self.cell_coords(row_index, col_index)})"
            )

            if req_name in obs_req_cols:
                col_desc = self.cell_content_desc(cell_content, row_index, col_index)
                agg.add_error(FileParseCodes.DUPLICATE_COL_HEADER, occurrence=col_desc)
            else:
                obs_req_cols.add(req_name)
                layout[req_name] = col_index
                if req_name != cell_content:
                    self.obs_col_names[req_name] = cell_content
        elif opt_name:
            logger.info(
                f'Found optional column "{opt_name}".'
                f"({self.cell_coords(row_index, col_index)})"
            )
            if opt_name in obs_opt_cols:
                col_desc = self.cell_content_desc(cell_content, row_index, col_index)
                agg.add_error(FileParseCodes.DUPLICATE_COL_HEADER, occurrence=col_desc)
            else:
                obs_opt_cols.add(opt_name)
                layout[opt_name] = col_index
                if cell_content != opt_name:
                    self.obs_col_names[opt_name] = cell_content
        else:
            col_desc = self.cell_content_desc(cell_content, row_index, col_index)
            non_header_vals.append(col_desc)

    def _raw_cell_value(self, cell):
        """
        Gets the raw cell value in whatever format it was stored in the file
        :param cell: the cell
        :return: the cell value, with leading and trailing whitespace stripped if the content
        was a string
        """
        if self._is_excel:
            val = cell.value
        else:
            val = cell

        if isinstance(val, string_types):
            return val.strip()

        return val

    def _get_raw_value(self, row, col_name):
        """
        Gets the raw value from the table, regardless of file format.
        :param row: an iterable of columns in this row of the file
        :param col_name: the canonical name of the column whose value should be read
        :return: the value, or None if the col with col_name isn't in the file
        """
        col_index = self.column_layout.get(col_name, None) if col_name else None

        if col_index is None:
            return None

        cell = row[col_index]
        return self._raw_cell_value(cell)

    @property
    def measurement_count(self):
        """
        Gets the number of valid measurements read from the file
        """
        # children must implement
        raise NotImplementedError()

    def obs_col_name(self, canonical_name):
        """
        Gets the observed column label from the file corresponding to the canonical name understood
        by the parser.
        :param canonical_name: the canonical column name.
        :return: the observed column name, or the canonical column name if the column wasn't found
            in the file.
        """
        return self.obs_col_names.get(canonical_name, canonical_name)

    # TODO: consider (optional) mtype ID pattern verification
    def _parse_and_verify_val(self, val, row_index, col_name):
        """
        Tests a parsed value, and logs an error message if it the observed value doesn't meet
        expectations. Verifies that:
        1) The value is present if required
        2) It's a scalar number, if required
        3) If it's a unit, that it falls within the set of supported units
        :param val: the value read from file
        :param row_index: the index of the file row the value is from
        :param col_name: the canonical name for this column, rather than the one observed in
        the file.
        :return: the value, if it was valid, or None if it failed one ore more validation criteria
        """
        importer = self.aggregator
        col_index = self.column_layout.get(col_name, None)

        # if parser is asking for a column not found in the file, end early.
        # if it's required, an error will already have been recorded
        if col_index is None:
            logger.warning(f'Column "{col_name}" not found in file')
            return None

        # if value is missing, but required, log an error
        if val is None:
            if col_name not in self.value_opt_cols:
                importer.add_error(
                    FileParseCodes.MISSING_REQ_VALUE,
                    subcategory=self.obs_col_name(col_name),
                    occurrence=self.cell_coords(row_index, col_index),
                )
            return val

        # if observed value is a string,
        if isinstance(val, string_types):
            if (not val) and col_name not in self.value_opt_cols:
                importer.add_error(
                    FileParseCodes.MISSING_REQ_VALUE,
                    subcategory=self.obs_col_name(col_name),
                    occurrence=self.cell_coords(row_index, col_index),
                )
                return None

            # parse / verify expected numeric values... when read from CSV, they'll have to be
            # parsed
            if col_name in self.numeric_cols:
                return self._parse_num(val, col_name, row_index, col_index)

            return val

        # assumption (for now) is that value is numeric. should work for both vector (
        # 1-dimensional) and numeric inputs
        if not isinstance(val, numbers.Number):
            importer.add_error(
                FileParseCodes.INVALID_VALUE,
                subcategory=self.obs_col_name(col_name),
                occurrence=self.cell_coords(row_index, col_index),
            )

        return val

    def _parse_num(self, token, col_name, row_index, col_index):
        try:
            return decimal.Decimal(token)
        except (decimal.InvalidOperation, decimal.Clamped):
            self.aggregator.add_error(
                FileParseCodes.INVALID_VALUE,
                subcategory=self.obs_col_name(col_name),
                occurrence=self.cell_content_desc(token, row_index, col_index),
            )
            return None

    def _parse_col_header(
        self, col_patterns, col_names, row_index, col_index, cell_content
    ):
        """
        Tests the cell content against the canonical column definitions, and extracts the
        canonical column name and optional observed units if it matches.
        :param col_patterns: the iterable of column name patterns to match against
        :param col_names: an iterable of canonical column names, parallel structure to
        col_patterns
        :param row_index: the row index currently being parsed
        :param col_index: the column index currently being parsed
        :param cell_content: the content of the current cell
        """
        # loop over all column patterns, even those already matched.  Otherwise we can't detect
        # duplicates.
        for col_pattern, col_name in zip(col_patterns, col_names):
            if col_pattern.match(cell_content):
                return col_name

        return None

    def _verify_layout(self, importer, header_row_index):
        pass  # children may optionally implement

    def _parse_row(self, cols_list, row_index):
        raise NotImplementedError()  # children must implement

    @property
    def series_data(self):
        raise NotImplementedError()  # children must implement

    @property
    def units(self):
        raise NotImplementedError()  # children must implement

    @property
    def mtypes(self):
        raise NotImplementedError()  # children must implement

    @property
    def line_or_assay_names(self):
        raise NotImplementedError()

    @staticmethod
    def cell_coords(row_index, col_index):
        col_letter = get_column_letter(col_index + 1)
        return f"{col_letter}{row_index+1}"

    def _verify_required_val(self, value, row_index, col_index, col_title):
        if value is None or (isinstance(value, string_types) and not value.strip()):
            self.aggregator.add_error(
                FileParseCodes.MISSING_REQ_VALUE,
                subcategory=col_title,
                occurrence=self.cell_coords(row_index, col_index),
            )

# TODO: in a later version, resolve with RawImportRecord from the older import...only significant
# differences here are that "kind" is removed, no deep copying, and units & src have been added.
# Best to keep things separate for now.  TODO: update this comment prior to commit
class MeasurementParseRecord(object):
    """
    A record resulting from parsing a single Measurement from an import file.  This object should
    be flexible enough to capture the full level of detail needed to construct a Measurement from
    any file, though many formats won't require this level of detail.

    Compare with RawImportRecord from the legacy import.  Major differences are:
    1. "Kind" is removed
    2. No deep copying -- client code retains control
    3. Units have been added
    4. src_id is added
    """

    def __init__(self, **kwargs):
        self.line_or_assay_name = kwargs.get("line_or_assay_name", None)
        self.mtype_name = kwargs.get("mtype_name", None)
        self.data = kwargs.get("data", None)
        self.metadata_by_name = kwargs.get("meta", None)
        self.units_name = kwargs.get("units_name", None)

        # data source for this measurement...often a row num if input is Excel
        self.src_id = kwargs.get("src_id", None)

    def to_json(self):
        return {
            "line_or_assay_name": self.line_or_assay_name,
            "measurement_name": self.mtype_name,
            "metadata_by_name": self.metadata_by_name,
            "units_name": self.units_name,
            "data": self.data,
        }

class GenericImportParser(TableParser):
    """
    Parser for EDD's "Generic" import file, a normalized, a simple, accessible,
    and machine-readable tabular format designed for automated data import.
    """

    def __init__(self, aggregator=None):

        super(GenericImportParser, self).__init__(
            req_cols=["Line Name", "Time", "Measurement Type", "Value", "Units"],
            numeric_cols=["Time", "Value"],
            aggregator=aggregator,
        )
        self._measurements = []
        self.unique_units = set()
        self.unique_mtypes = set()
        self.unique_line_or_assay_names = set()

    def _verify_layout(self, importer, header_row_index):
        pass

    def _parse_row(self, cells_list, row_index):
        # extract raw values from in-use cols in this row
        # TODO: should rename col header for clarity if this could actually be assay name
        line_or_assay_name = self._get_raw_value(cells_list, "Line Name")
        mtype = self._get_raw_value(cells_list, "Measurement Type")
        val = self._get_raw_value(cells_list, "Value")
        time = self._get_raw_value(cells_list, "Time")
        units = self._get_raw_value(cells_list, "Units")

        # skip the row entirely if no in-use column has a value in it
        any_value = _has_value(line_or_assay_name, mtype, val, time, units)
        if not any_value:
            return None

        # now that we've seen at least a single value in the row, do more rigorous parsing /
        # verification of the values
        line_or_assay_name = self._parse_and_verify_val(
            line_or_assay_name, row_index, "Line Name"
        )
        mtype = self._parse_and_verify_val(mtype, row_index, "Measurement Type")
        val = self._parse_and_verify_val(val, row_index, "Value")
        time = self._parse_and_verify_val(time, row_index, "Time")
        units = self._parse_and_verify_val(units, row_index, "Units")

        if isinstance(val, collections.Iterable):
            data = [time]
            for item in val:
                data.append(item)
        else:
            data = [time, val]

        m = MeasurementParseRecord(
            line_or_assay_name=line_or_assay_name,
            mtype_name=mtype,
            data=data,
            units_name=units,
            src_id=f"row {row_index+1}",
        )
        self._measurements.append(m)

        # track unique observed values
        if line_or_assay_name:
            self.unique_line_or_assay_names.add(line_or_assay_name)
        if val:
            self.unique_mtypes.add(mtype)
        if units:
            self.unique_units.add(units)

    @property
    def mtypes(self):
        return self.unique_mtypes

    @property
    def line_or_assay_names(self):
        return self.unique_line_or_assay_names

    @property
    def units(self):
        return self.unique_units

    @property
    def series_data(self):
        return self._measurements

    @property
    def measurement_count(self):
        return len(self._measurements)

    @property
    def has_all_times(self):
        """
        Tests whether the parsed file contained time values for all measurements.
        """
        return (
            True
        )  # overrides default False.  Time is a required column in this format.

    def has_all_units(self):
        """
        Tests whether the parsed file contained units for all measurements
        Partial units should be treated as a parse error.
        """
        return (
            True
        )  # overrides default False.  Units is a required column in this format


import tweepy
import config
import time
import datetime
import numpy as np
import os
import json
import pandas as pd

def rehydrate(directory_path):
    '''
    rehydrates the tweet ids in all txt files in the current directory
    tweets are saved into a separate file with the same name into jsonl format
    '''
    #Stores the current path so we can change back to it later
    cwd = os.getcwd()

    #Changes the path to the folder containing the files we wish to rehydrate
    os.chdir(directory_path)

    #Lists the files so we can iterate through them
    folder = os.listdir()

    for file in range(len(folder)):

        #Checks to make sure the jsonl file doesn't already exist for the politician
        politician_check = folder[file][:-4] + '.jsonl'
        if politician_check in folder:
            print("The data for " + folder[file][:-4] + " already exists.")
            continue

        #Checks to make sure file ends in a txt extension
        if folder[file][-4:] == '.txt':
            #Rehydrates the tweet ids using twarc
            os.system('twarc hydrate ' + folder[file] + ' > ' + folder[file][:-4] + '.jsonl')
            print("Data for " + folder[file][:-4] + " has been obtained.")
        else:
            continue

    #Changes the directory to the original path
    os.chdir(cwd)

def get_data(scientific_path, misinformation_path):
    '''
    setup for rehyrating tweets for both our scientific and misinformation groups
    '''
    rehydrate(scientific_path)
    rehydrate(misinformation_path)

def get_metrics(politician_file, filepath):
    '''
    obtains the engagement metrics for all the tweet ids in our txt files
    engagement metrics are saved into a separate file with the same name in csv format
    '''

    working_path = os.getcwd()
    os.chdir(filepath)

    check = politician_file + '.csv'

    #Checks to make sure the csv file does not already exist before calling on the API
    if check in os.listdir():
        print("The csv for " + politician_list + " already exists.")
        os.chdir(working_path)
        return

    #Opens the text file of tweet ids found in the filepath
    text_file = open(politician_file + ".txt", "r")
    lines = text_file.read().splitlines()
    text_file.close()

    #List containing dataframes consisting of 100 rows
    metrics_list = []

    for i in range(0, len(lines), 100):
        #Gets 100 tweet ids at a time and passes it to the call to the API
        #The API can only look up metrics for 100 tweets at a time
        partition = lines[i:i+100]
        partition_string = ','.join(partition)

        request = os.popen("curl 'https://api.twitter.com/2/tweets?ids=" + partition_string + "&tweet.fields=public_metrics' --header 'Authorization: Bearer " + config.bearer_token + "'").read()

        #Check for rate limit, if exceeded we sleep for 15 minutes to reset the limit
        #Then continues where it left off for the tweet ids
        if 'exceeded' in request:
            print("Rate Limit Exceeded. Sleeping for 15 Minutes.")
            time.sleep(60*15)
            request = os.popen("curl 'https://api.twitter.com/2/tweets?ids=" + partition_string + "&tweet.fields=public_metrics' --header 'Authorization: Bearer " + config.bearer_token + "'").read()

        #A string with json is returned from the API call, this converts it to Json
        to_json = json.loads(request)

        #Takes the data portion of the output and turns it into a dataframe
        #Appends the dataframe of 100 rows to a list containing the smaller dataframes
        metrics_data = pd.DataFrame.from_dict(to_json['data'])
        metrics_list.append(metrics_data)

    #Appends the small dataframes of 100 into a larger dataframe
    metrics = pd.concat(metrics_list, ignore_index=True)

    #Saves the metrics dataframe to a csv file so we can reduce the number of API calls we make
    metrics.to_csv(politician_file + '.csv', index=True)
    print("The csv for " + politician_file + " has been completed.")

    os.chdir(working_path)

    return metrics

def get_csvs(scientific_path, misinformation_path, scientific_list, misinformation_list):
    '''
    setup for obtaining engagement metrics for both our scientific and misinformation groups
    '''

    for i in range(len(scientific_list)):
        get_metrics(scientific_list[i], scientific_path)

    for i in range(len(misinformation_list)):
        get_metrics(misinformation_list[i], misinformation_path)


import requests

from django.conf import settings

def get_lcbo_auth_headers():
    return {
        "Authorization": "Token " + settings.LCBO_API_KEY,
    }

def get_closest_lcbo_stores(latitude, longitude, num, http_client=None):
    http_client = http_client or requests

    store_url = settings.LCBO_URL_BASE + "stores"

    headers = get_lcbo_auth_headers()

    params = {
        "lat": latitude,
        "lon": longitude,
        "per_page": num,
    }

    try:
        response = http_client.get(
            store_url,
            headers=headers,
            params=params,
        ).json()
    except requests.exceptions.ConnectionError:
        raise

    try:
        return response["result"]
    except KeyError:
        return []

def get_cheapest_drink_by_alcohol_volume(store_id, http_client=None):
    http_client = http_client or requests

    drink_url = settings.LCBO_URL_BASE + "products"

    headers = get_lcbo_auth_headers()

    params = {
        "page": 1,
        "per_page": 100,
        "order": "price_per_liter_of_alcohol_in_cents.asc",
        "store_id": store_id,
    }

    return _find_first_alcoholic_product(
        http_client=http_client,
        url=drink_url,
        headers=headers,
        params=params,
    )

def _find_first_alcoholic_product(http_client, url, headers, params):
    """
    Returns the first product in the response with a non-zero alcohol content.
    If not found, it fetches the next page until one is found.
    """
    try:
        response = http_client.get(
            url,
            headers=headers,
            params=params,
        ).json()
    except requests.exceptions.ConnectionError:
        raise

    for product in response["result"]:
        if product["alcohol_content"] > 0:
            return product

    if response["pager"]["is_final_page"]:
        raise Exception("This store has no alcoholic products")

    next_page_number = response["pager"]["next_page"]
    params["page"] = next_page_number
    return _find_first_alcoholic_product(http_client, url, headers, params)

def format_drink_json(product):
    product["price"] = product["price_in_cents"] / 100
    product["price_per_liter_of_alcohol"] = product["price_per_liter_of_alcohol_in_cents"] / 100

    return product