'''
Created on 2021-07-31

@author: wf
'''
import unittest
from tests.datasourcetoolbox import DataSourceTest
from corpus.lookup import CorpusLookup
from corpus.event import EventStorage
from lodstorage.query import Query
import copy
import json

class TestStatistics(DataSourceTest):
    '''
    test statistics
    '''

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.lookup=CorpusLookup()

    def setUp(self, debug:bool=False, profile:bool=True, **kwargs):
        DataSourceTest.setUp(self, debug=debug, profile=profile, **kwargs)
        self.lookup=TestStatistics.lookup
        pass

    def testIssue255(self):
        '''
        test issue 255 problem list
        '''
        show=False
        lookup=self.lookup
        sqlQuery="""select url,endDate-startdate+1 as duration from
event_orclone
where endDate is not Null and startDate is not Null
and duration<0 or  duration>7
order by duration"""
        query=Query(name="issue #255",query=sqlQuery,lang='sql')
        qlod=lookup.getLod4Query(query.query)
        prefix="https://confident.dbis.rwth-aachen.de/or/index.php?title="
        for tablefmt in ["mediawiki","github","latex"]:
            lod=copy.deepcopy(qlod)
            query.prefixToLink(lod, prefix, tablefmt)
            qdoc=query.documentQueryResult(lod,tablefmt=tablefmt)
            if show:
                print(qdoc)

    def testMultiQuery(self):
        '''
        test MultiQuery
        '''
        lookup=self.lookup
        multiQuery="""SELECT count(*) AS count,ordinal
        FROM {event}
        GROUP BY ordinal ORDER by 1"""
        dictOfLod=lookup.getDictOfLod4MultiQuery(multiQuery)
        debug=self.debug
        #debug=True
        if debug:
            jsonStr=json.dumps(dictOfLod, sort_keys=True, indent=2,default=str)
            print(jsonStr)

    def testStatistics(self):
        '''
        test statistics
        '''
        lookup=self.lookup
        qm=EventStorage.getQueryManager()
        self.assertIsNotNone(qm)
        self.assertTrue(len(qm.queriesByName)>1)
        showMarkup=False
        # showMarkup=True
        failCount=0
        for _name,query in qm.queriesByName.items():
            try:
                qlod=lookup.getLod4Query(query.query)
                for tablefmt in ["mediawiki","github","latex"]:
                    qdoc=query.documentQueryResult(qlod,tablefmt=tablefmt)
                    if showMarkup:
                        print(qdoc)
            except Exception as ex:
                print(f"query: {query.query} failed:\n{ex}")
                failCount+=1
        self.assertTrue(failCount<=3)
        pass

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()


from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from pathlib import Path
import pandas as pd
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from itertools import chain
import os
import time

def delete_old_file(file):
    if os.path.exists(file):
        os.remove(file)
    else:
        print("The file does not exist")

def configure_driver():
    # Add additional Options to the webdriver
    chrome_options = Options()
    # add the argument and make the browser Headless.
    chrome_options.add_argument("--headless")
    #finds and use local chromedriver
    d = Path(__file__).resolve().parents[0]
    driver = webdriver.Chrome(executable_path=str(d)+ "/chromedriver", options = chrome_options)
    return driver

def getData(driver,file,arrow_count):
    #Get Page
    driver.get(f"https://finance.yahoo.com/quote/%5EGSPC/history?period1=-1325635200&period2=1603929600&interval=1mo&filter=history&frequency=1mo&includeAdjustedClose=true")
    try:
        #Simulate Key Down to Load Entire Page
        actions = ActionChains(driver)
        actions.send_keys(Keys.ARROW_DOWN * arrow_count)
        actions.perform()
    except TimeoutException:
        print("TimeoutException: Element not found")
        return None

    #Create a parse tree of page sources after searching
    soup = BeautifulSoup(driver.page_source, "lxml")

    #Find the table element
    table = soup.find('table')

    #Seperate Headers
    headerList=[]
    table_headers = table.find_all('th')
    for th in table_headers:
        col = th.find_all('span')
        row = [i.text for i in col]
        headerList.append(row)

    headerList=list(chain.from_iterable(headerList))

    #Seperate Rows
    rowList=[]
    table_rows = table.find_all('tr')
    for tr in table_rows:
        td = tr.find_all('td')
        row = [i.text for i in td]
        rowList.append(row)

    #Emit CSV
    df = pd.DataFrame(rowList,columns=headerList)
    df.to_csv (file, index = False, header=True)
    return (df)

#Arrow Count Config (for test)
arrow_count=50000
#arrow_count=1
#FileName Config
file_name="data/SPHistory.csv"
#Get Start Time Timing
start_time = time.time()
#delete old file
delete_old_file(file_name)
# create the driver object.
driver = configure_driver()
#search_keyword = "Web Scraping"
df=getData(driver,file_name,arrow_count)
# close the driver.
driver.close()

#df= pd.read_csv(FileName)
print("Compete!!" +"\n\n" +
      "Rows Scraped:" + "\n" +
      str(df.count()) +  "\n\n"  +
      "Time Elapsed" +"\n" +
      "--- %s seconds ---" % round((time.time() - start_time),2)
        )


# -*- coding: utf-8 -*-
import sys
import multiprocessing
from multiprocessing import *
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import as_completed
import platform
import time
import cv2
from pprint import pprint

from utils.pretty_print import *

def load_images(count, image_filepaths):
    start_time = time.time()

    # images = [cv2.imread(f_input, cv2.IMREAD_COLOR) for f_input in image_list]
    loaded_images = [None] * count

    if count > len(image_filepaths):
        print_red(f"Error: load_images: {count} > {len(image_filepaths)}")

    cpu_count = multiprocessing.cpu_count()
    cpu_count = min(int(cpu_count * (3/4)), cpu_count-2)

    if True:
        with ThreadPoolExecutor(max_workers=min(cpu_count, count)) as executor:
            works = [[no, image_filepaths[no]] for no in range(count)]
            futures = [executor.submit(__load_image, w) for w in works]
            for future in concurrent.futures.as_completed(futures):
                (i, img) = future.result()
                loaded_images[i] = img

    else:
        worker_count = int(count / cpu_count)
        img_count = cpu_count * worker_count
        remaining_img_count = count - img_count

        with ProcessPoolExecutor(worker_count) as executor:
            futures = list()

            for i in range(0, img_count, worker_count):
                futures.append(executor.submit(__load_images, [[no, image_filepaths[no]] for no in range(i, i+worker_count)]))

            # Put the img in the list
            for future in as_completed(futures):
                i_imgs = future.result()
                for (no, img) in i_imgs:
                    loaded_images[no] = img
                    # print(f"loaded image no. {no}")

        if remaining_img_count > 0:
            i_imgs = __load_images([[no, image_filepaths[no]] for no in range(img_count, count)])
            for (no, img) in i_imgs:
                loaded_images[no] = img

    elapsed_time = time.time() - start_time
    print(f"loaded {count} images in {elapsed_time:.02f}s (cpu count={cpu_count})")
    # sys.exit()

    return loaded_images

def __load_images(i_fps):
    # List of [i, image_filepath]
    with ThreadPoolExecutor(len(i_fps)) as executor:
        futures = [executor.submit(__load_image, i_fp) for i_fp in i_fps]
        i_img =  [future.result() for future in futures]
        return i_img

def __load_image(i_fp):
    i, filepath = i_fp
    # print(f"load no. {i}: {filepath}")
    return i, cv2.imread(filepath, cv2.IMREAD_COLOR)


from __future__ import annotations

from typing import Any, Optional

import random

import rules
from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from loguru import logger
from model_utils.models import TimeStampedModel
from rules.contrib.models import RulesModelBase, RulesModelMixin

from .markov_utils import MarkovPOSText
from .rules import (  # is_character_owner,; is_group_owner_and_authenticated,
    is_owner,
    is_owner_or_public,
)
from .signals import markov_sentence_generated, quote_random_retrieved
from .utils import generate_unique_slug_for_model

MAX_QUOTES_FOR_RANDOM_SET = 50
MAX_QUOTES_FOR_RANDOM_GROUP_SET = 50

if hasattr(settings, "MAX_QUOTES_FOR_RANDOM_SET"):  # pragma: nocover
    MAX_QUOTES_FOR_RANDOM_SET = settings.MAX_QUOTES_FOR_RANDOM_SET

if hasattr(settings, "MAX_QUOTES_FOR_RANDOM_GROUP_SET"):  # pragma: nocover
    MAX_QUOTES_FOR_RANDOM_GROUP_SET = settings.MAX_QUOTES_FOR_RANDOM_GROUP_SET

class AbstractOwnerModel(models.Model):
    """
    Abstract model for representing an entity owned by a user with toggles for either allowing submissions for it
    and public access. Defaults to completely private by default.

    Attributes:
        public (bool): is this object public to any authenticated user? Default: False
        allow_submissions (bool): allow other users to submit child objects? Default: False. Not implemented yet.
        owner (User): The user that created this object.
    """

    public = models.BooleanField(
        default=False,
        help_text=_("Is this a public source available for any user to view?"),
    )
    allow_submissions = models.BooleanField(
        default=False, help_text=_("Allow submissions from other users?")
    )
    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)

    class Meta:
        abstract = True

# Create your models here.
class SourceGroup(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    An abstract group or source for a given set of quotes. Multiple sources, or Source objects, can belong to
    the same group. For example, a novel or series if you plan to quote the characters within individually.

    Attributes:
        id (int): Database primary key for the object.
        name (str): Human readable string to name the group. This will be converted to a slug prefix.
        description (str): A description of the group for convenience. Markdown can be used here for styling.
        description_rendered (str): The HTML representation of the description string. Generated automatically.
        owner (User): The user that created the group and therefore owns it.
        public (bool): Is this group public or private. Defaults to False.
        allow_submissions (bool): Allow other users to submit characters to this. Not yet implemented.
        slug (str): A unique slug to represent this group. Generated automatically from name.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    name = models.CharField(
        _("Source Name"),
        max_length=50,
        help_text=_(
            "A source for individuals making the quotes. Use as an abstract grouping."
        ),
        db_index=True,
    )
    description = models.TextField(
        help_text=_("Description for the source. You can style using Markdown."),
        null=True,
        blank=True,
    )
    description_rendered = models.TextField(
        help_text=_("Automatically generated from description"), null=True, blank=True
    )
    slug = models.SlugField(
        unique=True,
        max_length=70,
        editable=False,
        blank=True,
        help_text=_("Unique slug for this group."),
    )

    @cached_property
    def total_sources(self) -> int:
        return Source.objects.filter(group=self).count()

    @cached_property
    def markov_sources(self) -> int:
        return Source.objects.filter(group=self, allow_markov=True).count()

    @cached_property
    def total_quotes(self) -> int:
        return Quote.objects.filter(
            source__in=Source.objects.filter(group=self)
        ).count()

    @cached_property
    def markov_ready(self) -> bool:
        if (
            self.markov_sources > 0
            and Quote.objects.filter(
                source__in=self.source_set.filter(allow_markov=True)
            ).count()
            > 10
        ):
            return True
        return False

    def generate_markov_sentence(
        self, max_characters: Optional[int] = 280
    ) -> str | None:
        """
        Generate a markov sentence based on quotes from markov enabled characters for the group.

        :return: str or None
        """
        if self.markov_ready:
            logger.debug("Group is ready for markov sentences. Checking model...")
            mmodel = GroupMarkovModel.objects.get(group=self)
            if mmodel.data is None:
                logger.debug(
                    "Markov model for group is not generated yet! Generating..."
                )
                mmodel.generate_model_from_corpus()
            logger.debug("Loading text model...")
            text_model = MarkovPOSText.from_json(mmodel.data)
            logger.debug("Generating sentence...")
            sentence: str | None = text_model.make_short_sentence(
                max_chars=max_characters
            )
            if sentence is not None:
                logger.debug(f"Returning generated sentence: '{sentence}'")
                return sentence
        logger.debug("Group is not ready for markov requests yet!")
        return None

    def get_random_quote(
        self, max_quotes_to_process: Optional[int] = MAX_QUOTES_FOR_RANDOM_GROUP_SET
    ) -> Any:
        """
        Get a random quote object from any of the characters defined within the group.
        Prioritizes quotes that have been returned less often.

        :return: ``Quote`` object or None if no quotes are defined.
        """
        # TODO: Create Q object to filter for null datetimes or less than now.
        quotes = (
            Quote.objects.filter(source__in=self.source_set.all())
            .filter(
                models.Q(pub_date__isnull=True) | models.Q(pub_date__lte=timezone.now())
            )
            .select_related("stats")
            .order_by("stats__times_used")[:max_quotes_to_process]
        )
        if quotes.exists():
            quote = random.choice(list(quotes))
            quote_random_retrieved.send(
                type(quote.source), instance=quote.source, quote_retrieved=quote
            )
            return quote
        return None

    def refresh_from_db(self, *args, **kwargs):
        super().refresh_from_db(*args, **kwargs)
        cached_properties = ["total_sources", "markov_sources", "total_quotes"]
        for prop in cached_properties:
            try:
                del self.__dict__[prop]
            except KeyError:  # pragma: nocover
                pass

    def save(self, *args, **kwargs):
        if (
            not self.slug
        ):  # Once this slug is set, it does not change except through devil pacts
            logger.debug("Group is being saved and a slug was provided.")
            self.slug = generate_unique_slug_for_model(
                model_class=type(self), text=self.name
            )
        super().save(*args, **kwargs)

    def __str__(self):  # pragma: nocover
        return self.name

    class Meta:
        rules_permissions = {
            "add": rules.is_authenticated,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }
        ordering = ["name"]

class Source(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    An individual source to attribute the quote to in the system, such as a character from a podcast/book, or a specific
    author.

    Attributes:
        id (int): Database primary key for the object.
        name (str): Unique name of a character within a ``CharacterGroup`` for this entity.
        group (SourceGroup): The parent ``SourceGroup``.
        slug (str): Slug made up of a generated version of the character name and the group slug prefix.
        description (str): Description for the character. Markdown can be used for styling.
        description_rendered (str): HTML representation of the description for convenience. Automatically generated.
        allow_markov (bool): Allow markov quotes to be requested from this character? Default False.
        owner (User): The user that created and owns this character.
        public (bool): Is the character public to other users? Defaults to False.
        allow_submissions (bool): Allow other users to submit quotes for this character? Defaults to False.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    name = models.CharField(max_length=100, help_text=_("Name of the character"))
    slug = models.SlugField(
        max_length=250,
        help_text=_(
            "Global slug of the character, will be auto generated from name and group if not overridden."
        ),
        blank=True,
        editable=False,
        unique=True,
        db_index=True,
    )
    description = models.TextField(
        null=True,
        blank=True,
        help_text=_("Description of this character. You can style this with Markdown."),
    )
    description_rendered = models.TextField(
        null=True, blank=True, help_text=_("Automatically generated from description.")
    )
    allow_markov = models.BooleanField(
        default=False, help_text=_("Allow to be used in markov chains?")
    )
    group = models.ForeignKey(
        SourceGroup,
        on_delete=models.CASCADE,
        help_text=_("The group this character belongs to."),
    )

    @property
    def markov_ready(self) -> bool:
        """
        Conducts sanity checks to see if requesting a markov chain is feasible. Markov must be enabled for a character
        and there must be a sufficient corpus to generate a sentence from. Currently set at a minimum of 10 quotes.

        :return: bool
        """
        if self.allow_markov and Quote.objects.filter(source=self).count() > 10:
            return True
        return False

    def get_markov_sentence(self, max_characters: Optional[int] = 280) -> Optional[str]:
        """
        If valid, generate a markov sentence. If not, return None.

        :param max_characters: Optional maximum limit of characters in the return set. Default: 280
        :return: str or None
        """
        logger.debug("Checking to see if character is markov ready...")
        if self.markov_ready:
            logger.debug("It IS ready. Fetching markov model.")
            markov_model = SourceMarkovModel.objects.get(source=self)
            if not markov_model.data:
                logger.debug("No model defined yet, generating...")
                markov_model.generate_model_from_corpus()
            text_model = MarkovPOSText.from_json(markov_model.data)
            logger.debug("Markov text model loaded. Generating sentence.")
            sentence: str | None = text_model.make_short_sentence(
                max_chars=max_characters
            )
            if sentence is not None:
                markov_sentence_generated.send(type(self), instance=self)
                return sentence
        return None

    def get_random_quote(
        self, max_quotes_to_process: Optional[int] = MAX_QUOTES_FOR_RANDOM_SET
    ) -> Optional[Any]:
        """
        This actually not all that random. It's going to grab the quotes
        ordered ordered by how infrequently they've been returned, and then grab a random one
        in the set. But for our purposes, it's fine. If there aren't any quotes, it will return None.

        :return: ``Quote`` object or None
        """
        quotes_to_pick = (
            Quote.objects.filter(source=self)
            .filter(
                models.Q(pub_date__isnull=True) | models.Q(pub_date__lte=timezone.now())
            )
            .select_related("stats")
            .order_by("stats__times_used")[:max_quotes_to_process]
        )
        if quotes_to_pick.exists():
            # Select a random index in the result set.
            quote_to_return = random.choice(list(quotes_to_pick))
            quote_random_retrieved.send(
                type(self), instance=self, quote_retrieved=quote_to_return
            )
            return quote_to_return
        return None

    def __str__(self):  # pragma: nocover
        return self.name

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = generate_unique_slug_for_model(
                type(self), text=f"{self.group.slug} {self.name}"
            )
        super().save(*args, **kwargs)

    class Meta:
        rules_permissions = {
            "add": rules.is_authenticated,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }

class Quote(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    A quote from a given source.

    Attributes:
        id (int): Database primary key for the object.
        quote (str): The quote text to use. You can use Markdown for styling. Must be <= 280 characters for tweets
        quote_rendered (str): HTML rendered version of the quote field. Automatically generated.
        citation (str): Optional description of quote source, e.g. episode number or book title.
        citation_url (str): Optional accompanying URL for the citation.
        character (Source): The source of this quote.
        owner (User): The user that created and owns this quote.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    quote = models.CharField(
        max_length=280,  # Keep the base limit to 280 so that quotes are 'tweetable'
        help_text="Plain text representation of quote. You can use Markdown here.",
    )
    quote_rendered = models.TextField(
        null=True,
        blank=True,
        help_text=_("HTML rendered version of quote generated from quote plain text."),
    )
    source = models.ForeignKey(
        Source,
        on_delete=models.CASCADE,
        help_text=_("The source of this quote, i.e. name."),
    )
    citation = models.CharField(
        max_length=250,
        blank=True,
        null=True,
        help_text=_("Where is this quote from? Episode #, book?"),
    )
    citation_url = models.URLField(
        null=True, blank=True, help_text=_("URL for citation, if applicable.")
    )
    pub_date = models.DateTimeField(
        null=True,
        blank=True,
        help_text=_("When is the earliest time this should appear in random results?"),
    )

    def __str__(self):  # pragma: nocover
        return f"{self.source.name}: {self.quote}"

    class Meta:
        rules_permissions = {
            # "add": is_character_owner,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }

class SourceMarkovModel(TimeStampedModel):
    """
    The cached markov model for a given source. The database object for this is automatically created
    whenever a new character object is saved.

    Attributes:
        id (int): Database primary key for the object.
        source (Source): The character who the model is sourced from.
        data (json): The JSON representation of the Markov model created by ``markovify``.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    source = models.OneToOneField(Source, on_delete=models.CASCADE)
    data = models.JSONField(null=True, blank=True)

    def generate_model_from_corpus(self):
        """
        Collect all quotes attributed to the related character. Then
        create, compile, and save the model.
        """
        logger.debug("Generating text model. Fetching quotes.")
        quotes = Quote.objects.filter(source=self.source)
        # Don't bother generating model if there isn't data.
        if not quotes.exists():  # pragma: nocover
            logger.debug("There are no quotes. Returning None.")
            return
        logger.debug("Quotes retrieved! Forming into corpus.")
        corpus = " ".join(quote.quote for quote in quotes)
        logger.debug("Building text model.")
        text_model = MarkovPOSText(corpus)
        logger.debug("Compiling text model.")
        text_model.compile(inplace=True)
        logger.debug("Saving model as JSON.")
        self.data = text_model.to_json()
        self.save()
        logger.debug("Markov model populated to database.")

    def __str__(self):  # pragma: nocover
        return self.source.name

class GroupMarkovModel(TimeStampedModel):
    """
    The cached markov model for the entire group. It is made up of every quote from every markov enabled
    character within the group.

    Attributes:
        id (int): The database id of this object.
        group (SourceGroup): The OneToOne relationship to ``SourceGroup``
        data (json): The cached markov model.
        created (datetime): When the object was created.
        modified (datetime): When the object was last modified.
    """

    group = models.OneToOneField(
        SourceGroup,
        on_delete=models.CASCADE,
        help_text=_("The character group this model belongs to."),
    )
    data = models.JSONField(
        null=True, blank=True, help_text=_("The cached markov model as JSON.")
    )

    def generate_model_from_corpus(self):
        """
        Collect all quotes from markov enabled characters in this group and then compile the model and save it.
        """
        logger.debug(f"Gathering corpus for character group: {self.group.name}")
        quotes = Quote.objects.filter(
            source__in=Source.objects.filter(group=self.group, allow_markov=True)
        )
        if quotes.exists():
            if quotes.count() >= 10:
                logger.debug("Found sufficient quotes for a model!")
                corpus = " ".join(quote.quote for quote in quotes)
                logger.debug("Forming text model...")
                text_model = MarkovPOSText(corpus)
                logger.debug("Compiling text model...")
                text_model.compile(inplace=True)
                logger.debug("Saving compiled model to JSON...")
                self.data = text_model.to_json()
                self.save()
                logger.debug(
                    f"Finished building and saving group markov model for {self.group.name}!"
                )

class QuoteStats(TimeStampedModel):
    """
    A simple object used to track how often an individual quote is used.

    Attributes:
        id (int): The database primary key of this object.
        quote (Quote): The quote this stat relates to.
        times_used (int): The number of times this has been used by an service such as random quote.
        created (datetime): When this was created.
        modified (datetime): When this was last modified.
    """

    quote = models.OneToOneField(
        Quote,
        on_delete=models.CASCADE,
        related_name="stats",
        help_text=_("The Quote the stats related to."),
    )
    times_used = models.PositiveIntegerField(
        default=0, help_text=_("Times used for random quotes, etc.")
    )

    def __str__(self):  # pragma: nocover
        return f"Stats for Quote {self.quote.id}"

class GroupStats(TimeStampedModel):
    """
    An object for using to track usage stats for ``CharacterGroup``.

    Attributes:
        group (SourceGroup): The group this is collecting stats for.
        quotes_requested (int): The number of times a quote from this object or its children has been requested.
        quotes_generated (int): The number of times a markov quote has been generated for this or it's children.
    """

    group = models.OneToOneField(
        SourceGroup, related_name="stats", on_delete=models.CASCADE
    )
    quotes_requested = models.PositiveIntegerField(
        default=0, help_text=_("Number of time child quotes have been requested.")
    )
    quotes_generated = models.PositiveIntegerField(
        default=0,
        help_text=_("Number of times markov generated quotes have been requested."),
    )

class SourceStats(TimeStampedModel):
    """
    An object for using to track usage stats for ``Character``.

    Attributes:
        source (Source): The source this is collecting stats for.
        quotes_requested (int): The number of times a quote from this object or its children has been requested.
        quotes_generated (int): The number of times a markov quote has been generated for this or it's children.
    """

    source = models.OneToOneField(
        Source, related_name="stats", on_delete=models.CASCADE
    )
    quotes_requested = models.PositiveIntegerField(
        default=0, help_text=_("Number of time child quotes have been requested.")
    )
    quotes_generated = models.PositiveIntegerField(
        default=0,
        help_text=_("Number of times markov generated quotes have been requested."),
    )


# 导入tushare
import tushare as ts

# 初始化pro接口
pro = ts.pro_api('ca0af3044cc38461f8e4ae128c9edabc12bcab9f4628f5cf6b6d863a')

# 拉取数据
df = pro.index_dailybasic(**{
    "trade_date": "",
    "ts_code": "000001.SH",
    "start_date": "",
    "end_date": "",
    "limit": "",
    "offset": ""
}, fields=[
    "ts_code",
    "trade_date",
    "total_mv",
    "float_mv",
    "total_share",
    "float_share",
    "free_share",
    "turnover_rate",
    "turnover_rate_f",
    "pe",
    "pe_ttm",
    "pb"
])
print(df)

05-指数/06-大盘指数每日指标.py

06-大盘指数每日指标.py


from google.cloud import storage

import os
from pathlib import Path
import shutil
from time import time
from datetime import datetime

import subprocess
from zipfile import ZipFile,BadZipfile

import requests
import csv

import numpy as np

import logging

def list_gcs_directories(client, bucket, prefix,debug=False):
    # from https://github.com/GoogleCloudPlatform/google-cloud-python/issues/920
    iterator = client.list_blobs(bucket, prefix=prefix, delimiter='/')
    prefixes = set()
    for page in iterator.pages:
        prefixes.update(page.prefixes)
        if debug:
            break #only do one page
    ticker_list = [ticker[5:-1] for ticker in prefixes]
    return ticker_list

def setup_bucket():
    bucket_name = 'kaggle-sec-data'
    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    return bucket_name, storage_client, bucket

def get_all_tickers():
    resp = requests.get('https://www.sec.gov/include/ticker.txt')
    ticker_cik_text = resp.content.decode('utf-8')
    ticker_cik_list = list(csv.reader(ticker_cik_text.splitlines(), delimiter='\t'))

    ticker_list = []
    for item in ticker_cik_list:
        ticker_list.append(item[0].upper())

    return ticker_list

debug = True

if debug == True:
    os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="/Users/goldbloom/Dropbox/Side Projects/Edgar/Key/kaggle-playground-0f760ec0ebcd.json"
else:
    os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="./kaggle-playground-0f760ec0ebcd.json"

bucket_name, storage_client, bucket = setup_bucket()

sec_ticker_list = get_all_tickers()

gcs_ticker_list = list_gcs_directories(storage_client,bucket,'data/',False)

complete_ticker_list = []

for ticker in gcs_ticker_list:
    to_match = [f'{ticker} Income Statement.csv', f'{ticker} Balance Sheet.csv', f'{ticker} Cash Flow.csv']
    blobs = storage_client.list_blobs(bucket_name, prefix=f'data/{ticker}/timeseries/Clean Statements/')
    for blob in blobs:
        filename_and_path = blob.name
        filename = filename_and_path[filename_and_path.rfind('/')+1:]
        if filename in to_match:
            to_match.remove(filename)

    if len(to_match) == 0:
        complete_ticker_list.append(ticker)

print("Missing from GCS list")
gcs_missing = [ticker for ticker in sec_ticker_list if ticker not in gcs_ticker_list]
print(gcs_missing)
print("\n")

print("Not complete in GCS")
gcs_not_complete = [ticker for ticker in gcs_ticker_list if ticker not in complete_ticker_list]
print(gcs_not_complete)
print("\n")

print(f"SEC: {len(sec_ticker_list)}; GCS: {len(gcs_ticker_list)}; GCS complete: {len(complete_ticker_list)}")
print(f"Missing from GCS: {len(gcs_missing)}; Not complete in GCS: {len(gcs_not_complete)}")


import sys
sys.stdin = open('h.txt', 'r')

import sys
sys.setrecursionlimit(99999999)

from collections import deque
def bfs(a):
    visit[a] = True
    q = deque()
    q.append(a)
    while q:
        n = q.popleft()
        for i in range(len(G[n])):
            if visit[G[n][i]]:continue
            visit[G[n][i]] = True
            q.append(G[n][i])

def dfs(a):
    visit[a] = True
    for i in range(len(G[a])):
        if visit[G[a][i]]:continue
        dfs(G[a][i])

N, M = map(int, input().split())
arr = [list(map(int, input().split())) for _ in range(M)]
G = [[] for _ in range(N+1)]
visit = [False] * (N+1)
for i in range(M):
    G[arr[i][1]].append(arr[i][0])
for i in range(1, len(visit)):
    if visit[i]: continue
    dfs(i)
    print(i,end=' ')
print()


import requests
import openpyxl
from bs4 import BeautifulSoup

#enter the url here
# url1_indi = "https://www.flipkart.com/dinine-craft-wooden-2-pocket-key-holder-wall-home-decor-decorative-showpiece-6-cm/product-reviews/itmb6135a09b4d7d?pid=SHIGG8J7VQ7BHUYK&lid=LSTSHIGG8J7VQ7BHUYKODYETT&marketplace=FLIPKART"
url1_indi = input("Enter Comments URL Here: ")
page_01 = requests.get(url1_indi)
doc_01 = page_01.text
soup_01 = BeautifulSoup(doc_01,'html.parser')

string = "https://www.flipkart.com"
pages = soup_01.find_all(["a"],class_="ge-49M")

excel = openpyxl.Workbook()
sheet = excel.active
sheet.title = 'Page 1'

insert = []

for i in pages:
    url = string+i['href']
    # print(url)
    page = requests.get(url)
    doc = page.text
    soup = BeautifulSoup(doc,'html.parser')

    for temp in (soup.find_all("div",{ "class" : "col _2wzgFH K0kLPL"})) :
        rating = temp.find("div",{ "class" : "_3LWZlK _1BLPMq"}) #rating
        heading = temp.find("p",{ "class" : "_2-N8zT"}).get_text() #Heading
        comments = temp.find("div",{"class" : "t-ZTKy"}).get_text() #Comments
        comments = comments.removesuffix('READ MORE')
        update = ""
        if rating == None:
            bad_rating = temp.find("div",{ "class" : "_3LWZlK _1rdVr6 _1BLPMq"})
            if bad_rating == None:
                bad_rating = "NULL"
            else:
                bad_rating = bad_rating.get_text()

            update = update + (bad_rating) +"-"
        else:
            update = update + (rating.get_text()) +"-"

        update = update + (heading) +"-"
        update = update + (comments)
        insert.append(update)

sheet.append(insert)

sheet_name = input("Enter the excel sheet file name(No Spaces): ")
excel.save(sheet_name+".xlsx") #change the excel sheet name


from pytube import YouTube, Playlist
from pytube.cli import on_progress
import os

# VIDEO_URL = 'https://www.youtube.com/watch?v=D26bLJ9ut88'
# PLAYLIST_URL = 'https://www.youtube.com/watch?v=Ck3jX3wbyfY&list=PLmHzN0jU_yECKq_7W0DASIGejHHrMgKw0&ab_channel=CortesdoLutz%5BOFICIAL%5D'

while True:
    #Limpando console
    if os.name == "nt":
        os.system("cls")
    else:
        os.system("clear")

    #Escolher tipo de download
    answer = int(input("Deseja fazer download de um video ou playlist ? \n [1]Vídeo\n [2]Playlist\n Resposta:"))

    if answer == 1:
        url_video = input("Digite a url do vídeo: ")
        yt = YouTube(url_video, on_progress_callback=on_progress)
        yt.streams.get_highest_resolution().download()
        break
    elif answer == 2:
        url_playlist = input("Digite a url da playlist: ")
        playlist = Playlist(url_playlist)

        for video in playlist:
            yt = YouTube(video, on_progress_callback=on_progress)
            yt.streams.get_highest_resolution().download()
        break

print("Download complete")


from datetime import datetime
import argparse
from data import get_dataset

dout_dir = (
    "./out/"
    + datetime.now().strftime("%Y-%m-%d")
    + "/run_"
    + datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
)

default_args = dict(
    epochs=100,
    lr=0.03,
    random_seed=42,
    batch_size = 128,
    save_every = 100,
    out_dir=dout_dir,
    date_time_dir = None,
    exp_name="resnet_test",
    wd = 0.0,
    use_cuda=True,
    det_run=True,
    scale_inv = False,
    loglevel_info=False,
    model = 'resnet18',
    model_width = 64, #default for VGG
    half_prec=False,
    no_data_augm = True,
    batch_norm = False,
    dataset = 'cifar10',
    momentum = 0.9,
    first_decay = 0.5,
    lr_gamma_decay = 1.0,
    lr_flow=1e-4,
    flow_steps=100,
    flow_every = 50,
    wd_flow=0.0,
    wd_gamma_decay = 1.0,
    ckpt = None,
    config_file = None,
    radius = None
)

def configuration(args=None):
    parser = argparse.ArgumentParser(description="Exp Resnet")

    #--------------------------------------------------------------------------------
    # TRAINING OPTIONS
    #--------------------------------------------------------------------------------
    tgroup = parser.add_argument_group("Training options")
    tgroup.add_argument(
        "--epochs",
        type=int,
        metavar="N",
        help="Number of training iters. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--flow_steps",
        type=int,
        metavar="N",
        help="Number of gradient flow iters. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--flow_every",
        type=int,
        metavar="N",
        help="Lr is decayed at multiples of this number. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--batch_size",
        type=int,
        metavar="N",
        help="Batch size. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--lr",
        type=float,
        help="Learning rate of optimizer. Default: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--first_decay",
        type=float,
        help="Fraction of epochs when lr is first decayed: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--lr_gamma_decay",
        type=float,
        help="Decaying factor of the learning rate: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--lr_flow",
        type=float,
        help="Decaying the learning rate to this exact value: " +
        "%(default)s.",
    )
    tgroup.add_argument(
        "--wd_flow",
        type=float,
        help="Decaying the learning rate to this exact value: " +
        "%(default)s.",
    )
    tgroup.add_argument(
        "--wd_gamma_decay",
        type=float,
        help="Decaying factor of the weight decay: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--wd",
        type=float,
        help="Weight decay of optimizer. Default: " + "%(default)s.",
    )

    tgroup.add_argument(
        "--radius",
        type=float,
        help="Radius for PGD. Default: " + "%(default)s.",
    )
    tgroup.add_argument('--momentum', type=float, help="Momentum of optimizer. Default: " + "%(default)s.",)

    #--------------------------------------------------------------------------------
    # MODEL OPTIONS
    #--------------------------------------------------------------------------------
    modelgroup = parser.add_argument_group("Model options")
    modelgroup.add_argument('--model', choices=['vit_basic', 'vit_exp', 'vgg16', 'resnet18', 'resnet18_plain', 'resnet18_gn', 'resnet_tiny', 'resnet_tiny_gn', 'resnet34', 'resnet34_plain', 'resnet34preact', 'resnet34_gn', 'wrn28', 'lenet', 'cnn', 'fc', 'linear'], type=str)
    modelgroup.add_argument('--model_width', type=int, help='model width (# conv filters on the first layer for ResNets)')
    modelgroup.add_argument('--half_prec', action='store_true', help='if enabled, runs everything as half precision [not recommended]')
    modelgroup.add_argument('--batch_norm', action = 'store_true', help = 'if enabled, adds batch-norm layers')
    modelgroup.add_argument('--scale_inv', action = 'store_true', help = 'if enabled, makes the model scale invariant, only for resnets')

    #--------------------------------------------------------------------------------
    # MISCELLANEOUS OPTIONS
    #--------------------------------------------------------------------------------
    mgroup = parser.add_argument_group("Miscellaneous options")
    mgroup.add_argument(
        "--random_seed",
        type=int,
        metavar="N",
        help="Random seed. Default: %(default)s.",
    )
    mgroup.add_argument(
        "--save_every",
        type=int,
        metavar="N",
        help="Numer of iterations for saving model. Default: %(default)s.",
    )
    mgroup.add_argument(
        "--out_dir", type=str, help="Where to store the outputs of this simulation."
    )
    mgroup.add_argument("--exp_name", type=str, help="Name of the experiment.")

    mgroup.add_argument("--date_time_dir", type=str, help="Name of the date time dir.")

    #--------------------------------------------------------------------------------
    # EXPERIMENTS OPTIONS
    #--------------------------------------------------------------------------------
    expgroup = parser.add_argument_group("Experiments options")
    expgroup.add_argument(
        "--use_cuda",
        action="store_true",
        help="Use cuda",
    )
    expgroup.add_argument(
        "--det_run",
        action="store_true",
        help="Ensure deterministic behaviour",
    )
    expgroup.add_argument(
        "--loglevel_info",
        action="store_true",
        help="Info in the logger",
    )
    expgroup.add_argument(
        "--ckpt",
        type=str,
        help="Path for model checkpoint: " + "%(default)s.",
    )
    expgroup.add_argument(
        "--config_file",
        type=str,
        help="Path to specify config file: " + "%(default)s.",
    )
    expgroup.add_argument('--no_data_augm', action='store_true')

    #--------------------------------------------------------------------------------
    # DATA OPTIONS
    #--------------------------------------------------------------------------------
    datagroup = parser.add_argument_group("Dataset options")
    datagroup.add_argument(
        "--input_dim",
        type=int,
        help="Random seed. Default: %(default)s.",
    )
    datagroup.add_argument('--dataset', choices=get_dataset.datasets_dict.keys(), type=str)

    parser.set_defaults(**default_args)

    if args is None:
        return parser.parse_args()
    else:
        return parser.parse_args(args=args)


from flask import Flask
from flask import request
import subprocess
import logging
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s", "%d-%m-%y %H:%M:%S")

#convert approved ids to list
file = open("approvedIDs.txt", "r")
ids = file.read().split(".")

app = Flask(__name__)

def start():
    subprocess.call([r'D:/Minecraft servers/1.19/test/Run me.bat'])

@app.route("/minecraft/start", methods=["GET"])
def checkID():
    uid = request.args.get("uid")
    logging.info(f"UID: {uid}")

    if uid in ids:
        start()
        return "Success"

    else:
        return "Fail"


import matplotlib.pyplot as plt
import pandas as pd

from q51 import load_data
from q52 import create_model

train_x_vec = pd.read_table('files/train.feature.txt', header=None)
valid_x_vec = pd.read_table('files/valid.feature.txt', header=None)
test_x_vec = pd.read_table('files/test.feature.txt', header=None)

_, train_y = load_data('files/train.txt')
_, valid_y = load_data('files/valid.txt')
_, test_y = load_data('files/test.txt')

for index, penalty in enumerate(['l1', 'l2'], start=1):
    models = [
        create_model(train_x_vec, train_y, C=10**(-c), random_state=42, penalty=penalty)
        for c in range(-2, 6)
    ]
    plt.subplot(2, 1, index)
    plt.xscale('log')
    plt.title(f'{penalty.upper()}')
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(train_x_vec, train_y) for model in models],
        label='train',
        marker='o'
    )
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(valid_x_vec, valid_y) for model in models],
        label='valid',
        marker='o'
    )
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(test_x_vec, test_y) for model in models],
        label='test',
        marker='o'
    )
    plt.ylim([0, 1])
    plt.legend()

plt.show()


#coding:utf-8
from django.utils.safestring import mark_safe

def page(baseurl, totalnum, currentpage):
    perpage = 20
    pagenum = 11

    temp = divmod(totalnum, perpage)

    totalpage = temp[0]
    if temp[1]:
        totalpage += 1

    start = 1
    if currentpage>6:
         start = currentpage - 5
    end = currentpage + 5

    if end > totalpage:
        end = totalpage

    html = []
    for i in range(start, end):
        url = baseurl + str(i)
        if i == currentpage:
            html.append('' + str(i) + '')
        else:
            html.append(''+str(i)+'')

    return mark_safe(''.join(html))


import shutil
from pathlib import Path
import argparse

def main(args):
    get_size = lambda path: sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())

    root_paths = Path(args.root_dir).glob('*')

    for root_path in root_paths:
        for path in root_path.glob('*'):
            if not path.is_dir(): continue
            kb_size = get_size(path) / (1024 ** 2)

            if kb_size < 200.:
                shutil.rmtree(path)
                print(f"{path.parent.name}/{path.name}: {kb_size:.4f} kb (deleted)")
            else:
                print(f"{path.parent.name}/{path.name}: {kb_size:.4f} kb")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='root path')

    parser.add_argument('--root-dir', type=str,
                        default='/workspace/logs/cassava-leaf-disease-classification/',
                        help='root log directory')

    args = parser.parse_args()
    main(args)


from collections import deque
import numpy as np
import random
import heapq

class MaxHeapObj(object):
    def __init__(self, val):
        self.val = val

    def __lt__(self, other):
        return self.val > other.val

    def __eq__(self, other):
        return self.val == other.val

    def __str__(self):
        return str(self.val)

class MinHeap(object):
    def __init__(self):
        self.h = []

    def heappush(self, x):
        heapq.heappush(self.h, x)

    def heappop(self):
        return heapq.heappop(self.h)

    def __getitem__(self, i):
        return self.h[i]

    def __len__(self):
        return len(self.h)

    def remove(self, v):
        self.h.remove(v)

    def heaptop(self):
        if len(self.h) > 0:
            return self.h[0]
        else:
            return None

class MaxHeap(MinHeap):
    def heappush(self, x):
        heapq.heappush(self.h, MaxHeapObj(x))

    def heappop(self):
        return heapq.heappop(self.h).val

    def __getitem__(self, i):
        return self.h[i].val

    def remove(self, v):
        self.h.remove(MaxHeapObj(v))

class ReplayBuffer:
    def __init__(self, size, gamma):
        # init deque
        self.memory = deque(maxlen=size)
        # init heaps
        self.goodReward = MinHeap()
        self.badReward = MaxHeap()

        self.gamma = gamma

    def remember(self, state, action, reward, next_state):
        element = (reward, state, action, next_state)

        if len(self.memory) == self.memory.maxlen:
            vOut = self.memory[0]
            # remove oldest deque element from heaps
            if vOut >= self.goodReward.heaptop():
                self.goodReward.remove(vOut)
            else:
                self.badReward.remove(vOut)

        # add element to the relevant heap
        if (len(self.goodReward) > 0) and (element >= self.goodReward.heaptop()):
            self.goodReward.heappush(element)
        else:
            self.badReward.heappush(element)

        # balance heaps
        if abs(len(self.goodReward) - len(self.badReward)) > 1:
            if len(self.goodReward) > len(self.badReward):
                vOut = self.goodReward.heappop()
                self.badReward.heappush(vOut)
            else:
                vOut = self.badReward.heappop()
                self.goodReward.heappush(vOut)

        # add element to deque
        self.memory.append(element)

    def replay(self, actorMainModel, actorTargetModel, actorTrainFunc, actorWolpertingerFunc, actorUpdateEpsilonFunc,
               criticMainModel, criticTargetModel, criticGradientsFunc, stateNormalizeFunc, updateTargetModelsParams,
               trainSetSize):
        # Sample trainSet from  memory
        # trainSet = random.sample(self.memory, min(trainSetSize, len(self.memory)))
        trainSet = random.sample(self.goodReward.h, min(int(trainSetSize / 2), len(self.goodReward)))
        tempSet = random.sample(self.badReward.h, min(trainSetSize - len(trainSet), len(self.badReward)))
        trainSet.extend([t.val for t in tempSet])
        del tempSet

        trainState = []
        trainAction = []
        trainNextState = []
        trainReward = []
        for i, (state, action, reward, next_state) in enumerate(trainSet):
            trainState.append(stateNormalizeFunc(state))
            trainAction.append(action)
            trainNextState.append(stateNormalizeFunc(next_state))
            trainReward.append(reward)
        # TODO: check trainAction contains action vectors rather than action ID

        trainState = np.array(trainState)
        trainAction = np.array(trainAction)
        trainReward = np.expand_dims(np.array(trainReward), axis=1)
        trainNextState = np.array(trainNextState)

        # Calculate targets
        # # predict actor target model next state preferred continuous action
        # action = actorTargetModel.predict(trainNextState)
        # predict actor target model next state preferred discrete action
        action, _, _ = actorWolpertingerFunc(trainNextState, actorTargetModel, criticTargetModel)
        # TODO: it is not clear from paper if critic model should be MAIN or TARGET ???
        # predict critic target model (nextState, nextAction) Q value, i.e. future reward
        criticTargetPrediction = criticTargetModel.predict([trainNextState, action])
        # update future **discounted** reward
        target = trainReward + (self.gamma * criticTargetPrediction)

        # train (update) critic train model
        loss = criticMainModel.train_on_batch([trainState, trainAction], target)
        # train (update) actor train model
        actions_for_grad = actorMainModel.predict(trainState)
        grads = criticGradientsFunc(trainState, actions_for_grad)
        actorTrainFunc(trainState, grads)

        # update actor & critic target models weights
        updateTargetModelsParams()

        # update epsilon value
        actorUpdateEpsilonFunc()

        return loss


import os, time, datetime
from flask import flash, render_template, redirect, send_file, request, url_for
from flask_login import current_user, login_required, user_unauthorized
from google.cloud import storage
from app.admin.forms import CommentForm, UserForm
from app import db
from app.models import User, Image, Comment
from app.admin.forms import ImageForm
from app.main import bp
from config import basedir, GOOGLE_AUTH, BUCKET_NAME

@bp.route('/')
@bp.route('/index')
@login_required
def index():
    if not user_unauthorized:
        if current_user.is_admin:
            return redirect('admin.index')
    images = Image.query.filter_by(user_id= current_user.id).all()
    if len(images) > 0:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)

        for i in images:
            bolb = bucket.blob(i.src)
            bolb.make_public()
            i.file_path = bolb.public_url

    return render_template('index.html', user= current_user, images = images)

@bp.route('/image/<int:id>', methods=['GET'])
def image(id):
    image = Image.query.filter_by(id= id, user_id = current_user.id).one()
    if image:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)
        bolb = bucket.blob(image.src)
        image.file_path = bolb.public_url
    comments = Comment.query.filter_by(image_id=id).all()

    form = CommentForm(csrf_enabled= False)

    return render_template('main/image.html', user= current_user, image= image, comments= comments, form= form, action_url=url_for('main.add_comment', imgid= image.id), btn_label='Add Comment')

@bp.route('/images', methods= ['GET'])
@login_required
def images():
    images = Image.query.filter_by(user_id = current_user.id).all()
    if len(images) > 0:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)

        for i in images:
            bolb = bucket.blob(i.src)
            i.file_path = bolb.public_url
    return render_template('main/images.html', user= current_user, images= images)

@bp.route('/image/add', methods=['GET', 'POST'])
@login_required
def add_image():
    form = ImageForm(csrf_enabled= False)
    if form.validate_on_submit() and request.method == 'POST':
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
        bucket = gcs.bucket(BUCKET_NAME)

        image_data = form.image.data.stream.read()

        now = str(int(time.time()))
        n_filename = now + '_' + form.image.data.filename

        blob = bucket.blob(n_filename)

        blob.upload_from_string(image_data, content_type=form.image.data.mimetype)
        blob.make_public()
        image = Image(title= form.title.data, src= n_filename , user_id= current_user.id, description= form.description.data)
        db.session.add(image)
        db.session.commit()

        return redirect(url_for('main.images'))
    return render_template('main/add_image.html', user= current_user, form= form, action_url=url_for('main.add_image'), btn_label='Add Image')

@bp.route('/image/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit_image(id):
    image = Image.query.filter_by(id= id, user_id = current_user.id).one()
    form  = ImageForm(csrf_enabled= False, obj=image)
    if form.validate_on_submit() and request.method == 'POST':
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
        bucket = gcs.bucket(BUCKET_NAME)

        image_data = form.image.data.stream.read()

        now = str(int(time.time()))
        n_filename = now + '_' + form.image.data.filename

        blob = bucket.blob(n_filename)
        o_blob = bucket.blob(image.src)
        o_blob.delete()

        blob.upload_from_string(image_data, content_type=form.image.data.mimetype)
        blob.make_public()
        image.src = n_filename
        image.title = form.title.data

        image.description = form.description.data
        db.session.merge(image)
        db.session.commit()

        return redirect(url_for('main.images'))
    return render_template('main/add_image.html', user= current_user, form= form, action_url=url_for('main.edit_image', id= id), btn_label="Update Image")

@bp.route('/image/delete/<int:id>', methods=['GET'])
@login_required
def delete_image(id):
    image = Image.query.filter_by(id = id, user_id= current_user.id).one()
    gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
    bucket = gcs.bucket(BUCKET_NAME)
    blob = bucket.blob(image.src)
    blob.delete()
    db.session.delete(image)
    db.session.commit()
    # flash('Image Deleted!')
    return redirect(url_for('main.images'))

@bp.route('/image/download/<int:id>', methods=['GET'])
@login_required
def download_image(id):
    image = Image.query.filter_by(id = id).one()
    gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
    bucket = gcs.bucket(BUCKET_NAME)
    blob = bucket.blob(image.src)
    temppath = os.path.join(basedir, 'app/static/tempimg/')
    if not os.path.exists(temppath):
        os.mkdir(temppath)
    with open(os.path.join(temppath, blob.name), 'wb') as file:
        blob.download_to_file(file)
    return send_file(os.path.join(temppath, blob.name), attachment_filename=blob.name, mimetype='image/jpg')

@bp.route('/user/edit', methods=['GET', 'POST'])
@login_required
def edit_user():
    user = User.query.filter_by(id= current_user.id).one()
    form = UserForm(csrf_enabled= False, obj= user)
    if form.validate_on_submit() and request.method == 'POST':
        user.username = form.username.data
        user.set_password(form.username.data)
        user.address = form.address.data
        user.email = form.email.data

        db.session.merge(user)
        db.session.commit()

        # flash('User profile updated!')
        return redirect(url_for('main.index'))
    return render_template('main/edit_user.html',user= current_user, form= form, action_url=url_for('main.edit_user'), btn_label="Update User Profile")

@bp.route('/comment/add/<int:imgid>', methods=['POST'])
def add_comment(imgid):
    form = CommentForm(csrf_enabled= False)
    if form.validate_on_submit() and request.method == 'POST':
        comment = Comment(content= form.content.data, image_id=imgid, user_id= current_user.id, created_at= datetime.datetime.now())
        db.session.add(comment)
        db.session.commit()
        # flash('Comment Added')
    return redirect(url_for('main.image', id= imgid))


import re
from typing_extensions import Self
from flask import Flask, request, template_rendered, Blueprint
from flask import url_for, redirect, flash
from flask import render_template
from flask_login import LoginManager, UserMixin, login_user, logout_user, login_required, current_user
from datetime import datetime
from numpy import identity, product
import random, string
from sqlalchemy import null
from link import *
import math
from base64 import b64encode
from api.sql import *

store = Blueprint('bookstore', __name__, template_folder='../templates')

@store.route('/', methods=['GET', 'POST'])
@login_required
def bookstore():
    result = Product.count()
    count = math.ceil(result[0]/6)
    flag = 0

    if request.method == 'GET':
        if(current_user.role == 'manager'):
            flash('No permission')
            return redirect(url_for('manager.home'))

    if 'keyword' in request.args and 'page' in request.args:
        total = 0
        single = 1
        page = int(request.args['page'])
        start = (page - 1) * 6
        end = page * 6
        search = request.values.get('keyword')
        keyword = search

        cursor.prepare('SELECT * FROM PRODUCT WHERE PNAME LIKE :search')
        cursor.execute(None, {'search': '%' + keyword + '%'})
        book_row = cursor.fetchall()
        book_data = []
        final_data = []

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5],
            }
            book_data.append(book)
            total = total + 1

        if(len(book_data) < end):
            end = len(book_data)
            flag = 1

        for j in range(start, end):
            final_data.append(book_data[j])

        count = math.ceil(total/6)

        return render_template('bookstore.html', single=single, keyword=search, book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

    elif 'pid' in request.args:
        pid = request.args['pid']
        data = Product.get_product(pid)

        pname = data[1]
        price = data[2]
        sname = data[3]
        description = data[4]

        image = 'default.png'  #在此更改圖片
        if(data[5] != None):
            image = data[5]

        sdesc = '尚未新增品牌介紹'
        if(SUPPLIER.get_sDesc(sname) != None):
            sdesc = SUPPLIER.get_sDesc(sname)[0]

        product = {
            '商品編號': pid,
            '商品名稱': pname,
            '單價': price,
            '品牌': sname,
            '商品敘述': description,
            '商品圖片': image,
            '品牌介紹': sdesc
        }

        return render_template('product.html', data = product, user=current_user.name)

    elif 'page' in request.args:
        page = int(request.args['page'])
        start = (page - 1) * 6
        end = page * 6

        book_row = Product.get_all_product()
        book_data = []
        final_data = []

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }
            book_data.append(book)

        if(len(book_data) < end):
            end = len(book_data)
            flag = 1

        for j in range(start, end):
            final_data.append(book_data[j])

        return render_template('bookstore.html', book_data=final_data, user=current_user.name, page=page, flag=flag, count=count)

    elif 'keyword' in request.args:
        single = 1
        search = request.values.get('keyword')
        keyword = search
        cursor.prepare('SELECT * FROM PRODUCT WHERE PNAME LIKE :search')
        cursor.execute(None, {'search': '%' + keyword + '%'})
        book_row = cursor.fetchall()
        book_data = []
        total = 0

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }

            book_data.append(book)
            total = total + 1

        if(len(book_data) < 6):
            flag = 1

        count = math.ceil(total/6)

        return render_template('bookstore.html', keyword=search, single=single, book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

    else:
        book_row = Product.get_all_product()
        book_data = []
        temp = 0
        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }
            if len(book_data) < 6:
                book_data.append(book)

        return render_template('bookstore.html', book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

# 會員購物車
@store.route('/cart', methods=['GET', 'POST'])
@login_required # 使用者登入後才可以看
def cart():

    # 以防管理者誤闖
    if request.method == 'GET':
        if( current_user.role == 'manager'):
            flash('No permission')
            return redirect(url_for('manager.home'))

    # 回傳有 pid 代表要 加商品
    if request.method == 'POST':

        if "pid" in request.form :
            data = Cart.get_cart(current_user.id)

            if( data == None): #假如購物車裡面沒有他的資料
                time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                Cart.add_cart(current_user.id, time) # 幫他加一台購物車
                data = Cart.get_cart(current_user.id)

            cid = data[3] # 取得購物車編號(cId)
            pid = request.values.get('pid') # 使用者想要購買的東西
            # 檢查購物車裡面有沒有商品
            product = CONTAIN.check_product(cid, pid)
            # 取得商品價錢
            price = Product.get_product(pid)[2]

            # 如果購物車裡面沒有的話 把他加一個進去
            if(product == None):
                CONTAIN.add_product( {'cid': cid, 'pid':pid} )
            else:
                # 假如購物車裡面有的話，就多加一個進去
                amount = CONTAIN.get_amount(cid, pid)
                CONTAIN.update_product({'amount':amount+1, 'pid':pid, 'cid':cid})

        elif "delete" in request.form :
            pid = request.values.get('delete')
            cid = Cart.get_cart(current_user.id)[3]

            Member.delete_product(cid, pid)
            product_data = only_cart()

        elif "user_edit" in request.form:
            change_order()
            return redirect(url_for('bookstore.bookstore'))

        elif "buy" in request.form:
            change_order()
            return redirect(url_for('bookstore.order'))

        elif "order" in request.form:
            cid = Cart.get_cart(current_user.id)[3]
            total = CONTAIN.get_total(cid)

            time = str(datetime.now().strftime('%Y/%m/%d %H:%M:%S'))
            format = 'yyyy/mm/dd hh24:mi:ss'
            TRANSACTION.add_order( {'mid': current_user.id,
                                    'time':time,
                                    'cid':cid,
                                    'format':format,
                                    'payment':'轉帳',
                                    } )

            tno = TRANSACTION.get_tno(cid)[0][0]
            Cart.set_cart(cid, tno)

            Record.add_record(cid, tno)

            return render_template('complete.html', user=current_user.name)

    product_data = only_cart()

    if product_data == 0:
        return render_template('empty.html', user=current_user.name)
    else:
        return render_template('cart.html', data=product_data, user=current_user.name)

@store.route('/order')
def order():
    data = Cart.get_cart(current_user.id)
    cid = data[3]

    product_row = CONTAIN.get_record(cid)
    product_data = []

    for i in product_row:
        pname = Product.get_name(i[2])
        product = {
            '商品編號': i[2],
            '商品名稱': pname,
            '商品價格': Product.get_price(i[2]),
            '數量': i[1]
        }
        product_data.append(product)

    total = CONTAIN.get_total(cid)

    return render_template('order.html', data=product_data, total=total, user=current_user.name)

@store.route('/orderlist', methods=['GET', 'POST'])
def orderlist():
    #新增取消訂單的功能
    if 'delete' in request.values:
        tno = request.values.get('delete')
        data = TRANSACTION.delete_order(tno)

    if "oid" in request.args :
        pass

    user_id = current_user.id

    data = Member.get_order(user_id)
    orderlist = []

    for i in data:
        temp = {
            '訂單編號': i[3],
            '訂單總價': CONTAIN.get_total(i[2]),
            '訂單時間': i[1]
        }
        orderlist.append(temp)

    orderdetail_row = TRANSACTION.get_orderdetail()
    orderdetail = []

    for j in orderdetail_row:
        temp = {
            '訂單編號': j[0],
            '商品名稱': j[1],
            '商品單價': j[2],
            '訂購數量': j[3]
        }
        orderdetail.append(temp)

    return render_template('orderlist.html', data=orderlist, detail=orderdetail, user=current_user.name)

def change_order():
    data = Cart.get_cart(current_user.id)
    cid = data[3] # 使用者有購物車了，取得購物車編號
    product_row = CONTAIN.get_record(cid)

    for i in product_row:

        # i[0]：交易編號 / i[1]：商品編號 / i[2]：數量 / i[3]：價格
        # i[0]：cid / i[1]：商品數量 / i[2]：pid
        if int(request.form[i[2]]) != i[1]:
            CONTAIN.update_product({
                'amount':request.form[i[2]],
                'pid':i[2],
                'cid':cid,
                #'total':int(request.form[i[1]])*int(Product.get_price(i[2]))
            })
            print('change')

    return 0

def only_cart():

    count = Cart.check(current_user.id)

    if(count == None):
        return 0

    data = Cart.get_cart(current_user.id)
    cid = data[3]
    product_row = CONTAIN.get_record(cid)
    product_data = []

    for i in product_row:
        pid = i[2]
        pname = Product.get_name(pid)
        price = Product.get_price(pid)
        amount = i[1]

        product = {
            '商品編號': pid,
            '商品名稱': pname,
            '商品價格': price,
            '數量': amount
        }
        product_data.append(product)

    return product_data


from django.shortcuts import render
from .models import Nodes,Data,SearchData,Commodity,Order
from rest_framework import mixins,status,filters,viewsets
from .serializers import NodesSerializer,DataSerializer,SearchDataSerializer,OrderSerializer,CommoditySerializer
from rest_framework.pagination import PageNumberPagination
from django_filters.rest_framework import DjangoFilterBackend
from .filters import NodesFilter
from .mqtt import *
from rest_framework.decorators import action
from rest_framework.response import Response
from .CountModelMixin import CountModelMixin
from django.db.models import Count,F
from rest_framework.decorators import api_view
from itertools import chain
import time
import json
import datetime

# Create your views here.
class DataListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口允许被查看和修改
    """
    queryset = Data.objects.all().order_by('-recordTime')
    serializer_class = DataSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('nodeId', 'recordTime', 'val', 'safe', 'unit')
    filter_fields = ('nodeId', 'recordTime', 'val', 'safe', 'unit')

    """
    根据时间段返回list数据
    """
    @action(detail=False)
    def get(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())
        start_date = self.request.query_params.get('start_date', None)
        end_date = self.request.query_params.get('end_date', None)
        if start_date or end_date:
            queryset = queryset.filter(recordTime__range=[start_date, end_date])
            serializer = DataSerializer(queryset, many=True)
            return Response(serializer.data)

    """
    根据时间返回list数据
    """
    @action(detail=False)
    def getByTimescale(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        if timescale == 'month':
            queryset = Data.objects.filter(recordTime__month=num)
        elif timescale == 'year':
            queryset = Data.objects.filter(recordTime__year=num)
        serializer = DataSerializer(queryset, many=True)
        return Response(serializer.data)

    """
    根据时间尺度分割，并返回相应的数据，其中type有2种类型，默认为data，然后就是count
    """
    @action(detail=False)
    def segmentData(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        type = self.request.query_params.get('type', None)
        nodeId = self.request.query_params.get('nodeId', None)
        result = []
        if timescale == 'year':
            queryset = Data.objects.filter(recordTime__year=num)
            if nodeId!='all' and nodeId:
                queryset = queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,13):
                    result.insert(i, {'x': str(i)+'月', 'y': len(DataSerializer(queryset.filter(recordTime__month=i), many=True).data)})
            else:
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(queryset.filter(recordTime__month=i),many=True).data})
        elif timescale == 'month':
            queryset = Data.objects.filter(recordTime__year=2019,recordTime__month=num)
            if nodeId!='all' and nodeId:
                queryset = Data.objects.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,32):
                    result.insert(i, {'x': str(i)+'号', 'y': len(DataSerializer(queryset.filter(recordTime__day=i), many=True).data)})
            else:
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(queryset.filter(recordTime__day=i),many=True).data})
        return Response(result)

    """
    根据时间尺度返回安全率数值   type有三个值  count、safe、unsafe
    """
    @action(detail=False)
    def segmentSafe(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        type = self.request.query_params.get('type', None)
        nodeId = self.request.query_params.get('nodeId', None)
        result = []
        if timescale == 'year':
            last_range = datetime.datetime.now() - datetime.timedelta(days=365)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,13):
                    allNum = queryset.filter(recordTime__month=i).count()
                    temp = safeQueryset.filter(recordTime__month=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'月','y': y})
            elif type == 'safe':
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(safeQueryset.filter(recordTime__month=i),many=True).data})
            else:
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(unsafeQueryset.filter(recordTime__month=i),many=True).data})
        elif timescale == 'month':
            last_range = datetime.datetime.now() - datetime.timedelta(days=31)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,32):
                    allNum = queryset.filter(recordTime__day=i).count()
                    temp = safeQueryset.filter(recordTime__day=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'号','y': y})
            elif type == 'safe':
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(safeQueryset.filter(recordTime__day=i),many=True).data})
            else:
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(unsafeQueryset.filter(recordTime__day=i),many=True).data})
        elif timescale == 'today':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,25):
                    allNum = queryset.filter(recordTime__hour=i).count()
                    temp = safeQueryset.filter(recordTime__hour=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'点','y': y})
            elif type == 'safe':
                for i in range(1,25):
                    result.insert(i, {str(i)+'点': DataSerializer(safeQueryset.filter(recordTime__hour=i),many=True).data})
            else:
                for i in range(1,25):
                    result.insert(i, {str(i)+'点': DataSerializer(unsafeQueryset.filter(recordTime__hour=i),many=True).data})
        return Response(result)

    """
    返回三天的图标，用于linechart表格的展示

    order_by  '-' 从大到小,不加则为从小到大
    """
    @action(detail=False)
    def lineChartData(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        today_range = datetime.datetime.now() - datetime.timedelta(days=1)
        yesterday_range = datetime.datetime.now() - datetime.timedelta(days=2)
        today_queryset = Data.objects.filter(nodeId=nodeId,recordTime__range=[today_range,datetime.datetime.now()])
        yesterday_queryset = Data.objects.filter(nodeId=nodeId,recordTime__range=[yesterday_range,today_range])
        today_count = today_queryset.count()
        yesterday_count = yesterday_queryset.count()
        today_safe = today_queryset.filter(safe=True).values(x=F('recordTime'),y1=F('val'))
        today_unsafe = today_queryset.filter(safe=False).values(x=F('recordTime'),y2=F('val'))
        yesterday_safe = yesterday_queryset.filter(safe=True).values(x=F('recordTime'),y1=F('val'))
        yesterday_unsafe = yesterday_queryset.filter(safe=False).values(x=F('recordTime'),y2=F('val'))
        result= [{'name':'今天','cvr':today_count},{'name':'昨天','cvr':yesterday_count}]
        for i in today_safe:
            i['y2']=0
        for i in today_unsafe:
            i['y1']=0
        for i in yesterday_safe:
            i['y2']=0
        for i in yesterday_unsafe:
            i['y1']=0
        today = list(chain(today_safe,today_unsafe))
        yesterday = list(chain(yesterday_safe,yesterday_unsafe))
        #today = today_safe | today_unsafe
        #yesterday = yesterday_safe | yesterday_unsafe
        result[0]['shop']=sorted(today, key=lambda x:x['x'])
        result[1]['shop']=sorted(yesterday, key=lambda x:x['x'])
        return Response(result)

    @action(detail=False)
    def realTimeSafe(self, request, *args, **kwargs):
        """实时安全率

        根据时间尺度返回实时安全率数据
        gte greater      lt  less then

        Args:
            timescale: 时间尺度,可为hour,day,week,month,year

        Returns:
            返回实时安全率数值，以dict的形式

            {'safeRate': 15}
        """
        timescale = self.request.query_params.get('timescale',None)
        last_range = ''
        if timescale == 'hour':
            last_range = datetime.datetime.now() - datetime.timedelta(hours=1)
        elif timescale == 'day':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
        elif timescale == 'week':
            last_range = datetime.datetime.now() - datetime.timedelta(weeks=7)
        elif timescale == 'month':
            last_range = datetime.datetime.now() - datetime.timedelta(days=30)
        else:
            last_range = datetime.datetime.now() -datetime.timedelta(days=365)
        safeNum = Data.objects.filter(recordTime__gte=last_range,safe=True).count()
        allNum = Data.objects.filter(recordTime__gte=last_range).count()
        result= round(safeNum/allNum*100,2) if allNum!=0 else 0
        return Response({'safeRate': result})

    @action(detail=False)
    def countByTimescale(self, request, *args, **kwargs):
        """根据时间返回count值

        timescale可为年月类型，然后根据查询的时间数字，返回需要的count数据

        Args:
            timescale: 时间尺度可为年月
            num: 查询时间

        Returns:
            返回一个dict类型，count数值
            示例：

            {'count': 12}
        """
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        count = 0
        if timescale == 'month':
            count = Data.objects.filter(recordTime__year=2019,recordTime__month=num).count()
        elif timescale == 'year':
            count = Data.objects.filter(recordTime__year=num).count()
        elif timescale == 'today':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
            count = Data.objects.filter(recordTime__gte=last_range).count()
        elif timescale == 'hour':
            last_range = datetime.datetime.now() - datetime.timedelta(hours=1)
            count = Data.objects.filter(recordTime__gte=last_range).count()
        return Response({'count': count})

    """
    返回数据收集排名榜，做个名字加数字总和
    """
    @action(detail=False)
    def countRank(self, request, *args, **kwargs):
        limit = self.request.query_params.get('limit', None)
        type = self.request.query_params.get('type', None)
        result= []
        if type == 'data':
            queryset = Nodes.objects.annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            for i in queryset:
                result.append({'title': i.node_name,'total': i.num})
        elif type == 'safe':
            queryset = Nodes.objects.annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            safeQueryset = Nodes.objects.filter(data__safe=True).annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            for i in range(len(safeQueryset)):
                safeRate=safeQueryset[i].num/queryset[i].num*100 if queryset[i].num!=0 else 0
                result.append({'title': queryset[i].node_name,'total': safeRate})
        return Response(result)

    """
    返回指定id的所有count数据
    """
    @action(detail=False)
    def getCount(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        totalCount = Data.objects.filter(nodeId = nodeId).count()
        daily_range = datetime.datetime.now() - datetime.timedelta(days=1)
        dailyCount = Data.objects.filter(nodeId = nodeId, recordTime__gte=daily_range).count()
        return Response({'totalCollect': totalCount,'dailyCollect': dailyCount})

    """
    返回指定id的safe数据
    """
    @action(detail=False)
    def getSafeCount(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        totalCount = Data.objects.filter(nodeId=nodeId).count()
        totalSafeCount = Data.objects.filter(nodeId=nodeId, safe=True).count()
        daily_range = datetime.datetime.now() - datetime.timedelta(days=1)
        dailySafeCount = Data.objects.filter(nodeId=nodeId, recordTime__gte=daily_range).count()
        totalSafe = round(totalSafeCount / totalCount *100,2) if totalCount!= 0 else 0
        dailySafe = round(dailySafeCount / totalCount *100,2) if totalCount!=0 else 0
        return Response({'totalSafe': totalSafe,'dailySafe': dailySafe})

class OrderListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Order.objects.all().order_by('-amount')
    serializer_class = OrderSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('id', 'commodityId', 'amount')
    filter_fields = ('id', 'commodityId', 'amount')

class CommodityListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Commodity.objects.all().order_by('-y')
    serializer_class = CommoditySerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('id', 'x', 'type','y')
    filter_fields = ('id', 'x', 'type','y')

    @action(detail=False)
    def getSalesTypeData(self, request, *args, **kwargs):
        queryset = Commodity.objects.values('x','y','type')
        return Response(queryset)

class SearchDataListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = SearchData.objects.all().order_by('-keyword')
    serializer_class = SearchDataSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('keyword', 'count', 'status')
    filter_fields = ('keyword', 'count', 'status')

    @action(detail=False,methods=['POST'])
    def addHistory(self, request, *args, **kwargs):
        searchName = self.request.data.get('keyword')
        try:
            searchHistory = SearchData.objects.filter(keyword=searchName)
            if searchHistory.count() > 0:
                count = SearchData.objects.get(keyword=searchName).count + 1
                searchHistory.update(count=count)
                return Response({'msg:':'add history count successfully'},status=status.HTTP_200_OK)
            else:
                searchHistory = SearchData(keyword=searchName)
                searchHistory.save()
                return Response({'msg': 'add history data successfully'},status= status.HTTP_200_OK)
        except:
            searchHistory = SearchData(keyword=searchName)
            searchHistory.save()
            return Response({'msg': searchHistory},status= status.HTTP_200_OK)

 #           return Response({'msg':'error'},status= status.HTTP_400_BAD_REQUEST)

# 可选用的模型mixins.ListModelMixin, viewsets.GenericViewSet 自定义型
class NodesListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Nodes.objects.all().order_by('-node_type')
    serializer_class = NodesSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('node_name', 'node_type', 'minVal', 'maxVal', 'adcode','nodeId','subscribe')
    filter_fields = ('node_name', 'node_type', 'minVal', 'maxVal', 'adcode','nodeId','subscribe')

    @action(detail=True,methods=['POST'])
    def subscribe(self,request,*args,**kwargs):
        nodeId = self.request.data.get('nodeId')
        subscribe =self.request.data.get('subscribe')
        try:
            subscribeNode =Nodes.objects.get(nodeId=nodeId)
            subscribeNode.subscribe=subscribe
            subscribeNode.save()
            topics=list(Nodes.objects.filter(subscribe=True).values_list('nodeId',flat=True))
            for i,item in enumerate(topics):
                topics[i] = ("application/1/device/"+topics[i]+"/rx",0)
            topics.append(("gateway/+/rx",0))
            print(topics)
            #client.subscribe([("gateway/+/rx",0),("application/1/device/+/rx",0)])
            client.subscribe(topics)
            print("subscribing done")
            #client.loop_stop()
            #time.sleep(5)
            client.loop_start()
            return Response({'msg':'successfully change'},status=status.HTTP_200_OK)
        except:
            return Response({'msg':'error with nodeId'},status=status.HTTP_404_NOT_FOUND)

    @action(detail=True,methods=['POST'])
    def test(self,request,*args,**kwargs):
        tx = self.request.data.get('tx')
        nodeId = self.request.data.get('nodeId')
        msg=''
        topic= "application/1/device/"+nodeId+"/tx"
        print("tx with:"+topic)
        if tx == True:
            msg={"reference":"abcd34","confirmed":False,"fPort":10,"data":"MQEAAQE="}
        #else:
        #    msg={"reference":"abcd1234","confirmed":False,"fPort":10,"data":"MQEAAQA="}
        str_msg=json.dumps(msg)
        client.publish(topic,str_msg,0)
        node = Nodes.objects.get(nodeId=nodeId)
        node.tx=tx
        node.save()
        return Response({'msg':'tx change successfully'},status=status.HTTP_200_OK)


import os
from datetime import datetime
from users import *

def log_conversation(user_input, jarvis_response, log_dir='./Jarvis_Memory/conversation_history'):
    current_date = datetime.now().strftime("%Y-%m-%d")
    current_time = datetime.now().strftime("%H:%M:%S")

    log_file = f"{current_date}_conversation.log"
    log_file_path = os.path.join(log_dir, log_file)

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    with open(log_file_path, 'a', encoding='utf-8') as f:
        f.write("---- " + current_time + " ----\n")
        f.write(user_manager.current_user.name + ": " + user_input + "\n")
        f.write("Jarvis: " + jarvis_response + "\n\n")

def read_conversation(date, log_dir):
    log_file = f"{date}_conversation.log"
    log_file_path = os.path.join(log_dir, log_file)

    if not os.path.exists(log_file_path):
        return f"No conversation history found for {date}."

    with open(log_file_path, 'r', encoding='utf-8') as f:
        return f.read()


# -*- coding: utf-8 -*-

import os
import json
import copy
import random

from rich import print as rprint
from faker import Faker
from s3pathlib import S3Path, context
from mpire import WorkerPool
import pandas as pd
import awswrangler as wr

from learn_aws_glue_etl_programming.config import config
from learn_aws_glue_etl_programming.boto_ses import boto_ses

context.attach_boto_session(boto_ses)

s3path_prefix = S3Path(config.s3path_prefix, "ds02_nested_parquet")

fake = Faker()

n_files = 3
n_rows_per_file = 1000
n_message_lower = 1
n_message_upper = 3
n_tag_lower = 0
n_tag_upper = 5

def create_one(nth_file: int):
    start_id = 1 + (nth_file - 1) * n_rows_per_file
    end_id = start_id + n_rows_per_file
    data = [
        {
            "id": _id,
            "messages": [
                f"message {id + 1}"
                for id in range(random.randint(n_message_lower, n_message_upper))
            ],
            "tags": [
                {"key": "Name", "value": fake.name()}
                for _ in range(random.randint(n_tag_lower, n_tag_upper))
            ]
        }
        for _id in range(start_id, end_id)
    ]
    for dct in data:
        dct["data"] = copy.deepcopy(dct)

    df = pd.DataFrame(data, columns=["id", "messages", "tags", "data"])
    s3path = S3Path(s3path_prefix, f"{str(nth_file).zfill(3)}.parquet")
    print(f"dump to {s3path.console_url}")
    wr.s3.to_parquet(
        df,
        path=s3path.uri,
        index=False,
        dataset=False,
    )

    return data

def create_dataset():
    print(f"--- creating dataset {s3path_prefix.basename} ---")
    kwargs = [
        {"nth_file": ith_file}
        for ith_file in range(1, 1 + n_files)
    ]
    with WorkerPool(n_jobs=4) as pool:
        pool.map(create_one, kwargs)

if __name__ == "__main__":
    # create_one(nth_file=1)
    create_dataset()
    pass


import re
from collections import Counter

class Ingredient:
    quantity = None
    material = None

    def __init__(self, quantity, material):
        self.quantity = quantity
        self.material = material

    def get_material(self):
        return self.material

    def get_quantity(self):
        return self.quantity

class Reaction:
    inputs = []
    output = None
    INGREDIENT_PATTERN = r'[0-9]+ [A-Z]+'

    def __init__(self, string):
        (inputs, output) = string.split(' => ')
        inputs = re.findall(self.INGREDIENT_PATTERN, inputs)
        output = re.findall(self.INGREDIENT_PATTERN, output)[0]

        (output_quantity, output_material) = output.split(' ')
        self.output = Ingredient(int(output_quantity), output_material)

        inputs = [
            input.split(' ') for input in inputs
        ]
        self.inputs = [
            Ingredient(int(input[0]), input[1]) for input in inputs
        ]

    def get_output(self):
        return self.output

    def get_inputs(self):
        return self.inputs

    def consume_ingredient(self, ingredient, available_materials):
        if ingredient in self.inputs:
            available_materials.update({
                ingredient.get_material(): -1 * ingredient.get_quantity()
            })
        else:
            raise Exception(
                'Unrecognized ingredient: {} {}'.format(ingredient.get_quantity(), ingredient.get_material())
            )

    def produce(self, available_materials):
        available_materials.update({
            self.output.get_material(): self.output.get_quantity()
        })

class Production:
    available_reaction_dict = None
    available_materials = None
    raw_material = None
    raw_material_count = None

    def __init__(self, reactions, raw_material):
        self.available_reaction_dict = {
            reaction.get_output().get_material(): reaction
            for reaction in reactions
        }
        self.raw_material = raw_material
        self.available_materials = Counter()
        self.raw_material_count = 0

    def get_raw_material_count(self):
        return self.raw_material_count

    def get_available_material_count(self):
        return sum(self.available_materials.values())

    def get_available_materials(self):
        return self.available_materials

    def reset(self):
        self.available_materials = Counter()
        self.raw_material_count = 0

    def add_or_consume_material(self, quantity, material):
        self.available_materials.update({
            material: quantity
        })

    def multiply_all_available_materials(self, multiplier):
        self.available_materials = {
            key: value * multiplier
            for key, value in self.available_materials.items()
        }

    def produce_material(self, quantity, material, allow_raw_material_production=True):
        quantity_produced = 0
        while quantity_produced < quantity:
            reaction = self.available_reaction_dict[material]
            inputs = reaction.get_inputs()
            for ingredient in inputs:
                ingredient_quantity = ingredient.get_quantity()
                ingredient_material = ingredient.get_material()
                if self.available_materials[ingredient_material] < ingredient_quantity:
                    if ingredient_material == self.raw_material:
                        if allow_raw_material_production:
                            self.available_materials.update({
                                ingredient_material: ingredient_quantity
                            })
                            self.raw_material_count += ingredient_quantity
                        else:
                            return False
                    else:
                        success = self.produce_material(
                            ingredient_quantity - self.available_materials[ingredient_material],
                            ingredient_material,
                            allow_raw_material_production=allow_raw_material_production
                        )
                        if not success:
                            return False

                reaction.consume_ingredient(ingredient, self.available_materials)

            reaction.produce(self.available_materials)
            quantity_produced += reaction.get_output().get_quantity()

        return True

FUEL = 'FUEL'
ORE = 'ORE'
TOTAL_ORE_COUNT = 1000000000000

with open('input.txt', 'r') as f:
    reaction_strings = f.readlines()

reactions = [
    Reaction(reaction_string) for reaction_string in reaction_strings
]

# Part 1
p = Production(reactions, ORE)
p.produce_material(1, FUEL)
ore_to_produce_one_fuel = p.get_raw_material_count()
print(ore_to_produce_one_fuel)

# Part 2
min_fuel_produced = TOTAL_ORE_COUNT // ore_to_produce_one_fuel
p.reset()
p.produce_material(min_fuel_produced, FUEL)
print(p.get_raw_material_count())

# Attempt 2:
# leftover_ore = TOTAL_ORE_COUNT % ore_to_produce_one_fuel
# p.multiply_all_available_materials(min_fuel_produced)
# p.add_or_consume_material(leftover_ore, ORE)
#
# additional_fuel_produced = 0
# while True:
#     success = p.produce_material(1, FUEL, allow_raw_material_production=False)
#     if not success:
#         break
#     additional_fuel_produced += 1
#     print(additional_fuel_produced)
# print(min_fuel_produced)
# print(min_fuel_produced + additional_fuel_produced)

# Attempt 1:
# p = Production(reactions, ORE)
# fuel_per_period = 0
# while True:
#     p.produce_material(1, FUEL)
#     p.add_or_consume_material(-1, FUEL)
#     fuel_per_period += 1
#     if p.get_available_material_count() == 0:
#         break
#
# ore_per_period = p.get_raw_material_count()
# number_of_periods = (ORE_COUNT // ore_per_period)
# fuel_produced = number_of_periods * fuel_per_period
# leftover_ores = ORE_COUNT % ore_per_period
#
# p.reset()
# leftover_fuel_produced = 0
# while True:
#     p.produce_material(1, FUEL)
#     if p.raw_material_count > leftover_ores:
#         break
#     leftover_fuel_produced += 1
#
# fuel_produced += leftover_fuel_produced
# print(fuel_produced)