import os.path
import subprocess
import shutil
from copy import copy
from datetime import date, datetime
from pickle import dump, load
from glob import glob

from pelican.utils import slugify
import yaml
import pdfkit
from scribbler.PyPDF2 import PdfFileMerger

from .errors import ScribblerWarning, ScribblerError
from .content import ScribblerContent

class Notebook(object):
    """
    A class describing and managing a Scribbler notebook.
    """
    CONTENT_DIR = '.__content__'
    SETTINGS_FILE = 'notebook.yml'
    STORAGE_FILE = '.__notebook__.pkl'
    PELICANCONF_FILE = '.__pelicanconf__.py'
    NOTE_DIR = 'notes'
    APPE_DIR = 'appendices'
    STATIC_DIR = 'files'
    HTML_DIR = 'html'
    PDF_DIR = 'pdf'
    MASTER_PDF = 'FullNotebook.pdf'
    DEFAULT_SETTINGS = {
        'author': 'No Author',
        'notebook name': 'A Scribbler Notebook',
        'timezone': 'Etc/UCT',
        'language': 'en',
        'links': [],
        'email': '',
        'description': '',
        'address': False,
        'street address' : '',
        'city': '',
        'region': '',
        'postal': '',
        'country': '',
        'plugins': [],
        'markdown extensions': [],
        'bibfile': '',
        'filetypes': {},
        'paper': 'Letter',
    }
    PELICAN_MAPPING = {
        'author': 'AUTHOR',
        'notebook name': 'SITENAME',
        'timezone': 'TIMEZONE',
        'language': 'DEFAULT_LANG',
        'links': 'LINKS',
        'email': 'EMAIL',
        'description': 'DESCRIPTION',
        'address': 'ADDRESS',
        'street address': 'STREET_ADDRESS',
        'city': 'CITY',
        'region': 'REGION',
        'postal': 'POSTAL',
        'country': 'COUNTRY',
        'plugins': 'PLUGINS',
        'markdown extensions': 'MD_EXTENSIONS',
        'bibfile': 'PUBLICATIONS_SRC',
    }
    NO_MAPPING = ['paper', 'filetypes']
    PELICAN_PLUGINS = ['scribbler.render_math', 'scribbler.tipue_search',
                       'scribbler.neighbors', 'scribbler.pdf-img',
                       'scribbler.slugcollision','scribbler.pelican-cite',
                       'scribbler.figure-ref']
    MARKDOWN_PLUGINS = ['scribbler.figureAltCaption','superscript',
                        'markdown_checklist.extension','extra',
                        'subscript','MarkdownHighlight.highlight',
                        'codehilite(css_class=highlight)','del_ins',
                        'markdown_include.include(base_path={})',
                        'scribbler.plantuml']
    FILETYPES = {
        'jpg': 'images',
        'jpeg': 'images',
        'png': 'images',
        'gif': 'images',
        'eps': 'images',
        'svg': 'images',
        'pdf': 'pdfs',
        'ps': 'pdfs',
        'dvi': 'pdfs',
        'tar.gz': 'archives',
        'tar.bz': 'archives',
        'tar.bz2': 'archives',
        'tar.xz': 'archives',
        'tar': 'archives',
        'rar': 'archives',
        'zip': 'archives',
        'rtp': 'archives',
        'deb': 'archives',
        '*': 'attachments'
    }
    DEFAULT_PELICAN_SETTINGS = {
        'DELETE_OUTPUT_DIRECTORY': True,
        'OUTPUT_PATH': HTML_DIR,
        'PATH': CONTENT_DIR,
        'ARTICLE_PATHS': [NOTE_DIR],
        'PAGE_PATHS': [APPE_DIR],
        'STATIC_PATHS': [STATIC_DIR],
        'RELATIVE_URLS': True,
        'THEME': os.path.normpath(os.path.join(os.path.dirname(__file__), 'notebook-theme')),
        'DIRECT_TEMPLATES':  ['index', 'archives', 'tags', 'search'],
        'TYPOGRIFY': True,
        'DEFAULT_PAGINATION': 10,
        'DEFAULT_ORPHANS': 2,
        'PAGINATED_DIRECT_TEMPLATES': ('index', 'tag', 'archives', 'period_archives'),
        'YEAR': date.today().year,
        'AUTHOR_SAVE_AS': '',
        'CATEGORY_SAVE_AS': '',
        'ARTICLE_URL': NOTE_DIR + '/{slug}.html',
        'ARTICLE_SAVE_AS': NOTE_DIR + '/{slug}.html',
        'SLUGIFY_SOURCE': 'basename',
        'FEED_ALL_ATOM': None,
        'CATEGORY_FEED_ATOM': None,
        'AUTHOR_FEED_ATOM': None,
        'AUTHOR_FEED_RSS': None,
        'SITEURL': 'http://www.null.org',
        'MATH_JAX': {'message_style': 'none'},
        'CACHE_CONTENT': True,
        'CACHE_PATH': '.__cache__',
        #~ 'MONTH_ARCHIVE_SAVE_AS': '{date:%Y}/{date:%b}/index.html',
        #~ 'YEAR_ARCHIVE_SAVE_AS': '{date:%Y}/index.html',
    }
    PDF_SETTINGS = {
        'margin-top': '0.7in',
        'margin-right': '0.7in',
        'margin-bottom': '0.7in',
        'margin-left': '0.7in',
        'quiet': '',
        'print-media-type': '',
        'javascript-delay': '5000',
    }

    def __init__(self, name, location):
        """
        If location already exists, then try to read notebook
        information from its contents. This may override name.
        Otherwise, create basic contents for a notebook at location.
        """
        subdirs = [os.path.join(location, d) for d in
                    [Notebook.APPE_DIR, Notebook.HTML_DIR, Notebook.PDF_DIR,
                     Notebook.NOTE_DIR, Notebook.STATIC_DIR]
                  ]
        self.name = name
        self.location = os.path.abspath(location)
        self._settings = None
        self._pelican_settings = None
        self.notes = {}
        self.appendices = {}
        self.settings_mod_time = 0
        self.psettings_mod_time = 0
        for dirname in subdirs:
            try:
                os.mkdir(dirname)
            except:
                pass
        #~ self.output_path = os.path.join(location,'html')
        #~ self.content_path = os.path.join(location,'content')

    def __eq__(self, other):
        """
        Equality test, needed for unit testing.
        """
        try:
            return self.__dict__ == other.__dict__
        except AttributeError:
            return False

    @property
    def settings(self):
        '''
        Takes the YAML file containing settings for this notebook and
        returns a dictionary constructed by adjusting the  default
        settings with this information.
        '''
        yaml_time = os.path.getmtime(os.path.join(self.location,self.SETTINGS_FILE))
        if self._settings and yaml_time <= self.settings_mod_time:
            return self._settings
        settings_file = open(os.path.join(self.location,self.SETTINGS_FILE),'r')
        file_settings = yaml.load(settings_file)
        settings = copy(self.DEFAULT_SETTINGS)
        settings.update(file_settings)
        if isinstance(settings['description'], unicode):
            settings['description'] = str(settings['description'])
        for key, val in settings.iteritems():
            if key in self.DEFAULT_SETTINGS and not isinstance(val,type(self.DEFAULT_SETTINGS[key])):
                raise TypeError('Key "{}" in settings has value of type {}'.format(key,type(val).__name__))
        tmp = settings['filetypes']
        settings['filetypes'] = copy(self.FILETYPES)
        settings['filetypes'].update(tmp)
        settings['markdown extensions'].extend(self.MARKDOWN_PLUGINS)
        pplugins = copy(self.PELICAN_PLUGINS)
        pplugins[-1] = pplugins[-1].format(self.location)
        settings['plugins'] = pplugins + settings['plugins']
        settings['bibfile'] = os.path.join(self.location, settings['bibfile'])
        self._settings = settings
        self.settings_mod_time = yaml_time
        return settings

    @property
    def pelican_settings(self):
        '''
        Returns a dictionary with the settings to be provided for Pelican.
        '''
        self.settings # Called so that self.settings_mod_time will be updated
        if self._pelican_settings and self.settings_mod_time <= self.psettings_mod_time:
            return self._pelican_settings
        psettings = copy(self.DEFAULT_PELICAN_SETTINGS)
        psettings['OUTPUT_PATH'] = os.path.join(self.location,psettings['OUTPUT_PATH'])
        psettings['PATH'] = os.path.join(self.location,psettings['PATH'])
        for key, val in self.settings.iteritems():
            if key in self.PELICAN_MAPPING:
                psettings[self.PELICAN_MAPPING[key]] = self.settings[key]
            elif key not in self.NO_MAPPING:
                raise ScribblerWarning('Unrecognized setting: `{}`'.format(key))
        self._pelican_settings = psettings
        self.psettings_mod_time = self.settings_mod_time
        return psettings

    @property
    def pdf_settings(self):
        """
        Returns a dictionary with the settings to be handed to pdfkit.
        """
        settings = copy(self.PDF_SETTINGS)
        settings['page-size'] = self.settings['paper']
        return settings

    @property
    def storage_file(self):
        return os.path.join(self.location, self.STORAGE_FILE)

    def make_pelicanconf(self):
        """
        Create .pelicanconf.py from the notebook settings.
        """
        pfile = open(os.path.join(self.location, self.PELICANCONF_FILE), 'w')
        pfile.write('#!/usr/bin/env python\n'
                    '# -*- coding: utf-8 -*- #\n'
                    '#from __future__ import unicode_literals\n')
        for key, val in self.pelican_settings.iteritems():
            pfile.write('{} = {}\n'.format(key, repr(val)))
        pfile.close()

    def del_pelicanconf(self):
        """
        Delete pelicanconf file if exists. Returns True if did exist,
        False otherwise.
        """
        if os.path.isfile(os.path.join(self.location, self.PELICANCONF_FILE)):
            os.remove(os.path.join(self.location, self.PELICANCONF_FILE))
            if os.path.isfile(os.path.join(self.location, self.PELICANCONF_FILE+'c')):
                os.remove(os.path.join(self.location, self.PELICANCONF_FILE+'c'))
            return True
        else:
            return False

    @staticmethod
    def mkdirs(path):
        """
        If they do not exist, creates all directories needed for PATH.
        """
        try:
            os.makedirs(os.path.dirname(path))
        except:
            pass

    def get_destination(self, path, location=None):
        """
        Returns the location in which to place the file.
        """
        # TODO: note that there may be corner cases I have not thought of or properly accounted for here
        if path.strip().endswith(os.path.sep): path = os.path.dirname(path)
        filename = os.path.basename(path)
        if isinstance(location, str):
            loc = location
            if os.path.isdir(os.path.join(self.location,self.STATIC_DIR,loc)):
                loc = os.path.join(loc, filename)
            if os.path.isdir(path):
                loc += os.path.sep
        else:
            for ext, dest in self.settings['filetypes'].iteritems():
                if ext != '*' and filename.endswith('.' + ext):
                    loc = dest
                    break
            else:
                try:
                    loc = self.settings['filetypes']['*']
                except KeyError:
                    raise ScribblerError('Could not find location for '
                                         'type of file ' + filename)
            loc = os.path.join(loc, filename)
        return os.path.join(self.location, self.STATIC_DIR, loc)

    def copy_in(self, path, location=None, overwrite=False):
        """
        Copy the file at the specified path into the notebook contents
        If location is specified, place it there. Otherwise, place in
        default location for that filetypes.
        """
        dest = self.get_destination(path, location)
        if not overwrite and (os.path.isfile(dest) or os.path.isdir(dest)):
            raise OSError("File already exists at '{}'".format(dest))
        elif overwrite and os.path.isdir(dest):
            if os.path.islink(dest):
                os.remove(dest)
            else:
                shutil.rmtree(dest)
        self.mkdirs(dest)
        if os.path.isdir(path):
            shutil.copytree(path, dest)
        else:
            shutil.copy(path, dest)

    def link_in(self, path, location=None, overwrite=False):
        """
        Create hard link in the notebook contents to the file at the
        specified path. If location is specified, place link there.
        Otherwise, place in default locations for that filetype.
        """
        if os.path.isdir(path):
            raise OSError("File '{}' is a directory".format(path))
        dest = self.get_destination(path, location)
        if os.path.isfile(dest):
            if overwrite:
                os.remove(dest)
            else:
                raise OSError("File already exists at '{}'".format(dest))
        self.mkdirs(dest)
        os.link(path, dest)

    def symlink_in(self, path, location=None, overwrite=False):
        """
        Create symbolic link in the notebook contents to the file at the
        specified path. If location is specified, place link there.
        Otherwise, place in default locations for those filetypes.
        """
        dest = self.get_destination(path, location)
        if not overwrite and (os.path.isfile(dest) or os.path.isdir(dest)):
            raise OSError("File already exists at '{}'".format(dest))
        elif overwrite and os.path.isfile(dest):
            os.remove(dest)
        elif overwrite and os.path.isdir(dest):
            if os.path.islink(dest):
                os.remove(dest)
            else:
                shutil.rmtree(dest)
        self.mkdirs(dest)
        if os.path.isabs(path):
            rpath = path
        else:
            if dest.endswith(os.path.sep):
                rpath = os.path.relpath(os.path.abspath(path), dest[:len(os.path.sep)])
            else:
                rpath = os.path.relpath(os.path.abspath(path), os.path.dirname(dest))
        os.symlink(rpath, dest)

    def newnote(self, date, title, markup='md'):
        """
        Create a new note with the specified date and title. Returns the
        path to that note.
        """
        MARKUP_OPTIONS = {
            'md': "Title: {0}\nDate: {1}\nTags: \n\n",
            'rst': "{0}\n###################\n\n:date: {1}\n:tags: \n\n",
            'html': "\n\t\n\t\t<title>{0}</title>\n\t\t"
                    "<meta name='date' content='{1}' />\n\t\t"
                    "<meta name='tags' content=' ' />\n\t"
                    "\n\t\n\t\t\n\t\n\n",
        }
        try:
            datetime.strptime(date, '%Y-%m-%d %H:%M')
        except:
            raise ScribblerError('Incorrectly formatted date; date format '
                             'should be YYYY-MM-DD HH:mm')
        name = date.split()[0] + '-' + slugify(title)
        for ext in MARKUP_OPTIONS:
            if os.path.isfile(os.path.join(self.location,self.NOTE_DIR,
                              name + '.' + ext)):
                raise ScribblerError('Date and name produce slug the '
                                     'same as existing file ' + name +
                                     '.' + ext)
        basename = name + '.' + markup
        path = os.path.join(self.location, self.NOTE_DIR, basename)
        out = open(path, 'w')
        try:
            out.write(MARKUP_OPTIONS[markup].format(title, date))
            out.close()
        except KeyError as e:
            out.close()
            os.remove(path)
            raise e
        self.notes[basename] = ScribblerContent(title, date[0:10],
                                                os.path.join(self.NOTE_DIR, basename),
                                                self)
        self.save(self.storage_file)
        return path

    def newpage(self, title, markup='md'):
        """
        Create a new page ("appendix") with the specified title. Returns
        the path to the page file.
        """
        MARKUP_OPTIONS = {
            'md': "Title: {0}\n\n",
            'rst': "{0}\n###################\n\n",
            'html': "\n\t\n\t\t<title>{0}</title>\n\t"
                    "\n\t\n\t\t\n\t\n\n",
        }
        name = slugify(title)
        for ext in MARKUP_OPTIONS:
            if os.path.isfile(os.path.join(self.location,self.APPE_DIR,
                              name + '.' + ext)):
                raise ScribblerError('Date and name produce slug the '
                                     'same as existing file ' + name +
                                     '.' + ext)
        basename = name + '.' + markup
        path = os.path.join(self.location, self.APPE_DIR, basename)
        out = open(path, 'w')
        try:
            out.write(MARKUP_OPTIONS[markup].format(title))
            out.close()
        except KeyError as e:
            out.close()
            os.remove(path)
            raise e
        self.appendices[basename] = ScribblerContent(title, '????-??-??',
                                                     os.path.join(self.APPE_DIR,
                                                     basename), self)
        self.save(self.storage_file)
        return path

    def addnote(self, date, title, path, overwrite=False):
        """
        Add a record of a note in an existing file (PATH), not previously
        registered with Scribbler. Note has TITLE and DATE specified.
        """
        try:
            datetime.strptime(date, '%Y-%m-%d')
        except:
            raise ScribblerError('Incorrectly formatted date; date format '
                             'should be YYYY-MM-DD')
        basename = os.path.relpath(os.path.abspath(path), os.path.join(self.location, self.NOTE_DIR))
        if basename in self.notes and not overwrite:
            raise ScribblerError('Note with file `{}` already exists.'.format(basename))
        self.notes[basename] = ScribblerContent(title, date,
                                    os.path.join(self.NOTE_DIR, basename), self)
        self.save(self.storage_file)

    def addpage(self, title, path, overwrite=False):
        """
        Add a record of a page/appendix in an existing file (PATH), not
        previously registered with Scribbler. Has the specified TITLE.
        """
        basename = os.path.relpath(path, os.path.join(self.location, self.APPE_DIR))
        if basename in self.appendices and not overwrite:
            raise ScribblerError('Note with file `{}` already exists.'.format(basename))
        self.appendices[basename] = ScribblerContent(title, '????-??-??',
                                        os.path.join(self.APPE_DIR, basename), self)
        self.save(self.storage_file)

    def list_contents(self):
        """
        Return a list (format to be determined) detailing the notes,
        appendices, and possibly static content stored in the notebook's
        content directory.
        """
        raise NotImplementedError() #TODO: Add this feature

    def build(self, debug=False):
        """
        Run Pelican to produce the HTML for this notebook. Then produce
        the PDF pages. Passes `--debug` flag to Pelican if DEBUG is true.
        """
        self.make_pelicanconf()
        content = os.path.join(self.location, self.CONTENT_DIR)
        if not os.path.isdir(content):
            os.mkdir(content)
            os.symlink(os.path.join(self.location, self.NOTE_DIR),
                       os.path.join(content, self.NOTE_DIR))
            os.symlink(os.path.join(self.location, self.APPE_DIR),
                       os.path.join(content, self.APPE_DIR))
            os.symlink(os.path.join(self.location, self.STATIC_DIR),
                       os.path.join(content, self.STATIC_DIR))
        print('Producing HTML files...')
        call = ['pelican','-s',os.path.join(self.location,self.PELICANCONF_FILE)]
        if debug:
            call.append('--debug')
        subprocess.call(call)
        #~ self.del_pelicanconf()
        self.update()
        #~ shutil.rmtree(content)
        print('Producing PDF files...')
        if not os.path.isdir(os.path.join(self.location, self.PDF_DIR)):
            os.mkdir(os.path.join(self.location, self.PDF_DIR))
        master = PdfFileMerger(strict=False)
        master.addMetadata({u'/Title': self.settings['notebook name'],
                            u'/Author': self.settings['author']})
        src = os.path.join(self.location, self.HTML_DIR, 'index.html')
        dest = os.path.join(self.location, self.PDF_DIR, 'titlepage.pdf')
        pdfkit.from_file(src, dest, options=self.pdf_settings)
        master.append(dest)
        for note in sorted(self.notes.values(), key=lambda n: n.slug):
            if note.pdf_date < note.src_date:
                note.make_pdf()
                note.update()
            master.append(os.path.join(self.location, note.pdf_path),
                          note.date + ': ' + note.name)
        for appe in sorted(self.appendices.values(), key=lambda a: a.slug):
            if appe.pdf_date < appe.src_date:
                appe.make_pdf()
                appe.update()
            master.append(os.path.join(self.location, appe.pdf_path),
                          'Appendix: ' + appe.name)
        master.write(os.path.join(self.location, self.PDF_DIR, self.MASTER_PDF))
        print('Done.')
        self.update()

    def update(self):
        """
        Update the list of the content stored in this notebook.
        """
        notes = copy(self.notes)
        for root, dirs, files in os.walk(os.path.join(self.location, self.NOTE_DIR)):
            for f in files:
                if f in notes:
                    self.notes[f].update()
                    del notes[f]
                else:
                    if f.endswith('~') or f.startswith('.') or f.startswith('#'):
                        continue
                    path = os.path.join(os.path.relpath(root, self.location), f)
                    self.notes[f] = ScribblerContent('Unknown', '????-??-??',
                                                     path, self)
        for n in notes:
            del self.notes[n]
        appen = copy(self.appendices)
        for root, dirs, files in os.walk(os.path.join(self.location, self.APPE_DIR)):
            for f in files:
                if f in appen:
                    self.appendices[f].update()
                    del appen[f]
                else:
                    if f.endswith('~') or f.startswith('.') or f.startswith('#'):
                        continue
                    path = os.path.join(os.path.relpath(root, self.location), f)
                    self.appendices[f] = ScribblerContent('Unknown', '????-??-??',
                                                          path, self)
        for a in appen:
            del self.appendices[a]
        self.save(self.storage_file)

    def save(self, path):
        """
        Asks notebook to pickle itself and stores it in the specified path.
        """
        out = open(path,'w')
        dump(self, out)
        out.close()

def create_notebook(name, location):
    """
    Looks to see if a notebook exists in this location. If so, loads
    and returns it. Otherwise, creates one there and then returns a
    corresponding object.
    """
    if os.path.isdir(location):
        if not os.path.isfile(os.path.join(location, Notebook.SETTINGS_FILE)):
            with open(os.path.join(location, Notebook.SETTINGS_FILE), 'w') as f:
                f.write("# Notebook configuration file\n")
                f.write("notebook name: {}".format(name))
        try:
            infile = open(os.path.join(location, Notebook.STORAGE_FILE), 'r')
            nb = load(infile)
            nb.location = os.path.abspath(location)
        except:
            nb = Notebook(name, location)
        nb.update()
    else:
        os.mkdir(location)
        with open(os.path.join(location, Notebook.SETTINGS_FILE), 'w') as f:
            f.write("# Notebook configuration file\n")
            f.write("notebook name: {}".format(name))
        nb = Notebook(name, location)
    nb.save(nb.storage_file)
    return nb


#!/usr/bin/python
"""
selfcheck
This command is used to check if external tools are present on the system.
"""
import os
import fritutils

logger = fritutils.fritlog.loggers['selfcheckLog']

def selfcheck():
    logger.info('Starting selfcheck')
    for tool in fritutils.fritglobals.toolbox:
        if tool.check():
            fritutils.termout.printNormal('Tool "%s" found' % tool)
        else:
            fritutils.termount.printWarning('Tool "%s" NOT found' % tool)


from operator import itemgetter
flag = False
while True:
    try:

        l = list(input())
        if l[0] == '':
            break
        if flag:
            print()
        flag = True

        d = {}
        lset = l.copy()
        lset = set(lset)
        lres = []

        for i in lset:
            d['ord'] = ord(i)
            d['contador'] = l.count(i)
            lres.append(d.copy())

        lres = sorted(lres, key=itemgetter('ord'), reverse=True)
        lres = sorted(lres, key=itemgetter('contador'))

        for i in lres:
            print(i['ord'], i['contador'])

    except EOFError:
        break


#!/usr/bin/python3

# COMP3311 22T3 Assignment 2
# Print info about one movie; may need to choose

import sys
import psycopg2
import helpers

### Globals

db = None
usage = f"Usage: {sys.argv[0]} 'PartialMovieName'"

### Command-line args

if len(sys.argv) < 2:
   print(usage)
   exit(1)

# process the command-line args ...
movie = sys.argv[1].lower()
pattern = f"%{movie}%"
### Queries

qryMovieExist = """
   select m.title, m.year, m.id
   from Movies m
   where lower(title) like %s
   order by m.title, m.year
   ;
"""

#qry only with actor and actress
qryAct = """
   select p.name, pr.job, pr.id
   from Principals pr
   join People p on p.id = pr.person
   where pr.movie = %s
   order by pr.ord
   ;
"""

#qry except actor and actress
qryRole = """
   select role
   from Playsrole
   where inMovie = %s
   ;
"""

### Manipulating database
db = psycopg2.connect("dbname=ass2")
cur = db.cursor()

try:
   cur.execute(qryMovieExist, [pattern])
   movies = cur.fetchall()

   numMovie = 0
   if not movies:
      print(f"No movie matching: '{sys.argv[1]}'")
      exit(0)
   else:
      numMovie = len(movies)

   # only 1 movie match
   if numMovie == 1:
      for m in movies:
         print(f"{m[0]} ({m[1]})")
         cur.execute(qryAct, [m[2]])
         actPri = cur.fetchall()
         numRoles = len(actPri)
         for a in actPri:
            if(a[1] == "self" or a[1] == "actor" or a[1] == "actress"):
               cur.execute(qryRole, [a[2]])
               roles = cur.fetchone()
               if not roles:
                  print(f"{a[0]} plays ???")
               else:
                  print(f"{a[0]} plays {roles[0]}")
            else:
               print(f"{a[0]}: {a[1]}")
   else :
      counter = 0
      for m in movies:
         counter += 1
         print(f"{counter}. {m[0]} ({m[1]})")
         cur.execute(qryAct, [m[2]])
         actPri = cur.fetchall()
         numRoles = len(actPri)

      chooseMovie = input("Which movie? ")
      counter = 0
      for m in movies:
         counter += 1
         if (counter == int(chooseMovie)):
            print(f"{m[0]} ({m[1]})")
            cur.execute(qryAct, [m[2]])
            actPri = cur.fetchall()
            numRoles = len(actPri)
            for a in actPri:
               if(a[1] == "self" or a[1] == "actor" or a[1] == "actress"):
                  cur.execute(qryRole, [a[2]])
                  roles = cur.fetchone()
                  if not roles:
                     print(f"{a[0]} plays ???")
                  else:
                     print(f"{a[0]} plays {roles[0]}")
               else:
                  print(f"{a[0]}: {a[1]}")
         #else:
               #print("No such movie")

except Exception as err:
   print("DB error: ", err)

finally:
   if db:
      db.close()


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.11.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Batch Gradient Descent With a Single Neuron
#
# *Tue Sep 14, Week 3, Lecture 5*
#
# ## Why Python and why Jupyter Notebooks?
#
# - Python is not an ideal langauge for this
# - Jupyter is not an ideal environment for htis
# - They are, however, in my opinion the **best** we have
#
# ## Notes
#
# - In this example, I am not going to do fancy imports
# - I want you to see the layout of torch and torchvision

# %%
import torch
import torchvision
import pandas as pd

# %%
torch.rand(5)

# %%
torch.rand(5).shape

# %%
torch.rand(5, 25)

# %%
torch.rand(5, 25).shape

# %%
X = torch.rand(5, 12)
Y = torch.rand(12, 16)

# %%
X.shape, Y.shape

# %%
X

# %%
X @ Y

# %%
Z = X @ Y

# %%
X.shape, Y.shape

# %%
Z.shape

# %%
Y @ Z

# %%
# MNIST : hello world
# EMNIST : extended with letters in addition to digits
# KMNIST : Kuzushiji, Japanese characters
# QMNIST : newer MNIST with better source information

data_path = "../data/"

mnist_transforms = torchvision.transforms.Compose(
    [
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize((0.1307,), (0.3081,)),
    ]
)

train_dataset = torchvision.datasets.MNIST(
    root=data_path, train=True, download=True, transform=mnist_transforms
)

# %%
# batch_size, shuffle
train_loader = torch.utils.data.DataLoader(train_dataset)

# %%
image, label = next(iter(train_loader))

# %%
image.shape

# %%
label.shape, label

# %%
image

# %%
mnist_avg, mnist_std = 0.130, 0.3081

image = torch.as_tensor((image * mnist_std + mnist_avg) * 255, dtype=torch.uint8)
image

# %%
image_df = pd.DataFrame(image.squeeze().numpy())
image_df

# %%
image_df.style.set_properties(**{'font-size':'6pt'}).background_gradient('Greys')

# %%
import matplotlib.pyplot as plt

# %%
num_to_show = 8

fig, axes = plt.subplots(1, num_to_show, figsize=(12, 12))

for axis, (image, label) in zip(axes, train_loader):
    axis.imshow(image.squeeze(), cmap="Greys")
    axis.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)
    axis.set_title(f"Label: {label[0]}")

# %% [markdown]
# # A Single Neuron Model

# %%
# Simple single neuron model; sometimes called logistic regression

num_pixels = 28 * 28

# Neuron parameters
weights = torch.randn(num_pixels, 1) * 0.01
bias = torch.zeros(1)

# %%
weights.shape, bias.shape, image[0].shape, image[0].view(num_pixels, 1).shape

# %%
x = image[0]

# %%
weights @ x

# %%
x = image[0].view(num_pixels, 1)

# %%
weights @ x

# %%
weights.T @ x

# %%
# from torch import FloatTensor, FloatType

def linear(w, b, x):
    return w.T @ x + b

# %%
linear(weights, bias, x)

# %%
def sigmoid(z):
    return 1 / (1 + torch.exp(-z))

# %%
sigmoid(linear(weights, bias, x))

# %% [markdown]
# # Let's Focus on Binary Classification

# %%
# Get down to two classes (let them pick)

class1 = 7
class2 = 1

idx1 = [i for i, t in enumerate(train_dataset.targets) if t == class1]
idx2 = [i for i, t in enumerate(train_dataset.targets) if t == class2]

idxs = idx1 + idx2
train_size = len(idxs)

train_loader2 = torch.utils.data.DataLoader(
    train_dataset, sampler=torch.utils.data.SubsetRandomSampler(idxs)
)

# %%
num_to_show = 8

fig, axes = plt.subplots(1, num_to_show, figsize=(12, 12))

for axis, (image, label) in zip(axes, train_loader2):
    axis.imshow(image.squeeze(), cmap="Greys")
    axis.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)
    axis.set_title(f"Label: {label[0]}")

# %% [markdown]
# # Using Our Derived Equations

# %%
from time import time

# %%
# Optimization, binary cross-entropy loss (Log loss, aka logistic loss or cross-entropy loss)
# defined as the negative log-likelihood of a logistic model that returns y_pred probabilities for its training data y_true

# compare with: subtraction, MSE, abs

def target_to_sigmoid(target):
    new_target = torch.zeros_like(target)
    new_target[target == class1] = 1
    return new_target

# Neuron parameters
num_pixels = 28 * 28
weights = torch.randn(num_pixels, 1) * 0.01
bias = torch.zeros(1)

num_epochs = 4
learning_rate = 0.01

for epoch in range(num_epochs):

    cost = 0
    weights_derivatives = torch.zeros_like(weights)
    bias_derivative = torch.zeros_like(bias)

    start = time()

    for image, target in train_loader2:

        image = image.view(num_pixels, 1)
        target = target_to_sigmoid(target)

        prediction = sigmoid(linear(weights, bias, image))

        loss = target * torch.log(prediction) + (1 - target) * torch.log(1 - prediction)
        cost -= loss

        dz = prediction - target
        weights_derivatives += image * dz
        bias_derivative += dz.squeeze()

    weights_derivatives /= len(train_loader2)
    bias_derivative /= len(train_loader2)

    weights -= learning_rate * weights_derivatives
    bias -= learning_rate * bias_derivative

    print(
        f"{epoch+1:>2}/{num_epochs}, Cost={cost[0][0]:0.1f}, Time={time()-start:0.1f}s"
    )

# %% [markdown]
# # Let's Compute All At Once (Batch GD)

# %%
# Set the batch size to be equal to the size of the training dataset
train_loader2All = torch.utils.data.DataLoader(
    train_dataset,
    sampler=torch.utils.data.SubsetRandomSampler(idxs),
    batch_size=train_size,
)

# %%
# Vectorize (operate on all images at once)

# Neuron parameters
num_pixels = 28 * 28
weights = torch.randn(num_pixels, 1) * 0.01
bias = torch.zeros(1)

num_epochs = 4
learning_rate = 0.01

for epoch in range(num_epochs):

    weights_derivatives = torch.zeros_like(weights)
    bias_derivative = torch.zeros_like(bias)

    start = time()

    images, targets = next(iter(train_loader2All))
    # print(len(images), images.shape, images.view(train_size, num_pixels))

    images = images.view(train_size, num_pixels)
    targets = target_to_sigmoid(targets)

    predictions = sigmoid(linear(weights, bias, images.T))
    # print(predictions.shape, targets.shape)

    loss = targets * torch.log(predictions) + (1 - targets) * torch.log(1 - predictions)
    cost = -loss.sum()

    dZ = predictions - targets
    weights_derivatives = (1 / train_size) * (dZ @ images)
    bias_derivative = (1 / train_size) * dZ.sum()

    weights -= learning_rate * weights_derivatives.T
    bias -= learning_rate * bias_derivative

    print(f"{epoch+1:>2}/{num_epochs}, Cost={cost:0.1f}, Time={time()-start:0.1f}s")

# %% [markdown]
# # Is This Better? Need a Metric

# %%
# Create the validation dataset

valid_dataset = torchvision.datasets.MNIST(
    root=data_path, train=False, download=True, transform=mnist_transforms
)

idx1 = [i for i, t in enumerate(valid_dataset.targets) if t == class1]
idx2 = [i for i, t in enumerate(valid_dataset.targets) if t == class2]

valid_idxs = idx1 + idx2
valid_size = len(valid_idxs)

print(valid_size)

valid_loader2All = torch.utils.data.DataLoader(
    valid_dataset,
    sampler=torch.utils.data.SubsetRandomSampler(valid_idxs),
    batch_size=valid_size,
)

# %%
# Add validation metric

# Neuron parameters
num_pixels = 28 * 28
weights = torch.randn(num_pixels, 1) * 0.01
bias = torch.zeros(1)

num_epochs = 4
learning_rate = 0.01

# Validate with accuracy
valid_images, valid_targets = next(iter(valid_loader2All))
valid_images = valid_images.view(valid_size, num_pixels)
valid_targets = target_to_sigmoid(valid_targets)

valid_predictions = sigmoid(linear(weights, bias, valid_images.T))

accuracy = 1 - (torch.round(valid_predictions) - valid_targets).abs().sum() / valid_size

print(f"Accuracy={accuracy:.2f}")

for epoch in range(num_epochs):

    weights_derivatives = torch.zeros_like(weights)
    bias_derivative = torch.zeros_like(bias)

    start = time()

    images, targets = next(iter(train_loader2All))
    images = images.view(train_size, num_pixels)
    targets = target_to_sigmoid(targets)

    predictions = sigmoid(linear(weights, bias, images.T))

    loss = targets * torch.log(predictions) + (1 - targets) * torch.log(1 - predictions)
    cost = -loss.sum()

    dZ = predictions - targets
    weights_derivatives = (1 / train_size) * (dZ @ images)
    bias_derivative = (1 / train_size) * dZ.sum()

    weights -= learning_rate * weights_derivatives.T
    bias -= learning_rate * bias_derivative

    # Validate with accuracy
    valid_images, valid_targets = next(iter(valid_loader2All))
    valid_images = valid_images.view(valid_size, num_pixels)
    valid_targets = target_to_sigmoid(valid_targets)

    valid_predictions = sigmoid(linear(weights, bias, valid_images.T))

    accuracy = (
        1 - (torch.round(valid_predictions) - valid_targets).abs().sum() / valid_size
    )

    print(
        f"{epoch+1:>2}/{num_epochs}, Cost={cost:0.1f}, Accuracy={accuracy:.2f}, Time={time()-start:0.1f}s"
    )

# %%
valid_predictions.shape

# %%
valid_targets.shape

# %%
valid_predictions[:10]

# %%
torch.round(valid_predictions)

# %%
(torch.round(valid_predictions) - valid_targets).sum()

# %%
(torch.round(valid_predictions) - valid_targets).sum()/valid_size

# %%
1 - (torch.round(valid_predictions) - valid_targets).sum()/valid_size

# %%
x = 1 - (torch.round(valid_predictions) - valid_targets).sum()/valid_size

# %%
f"Accuracy={x:.2f}"


from pyrogram import filters
from pyrogram import __version__ as pyro_vr
from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup, Message

from .. import (Venom1, Venom2, Venom3, Venom4,
                Venom5, Venom6, Venom7, Venom8,
                Venom9, Venom10, Venom11, Venom12,
                Venom13, Venom14, Venom15, HNDLR,
                SUDO_USERS, vcbot, ALIVE_PIC, __version__)

Array = ALIVE_PIC or "https://te.legra.ph/file/045d419efd37fd56e8562.jpg"

@vcbot.on_message(filters.user(SUDO_USERS) & filters.command(["alive"], prefixes=HNDLR))
async def _Alive(_, e: Message):
    ids = 0
    try:
        if Venom1:
            ids += 1
        if Venom2:
            ids += 1
        if Venom3:
            ids += 1
        if Venom4:
            ids += 1
        if Venom5:
            ids += 1
        if Venom6:
            ids += 1
        if Venom7:
            ids += 1
        if Venom8:
            ids += 1
        if Venom9:
            ids += 1
        if Venom10:
            ids += 1
        if Venom11:
            ids += 1
        if Venom12:
            ids += 1
        if Venom13:
            ids += 1
        if Venom14:
            ids += 1
        if Venom15:
            ids += 1
        Array_msg = f"𝗧𝗥𝗢𝗡𝗜𝗫 𝗥𝗔𝗜𝗗𝗘𝗥 𝗛𝗘𝗥𝗘. 🔥 \n\n"
        Array_msg += f"◈ ━━━━━━ ◆ ━━━━━━ ◈ \n"
        Array_msg += f"► Tʀᴏɴɪx ᴠᴇʀsɪᴏɴ : `{__version__}` \n"
        Array_msg += f"► Aᴄᴛɪᴠᴇ IDs : `{ids}` \n"
        Array_msg += f"► ᴘʏʀᴏ ᴠᴇʀsɪᴏɴ : `{pyro_vr}` \n"
        Array_msg += f"◈ ━━━━━━ ◆ ━━━━━━ ◈ \n\n"
        await e.reply_photo(
        photo=Array,
        caption=Array_msg,
        reply_markup=InlineKeyboardMarkup(
                [[
                    InlineKeyboardButton(
                        "• Contact •", url="https://t.me/P_4_PEEYUSH")
                ], [
                    InlineKeyboardButton(
                        "• Repo •", url="tg://settings")
                ]],
        ),
    )
    except Exception as lol:
        Array_msg = f"𝗧𝗥𝗢𝗡𝗜𝗫 𝗥𝗔𝗜𝗗𝗘𝗥 𝗛𝗘𝗥𝗘. 🔥 \n\n"
        Array_msg += f"◈ ━━━━━━ ◆ ━━━━━━ ◈ \n"
        Array_msg += f"► Tʀᴏɴɪx ᴠᴇʀsɪᴏɴ : `{__version__}` \n"
        Array_msg += f"► ᴘʏʀᴏ ᴠᴇʀsɪᴏɴ : `{pyro_vr}` \n"
        Array_msg += f"► Sᴜᴘᴘᴏʀᴛ : [Jᴏɪɴ.](tg://settings)\n"
        Array_msg += f"◈ ━━━━━━ ◆ ━━━━━━ ◈ \n\n"
        await e.reply_photo(
        photo=Array,
        caption=Array_msg,
        reply_markup=InlineKeyboardMarkup(
            [
                [
                    InlineKeyboardButton("• Contact •", url="https://t.me/P_4_PEEYUSH"),
                ],
                [
                    InlineKeyboardButton("• Repo •", url="tg://settings"),
                ],
            ],
        ),
    )


# But there's a few catches. First, the bombs self-replicate via one of two distinct processes:
# Every Mach bomb retrieves a sync unit from a Facula bomb; for every Mach bomb, a Facula bomb is created;
# Every Facula bomb spontaneously creates a Mach bomb.

# For example, if you had 3 Mach bombs and 2 Facula bombs,
# they could either PRODUCE
# 3 Mach bombs and 5 Facula bombs, or
# 5 Mach bombs and 2 Facula bombs.
# The replication process can be changed each cycle.

# NOTE: Man starter med 1 mach bombe og 1 fach bombe

# TODO: finde ud af hvor mange layers jeg skal generate ca
# TODO: make check in layer creation:
# check if the x,y or y,x exists in layer; = is in layer currently created
# if x,y or y,x is less than all in next layer; = is not possible
def solution(x, y):
    root = [[(1,1)]]
    curr_l = sum(root[0][0])
    int_x = int(x)
    int_y = int(y)
    depth = int_x
    if int_y > int_x:
        depth = int_y
    for each in range(depth):
        root.append(create_layer(root))
        root.remove(root[0])
        # print(root)
        if (int_x,int_y) in root[-1] or (int_y,int_x) in root[-1]:
            return(str(each+1))
        # for e in root[-1]:
        #     if sum(e) > curr_l:
        #         curr_l = sum(e)
        # if curr_l > int_x + int_y:
    return "impossible"
            # if e > (int_x,int_y) < e or (int_y,int_x) < e:
            #     return "impossible"
    # in_tree = check_in_tree(root, int_x, int_y)
    # if not in_tree:
    #     return "impossible"
    # else:
    #     return str(in_tree)

def check_in_tree(root, x,y):
    check = (x,y)
    check_flipped = (y,x)
    i = 0
    for layer in root:
        i-=-1
        if check in layer or check_flipped in layer:
            return i
    return False

def create_layer(root):
    # 1 mach cycle
    # 1 facu cycle
    layer = root[-1]
    len_layer = len(layer)
    temp = []
    # s = 0
    for i in range(len_layer):
        # facula bomb creation - Mach cycle
        temp.append(M_cycle(layer[i]))
        # mach bomb creation Facule cycle
        temp.append(F_cycle(layer[i]))
    # for each in temp:
    #     s = s + sum(each)
    # print(s)
    # print(layer)
    return temp

def M_cycle(b_in):
    m_in = b_in[0]
    f_in = b_in[1]
    f_out = m_in + f_in
    return m_in,f_out

def F_cycle(b_in):
    m_in = b_in[0]
    f_in = b_in[1]
    m_out = f_in + m_in
    return m_out, f_in

r0 = [(1,1)]
d1 = [[(1,1)],[(1,2),(2,1)]]
d2 = [[(1,1)],[(1,2),(2,1)],[(1,3),(3,2),(2,3),(3,1)]]

if __name__ == '__main__':
    print(solution('4', '7')) # 4
    # 0 M=1 F=1
    # 1 1 2 M_cycle creates 1 F
    # 2 1 3 M_cycle creates 1 F
    # 3 4 3 F_cycle creates 3 M
    # 4 4 7 M_cycle creates 4 F
    print(solution('2', '1')) # 1
    # 0 M=1 F=1
    # 1 2 1 F_cycle creates 1 M
    print(solution('7','19'))
    print(solution('6','9'))
    print(solution('90','100'))
    # print(solution('98431','8730912'))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Bot's work modules
import replies    # send cliched message to a user.
import filesoper  # save/load operations with files.
import lenta_api  # requests to lenta.com

import os
import telegram
from telegram import (
    InlineKeyboardButton,
    InlineKeyboardMarkup
)
from telegram.ext import (
    Updater,
    CommandHandler,
    MessageHandler,
    Filters,
    CallbackQueryHandler,
    ConversationHandler
)
from telegram.error import Unauthorized
import logging

import threading
import itertools
import time

from dotenv import load_dotenv   # for passing telegram token from .env
load_dotenv()

SITE_WHITELIST = [
    'https://lenta.com/product/'
]
USE_PROXY = False
CHECK_PRICE_PERIOD = 500  # minutes
CITY, TYPE_STORE_NAME, CHOOSE_STORE, CHOICE_FIN = range(4)
GOOD_LIST, DELGOOD = range(2)

#  log messages format
logFormatter = logging.Formatter("%(asctime)s - [%(threadName)-12.12s] - [%(levelname)-5.5s] - %(message)s")
logger = logging.getLogger()

# logging level ('INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL/FATAL')
logger.setLevel('INFO')

# save logs in ./logs dir.
if not os.path.isdir('./logs'):
    os.mkdir('logs')
fileHandler = logging.FileHandler("{0}/{1}.log".format("logs", "lentaBot"))
fileHandler.setFormatter(logFormatter)
logger.addHandler(fileHandler)

# echo logs in console
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
logger.addHandler(consoleHandler)

class LentaBot:
    """
    Allows you to track discounts on different goods by sending them to this bot.
    """

    # Put your telegram token in .env file
    TOKEN = os.environ.get('TELEGRAM_API_TOKEN')

    def __init__(self):
        print("\033[92m LentaBot started \033[0m")

        PROXY_URL = None
        if USE_PROXY:
            PROXY_URL = {'proxy_url': 'socks5h://192.168.1.114:9050/'}    # use proxy if tg is blocked in your country.

        updater = Updater(self.TOKEN,
                          request_kwargs=PROXY_URL,
                          use_context=True)

        CITIES_URL = "https://lenta.com/api/v1/cities"
        STORES_URL = "https://lenta.com/api/v1/stores"
        self.HEADERS = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"
        }
        self.stores_dict = []
        self.cities_dict = []
        while self.stores_dict == [] and self.cities_dict == []:  # if site is updating, so we can't get the cities or stores.
            self.stores_dict = lenta_api.get_json_from_url(STORES_URL, self.HEADERS)
            self.cities_dict = lenta_api.get_json_from_url(CITIES_URL, self.HEADERS)
            time.sleep(5)

        self.GOODS_PER_MESSAGE = 5  # amount of customer's goods shows in one message.
        self.GOODS_DATA_LOCATION = './goods_data.json'
        self.USERS_STORES_LOCATION = './users_stores.json'

        self.user_pos = {}
        self.first_time = True

        self.json_goods_data = filesoper.read_json(self.GOODS_DATA_LOCATION)
        self.users_stores = filesoper.read_json(self.USERS_STORES_LOCATION)

        dp = updater.dispatcher

        good_handler = ConversationHandler(
            [CommandHandler('goods', self.send_user_goods)], {
                GOOD_LIST: [
                    CallbackQueryHandler(self.forward_arrow, pattern="^" + "next" + "$"),
                    CallbackQueryHandler(self.back_arrow, pattern="^" + "previous" + "$"),
                    CallbackQueryHandler(self.good_handler_end, pattern="^" + "exit" + "$"),
                    CallbackQueryHandler(self.confirm_del_action_keyboard, pattern="^" + "[0-9]|[a-z]*" + "-")
                ],
                DELGOOD: [
                    CallbackQueryHandler(self.handle_del_good, pattern="^" + "[0-9]|[a-z]*" + "-"),
                    CallbackQueryHandler(self.handle_back_to_list, pattern="cancel")
                ]
            },
            fallbacks=[CommandHandler('goods', self.send_user_goods)]
        )

        store_handler = ConversationHandler(
            entry_points=[CommandHandler(['start', 'choose_store'], self.type_city_request)],
            states={
                CITY:            [MessageHandler(Filters.text, self.check_city_req)],
                TYPE_STORE_NAME: [CallbackQueryHandler(self.type_store_request)],
                CHOOSE_STORE:    [MessageHandler(Filters.text, self.check_store_req)],
                CHOICE_FIN:      [CallbackQueryHandler(self.choose_end, pattern="^" + "\d{4}" + "$")]
            },
            fallbacks=[CommandHandler('choose_store', self.type_city_request)]
        )

        dp.add_handler(CommandHandler('help', replies.manual_msg))
        dp.add_handler(CommandHandler('my_store', self.check_user_store))

        dp.add_handler(store_handler)
        dp.add_handler(good_handler)

        dp.add_handler(MessageHandler(Filters.text, self.main))

        dp.add_error_handler(self.error_handler)
        updater.start_polling()
        updater.idle()

    # inline keyboard
    def build_menu(self, buttons, n_cols, header_buttons=None, footer_buttons=None):
        """
        pattern for creating buttons grid.
        """
        menu = [buttons[i:i + n_cols] for i in range(0, len(buttons), n_cols)]
        if header_buttons:
            menu.insert(0, [header_buttons])
        if footer_buttons:
            menu.append([footer_buttons])
        return menu

    # city/store choose
    def type_city_request(self, update, context):
        context.bot.send_message(
            update.message.chat_id,
            text="В каком Вы городе?",
        )
        return CITY

    # city/store choose (search req in dict)
    def search_requested(self, required_place, places_dict):
        """
        Search a city or a store by typing it's name or it's short form.
        required_place: (string) 'Санкт-Петербург' or 'Сан' or 'Выборгский р-н'
        places_dict: (dict)
            {"id":"spb",
             "name":"Санкт-Петербург",
             "lat":59.939095,
             "long":30.315868,
             "mediumStoreConcentration":true,
             "highStoreConcentration":true
            }...
        """
        logger.debug("search_requested: required_place is %s", required_place)
        relevant_places = {}
        for num, city in enumerate(places_dict):
            if required_place.lower() in places_dict[num]['name'].lower():
                relevant_places.update({num: places_dict[num]})
        return relevant_places

    # city/store choose
    def check_city_req(self, update, context):
        message_text = update.message.text
        logger.info("kearch_city: message_text is %s", message_text)
        required_cities = self.search_requested(message_text, self.cities_dict)
        if not required_cities:
            replies.request_not_found_msg(update, context)
            return ConversationHandler.END

        self.choose_city(update, context, required_cities)
        return TYPE_STORE_NAME

    # city/store choose (inline)
    def choose_city(self, update, context, cities_dict=None):
        """
        Buttons with option to choose your city from list.
        """
        cities_dict = cities_dict if cities_dict else self.cities_dict
        button_list = []
        for city in cities_dict:
            logger.info("choose city: %s", city)
            logger.info("choose city: CITIES: %s", cities_dict[city])
            name = cities_dict[city]["name"]
            short_name = cities_dict[city]["id"]
            button_list.append(InlineKeyboardButton(name, callback_data="store " + short_name))

        # query = update.callback_query
        reply_markup = InlineKeyboardMarkup(self.build_menu(button_list, n_cols=3))
        update.message.reply_text('Выбирите город', reply_markup=reply_markup)

    # city/store choose
    def type_store_request(self, update, context):
        query = update.callback_query
        logger.info("type store request")
        context.bot.edit_message_text(
            chat_id=query.message.chat_id,
            message_id=query.message.message_id,
            text="На какой улице Ваш магазин?",
        )
        return CHOOSE_STORE

    # city/store choose
    def check_store_req(self, update, context):
        message_text = update.message.text
        required_stores = self.search_requested(message_text, self.stores_dict)
        if not required_stores:
            replies.request_not_found_msg(update, context)
            return ConversationHandler.END

        self.choose_store(update, context, required_stores)
        return CHOICE_FIN

    # city/store choose
    def choose_store(self, update, context, stores_dict=None):
        """
        Buttons with option to choose your store from list.
        """
        stores_dict = stores_dict if stores_dict else self.stores_dict
        button_list = []

        for store in stores_dict:
            name = stores_dict[store]["name"]
            store_id = stores_dict[store]["id"]
            button_list.append(InlineKeyboardButton(name, callback_data=store_id))

        reply_markup = InlineKeyboardMarkup(self.build_menu(button_list, n_cols=3))

        context.bot.send_message(
            chat_id=update.message.chat_id,
            text="Выбирите магазин.",
            reply_markup=reply_markup
        )

    # city/store choose
    def choose_end(self, update, context):
        """
        Remove buttons after choosing a store.
        """
        query = update.callback_query

        for lists in query.message.reply_markup.inline_keyboard:
            for element in lists:
                if query.data in element['callback_data']:
                    store_name = element["text"]
                    break

        self.users_stores = {query.message.chat_id: [query.data, store_name]}
        filesoper.write_json(self.users_stores, self.USERS_STORES_LOCATION)

        bot = context.bot
        bot.edit_message_text(
            chat_id=query.message.chat_id,
            message_id=query.message.message_id,
            text="Ваш магазин '%s'\nСкиньте ссылку на товар" % store_name
        )
        return ConversationHandler.END

    # user's goods inline
    def back_arrow(self, update, context):
        """
        Create back arrow buton.
        """
        query = update.callback_query
        user_id = query.message.chat_id
        new_pos = self.user_pos[user_id] - self.GOODS_PER_MESSAGE

        if new_pos >= 0:
            self.user_pos[user_id] = new_pos
        else:
            self.user_pos[user_id] = 0

        self.send_user_goods(update, context)

    # user's goods inline
    def forward_arrow(self, update, context):
        """
        Create forward arrow buton.
        """
        query = update.callback_query
        user_id = query.message.chat_id
        new_pos = self.user_pos[user_id] + self.GOODS_PER_MESSAGE
        if len(self.json_goods_data[user_id]) > new_pos:
            self.user_pos[user_id] = new_pos
        self.send_user_goods(update, context)

    # user's goods inline
    def make_arrows_list(self, update, context):
        """
        Combine arrows and exit button in inline keyboard.
        """
        left_arrow = u"\u2190"
        right_arrow = u"\u2192"
        exit_cross = "CLOSE"

        arrows_list = [InlineKeyboardButton(left_arrow, callback_data="previous"),
                       InlineKeyboardButton(exit_cross, callback_data="exit"),
                       InlineKeyboardButton(right_arrow, callback_data="next")]
        return arrows_list

    # user's goods inline
    def create_inline_del_buttons(self, update, context):
        """
        Create crosses in the bottom of inline keyboard.
        Link them to their goods.
        Return: [list] InlineKeyboard
        """
        query = update.callback_query
        if query is not None:
            user_id = query.message.chat_id
        else:
            user_id = update.message.chat_id
        del_buttons_list = []

        start = self.user_pos[user_id]
        end = self.user_pos[user_id] + self.GOODS_PER_MESSAGE
        sliced_dict = dict(itertools.islice(self.json_goods_data[user_id].items(), start, end))

        for num, url in enumerate(sliced_dict):
            short_name = sliced_dict[url]["title"].split(" ")[0]
            splited_name = url.rsplit("/", 2)[-2]
            splited_name = splited_name.rsplit("-")[-4:]
            url_part = "-".join(splited_name)

            num = str(num + 1)
            callback = url_part + " " + short_name
            del_buttons_list.append(InlineKeyboardButton(num + " " + u"\u274C", callback_data=callback))
        return del_buttons_list

    # user's goods inline
    def goods_text(self, update, context):
        """
        Creates text from goods' names, links, and their amount.
        """
        query = update.callback_query
        if query is not None:
            user_id = query.message.chat_id
        else:
            user_id = update.message.chat_id
        goods_text = ""

        start = self.user_pos[user_id]
        end = self.user_pos[user_id] + self.GOODS_PER_MESSAGE

        if user_id not in self.json_goods_data.keys():
            return None

        sliced_dict = dict(itertools.islice(self.json_goods_data[user_id].items(), start, end))
        end = start + len(sliced_dict)
        title_page_text_num = "Результаты %s-%s из %s \n" % (start + 1,
                                                             end,
                                                             str(len(self.json_goods_data[user_id])))

        for num, url in enumerate(sliced_dict):
            title = self.json_goods_data[user_id][url]["title"]
            price = self.json_goods_data[user_id][url]["price"]
            promoDate = self.json_goods_data[user_id][url]["promoDate"]

            goods_text = goods_text + str(num + 1) + ". " \
                                    + '' + title + ' ' \
                                    + " тек. ц: " + str(price) + ' р. ' + promoDate + "\n"

        if goods_text == "":
            return None
        goods_text = title_page_text_num + goods_text
        return goods_text

    # user's goods inline
    def create_goods_keyboard(self, update, context):
        """
        Combine delete buttons, arrow buttons, and text
        """
        del_buttons = self.build_menu(self.create_inline_del_buttons(update, context), n_cols=5)
        del_buttons.append(self.make_arrows_list(update, context))
        reply_markup = InlineKeyboardMarkup(del_buttons)
        return reply_markup

    # user's goods inline
    def send_user_goods(self, update, context):
        """
        When /goods command received.
        Send inline keyboard with user's goods.
        """
        query = update.callback_query

        if query is not None:
            user_id = query.message.chat_id
        else:
            user_id = update.message.chat_id
            self.user_pos[user_id] = 0

        self.user_pos.setdefault(user_id, 0)
        goods_list_text = self.goods_text(update, context)
        if goods_list_text is None:
            text = "У Вас нет отслеживаемых товаров"
            reply_markup = None
        else:
            text = self.goods_text(update, context)
            reply_markup = self.create_goods_keyboard(update, context)

        if update.message:
            update.effective_message.reply_text(text,
                                                reply_markup=reply_markup,
                                                disable_web_page_preview=True,
                                                parse_mode='HTML')

        else:
            try:
                context.bot.edit_message_text(text,
                                              chat_id=query.message.chat_id,
                                              message_id=query.message.message_id,
                                              reply_markup=reply_markup,
                                              disable_web_page_preview=True,
                                              parse_mode='HTML')
            except telegram.error.BadRequest:  # if there is no message to edit
                context.bot.answer_callback_query(update.callback_query.id,
                                                  text="there's no {} page.".format(query.data))
        return GOOD_LIST

    # user's goods inline
    def confirm_del_action_keyboard(self, update, context):
        """
        Buttons to confirm or reject deleting good from your list.

        """
        query = update.callback_query
        url_part_with_name = query.data.split(" ")
        url_part = url_part_with_name[0]
        short_name = url_part_with_name[1]

        keyboard = [[InlineKeyboardButton("Отмена", callback_data="cancel"),
                     InlineKeyboardButton("Удалить", callback_data=url_part)]]

        reply_markup = InlineKeyboardMarkup(keyboard)
        context.bot.edit_message_text("Вы уверены что хотите удалить {}?".format(short_name),
                                      chat_id=query.message.chat_id,
                                      message_id=query.message.message_id,
                                      reply_markup=reply_markup)
        return DELGOOD

    # user's goods inline
    def handle_back_to_list(self, update, context):
        """
        Go back to goods list with inline keyboard after deletion a good.
        """
        self.send_user_goods(update, context)
        return GOOD_LIST

    # user's goods inline
    def handle_del_good(self, update, context):
        """
        Deleting good from json_goods_data.
        After delete confirmation.
        """
        query = update.callback_query
        user_id = query.message.chat_id

        url_part = query.data
        for url in self.json_goods_data[user_id]:
            if url_part in url:
                del self.json_goods_data[user_id][url]
                filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)
                break

        self.send_user_goods(update, context)
        return GOOD_LIST

    # user's goods inline
    def good_handler_end(self, update, context):
        """
        Replace inline keyboard and goods info with default phrase.
        """
        query = update.callback_query

        bot = context.bot
        bot.edit_message_text(
            chat_id=query.message.chat_id,
            message_id=query.message.message_id, text="скиньте ссылку на товар"
        )

        return ConversationHandler.END

    # check store.
    def check_user_store(self, update, context):
        """ Check your current store name """

        if update.message.chat_id not in self.users_stores:
            store = "Не определён. Выберите его /choose_store"
        else:
            store = self.users_stores[update.message.chat_id][1]  # get store name from dict
        text = "Ваш магазин - %s" % (store)
        context.bot.send_message(update.message.chat_id, text=text)

    # url append store num
    def add_store_in_url(self, user_id, url):
        """
        Removes storeId from url if exists.
        Place user's store in the end of the url.
        """

        clear_url = url.rsplit('?', 1)[0]
        store_id = self.users_stores.get(user_id, ["0005"])[0]  # get store id
        url_with_store_id = clear_url + "?StoreId=" + store_id
        return url_with_store_id

    # change goods info
    def update_goods_data(self, user_id, url, good_info, repeatNotif=True):
        """
        Save current good info (or updates it's state) in self.json_goods_data
        Calls save data in file function.
        """
        title = good_info["title"]
        price = good_info["price"]
        isPromoForCardPrice = good_info["isPromoForCardPrice"]
        promoDate = good_info["promoDate"]
        self.json_goods_data.setdefault(user_id, {})
        self.json_goods_data[user_id].update({url: {"title": title,
                                                    "price": price,
                                                    "isPromoForCardPrice": isPromoForCardPrice,
                                                    "promoDate": promoDate,
                                                    "repeatNotif": repeatNotif}})

    # check discounts interval decorator.
    def setInterval(interval):
        """ decorator for periodic checking price """
        def decorator(function):
            def wrapper(*args, **kwargs):
                stopped = threading.Event()

                def loop():  # executed in another thread
                    while not stopped.wait(interval):  # until stopped
                        function(*args, **kwargs)

                t = threading.Thread(target=loop)
                t.daemon = True  # stop if the program exits
                t.start()
                return stopped
            return wrapper
        return decorator

    @setInterval(CHECK_PRICE_PERIOD * 60)  # seconds *  60 = minutes
    def check_discount_cycle(self, context):
        for user_id, url in self.iter_goods():
            new_good_info = self.get_new_good_info(url)
            good_title = new_good_info['title']
            if new_good_info == "not_available":
                logger.info(f"Site is not available! for {good_title}")
                break
            if new_good_info == "not_found":
                logger.info(f"Site is not found for {good_title}")
                self.good_not_found(context, user_id, url)
                break

            if not self.is_discount(new_good_info):
                logger.info(f"No new discounts for {good_title}")
                self.update_goods_data(user_id, url, new_good_info, repeatNotif=True)
                filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)
                continue

            if self.is_repeat_notif(user_id, url):
                logger.info(f"Found old discount for {good_title}")
                self.old_discount(user_id, url, new_good_info)
            else:
                logger.info(f"Found new discount for {good_title}")
                self.new_discount(user_id, url, new_good_info)

    def iter_goods(self):
        for user_id in list(self.json_goods_data.keys()):
            for url in list(self.json_goods_data[user_id].keys()):
                yield user_id, url

    def get_new_good_info(self, url):
        response = lenta_api.get_response(url, self.HEADERS)
        response_status = response.status_code
        response_text = response.text
        logger.debug("response_status: %d ", response_status)
        if response_status in (500, 501, 502):
            return "not_available"
        elif response_status == 404:
            return "not_found"
        return lenta_api.fetch_good_info(response_text)

    def is_repeat_notif(self, user_id, url):
        repeatNotif = self.json_goods_data[user_id][url]["repeatNotif"]
        # prev_isPromoForCardPrice = self.json_goods_data[user_id][url]["isPromoForCardPrice"]
        return repeatNotif

    def is_discount(self, new_good_info):
        """
        Returns 'True' if discound is found.
        """
        isPromoForCardPrice = new_good_info["isPromoForCardPrice"]
        if isPromoForCardPrice:  # disount avaliable
            return True
        return False

    def new_discount(self, context, user_id, url, new_good_info):
        self.update_goods_data(user_id, url, new_good_info, repeatNotif=False)
        filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)
        title = new_good_info['title']
        price = new_good_info['price']
        promoDate = new_good_info['pomoDate']
        try:
            replies.new_discount_msg(context, user_id, title, price, promoDate)
        except telegram.error.BadRequest:
            logger.error("The User has blocked the Bot!")

    def old_discount(self, user_id, url, new_good_info):
        self.update_goods_data(user_id, url, new_good_info, repeatNotif=True), filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)

    def good_not_found(self, context, user_id, url):
        good_title = self.json_goods_data[user_id][url]["title"]
        not_fount_phrase = ". Ссылка на товар больше не действительна!"
        if not_fount_phrase in good_title:  # already notified about this issue.
            return
        replies.good_not_found_msg(context, user_id, good_title)
        losted_good_info = {"title": good_title + not_fount_phrase, "price": 0, "promoDate": "", "isPromoForCardPrice": False}
        self.update_goods_data(user_id, url, losted_good_info)
        filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)

    def main(self, update, context):
        """
        Handles every text message.
        Checks if it's a link to Lenta's catalog.
        Saves this link and good info.
        """
        url = update.message.text
        user_id = update.message.chat_id

        if self.first_time is True:
            self.first_time = False
            logger.info("check discount init")
            self.check_discount_cycle(context)

        if not url.startswith(SITE_WHITELIST[0]):
            replies.not_valid_msg(update, context)
            return

        if (user_id in self.json_goods_data) and (url in self.json_goods_data[user_id]):
            context.bot.send_message(update.message.chat_id, text="Данный товар уже добавлен в список желаемых")
            return

        url = self.add_store_in_url(user_id, url)
        good_info = self.get_new_good_info(url)
        if not good_info:
            replies.not_valid_msg(update, context)
            return
        self.update_goods_data(user_id, url, good_info)
        filesoper.write_json(self.json_goods_data, self.GOODS_DATA_LOCATION)

        context.bot.send_message(update.message.chat_id, text="%s по цене %s руб добавлено" % (good_info["title"], good_info["price"]))

    def error_handler(self, update, context):
        """
        Error handler.
        """
        logger.error('Update "%s" caused error "%s"' % (update, context.error))

if __name__ == '__main__':
    Bot = LentaBot()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan  7 17:33:56 2020

@author: yumin
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 19 11:08:39 2019

@author: yumin
"""

def collect_GCM():
    import os
    import glob
    import shutil
    variable = 'pr' #'tasmin' #'tasmax'
    datapath = '/home/yumin/Downloads/CMIP5/CommonGrid/'
    #savepath = '/home/yumin/myProgramFiles/DATA/GCM/GCMdata/'+variable+'/raw/'
    savepath = '/home/yumin/myProgramFiles/DATA/GCM/GCMdata/ppt/35channels/rawdata/pr/'
    if not os.path.exists(savepath):
        os.makedirs(savepath)

    pathlist = glob.glob(datapath+'**/*.nc', recursive=True)
    filepaths = [f for f in pathlist if 'historical' in f and variable in f and f.endswith('.nc')]
    filepaths = sorted(filepaths)

    for f in filepaths:
        newf = savepath+f.split('/')[-1]
        shutil.copyfile(f,newf)
#collect_GCM()

#%%
def readGCM():
    #### generate whole USA map, saved on 10/22/2019
    #### read and process NASA GCM data and aglined
    import numpy as np
    from netCDF4 import Dataset
    from ncdump import ncdump
    from os import listdir
    from os.path import isfile, join
    import os

    prefix = '/home/yumin/Desktop/DS/DATA/'
    variablename = 'pr' #['pr' 'tas' 'tasmax' 'tasmin'] 'pr_37models'
    filepath = prefix+'GCM/GCMdata/ppt/35channels/rawdata/'+variablename+'/'
    savepath0 = prefix+'GCM/GCMdata/ppt/35channels/processeddata_26by59_points/'
    savepath = savepath0+variablename+'/'
    # if not os.path.exists(savepath0):
    #     os.makedirs(savepath0)
    if savepath and not os.path.exists(savepath):
        os.makedirs(savepath)

    filenames = [f for f in listdir(filepath) if isfile(join(filepath,f))]

    kk = 0
    values_gcms = []
    for filename in filenames:
        #filename = 'regridded_1deg_pr_amon_inmcm4_historical_r1i1p1_195001-200512.nc'
        dataset = Dataset(filepath+filename,mode='r')
        #dataset = Dataset(filepath+filename,mode='r',format="NETCDF3")
        #dataset = Dataset(filename,mode='r')
        if kk==0:
            nc_attrs, nc_dims, nc_vars = ncdump(dataset)
            kk += 1

        # original longitude is from 0.5 to 359.5 by 1, 360 points
        # original latitude is from -89.5 to 89.5 by 1, 180 points
        # whole USA longitude from [230.5E,304.5E] by 1, 75 points
        # whole USA latitude from [20.5N, 49.5N] by 1, 30 points
        # whole USA longitude from [235.5E,293.5E] by 1, 59 points
        # whole USA latitude from [24.5N, 49.5N] by 1, 26 points

        # original month from 195001 to 200512, 672 points
        # month from 200001 to 200412, 60 months

        time = dataset.variables['time'][:] # 195001 - 200512
        #lats = dataset.variables['latitude'][110:140] # [20.5N, 49.5N]
        #lons = dataset.variables['longitude'][230:305] # [230.5E, 304.5E]
        lats = dataset.variables['latitude'][114:140] # [24.5N, 49.5N]
        lons = dataset.variables['longitude'][235:294] # [235.5E, 293.5E]
        #### whole USA
        ## monthly mean precipitation, unit: mm/day
        #value1_gcm = dataset.variables[variablename][:,110:140,230:305]#[month,lat,lon] 195001-200512, totally
        value1_gcm = dataset.variables[variablename][:,114:140,235:294]#[month,lat,lon] 195001-200512, totally
        #value2_gcm = np.ma.filled(value1_gcm,-1.0e-8)
        value2_gcm = np.ma.filled(value1_gcm,0)
        (Nmon,Nlat,Nlon) = value2_gcm.shape # [672,26,59]
        value_gcm = np.zeros((Nmon,Nlat,Nlon)) # [672,26,59]
        for t in range(Nmon):
            value_gcm[t,:,:] = np.flipud(value2_gcm[t,:,:]) # lats from [49.5N,24.5N]

        #### retangular USA
        ## monthly mean precipitation, unit: mm/day
        #precipitation = dataset.variables['pr'][600:660,123:140,245:278]#[month,lat,lon]
        #prmean_month_gcm = np.ma.filled(precipitation,np.nan)

        if np.isnan(np.sum(value_gcm)):
            print(filename + 'has NAN!\n')
            break
        savename = filename.replace('.nc','_USA.npy')
        np.save(savepath+savename,value_gcm)
        values_gcms.append(value_gcm)

    values_gcms = np.stack(values_gcms,axis=0) # [Ngcm,Nmon,Nlat,Nlon]
    print('values_gcms.shape={}'.format(values_gcms.shape))

    time = np.array(time)
    #### whole USA
    # latitude from [20.5N, 49.5N] by 1, 30 points
    # latitude from [24.5N, 49.5N] by 1, 26 points
    #lats_gcm1 = dataset.variables['latitude'][110:140]
    lats_gcm1 = dataset.variables['latitude'][114:140]
    lats_gcm = np.flipud(lats_gcm1) # lats from [49.5N,24.5N]
    # longitude from [230.5E, 304.5E] by 1, 75 points
    # longitude from [234.5E, 295.5E] by 1, 62 points
    #lons_gcm = dataset.variables['longitude'][230:305]
    lons_gcm = dataset.variables['longitude'][235:294]
    lats_gcm = np.array(lats_gcm)
    lons_gcm = np.array(lons_gcm)
    #print('lats_gcm={},\nlons_gcm={}'.format(lats_gcm,lons_gcm))
    np.save(savepath0+'time_gcm.npy',time)
    np.save(savepath0+'lats_gcm.npy',lats_gcm)
    np.save(savepath0+'lons_gcm.npy',lons_gcm)
    np.save(savepath0+'35gcms_prmean_monthly_1by1_195001-200512_USA.npy',values_gcms)

#readGCM()

#%%
def readElevation():
    #### read the PRISM elevation data and aglined with GCM
    import numpy as np
    from netCDF4 import Dataset
    import os
    from ncdump import ncdump

    #%%
    prefix = '/home/yumin/Desktop/DS/DATA/'
    #prefix = '/scratch/wang.zife/YuminLiu/'
    #datapath = prefix+'myPythonFiles/Downscaling/data/Climate/PRISMdata/raw/wcs_4km_prism.nc'
    datapath = prefix+'PRISM/PRISMdata/elevation/raw/wcs_4km_prism.nc'
    savepath = prefix+'PRISM/PRISMdata/elevation/processeddata/' #None
    dataset = Dataset(datapath,mode='r')
    nc_attrs, nc_dims, nc_vars = ncdump(dataset)

    ## orginal latitude from [49.9375N, 24.1041N], by 1/24 (~0.04), 621 points
    ## orginal longitude from [-125.021, -66.521], by 1/24 (~0.04), 1405 points
    ## orginal longitude from [234.979E, 293.479E], by 1/24 (~0.04), 1405 points
    lats_o = dataset.variables['lat'][:]
    lons_o = dataset.variables['lon'][:]+360
    elevation_o  = dataset.variables['Band1'][:]
    print('elevation_o.min()={}, elevation_o.max()={}'.format(elevation_o.min(),elevation_o.max()))
    #import matplotlib.pyplot as plt
    #fig = plt.figure()
    #plt.imshow(elevation_o)
    #plt.show()

    #%% align with GCM grid
    ## latitude from [49.9167N,24.125N] by 1/24 (~0.04), 620 points
    ## longitude from [235E,293.458E] by 1/24 (~0.04), 1404 points
    lats = np.zeros(len(lats_o)-1,)
    lons = np.zeros(len(lons_o)-1,)
    for i in range(len(lats_o)-1):
        lats[i] = 0.5*(lats_o[i]+lats_o[i+1])
    for i in range(len(lons_o)-1):
        lons[i] = 0.5*(lons_o[i]+lons_o[i+1])
    (Nlat,Nlon) = elevation_o.shape
    elevation = np.zeros((Nlat-1,Nlon-1)) # [620,1404]
    for i in range(Nlat-1):
        for j in range(Nlon-1):
            elevation[i,j] = 0.25*(elevation_o[i,j]+elevation_o[i+1,j]
                                      +elevation_o[i,j+1]+elevation_o[i+1,j+1])

    if np.isnan(np.sum(elevation)):
        print("Error! nan found!\n")
    print('elevation.min()={}, elevation.max()={}'.format(elevation.min(),elevation.max()))
    #%%
    #if savepath and not os.path.exists(savepath):
    #    os.makedirs(savepath)
    #np.save(savepath+'lons_prism.npy',lons)
    #np.save(savepath+'lats_prism.npy',lats)
    #np.save(savepath+'prism_elevation_USA.npy',elevation)
    #import matplotlib.pyplot as plt
    #fig = plt.figure()
    #plt.imshow(elevation)
    #plt.show()

    #%%
    def savedata(savepath,savename,lats,lons,data):
        if savepath and not os.path.exists(savepath):
            os.makedirs(savepath)
        #np.save(savepath+'lons_prism.npy',lons)
        #np.save(savepath+'lats_prism.npy',lats)
        np.save(savepath+'prism_elevation_{}_USA.npy'.format(savename),data)
    ## latitude from [49.5N,24.5N]
    ## longitude from [235.5E,293.5E]
    lats_24 = lats[10:611]
    lons_24 = lons[12:]
    elevation_24 = elevation[10:611,12:] # [lat,lon]
    #elevation_24[elevation_24<0] = 0.0
    savedata(savepath+'prism_prmean_monthly_0.04by0.04/','0.04by0.04',lats_24,lons_24,elevation_24)
    import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(elevation_24)
    plt.savefig(savepath+'prism_prmean_monthly_0.04by0.04/elevation_prism_24.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    ## GCM is 26 by 59
    ## 8x is 208 by 472
    ## 4x is 104 by 236
    ## 2x is 52 by 118
    from skimage.transform import resize
    lats_8 = np.linspace(lats_24[0],lats_24[-1],num=208)
    lons_8 = np.linspace(lons_24[0],lons_24[-1],num=472)
    elevation_8 = resize(elevation_24,(208,472),order=1,preserve_range=True)
    #elevation_8[elevation_8<0] = 0.0
    savedata(savepath+'prism_prmean_monthly_0.125by0.125/','0.125by0.125',lats_8,lons_8,elevation_8)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(elevation_8)
    plt.savefig(savepath+'prism_prmean_monthly_0.125by0.125/elevation_prism_8.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_4 = np.linspace(lats_24[0],lats_24[-1],num=104)
    lons_4 = np.linspace(lons_24[0],lons_24[-1],num=236)
    elevation_4 = resize(elevation_24,(104,236),order=1,preserve_range=True)
    #elevation_4[elevation_4<0] = 0.0
    savedata(savepath+'prism_prmean_monthly_0.25by0.25/','0.25by0.25',lats_4,lons_4,elevation_4)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(elevation_4)
    plt.savefig(savepath+'prism_prmean_monthly_0.25by0.25/elevation_prism_4.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_2 = np.linspace(lats_24[0],lats_24[-1],num=52)
    lons_2 = np.linspace(lons_24[0],lons_24[-1],num=118)
    elevation_2 = resize(elevation_24,(52,118),order=1,preserve_range=True)
    #elevation_2[elevation_2<0] = 0.0
    savedata(savepath+'prism_prmean_monthly_0.5by0.5/','0.5by0.5',lats_2,lons_2,elevation_2)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(elevation_2)
    plt.savefig(savepath+'prism_prmean_monthly_0.5by0.5/elevation_prism_2.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_1 = np.linspace(lats_24[0],lats_24[-1],num=26)
    lons_1 = np.linspace(lons_24[0],lons_24[-1],num=59)
    elevation_1 = resize(elevation_24,(26,59),order=1,preserve_range=True)
    #elevation_1[elevation_1<0] = 0.0
    savedata(savepath+'prism_prmean_monthly_1.0by1.0/','1.0by1.0',lats_1,lons_1,elevation_1)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(elevation_1)
    plt.savefig(savepath+'prism_prmean_monthly_1.0by1.0/elevation_prism_1.png',dpi=1200,bbox_inches='tight')
    #plt.show()

#readElevation()

#%%
def zip_to_nc():
    '''
    read the original .zip file, unpack to .bil file, and then save to .nc file
    '''
    import os
    import zipfile
    import xarray as xr
    from readbilfile import BilFile
    prefix = '/home/yumin/Desktop/DS/DATA/'
    root_path = prefix+'PRISM/monthly/ppt/'

    datapaths= []
    for i, (root,dirs,files) in enumerate(os.walk(root_path)):
        #if i>=60: break
        for file in files:
            if file.endswith('all_bil.zip'):
                datapaths.append([root,file])
    datapaths = sorted(datapaths)
    #datapaths = datapaths[:5]
    xray_data_all = []
    for i, (datapath,filename) in enumerate(datapaths):
        year = 1950+i
        if not str(year) in filename:
            print('Error: {} year not in order!'.format(year))

        temppath = root_path+'temppath/'+datapath.split('/')[-1]
        if not os.path.exists(temppath):
            os.makedirs(temppath)
        savepath = root_path+'processed/'+datapath.split('/')[-1]
        if not os.path.exists(savepath):
            os.makedirs(savepath)
        #xray_data = []
        with zipfile.ZipFile(datapath+'/'+filename, 'r') as zf:
            zf.extractall(temppath+'/'+filename.replace('.zip',''))

        filepath = temppath+'/'+filename.replace('.zip','')

    #    bilhdrfiles = []
    #    for f in os.listdir(filepath):
    #        if f.endswith('.bil'):# or f.endswith('.hdr'):
    #            if len(f.split('_')[-2])==6:
    #                bilhdrfiles.append(f)
        bilhdrfiles = [f for f in os.listdir(filepath) if len(f.split('_')[-2])==6 and f.endswith('.bil')]
        bilhdrfiles = sorted(bilhdrfiles)
        #bilhdrfiles = bilhdrfiles[:1]
        #print('bilhdrfiles={}'.format(bilhdrfiles))
        #xray_data = []
        for bilfile in bilhdrfiles:
            b = BilFile(filepath+'/'+bilfile)
            #nodatavalue, data = b.get_array()

            #print('nodatavalue={}, data={}'.format(nodatavalue, data))

            dr = b.bil_to_xray()
            if dr.shape[1:]!=(621,1405):
                print('error: {} dr.shape not the same!'.format(dr.shape))
            #print('dr.shape={}'.format(dr.shape))
            #print('dr={}'.format(dr))
            #xray_data.append(dr)
            xray_data_all.append(dr)
        #if len(xray_data)!=12:
        #    print('Error: {} less than 12 months!'.format(bilfile))

        #ds = xr.Dataset({'ppt': xr.concat(xray_data, dim='time')})
        #save_nc_file = savepath+'/'+filename.replace('_all_bil.zip','_monthly.nc')
        #ds.to_netcdf(save_nc_file, format='NETCDF4')

    ds_all = xr.Dataset({'ppt': xr.concat(xray_data_all, dim='time')})
    save_nc_file_all = root_path+'/'+filename.replace('_2005_all_bil.zip','_1950-2005_monthly_total.nc')
    ds_all.to_netcdf(save_nc_file_all, format='NETCDF4')

#zip_to_nc()

#%%
def readPRISM():
    #### read the PRISM data and aglined with GCM
    import numpy as np
    from netCDF4 import Dataset
    import os
    from ncdump import ncdump

    prefix = '/home/yumin/Desktop/DS/DATA/'
    filepath = prefix+'PRISM/monthly/ppt/PRISM_ppt_stable_4kmM3_1950-2005_monthly_total.nc'
    savepath = prefix+'PRISM/PRISMdata/ppt/processeddata/prism_prmean_monthly_0.04by0.04_original_grid/'
    #%%
    dataset = Dataset(filepath,mode='r')
    nc_attrs, nc_dims, nc_vars = ncdump(dataset)

    # original latitude from 49.9375N to 24.1042N by 1/24 (~0.04), 621 points
    # latitude from N to N by ,  points
    lats_o = dataset.variables['lat'][:]
    # original longitude from -125.021 to -66.5208 by 1/24 (~0.04), 1405 points
    # original longitude from 234.979E to 293.479E by 1/24 (~0.04), 1405 points
    lons_o = dataset.variables['lon'][:]+360
    # time from 195001 to 200512 by 1, 672 points
    time = dataset.variables['time']#
    time = list(time)

    precipitation_o = dataset.variables['ppt'][:] #total precipitation, [month,lat,lon], mm/month
    prmean_month = np.ma.filled(precipitation_o,0)/30.0 # daily mean precipitation, unite: mm/day

    #%% align with GCM grid
    ## latitude from [49.9167N,24.125N] by 1/24 (~0.04), 620 points
    ## longitude from [235E,293.458E] by 1/24 (~0.04), 1404 points
    lats = np.zeros(len(lats_o)-1,)
    lons = np.zeros(len(lons_o)-1,)
    for i in range(len(lats_o)-1):
        lats[i] = 0.5*(lats_o[i]+lats_o[i+1])
    for i in range(len(lons_o)-1):
        lons[i] = 0.5*(lons_o[i]+lons_o[i+1])
    (Nmon,Nlat,Nlon) = prmean_month.shape
    prmean_month_prism = np.zeros((Nmon,Nlat-1,Nlon-1)) # [672, 117, 297],[672, 101, 245]
    for t in range(Nmon):
        for i in range(Nlat-1):
            for j in range(Nlon-1):
                prmean_month_prism[t,i,j] = 0.25*(prmean_month[t,i,j]+prmean_month[t,i+1,j]
                                          +prmean_month[t,i,j+1]+prmean_month[t,i+1,j+1])

    prmean_month_prism[prmean_month_prism<0] = 0
    #if np.isnan(np.sum(prmean_month_prism)):
    #    print("Error! nan found!\n")

    #%%
    #if savepath and not os.path.exists(savepath):
    #    os.makedirs(savepath)
    #np.save(savepath+'lons_prism.npy',lons)
    #np.save(savepath+'lats_prism.npy',lats)
    #np.save(savepath+'time_prism.npy',time)
    #np.save(savepath+'prism_prmean_monthly_195001-200512_USA.npy',prmean_month_prism)
    #import matplotlib.pyplot as plt
    #fig = plt.figure()
    #plt.imshow(prmean_month_prism[0,:,:])
    #plt.show()

#%% downsample to 8x, 4x and 2x resolutions with respect to GCM resolution
def downsamplePRISM():
    import numpy as np
    import os

    def savedata(savepath,savename,lats,lons,time,data):
        if savepath and not os.path.exists(savepath):
            os.makedirs(savepath)
        np.save(savepath+'lons_prism.npy',lons)
        np.save(savepath+'lats_prism.npy',lats)
        np.save(savepath+'time_prism.npy',time)
        np.save(savepath+'prism_prmean_monthly_{}_195001-200512_USA.npy'.format(savename),data)

    prefix = '/home/yumin/Desktop/DS/DATA/'
    #prefix = '/scratch/wang.zife/YuminLiu/'
    #datapath = prefix+'myPythonFiles/Downscaling/data/Climate/PRISMdata/processeddata/prism_prmean_monthly_0.04by0.04_original_grid/'
    datapath = prefix+'PRISM/PRISMdata/ppt/processeddata/prism_prmean_monthly_0.04by0.04_original_grid/'
    savepath = prefix+'PRISM/PRISMdata/ppt/processeddata/'
    lats = np.load(datapath+'lats_prism.npy')
    lons = np.load(datapath+'lons_prism.npy')
    time = np.load(datapath+'time_prism.npy')
    prmean_month_prism = np.load(datapath+'prism_prmean_monthly_195001-200512_USA.npy')

    ## latitude from [49.5N,24.5N]
    ## longitude from [235.5E,293.5E]
    ### longitude from [235.5E,292.5E]
    lats_24 = lats[10:611]
    lons_24 = lons[12:]
    prmean_month_prism_24 = np.transpose(prmean_month_prism[:,10:611,12:],axes=(1,2,0)) # [lat,lon,month]
    #lons_8 = lons[12:1381]
    #prmean_month_prism_24 = prmean_month_prism[:,10:611,12:1381]
    del lats, lons, prmean_month_prism

    savedata(savepath+'prism_prmean_monthly_0.04by0.04/','0.04by0.04',lats_24,lons_24,time,
             np.transpose(prmean_month_prism_24,axes=(2,0,1))) #[month,lat,lon]
    import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(np.sum(prmean_month_prism_24,axis=2))
    plt.savefig(savepath+'prism_prmean_monthly_0.04by0.04/prmean_month_prism_24.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    ## GCM is 26 by 59
    ## 8x is 208 by 472
    ## 4x is 104 by 236
    ## 2x is 52 by 118
    from skimage.transform import resize
    lats_8 = np.linspace(lats_24[0],lats_24[-1],num=208)
    lons_8 = np.linspace(lons_24[0],lons_24[-1],num=472)
    prmean_month_prism_8 = resize(prmean_month_prism_24,(208,472),order=1,preserve_range=True) #[lat,lon,month]
    prmean_month_prism_8 = np.transpose(prmean_month_prism_8,axes=(2,0,1)) #[month,lat,lon]
    savedata(savepath+'prism_prmean_monthly_0.125by0.125/','0.125by0.125',lats_8,lons_8,time,prmean_month_prism_8)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(np.sum(prmean_month_prism_8,axis=0))
    plt.savefig(savepath+'prism_prmean_monthly_0.125by0.125/prmean_month_prism_8.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_4 = np.linspace(lats_24[0],lats_24[-1],num=104)
    lons_4 = np.linspace(lons_24[0],lons_24[-1],num=236)
    prmean_month_prism_4 = resize(prmean_month_prism_24,(104,236),order=1,preserve_range=True) #[lat,lon,month]
    prmean_month_prism_4 = np.transpose(prmean_month_prism_4,axes=(2,0,1)) #[month,lat,lon]
    savedata(savepath+'prism_prmean_monthly_0.25by0.25/','0.25by0.25',lats_4,lons_4,time,prmean_month_prism_4)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(np.sum(prmean_month_prism_4,axis=0))
    plt.savefig(savepath+'prism_prmean_monthly_0.25by0.25/prmean_month_prism_4.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_2 = np.linspace(lats_24[0],lats_24[-1],num=52)
    lons_2 = np.linspace(lons_24[0],lons_24[-1],num=118)
    prmean_month_prism_2 = resize(prmean_month_prism_24,(52,118),order=1,preserve_range=True) #[lat,lon,month]
    prmean_month_prism_2 = np.transpose(prmean_month_prism_2,axes=(2,0,1)) #[month,lat,lon]
    savedata(savepath+'prism_prmean_monthly_0.5by0.5/','0.5by0.5',lats_2,lons_2,time,prmean_month_prism_2)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(np.sum(prmean_month_prism_2,axis=0))
    plt.savefig(savepath+'prism_prmean_monthly_0.5by0.5/prmean_month_prism_2.png',dpi=1200,bbox_inches='tight')
    #plt.show()

    lats_1 = np.linspace(lats_24[0],lats_24[-1],num=26)
    lons_1 = np.linspace(lons_24[0],lons_24[-1],num=59)
    prmean_month_prism_1 = resize(prmean_month_prism_24,(26,59),order=1,preserve_range=True) #[lat,lon,month]
    prmean_month_prism_1 = np.transpose(prmean_month_prism_1,axes=(2,0,1)) #[month,lat,lon]
    savedata(savepath+'prism_prmean_monthly_1.0by1.0/','1.0by1.0',lats_1,lons_1,time,prmean_month_prism_1)
    #import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.imshow(np.sum(prmean_month_prism_1,axis=0))
    plt.savefig(savepath+'prism_prmean_monthly_1.0by1.0/prmean_month_prism_1.png',dpi=1200,bbox_inches='tight')
    #plt.show()

#downsamplePRISM()

#%%
def mergexy(resolution='0.125'):
    import os
    import numpy as np
    #import seaborn as sns
    #import matplotlib.pyplot as plt
    from skimage.transform import resize

    prefix = '/home/yumin/Desktop/DS/DATA/'
    #prefix = '/scratch/wang.zife/YuminLiu/'
    climatologypath = '../data/Climate/PRISM_GCMdata/ppt/35channels/prism_gcm_log1p_prmean_monthly_{}by{}_195001-199912_USA_month_1to600.npy'.format(resolution,resolution)
    gcmpath = prefix+'GCM/GCMdata/ppt/35channels/processeddata_26by59_points/35gcms_prmean_monthly_1by1_195001-200512_USA.npy'
    #resolution = '0.5'
    prismpath = prefix+'PRISM/PRISMdata/ppt/processeddata/prism_prmean_monthly_{}by{}/'.format(resolution,resolution)
    prfilename = 'prism_prmean_monthly_{}by{}_195001-200512_USA.npy'.format(resolution,resolution)
    elevationfilename = 'prism_elevation_{}by{}_USA.npy'.format(resolution,resolution)
    savepath = '../data/Climate/PRISM_GCMdata/ppt/35channels/{}by{}/'.format(resolution,resolution)
    #if savepath and not os.path.exists(savepath):
    #    os.makedirs(savepath)
    climatologydata = np.load(climatologypath) # [12,Nlat,Nlon]
    gcmdata = np.load(gcmpath) # [Ngcm,Nmon,Nlat,Nlon], [0.0, 32.99746322631836]
    prismdata = np.load(prismpath+prfilename) # [Nmon,Nlat,Nlon] [0.0, 67.89530826556293]
    elevation = np.load(prismpath+elevationfilename) #[Nlat,Nlon] [-74.09817504882812, 3677.848941389869]
    elevation[elevation<=0] = 0.0 #[Nlat,Nlon] [0.0, 3677.848941389869]
    print('gcmdata.shape={}'.format(gcmdata.shape))
    print('prismdata.shape={}'.format(prismdata.shape))
    print('elevation.shape={}'.format(elevation.shape))
    print('gcmdata.min()={}, gcmdata.max()={}'.format(gcmdata.min(), gcmdata.max()))
    print('prismdata.min()={}, prismdata.max()={}'.format(prismdata.min(), prismdata.max()))
    print('elevation.min()={}, elevation.max()={}'.format(elevation.min(), elevation.max()))

    #ind = np.unravel_index(np.argmin(elevation, axis=None), elevation.shape)
    #print(ind)
    #import matplotlib.pyplot as plt
    #fig = plt.figure()
    #plt.imshow(elevation)
    #plt.show()

    MaskMapUSA = np.sum(prismdata,axis=0) #[Nlat,Nlon]
    MaskMapUSA[MaskMapUSA<=0] = 0.0
    MaskMapUSA[MaskMapUSA>0] = 1.0 #[0.0, 1.0]
    MaskMapUSAsmall = resize(image=MaskMapUSA,output_shape=gcmdata.shape[2:],preserve_range=True)
    print('MaskMapUSA.shape={}'.format(MaskMapUSA.shape))
    print('MaskMapUSAsmall.shape={}'.format(MaskMapUSAsmall.shape))
    print('MaskMapUSA.min()={}, max()={}'.format(MaskMapUSA.min(),MaskMapUSA.max()))
    print('MaskMapUSAsmall.min()={}, max()={}'.format(MaskMapUSAsmall.min(),MaskMapUSAsmall.max()))

    elevation[MaskMapUSA<=0] = 0.0 # [0.0, 3677.848941389869]
    elevation = elevation[np.newaxis,...] #[1,Nlat,Nlon]
    elevation = np.log(1.0+elevation) # [0,3676.5424662662012]-->[0.0, 8.210]
    #np.savez(savepath+'MaskMapUSA_0.125by0.125_MaskMapUSA_1.0by1.0_elevation_0.125by0.125.npz',
    #         MaskMapUSA=MaskMapUSA,MaskMapUSAsmall=MaskMapUSAsmall,elevation=elevation)

    for mon in range(prismdata.shape[0]):
        gcms = gcmdata[:,mon,:,:] #[Ngcm,Nlat,Nlon], mm/day
        #prism = prismdata[[mon],:,:] #[1,Nlat,Nlon], mm/day
        prism = prismdata[mon,:,:] #[Nlat,Nlon], mm/day

        for ngcm in range(gcms.shape[0]):
            gcms[ngcm,:,:][MaskMapUSAsmall<=0] = 0.0
        prism[MaskMapUSA<=0] = 0.0
        prism = prism[np.newaxis,...] #[1,Nlat,Nlon], mm/day
        #X = gcmdata
        #y = cpcdata

        gcms = np.log(1.0+gcms) ##[Ngcm,Nlat,Nlon], [0, 63.02] --> [0,3.737655630962239]
        prism = np.log(1.0+prism) ##[1,Nlat,Nlon],[0,41.0] --> [0,4.159132957458496]
        #print('gcms.max={}'.format(max(gcms.flatten())))
        #print('y.max={}'.format(max(y.flatten())))
        climatology = climatologydata[[mon%12],:,:] #[1,Nlat,Nlon]

        ### normalized to [0.0,1.0]
        #gcms = gcms/5.0
        #prism = prism/5.0
        #climatology = climatology/5.0

        if mon<9:
            savename = 'prism_gcm_log1p_prmean_monthly_{}to1.0_195001-200512_USA_month00{}'.format(resolution,mon+1)
        elif mon<99:
            savename = 'prism_gcm_log1p_prmean_monthly_{}to1.0_195001-200512_USA_month0{}'.format(resolution,mon+1)
        else:
            savename = 'prism_gcm_log1p_prmean_monthly_{}to1.0_195001-200512_USA_month{}'.format(resolution,mon+1)
        folder = None
        if mon<600:
            folder = 'train/'
        elif mon<636:
            folder = 'val/'
        else:
            folder = 'test/'
        if savepath and not os.path.exists(savepath+folder):
            os.makedirs(savepath+folder)
        #np.savez(savepath+savename+'.npz',gcms=gcms,prism=prism)
        np.savez(savepath+folder+savename+'.npz',gcms=gcms,prism=prism,elevation=elevation,climatology=climatology)
        # if mon==0:
        #     fig = plt.figure()
        #     sns.distplot(gcms.flatten())
        #     plt.title('GCM')
        #     plt.show()
        #     fig = plt.figure()
        #     sns.distplot(prism.flatten())
        #     plt.title('PRISM')
        #     plt.show()

#mergexy()

if __name__=='__main__':
    ##zip_to_nc()
    ##readPRISM()
    #print('processing downsamplePRISM...')
    #downsamplePRISM()
    #print('downsamplePRISM Done!')
    #print('processing readElevation...')
    #readElevation()
    #print('readElevation Done!')
    ##readGCM()

    resolutions = ['0.125','0.25','0.5']
    #resolutions = ['0.5']
    for res in resolutions:
        print('processing mergexy(resolution={})'.format(res))
        mergexy(resolution=res)
        print('mergexy(resolution={}) Done!'.format(res))

    print('All Jobs Done!')