import os
import re

import flask
from flask import (Blueprint, current_app, flash, g, redirect, render_template,
                   request, session, url_for)
from flask.cli import with_appcontext
from werkzeug.exceptions import abort

from app.auth import login_required
from app.db import get_db

bp = Blueprint('rating', __name__)

@bp.before_request
def csrf_protect():
    if request.method == "POST":
        token = str(session.get('_csrf_token', None))

        form_token = request.form.get('_csrf_token')

        if not token or token != form_token:
            abort(403)

# @bp.after_request
# def make_nav_active():
#     pass

def select_images():
    db = get_db()

    current_group = db.execute(
        'SELECT * FROM current_group'
    )

    curr_group = None
    for group in current_group:
        curr_group = group['current_group_name']

    images = db.execute(
        'SELECT DISTINCT *'
        ' FROM images'
        ' WHERE images.group_status = ?'
        ' ORDER BY RANDOM()'
        ' LIMIT 6',
        (curr_group, )
    )
    return images

@bp.route('/sortable1', methods=['GET'])
@login_required
def sortable1():
    return render_template('rating/sortable.html', images=select_images())

@bp.route('/sortable1', methods=['POST'])
@login_required
def sortable11():
    if not request.form['ratings']:
        flash("Error while selecting images.")
        return redirect(url_for('rating.sortable1'))

    ratings = request.form['ratings']
    int_ratings = [int(i) for i in ratings.split()]

    if not len(int_ratings) == 6:
        flash("Error while posting images.")
        return redirect(url_for('rating.sortable1'))

    db = get_db()
    message = None

    db.execute(
        'INSERT INTO rating (rated_first, rated_second, rated_third, rated_forth, rated_fifth, rated_sixth, author_id) VALUES (?, ?, ?, ?, ?, ?, ?)',
        (int_ratings[0], int_ratings[1], int_ratings[2], int_ratings[3], int_ratings[4], int_ratings[5], g.user['id'])
    )
    db.commit()

    return redirect(url_for('rating.sortable1'))

@bp.route("/sortable2", methods=['GET'])
@login_required
def sortable2():
    return render_template("rating/sortable2.html", images=select_images())

@bp.route("/sortable2", methods=['POST'])
@login_required
def sortable22():
    if not request.form['ratings']:
        flash("Error while selecting images.")
        return redirect(url_for('rating.sortable2'))

    ratings = request.form['ratings']
    int_ratings = [int(i) for i in ratings.split()]

    if not len(int_ratings) == 2:
        flash("Error while posting images.")
        return redirect(url_for('rating.sortable2'))

    db = get_db()
    message = None

    db.execute(
        'INSERT INTO selection (rated_first, rated_second, author_id) VALUES (?, ?, ?)',
        (int_ratings[0], int_ratings[1], g.user['id'])
    )
    db.commit()

    return redirect(url_for('rating.sortable2'))

@bp.route("/sortable3", methods=['GET'])
@login_required
def sortable3():
    return render_template("rating/sortable3.html", images=select_images())

@bp.route("/sortable3", methods=['POST'])
@login_required
def sortable33():
    if request.form['ratings'] and request.form['type']:
        if request.form['type'] == 'sortable':
            ratings = request.form['ratings']
            int_ratings = [int(i) for i in ratings.split()]

            db = get_db()
            message = None

            db.execute(
                'INSERT INTO rating (rated_first, rated_second, rated_third, rated_forth, rated_fifth, rated_sixth, author_id) VALUES (?, ?, ?, ?, ?, ?, ?)',
                (int_ratings[0], int_ratings[1], int_ratings[2], int_ratings[3], int_ratings[4], int_ratings[5], g.user['id'])
            )
            db.commit()
            # last_id = db.last_insert_rowid()

            return redirect(url_for('rating.sortable3'))
        elif request.form['type'] == 'select':
            ratings = request.form['ratings']
            int_ratings = [int(i) for i in ratings.split()]

            db = get_db()
            message = None

            db.execute(
                'INSERT INTO selection (rated_first, rated_second, author_id) VALUES (?, ?, ?)',
                (int_ratings[0], int_ratings[1], g.user['id'])
            )
            db.commit()

            return redirect(url_for('rating.sortable3'))
        else:
            ratings = request.form['ratings']
            int_ratings = [int(i) for i in ratings.split()]
            amount = request.form['amount']

            db = get_db()
            message = None

            db.execute(
                'INSERT INTO amount (image_rated, rating, author_id) VALUES (?, ?, ?)',
                (int_ratings[0], amount, g.user['id'])
            )
            db.commit()

            return redirect(url_for('rating.sortable3'))

    flash("Error while selecting images.")
    return redirect(url_for('rating.sortable3'))

@bp.route("/sortable4", methods=['GET'])
@login_required
def sortable4():
    return render_template("rating/sortable4.html", images=select_images())

@bp.route("/sortable4", methods=['POST'])
@login_required
def sortable44():
    if not request.form['ratings']:
        flash("Error while selecting images.")
        return redirect(url_for('rating.sortable4'))

    ratings = request.form['ratings']
    int_ratings = [int(i) for i in ratings.split()]
    amount = request.form['amount']

    db = get_db()
    message = None

    db.execute(
        'INSERT INTO amount (image_rated, rating, author_id) VALUES (?, ?, ?)',
        (int_ratings[0], amount, g.user['id'])
    )
    db.commit()

    return redirect(url_for('rating.sortable4'))


from mpi4py import MPI
from helper import profiler, lab2args, lab2out, distribute_tasks_per_processes, Lab2Tags
import os, sys, math

def calc_equation(variable_index:int, equations:list, last_values:list) -> float:
    b = equations[variable_index][-1]
    all_except_target = sum(a * last_values[i] for i, a in enumerate(equations[variable_index][:-1]) if i != variable_index)
    delim = equations[variable_index][variable_index]
    return 1.0 * (b - all_except_target) / delim

def need_continue_calculation(old_variables:list, new_variables:list, threshold:float) -> bool:
    for a, b in zip(old_variables, new_variables):
        if math.fabs(a * a - b * b) > threshold * threshold:
            return True
    return False

def work() -> None:
    try:
        rank = None
        comm = MPI.COMM_WORLD # type: MPI.Comm
        rank, size = comm.Get_rank(), comm.Get_size()

        if size == 1:
            raise Exception("You should start me with more that one process")

        arguments = lab2args(load_data=True)
        current_variables = arguments.start_equation_results
        count_of_free_processes = size - 1
        equations, variables = arguments.coefficients.shape
        startup_info = distribute_tasks_per_processes(equations, count_of_free_processes)

        if rank == 0:
            print("Common startup info is %s" % (startup_info,))
            sys.stdout.flush()

        common_info = {'finish': 0}  # first - need finish flag
        mine_startup_info = startup_info[rank - 1] if rank != 0 else None
        iter_count = 0

        with profiler("Full proc task at %s" % MPI.Get_processor_name()):
            while common_info['finish'] != 1:
                # Next. Calculate

                if rank != 0:
                    results = []
                    if mine_startup_info is None:
                        pass
                    else:
                        results = [0] * (mine_startup_info[1] - mine_startup_info[0] + 1)
                        for i, idx in enumerate(range(mine_startup_info[0], mine_startup_info[1] + 1)):
                            results[i] = calc_equation(idx, arguments.coefficients, current_variables)
                    comm.send(results, dest=0, tag=Lab2Tags.TEMP_RESULTS)

                if rank == 0:
                    iter_count += 1
                    temp_results_per_process = list(comm.recv(source=proc+1, tag=Lab2Tags.TEMP_RESULTS) for proc in range(count_of_free_processes))
                    new_variables = list(item for sublist in temp_results_per_process for item in sublist)

                    if not need_continue_calculation(current_variables, new_variables, arguments.threshold):
                        common_info['finish'] = 1
                        print("Calculation finished. Iteration count is: %d" % iter_count)

                    current_variables = new_variables

                current_variables = comm.bcast(current_variables, 0)
                common_info = comm.bcast(common_info, 0)

        if rank == 0:
            lab2out(current_variables, arguments.out_file)

    except Exception as exc:
        if not isinstance(rank, int):
            rank = 99
        print("[R#%02d] Exception at %s" % (rank, exc))
        exc_type, exc_obj, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        print(exc_type, fname, exc_tb.tb_lineno)

if __name__ == '__main__':
    work()


from __future__ import absolute_import, division, print_function

import json
import logging
import math
import collections
import linecache
import numpy as np
from io import open
from tqdm import tqdm
import torch

# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
# from utils_squad_evaluate import find_all_best_thresh_v2, make_qid_to_has_ans, get_raw_scores

logger = logging.getLogger(__name__)

from typing import Dict, List

try:
    import marisa_trie
except ModuleNotFoundError:
    pass

class Trie(object):
    def __init__(self, sequences: List[List[int]] = []):
        self.trie_dict = {}
        self.len = 0
        if sequences:
            for sequence in sequences:
                Trie._add_to_trie(sequence, self.trie_dict)
                self.len += 1

        self.append_trie = None
        self.bos_token_id = None

    def append(self, trie, bos_token_id):
        self.append_trie = trie
        self.bos_token_id = bos_token_id

    def add(self, sequence: List[int]):
        Trie._add_to_trie(sequence, self.trie_dict)
        self.len += 1

    def get(self, prefix_sequence: List[int]):

        return Trie._get_from_trie(
            prefix_sequence, self.trie_dict, self.append_trie, self.bos_token_id
        )

    @staticmethod
    def load_from_dict(trie_dict):
        trie = Trie()
        trie.trie_dict = trie_dict
        trie.len = sum(1 for _ in trie)
        return trie

    @staticmethod
    def _add_to_trie(sequence: List[int], trie_dict: Dict):
        if sequence:
            if sequence[0] not in trie_dict:
                trie_dict[sequence[0]] = {}
            Trie._add_to_trie(sequence[1:], trie_dict[sequence[0]])

    @staticmethod
    def _get_from_trie(
        prefix_sequence: List[int],
        trie_dict: Dict,
        append_trie=None,
        bos_token_id: int = None,
    ):
        if len(prefix_sequence) == 0:
            output = list(trie_dict.keys())
            if append_trie and bos_token_id in output:
                output.remove(bos_token_id)
                output += list(append_trie.trie_dict.keys())
            return output
        elif prefix_sequence[0] in trie_dict:
            return Trie._get_from_trie(
                prefix_sequence[1:],
                trie_dict[prefix_sequence[0]],
                append_trie,
                bos_token_id,
            )
        else:
            if append_trie:
                return append_trie.get(prefix_sequence)
            else:
                return []

    def __iter__(self):
        def _traverse(prefix_sequence, trie_dict):
            if trie_dict:
                for next_token in trie_dict:
                    yield from _traverse(
                        prefix_sequence + [next_token], trie_dict[next_token]
                    )
            else:
                yield prefix_sequence

        return _traverse([], self.trie_dict)

    def __len__(self):
        return self.len

    def __getitem__(self, value):
        return self.get(value)

class MarisaTrie(object):
    def __init__(
        self,
        sequences: List[List[int]] = [],
        cache_fist_branch=True,
        max_token_id=256001,
    ):

        self.int2char = [chr(i) for i in range(min(max_token_id, 55000))] + (
            [chr(i) for i in range(65000, max_token_id + 10000)]
            if max_token_id >= 55000
            else []
        )
        self.char2int = {self.int2char[i]: i for i in range(max_token_id)}

        self.cache_fist_branch = cache_fist_branch
        if self.cache_fist_branch:
            self.zero_iter = list({sequence[0] for sequence in sequences})
            assert len(self.zero_iter) == 1
            self.first_iter = list({sequence[1] for sequence in sequences})

        self.trie = marisa_trie.Trie(
            "".join([self.int2char[i] for i in sequence]) for sequence in sequences
        )

    def get(self, prefix_sequence: List[int]):
        if self.cache_fist_branch and len(prefix_sequence) == 0:
            return self.zero_iter
        elif (
            self.cache_fist_branch
            and len(prefix_sequence) == 1
            and self.zero_iter == prefix_sequence
        ):
            return self.first_iter
        else:
            key = "".join([self.int2char[i] for i in prefix_sequence])
            return list(
                {
                    self.char2int[e[len(key)]]
                    for e in self.trie.keys(key)
                    if len(e) > len(key)
                }
            )

    def __iter__(self):
        for sequence in self.trie.iterkeys():
            yield [self.char2int[e] for e in sequence]

    def __len__(self):
        return len(self.trie)

    def __getitem__(self, value):
        return self.get(value)

class DummyTrieMention(object):
    def __init__(self, return_values):
        self._return_values = return_values

    def get(self, indices=None):
        return self._return_values

class DummyTrieEntity(object):
    def __init__(self, return_values, codes):
        self._return_values = list(
            set(return_values).difference(
                set(
                    codes[e]
                    for e in (
                        "start_mention_token",
                        "end_mention_token",
                        "start_entity_token",
                    )
                )
            )
        )
        self._codes = codes

    def get(self, indices, depth=0):
        if len(indices) == 0 and depth == 0:
            return self._codes["end_mention_token"]
        elif len(indices) == 0 and depth == 1:
            return self._codes["start_entity_token"]
        elif len(indices) == 0:
            return self._return_values
        elif len(indices) == 1 and indices[0] == self._codes["end_entity_token"]:
            return self._codes["EOS"]
        else:
            return self.get(indices[1:], depth=depth + 1)


import pytest

from byte109.dict_lookups import get_workout_motd, CHILL_OUT, INVALID_DAY

@pytest.mark.parametrize("day, expected", [
    ('Monday', 'Go train Chest+biceps'),
    ('monday', 'Go train Chest+biceps'),
    ('Tuesday', 'Go train Back+triceps'),
    ('TuEsdAy', 'Go train Back+triceps'),
    ('Wednesday', 'Go train Core'),
    ('wednesdaY', 'Go train Core'),
    ('Thursday', 'Go train Legs'),
    ('Friday', 'Go train Shoulders'),
    ('Saturday', CHILL_OUT),
    ('Sunday', CHILL_OUT),
    ('sundAy', CHILL_OUT),
    ('nonsense', INVALID_DAY),
    ('monday2', INVALID_DAY),
])
def test_get_workout_valid_case_insensitive_dict_lookups(day, expected):
    assert get_workout_motd(day) == expected


#!/usr/bin/python3
"""
Usage: ./0-select_states.py
    <MySQL username>
    <MySQL password>
    <MySQL database>
(all arguments are required, and are
assumed to be there, without any more arguments)

Logs in database in 'localhost' with
the REQUIRED MySQL: 'username', 'password', 'database
name' in the above shell arguments, in port 3306.

Definition:
-----------------------------------
states (
    id INT NOT NULL AUTO_INCREMENT,
    name VARCHAR(256) NOT NULL,
    PRIMARY KEY (id)
)
-----------------------------------

Then, it dispays all the rows from the table 'states'
(hopefully) in the supplied database in the last
shell argument, ordered by the 'id' value.
"""
import MySQLdb

if __name__ == "__main__":
    from sys import argv
    database = MySQLdb.connect("localhost", *argv[1:], 3306)

    with database.cursor() as cursor:
        cursor.execute("SELECT * FROM states ORDER BY id ASC")
        database.commit()
        result = cursor.fetchall()
        if result:
            # an empty result shouldn't print anything,
            # not even a new line.
            print(*result, sep="\n")


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 10 16:51:24 2018

@author: sofyan.fadli
"""

import cx_Oracle
import base64

db = cx_Oracle.connect('sofian/sofian@192.168.9.105:1521/orcl')
cur = db.cursor()
cur.arraysize=50

# my_dict = {"nik" : 760447, "no_sim" : 134417} , "no_sim" : 134417
data = {"nama" : "anna solana hamami kadarachman", "no_sim" : None, "nik" : 676799}
new_data = data
for key in list(data):
    if data[key] == None:
        new_data.pop(key, None)

query = "SELECT * FROM data_sim WHERE "
keyword = []
values = []
row = []

# Iterasi key dan value-nya
for key, value in new_data.items():
    keyword.append(key + " = :"+ key)

query = query + " AND ".join(keyword)
print(query)
result = cur.execute(query, new_data)
row = result.fetchone()
img = row[4].read()
print(type(img))
print(img)


import re

from nltk import sent_tokenize

class MacIntyreContractions:
    """
    List of contractions adapted from Robert MacIntyre's tokenizer.
    """

    CONTRACTIONS2 = [
        r"(?i)\b(can)(?#X)(not)\b",
        r"(?i)\b(d)(?#X)('ye)\b",
        r"(?i)\b(gim)(?#X)(me)\b",
        r"(?i)\b(gon)(?#X)(na)\b",
        r"(?i)\b(got)(?#X)(ta)\b",
        r"(?i)\b(lem)(?#X)(me)\b",
        r"(?i)\b(mor)(?#X)('n)\b",
        r"(?i)\b(wan)(?#X)(na)\s",
    ]
    CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
    CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b", r"(?i)\b(wha)(t)(cha)\b"]

class TreebankWordTokenizer:
    """
    The Treebank tokenizer uses regular expressions to tokenize text as in
    Penn Treebank. This is the method. It assumes that thetext has already been
    segmented into sentences.
    This tokenizer performs the following steps:
    - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
    - treat most punctuation characters as separate tokens
    - split off commas and single quotes, when followed by whitespace
    - separate periods that appear at the end of line
    """

    # starting quotes
    STARTING_QUOTES = [
        (re.compile(u'([«“‘„]|[`]+)', re.U), r' \1 '),
        (re.compile(r'^\"'), r'``'),
        (re.compile(r'(``)'), r' \1 '),
        (re.compile(r"([ \(\[{<])(\"|\'{2})"), r'\1 `` '),
    ]

    # punctuation
    PUNCTUATION = [
        (re.compile(r'([^\.])(\.)([\]\)}>"\'' u'»”’ ' r']*)\s*$', re.U), r'\1 \2 \3 '),
        (re.compile(r'([:,])([^\d])'), r' \1 \2'),
        (re.compile(r'([:,])$'), r' \1 '),
        (re.compile(r'\.\.\.'), r' ... '),
        (re.compile(r'[;@#$%&]'), r' \g<0> '),
        (
            re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
            r'\1 \2\3 ',
        ),  # Handles the final period.
        (re.compile(r'[?!]'), r' \g<0> '),
        (re.compile(r"([^'])' "), r"\1 ' "),
    ]

    # Pads parentheses
    PARENS_BRACKETS = (re.compile(r'[\]\[\(\)\{\}\<\>]'), r' \g<0> ')

    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
    CONVERT_PARENTHESES = [
        (re.compile(r'\('), '-LRB-'),
        (re.compile(r'\)'), '-RRB-'),
        (re.compile(r'\['), '-LSB-'),
        (re.compile(r'\]'), '-RSB-'),
        (re.compile(r'\{'), '-LCB-'),
        (re.compile(r'\}'), '-RCB-'),
    ]

    DOUBLE_DASHES = (re.compile(r'--'), r' -- ')

    # ending quotes
    ENDING_QUOTES = [
        (re.compile(u'([»”’]+)', re.U), r' \1 '),
        (re.compile(r'"'), " '' "),
        (re.compile(r'(\S)(\'\')'), r'\1 \2 '),
        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
    ]

    # List of contractions adapted from Robert MacIntyre's tokenizer.
    _contractions = MacIntyreContractions()
    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))

    def tokenize(self, text, convert_parentheses=False, return_str=False):
        for regexp, substitution in self.STARTING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp, substitution in self.PUNCTUATION:
            text = regexp.sub(substitution, text)

        # Handles parentheses.
        regexp, substitution = self.PARENS_BRACKETS
        text = regexp.sub(substitution, text)
        # Optionally convert parentheses
        if convert_parentheses:
            for regexp, substitution in self.CONVERT_PARENTHESES:
                text = regexp.sub(substitution, text)

        # Handles double dash.
        regexp, substitution = self.DOUBLE_DASHES
        text = regexp.sub(substitution, text)

        # add extra space to make things easier
        text = " " + text + " "

        for regexp, substitution in self.ENDING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp in self.CONTRACTIONS2:
            text = regexp.sub(r' \1 \2 ', text)
        for regexp in self.CONTRACTIONS3:
            text = regexp.sub(r' \1 \2 ', text)

        # We are not using CONTRACTIONS4 since
        # they are also commented out in the SED scripts
        # for regexp in self._contractions.CONTRACTIONS4:
        #     text = regexp.sub(r' \1 \2 \3 ', text)

        return text if return_str else text.split()

_treebank_word_tokenizer = TreebankWordTokenizer()

def word_tokenize(text, language='english', preserve_line=False):
    sentences = [text] if preserve_line else sent_tokenize(text, language)
    return [token for sent in sentences
            for token in _treebank_word_tokenizer.tokenize(sent)]

__all__ = ['word_tokenize']


from scrapy.http import HtmlResponse
from scrapy.selector import Selector
url ='https://www.cryptocompare.com/coins/{}/overview'

def crawl_data(token):
    full_path = url.format(token)
    response = HtmlResponse(url=full_path)
    Selector(response=response)

crawl_data("BTC")


from pathlib import Path
import shutil
import numpy as np
import scipy
import torch
import torch.sparse
from transformers import AutoTokenizer, DataCollatorWithPadding
from utils import DataCollatorForLanguageModeling
from datasets import load_dataset, load_from_disk
from torch.utils.data import DataLoader, WeightedRandomSampler
from scipy.sparse import csr_matrix, coo_matrix
from lightning import LightningDataModule
from typing import List
import pickle
import numpy as np

class DynamicBatchModule(LightningDataModule):
    def __init__(
        self,
        batch_size: List[int] | int,
        effective_batch_size: int | None,
    ):
        super().__init__()
        if isinstance(batch_size, int):
            batch_size = [batch_size]
        self.batch_sizes = batch_size
        self.effective_batch_size = effective_batch_size
        self.batch_size = batch_size[0]

    def adjust_grad_accum(self):
        if self.effective_batch_size is None:
            return

        devices = self.trainer.num_devices * self.trainer.num_nodes
        new_accumulated_batches = self.effective_batch_size // (
            self.batch_size * devices
        )
        if new_accumulated_batches == 0:
            new_accumulated_batches = 1

        if (
            new_accumulated_batches * (self.batch_size * devices)
            != self.effective_batch_size
        ):
            raise ValueError("Inconsistent effective batch size")

        self.trainer.accumulate_grad_batches = new_accumulated_batches
        print(f"Next accumulated batches: {new_accumulated_batches}")

    def adjust_batch_size(self):
        if self.trainer is None:
            return

        new_batch_size = (
            self.batch_sizes[self.trainer.current_epoch]
            if self.trainer.current_epoch < len(self.batch_sizes)
            else self.batch_sizes[-1]
        )
        if self.batch_size != new_batch_size:
            self.batch_size = new_batch_size

        # If we won't have new batch size, we don't need to reload the dataloader
        if self.trainer.current_epoch + 1 < len(self.batch_sizes):
            self.trainer.reload_dataloaders_every_n_epochs = 1
        else:
            print("Won't reload dataloaders")
            self.trainer.reload_dataloaders_every_n_epochs = 0

        print(f"Next batch size: {self.batch_size}")

    def _train_dataloader(self) -> DataLoader:
        raise NotImplementedError()

    def _val_dataloader(self) -> DataLoader:
        raise NotImplementedError()

    def _test_dataloader(self) -> DataLoader:
        raise NotImplementedError()

    def _predict_dataloader(self) -> DataLoader:
        raise NotImplementedError()

    def train_dataloader(self):
        self.adjust_batch_size()
        self.adjust_grad_accum()
        return self._train_dataloader()

    def val_dataloader(self):
        self.adjust_batch_size()
        self.adjust_grad_accum()
        return self._val_dataloader()

    def test_dataloader(self):
        self.adjust_batch_size()
        self.adjust_grad_accum()
        return self._test_dataloader()

    def predict_dataloader(self):
        self.adjust_batch_size()
        self.adjust_grad_accum()
        return self._predict_dataloader()

class NewsDataModule(DynamicBatchModule):
    def __init__(
        self,
        column,
        num_classes,
        tokenizer,
        cache_dir,
        effective_batch_size: int | None,
        max_length=512,
        batch_size: List | int = 12,
        pin_memory=False,
        limit=None,
        num_proc: int = 4,
        pad_mode=True,
        trunc_type="start",
        train_split="train",
        val_split="validation",
        test_split="test",
        predict_split="test",
        time_weighting=False,
        time_weighting_factor=1.0,
    ):
        super().__init__(batch_size, effective_batch_size)
        self.num_proc = num_proc
        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer, use_fast=True)
        self.tokenizer_name = tokenizer
        self.max_length = max_length
        self.column = column
        self.trunc_type = trunc_type
        self.pin_memory = pin_memory

        self.data_collator = DataCollatorWithPadding(
            tokenizer=self.tokenizer, padding=pad_mode, max_length=max_length
        )
        self.cache_dir = Path(cache_dir)
        self.limit = limit
        self.pad_mode = pad_mode
        self.num_classes = num_classes
        self.train_split = train_split
        self.val_split = val_split
        self.test_split = test_split
        self.predict_split = predict_split
        self.time_weighting = time_weighting
        self.time_weighting_factor = time_weighting_factor
        self.weights = None

    def path(self, split):
        return (
            self.cache_dir
            / self.column
            / self.tokenizer_name
            / str(self.max_length)
            / self.trunc_type
            / split
        )

    def __truncate_tokenized(self, tokenized):
        if self.trunc_type == "start":
            truncated = tokenized[: self.max_length]
        elif self.trunc_type == "end":
            truncated = tokenized[-self.max_length :]
        # CLS and SEP tokens
        truncated[0] = tokenized[0]
        truncated[-1] = tokenized[-1]
        return truncated

    def tokenize(self, examples):
        content = examples["content"]
        tokenized = self.tokenizer(content, truncation=False, padding=False)
        for key in ["input_ids", "attention_mask"]:
            tokenized[key] = [self.__truncate_tokenized(x) for x in tokenized[key]]
        return tokenized

    def prepare_dst(self, split):
        if self.path(split).exists():
            # No cache invalidation but don't wanna bother
            return

        dataset = load_dataset(str("hynky/czech_news_dataset"), split=split)
        dataset = dataset.rename_column(self.column, "labels")
        dataset = dataset.filter(
            lambda batch: [x != 0 for x in batch["labels"]],
            batched=True,
            keep_in_memory=True,
        )

        if self.limit:
            dataset = dataset.select(range(self.limit))
        dataset = dataset.remove_columns(
            set(dataset.column_names) - {"content", "labels"}
        )
        dataset = dataset.map(
            self.tokenize,
            batched=True,
        )
        # Remove "Nones"
        # Map to 0-indexed labels
        dataset = dataset.map(
            lambda batch: {"labels": [x - 1 for x in batch["labels"]]},
            batched=True,
            keep_in_memory=True,
        )
        dataset.set_format("pt", columns=["input_ids", "attention_mask", "labels"])

        print("Saving")
        dataset.save_to_disk(self.path(split), num_proc=self.num_proc)

    def get_weights_name(self, split):
        return self.path(split) / f"weights-{self.time_weighting_factor}.pkl"

    def prepare_weights(self, split):
        if (
            not (self.time_weighting and split == "train")
            or (self.get_weights_name(split).exists())
        ):
            return

        dataset = load_dataset(str("hynky/czech_news_dataset"), split=split)
        dataset = dataset.rename_column(self.column, "labels")
        dataset = dataset.filter(
            lambda batch: [x != 0 for x in batch["labels"]],
            batched=True,
            keep_in_memory=True,
        )
        dates = dataset["date"]
        min_date = min(dates)
        dates = np.array([(x - min_date).days for x in dates])
        normalized_dates = dates / np.max(dates) * self.time_weighting_factor
        self.weights = np.exp(normalized_dates) / np.sum(np.exp(normalized_dates))
        # save weights to file
        with open(self.get_weights_name(split), "wb") as f:
            pickle.dump(self.weights, f)

    def prepare_split(self, split):
        self.prepare_dst(split)
        self.prepare_weights(split)

    def load_split(self, split):
        dataset = load_from_disk(
            self.path(split),
        )
        if split == "train" and self.time_weighting:
            with open(self.get_weights_name(split), "rb") as f:
                self.weights = pickle.load(f)

        return dataset

    def prepare_data(self):
        self.prepare_split(self.train_split)
        self.prepare_split(self.val_split)
        self.prepare_split(self.test_split)

    def setup(self, stage=None):
        if stage == "fit" or stage is None:
            self.train_dataset = self.load_split(self.train_split)
            self.val_dataset = self.load_split(self.val_split)

        if stage == "validate" or stage is None:
            self.val_dataset = self.load_split(self.val_split)

        if stage == "test" or stage is None:
            self.test_dataset = self.load_split(self.test_split)

        if stage == "predict" or stage is None:
            self.predict_dataset = self.load_split(self.predict_split)

    def create_dataloader(self, dataset, shuffle=False, weights=None):
        sampler = None
        if weights is not None:
            sampler = WeightedRandomSampler(self.weights, len(self.weights))
            shuffle = False

        return DataLoader(
            dataset,
            batch_size=self.batch_size,
            num_workers=self.num_proc,
            collate_fn=self.data_collator,
            pin_memory=self.pin_memory,
            shuffle=shuffle,
            sampler=sampler,
        )

    def _train_dataloader(self):
        return self.create_dataloader(
            self.train_dataset, shuffle=True, weights=self.weights
        )

    def _val_dataloader(self):
        return self.create_dataloader(self.val_dataset, shuffle=True)

    def _test_dataloader(self):
        return self.create_dataloader(self.test_dataset)

    def _predict_dataloader(self):
        return self.create_dataloader(self.predict_dataset)

class SparseDataset:
    """
    Custom Dataset class for scipy sparse matrix
    """

    def __init__(
        self,
        data: np.ndarray | coo_matrix | csr_matrix,
        targets: np.ndarray | coo_matrix | csr_matrix,
    ):

        # Transform data coo_matrix to csr_matrix for indexing
        if type(data) == coo_matrix:
            self.data = data.tocsr()
        else:
            self.data = data

        # Transform targets coo_matrix to csr_matrix for indexing
        if type(targets) == coo_matrix:
            self.targets = targets.tocsr()
        else:
            self.targets = targets

    def __getitem__(self, index):
        return self.data[index], self.targets[index]

    def __len__(self):
        return self.data.shape[0]

def sparse_coo_to_tensor(coo: scipy.sparse.coo_matrix):
    """
    Transform scipy coo matrix to pytorch sparse tensor
    """
    values = coo.data
    indices = np.vstack((coo.row, coo.col))
    shape = coo.shape

    i = torch.LongTensor(indices)
    v = torch.FloatTensor(values)
    s = torch.Size(shape)

    return torch.sparse_coo_tensor(i, v, s)

def sparse_batch_collate(batch: list):
    """
    Collate function which to transform scipy coo matrix to pytorch sparse tensor
    """
    data_batch, targets_batch = zip(*batch)
    if type(data_batch[0]) == scipy.sparse.csr_matrix:
        data_batch = scipy.sparse.vstack(data_batch).tocoo()
        data_batch = sparse_coo_to_tensor(data_batch)
    else:
        data_batch = torch.FloatTensor(data_batch)

    if type(targets_batch[0]) == scipy.sparse.csr_matrix:
        targets_batch = scipy.sparse.vstack(targets_batch).tocoo()
        targets_batch = sparse_coo_to_tensor(targets_batch)
    else:
        targets_batch = torch.LongTensor(targets_batch)
    return {"labels": targets_batch, "input_ids": data_batch}

class NewsTfidfDataModule(LightningDataModule):
    def __init__(
        self,
        column,
        num_classes,
        num_features,
        data_path: str,
        batch_size=12,
        num_proc: int = 4,
    ):
        super().__init__()
        self.batch_size = batch_size
        self.num_proc = num_proc
        self.column = column
        self.data_path = Path(data_path)
        self.dst_path = "hynky/czech_news_dataset"
        self.num_classes = num_classes
        self.num_features = num_features

    def load_split(self, split):
        labels = np.array(load_dataset(self.dst_path, split=split)[self.column]) - 1
        sparse_input = scipy.sparse.load_npz(
            self.data_path / f"{split}_tfidf_with_metadata.npz"
        )
        if sparse_input.shape[0] != len(labels):
            limit = min(sparse_input.shape[0], len(labels))
            sparse_input = sparse_input[:, :limit]
            labels = labels[:limit]
        keep_idxs = np.where(labels != -1)[0]
        labels = labels[keep_idxs]
        sparse_input = sparse_input[keep_idxs, :]
        return SparseDataset(sparse_input, labels)

    def setup(self, stage=None):
        if stage == "fit" or stage is None:
            self.train_dataset = self.load_split("validation")
            self.val_dataset = self.load_split("validation")
        if stage == "test" or stage is None:
            self.test_dataset = self.load_split("validation")

    def create_dataloader(self, dataset, shuffle=False):
        return DataLoader(
            dataset,
            batch_size=self.batch_size,
            num_workers=self.num_proc,
            collate_fn=sparse_batch_collate,
            shuffle=shuffle,
        )

    def train_dataloader(self):
        return self.create_dataloader(self.train_dataset, shuffle=True)

    def val_dataloader(self):
        return self.create_dataloader(self.val_dataset)

    def test_dataloader(self):
        return self.create_dataloader(self.test_dataset)

class NewsDataModuleForLM(DynamicBatchModule):
    def __init__(
        self,
        tokenizer,
        cache_dir,
        batch_size: List[int] | int,
        effective_batch_size: int | None,
        max_length=512,
        mlm=0.15,
        pin_memory=True,
        limit=None,
        num_proc: int = 4,
    ):
        super().__init__(batch_size, effective_batch_size)
        self.num_proc = num_proc
        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer, use_fast=True)
        self.max_length = max_length
        self.cache_dir = Path(cache_dir)
        self.max_tokenizer_id = max(self.tokenizer.get_vocab().values())
        self.pin_memory = pin_memory
        self.mlm = mlm
        self.limit = limit

    def tokenize_function(self, examples):
        result = self.tokenizer(examples["content"])
        if self.tokenizer.is_fast:
            result["word_ids"] = [
                result.word_ids(i) for i in range(len(result["input_ids"]))
            ]
        return result

    def chunkify(self, examples):
        concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
        total_length = len(concatenated_examples[list(examples.keys())[0]])
        total_length = (total_length // self.max_length) * self.max_length
        result = {
            k: [
                t[i : i + self.max_length]
                for i in range(0, total_length, self.max_length)
            ]
            for k, t in concatenated_examples.items()
        }

        result["labels"] = result["input_ids"].copy()
        return result

    def prepare_split(self, split):
        name = "LM"
        split_cache_dir = self.cache_dir / name / str(self.max_length) / split
        if split_cache_dir.exists():
            # No cache invalidation but don't wanna bother
            return

        dataset = load_dataset(str("hynky/czech_news_dataset"), split=split)
        if self.limit:
            dataset = dataset.select(range(self.limit))
        dataset = dataset.remove_columns(set(dataset.column_names) - {"content"})

        dataset = dataset.map(
            self.tokenize_function,
            batched=True,
            keep_in_memory=True,
            remove_columns=["content"],
        )
        dataset = dataset.map(self.chunkify, batched=True, keep_in_memory=True)
        dataset.set_format("pt", columns=["input_ids", "attention_mask", "labels"])
        dataset.save_to_disk(str(split_cache_dir), num_proc=self.num_proc)

    def load_split(self, split):
        dataset = load_from_disk(self.cache_dir / "LM" / str(self.max_length) / split)
        return dataset

    def prepare_data(self):
        self.prepare_split("train")
        self.prepare_split("validation")
        self.prepare_split("test")

    def setup(self, stage=None):
        if stage == "fit" or stage is None:
            self.train_dataset = self.load_split("train")
            self.val_dataset = self.load_split("validation")
        if stage == "validate" or stage is None:
            self.val_dataset = self.load_split("validation")
        if stage == "test" or stage is None:
            self.test_dataset = self.load_split("test")

    def create_dataloader(self, dataset):
        collator = DataCollatorForLanguageModeling(
            self.tokenizer, max_input_id=self.max_tokenizer_id, mlm_probability=self.mlm
        )
        return DataLoader(
            dataset,
            batch_size=self.batch_size,
            pin_memory=self.pin_memory,
            num_workers=self.num_proc,
            collate_fn=collator,
            shuffle=False,
        )

    def _train_dataloader(self):
        return self.create_dataloader(self.train_dataset)

    def _val_dataloader(self):
        return self.create_dataloader(self.val_dataset)

    def _test_dataloader(self):
        return self.create_dataloader(self.test_dataset)


import os
import re
import cv2
import json
import pickle
import numpy as np

from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV

from atlasbuggy.camera.pipeline import Pipeline

from .localbinarypatterns import LocalBinaryPatterns

class TexturePipeline(Pipeline):
    def __init__(self, enabled=True, log_level=None):
        super(TexturePipeline, self).__init__(enabled, log_level)

        self.file_extension = ".jpg"
        self.database_file_path = "naboris/texture/training.json"
        self.model_pickle_path = "naboris/texture/models/texture.pkl"
        self.training_image_path = "naboris/texture/training_images"

        with open(self.database_file_path) as training_file:
            self.training_data = json.load(training_file)

        self.results_service_tag = "results"
        self.add_service(self.results_service_tag)

        self.texture_service_tag = "texture"
        self.add_service(self.texture_service_tag, self.post_texture_images)

        self.desc = LocalBinaryPatterns(24, 8)
        self.model = LinearSVC(C=500.0, random_state=20)
        self.classifier = CalibratedClassifierCV(self.model)
        self.load_model()

        self.cropped = None
        self.cropped_label = ""
        self.cropped_num = 0

        self.prediction_labels = sorted(list(self.training_data.keys()))

        self.offset = 100

    def load_model(self):
        with open(self.model_pickle_path, 'rb') as model:
            self.classifier = pickle.load(model)

    def train(self):
        labels = []
        data = []

        for label, num_images in self.training_data.items():
            print("loading '%s'. %s images" % (label, num_images))
            dir_path = os.path.join(self.training_image_path, label)
            file_names = os.listdir(dir_path)
            if len(file_names) != num_images:
                self.logger.warning(
                    "Number of images (%s) doesn't match number in json (%s)" % (len(file_names), num_images)
                )

            for file_name in file_names:
                if file_name.endswith(self.file_extension):
                    path = os.path.join(dir_path, file_name)

                    if not os.path.isfile(path):
                        raise FileNotFoundError(path)
                    labels.append(label)
                    image = cv2.imread(path)
                    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
                    gray = cv2.equalizeHist(gray)
                    hist, _ = self.desc.describe(gray)
                    data.append(hist)
            print("done")

        print("fitting...")
        self.classifier.fit(data, labels)
        print("done")
        with open("naboris/texture/models/texture.pkl", 'wb') as model:
            pickle.dump(self.classifier, model)

    def write_training_image(self, category, image):
        self.training_data[category] += 1
        count = self.training_data[category]

        image_dir = os.path.join(self.training_image_path, category)
        if not os.path.isdir(image_dir):
            os.mkdir(image_dir)

        path = os.path.join(image_dir, "%s-%s%s" % (category, count, self.file_extension))

        cv2.imwrite(path, image)
        self.logger.info("Wrote: %s" % path)

    def post_texture_images(self, data):
        frame, texture = data
        return frame.copy(), texture.copy()

    def get_crop_points(self, frame):
        height, width = frame.shape[0:2]
        y1 = height - self.offset * 2
        y2 = height

        x1 = width // 2 - self.offset
        x2 = width // 2 + self.offset

        return y1, y2, x1, x2

    def crop_frame(self, frame):
        y1, y2, x1, x2 = self.get_crop_points(frame)
        return frame[y1: y2, x1: x2]

    def pipeline(self, frame):
        self.cropped = self.crop_frame(frame)
        gray = cv2.cvtColor(self.cropped, cv2.COLOR_BGR2GRAY)
        gray = cv2.equalizeHist(gray)
        hist, lbp = self.desc.describe(gray)

        # lbp = np.uint8(lbp)
        # lbp = cv2.equalizeHist(lbp)
        prediction = self.classifier.predict_proba(hist.reshape(1, -1))
        prediction = prediction.squeeze()

        max_index = np.argmax(prediction)
        prediction_label = self.prediction_labels[max_index]

        y1, y2, x1, x2 = self.get_crop_points(frame)
        # frame.setflags(write=1)
        # frame[y1: y2, x1: x2] = cv2.cvtColor(lbp, cv2.COLOR_GRAY2BGR)

        cv2.rectangle(frame, (x1 - 1, y1 - 1), (x2 + 1, y2 + 1), (255, 0, 0))

        self.post((prediction_label, prediction[max_index]), self.results_service_tag)

        text_y = 30
        num_line_elements = 3
        for index in range(0, len(prediction), num_line_elements):
            line_predictions = prediction[index: index + num_line_elements]
            line_map = map(lambda value: "%0.3f" % (100 * value), line_predictions)
            predictions_string = ", ".join(line_map)

            if index < num_line_elements:
                line = "%s %s" % (prediction_label, predictions_string)
            else:
                line = predictions_string

            cv2.putText(frame, line, (10, text_y), cv2.FONT_HERSHEY_SIMPLEX,
                        0.75, (0, 0, 255), 2)
            text_y += 25

        self.post((frame, self.cropped), self.texture_service_tag)

        return frame

    def stop(self):
        with open(self.database_file_path, 'w+') as training_file:
            json.dump(self.training_data, training_file)