from __future__ import annotations

import datetime
from dataclasses import dataclass, field
import random
from typing import List, Dict, Any

from chess_manager.M import turn_model

MAX_STR_LEN = 122

def check_valid_int(user_int_input: Any) -> bool:
    """ Vérifie si l'input de l'utilisateur est un int valide. """
    try:
        int(user_int_input)
        return True
    except ValueError:
        return False

def check_valid_str(user_str_input: Any) -> bool | str:
    """ Vérifie si l'input de l'utilisateur est un str valide. """
    if not 0 < len(user_str_input) < MAX_STR_LEN:
        return f"You must enter from 1 to {MAX_STR_LEN} char."
    return True

TOURNAMENT_FORM_VALIDATOR: Dict = {
    'name': check_valid_str,
    'place': check_valid_str,
    'description': check_valid_str,
    'turn_nbr': check_valid_int,
    'player_nbr': check_valid_int
}

def _shuffle_player_list(player_list: List) -> List:
    """Retourne une nouvelle liste mélangée"""
    randomised_list = player_list[:]
    random.shuffle(randomised_list)
    return randomised_list

def _order_player_by_score(full_player_data: List) -> List:
    full_player_data.sort(key=lambda individual_player_data: individual_player_data[1], reverse=True)
    return full_player_data

def _reset_player_pairing(ordered_player_data: List) -> List:
    """Visite chaque joueur de la liste reçue et en reinitialise le compteur d'adversaires"""
    for player_data in ordered_player_data:
        player_data[0].clear_player_pairing()
    return ordered_player_data

def _make_player_pair(ordered_player_data: List) -> List:
    """
    Reçoit une liste de (joueurs, score) et retourne une liste de paire (joueurs, score) classé par score
    décroissant. Si tous les joueurs ont déjà joué les uns contre les autres, les historiques de pairage sont
    réinitialisés
    """
    player_pairs = list()
    working_list = ordered_player_data[:]

    adversary_index = 0
    current_pairing_player = working_list.pop(0)

    while len(working_list) > 0:
        if current_pairing_player[0].has_played_against(working_list[adversary_index][0]):
            adversary_index += 1
            if adversary_index >= len(working_list):
                print("Cannot make more player pair without player playing each other again")
                return _make_player_pair(_reset_player_pairing(ordered_player_data[:]))
            continue
        player_pairs.append((current_pairing_player, working_list.pop(adversary_index)))
        adversary_index = 0

        if len(working_list) < 1:
            break
        current_pairing_player = working_list.pop(0)
    return player_pairs

@dataclass
class TournamentM:
    """Représentation d'un tournoi d'échec"""
    name: str
    place: str
    turn_nbr: int = 4
    description: str = "No description"
    player_nbr: int = 0
    players: List = field(default_factory=list)
    turn_list: List = field(default_factory=list)
    start_date: str | None = None
    end_date: str | None = None
    tournament_id: int = -1

    def __post_init__(self) -> None:
        if self.start_date is None:
            self.start_date = datetime.datetime.now().strftime("%d/%m/%y %H:%M")

    def register_turn(self, turn: turn_model.TurnM) -> None:
        self.turn_list.append(turn)

    def get_current_turn_nbr(self) -> int:
        return len(self.turn_list)

    @property
    def is_finished(self) -> bool:
        if self.end_date is not None:
            return True
        if self.get_current_turn_nbr() < self.turn_nbr:
            return False
        return self.turn_list[-1].finished

    def end_tournament(self) -> None:
        if self.end_date is not None:
            return
        self.end_date = datetime.datetime.now().strftime("%d/%m/%y %H:%M")

    def get_next_turn_player_pair(self) -> List:
        if self.get_current_turn_nbr() > 0:
            return _make_player_pair(_order_player_by_score(self.turn_list[-1].get_turn_data()))
        player_list = _shuffle_player_list(self.players)
        player_data = [[player, 0] for player in player_list]
        return _make_player_pair(_order_player_by_score(player_data))

    def from_obj_to_dict(self) -> Dict:
        return {'name': self.name,
                'place': self.place,
                'turn_nbr': int(self.turn_nbr),
                'description': self.description,
                'player_nbr': int(self.player_nbr),
                'players': [player.player_id for player in self.players],
                'turn_list': [turn.turn_id for turn in self.turn_list],
                'start_date': self.start_date,
                'end_date': self.end_date,
                'tournament_id': self.tournament_id,
                }


from django.contrib.auth.models import Group
from mf.crud.models import Dolar, HistoryOperations, Permisology
from mf.user.models import User
from django.utils import timezone
from datetime import date, datetime, timedelta

def convertToDecimalFormat(n):
    return n.replace('.', '').replace(',', '.')

def get_dollar():
    data = []
    try:
        dolar1 = Dolar.objects.using('default').get(pk=1)
        dl1 = float(dolar1.dolar)
    except:
        new_dolar1 = Dolar()
        new_dolar1.dolar = '1000000'
        new_dolar1.save()

        dolar1 = Dolar.objects.using('default').get(pk=1)
        dl1 = float(dolar1.dolar)

    try:
        dolar2 = Dolar.objects.using('default').get(pk=2)
        dl2 = float(dolar2.dolar)
    except:
        new_dolar2 = Dolar()
        new_dolar2.dolar = '1200000'
        new_dolar2.save()

        dolar1 = Dolar.objects.using('default').get(pk=1)
        dl1 = float(dolar1.dolar)

    data = {
        'dolar1': dl1,
        'dolar2': dl2
    }

    return data

def ValidatePermissions(perms, requestGroup):
    autorized = False
    try:
        permsRequired = perms
        pk = requestGroup.id

        group = Group.objects.get(pk=pk)
        permsRequired = perms

        for p in permsRequired:
            if not group.permissions.filter(codename=p).exists():
                autorized = False
                break
            else:
                autorized = True
    except:
        autorized = False
    return autorized

def RegisterOperation(db, user, action):
    date = timezone.localtime(timezone.now())
    result = 0
    try:
        h = HistoryOperations()
        h.datejoined = date.strftime('%Y-%m-%d | %H:%M:%S %p')
        h.userSession_id = user
        h.description = action
        h.save()
    except:
        result = 1
    return result

def get_q_events_today():
    data = 0
    try:
        start = date.today()
        end = start + timedelta(days=7)
        start_date = start.strftime('%Y-%m-%d')
        end_date = end.strftime('%Y-%m-%d')
        total = 0

        search = Permisology.objects.all()
        if len(start_date) and len(end_date):
            search = search.filter(day__range=[start_date, end_date])
            for s in search:
                total = int(total) + 1
            data = total
    except:
        pass
    return data

def get_events_today():
    data = []
    total = 0
    start = date.today()
    end = start + timedelta(days=7)
    start_date = start.strftime('%Y-%m-%d')
    end_date = end.strftime('%Y-%m-%d')

    search = Permisology.objects.all()
    if len(start_date) and len(end_date):
        search = search.filter(day__range=[start_date, end_date])
    for s in search:
        data.append(
            {
                'name': s.name,
                'description': s.description,
                'day': s.day.strftime('%Y-%m-%d'),
            }
        )
    return data


# -*- coding: utf-8 -*-
"""
Created on Fri Dec  4 11:20:09 2020

@author: KANNAN
"""

from flask import Flask, render_template, request
import emoji
#import sklearn
import pickle

model = pickle.load(open("diabetes_logreg.pkl", "rb"))

app = Flask(__name__)

@app.route('/')

def home():
    return render_template('Diabetes.html')

@app.route('/predict', methods = ["GET", "POST"])

def Diabetes():
    if request.method == "POST":

        Glucose = float(request.form["Glucose"])
        if (Glucose > 0 and Glucose < 140):
            Glucose_Prediabetes = 0
            Glucose_Diabetes = 0
        elif(Glucose >= 140 and Glucose < 200):
            Glucose_Prediabetes = 1
            Glucose_Diabetes = 0
        else:
            Glucose_Prediabetes = 0
            Glucose_Diabetes = 1

        BloodPressure = float(request.form["BloodPressure"])
        if (BloodPressure > 0 and BloodPressure < 80):
            BloodPressure_Hyper_St1 = 0
            BloodPressure_Hyper_St2 = 0
            BloodPressure_Hyper_Emer = 0
        elif (BloodPressure >=80 and BloodPressure < 90):
            BloodPressure_Hyper_St1 = 1
            BloodPressure_Hyper_St2 = 0
            BloodPressure_Hyper_Emer = 0
        elif (BloodPressure >= 90 and BloodPressure < 120):
            BloodPressure_Hyper_St1 = 0
            BloodPressure_Hyper_St2 = 1
            BloodPressure_Hyper_Emer = 0
        else:
            BloodPressure_Hyper_St1 = 0
            BloodPressure_Hyper_St2 = 0
            BloodPressure_Hyper_Emer = 1

        BMI = float(request.form["BMI"])
        if (BMI > 0 and BMI < 18.5):
            BMI_Normal = 0
            BMI_Overweight = 0
            BMI_Obese = 0
        elif (BMI >= 18.5 and BMI < 24.9):
            BMI_Normal = 1
            BMI_Overweight = 0
            BMI_Obese = 0
        elif (BMI >= 24.9 and BMI < 29.9):
            BMI_Normal = 0
            BMI_Overweight = 1
            BMI_Obese = 0
        else:
            BMI_Normal = 0
            BMI_Overweight = 0
            BMI_Obese = 1

        Insulin = float(request.form["Insulin"])
        if (Insulin >= 100 and Insulin <= 126):
            Insulin_Normal = 1
        else:
            Insulin_Normal = 0

        Pregnancies = float(request.form["Pregnancies"])
        Pregnancies = (Pregnancies - 3.874593) / 3.443637

        SkinThickness = float(request.form["SkinThickness"])
        SkinThickness = (SkinThickness - 29.180782) / 8.94289800

        DiabetesPedigreeFunction = float(request.form["DiabetesPedigreeFunction"])
        DiabetesPedigreeFunction = (DiabetesPedigreeFunction - 0.466471) / 0.333203

        Age = float(request.form["Age"])
        Age = (Age - 33.594463) / 12.016168

        prediction = model.predict([[
            Pregnancies,
            SkinThickness,
            DiabetesPedigreeFunction,
            Age,
            BMI_Normal,
            BMI_Overweight,
            BMI_Obese,
            BloodPressure_Hyper_St1,
            BloodPressure_Hyper_St2,
            BloodPressure_Hyper_Emer,
            Glucose_Prediabetes,
            Glucose_Diabetes,
            Insulin_Normal
            ]])

        output = prediction[0]
        if output == 0:
            text = "You are Healthy!!"+"\U0001F603"
        else:
            text = "You have Diabetes"+"\U0001F61E"

        return render_template('Diabetes.html',prediction_text = text)

    return render_template('Diabetes.html')

if __name__ == '__main__':
    app.run()


"""PDB dataset loader."""
import tree
import numpy as np
import torch
import pandas as pd
import logging
import random
import functools as fn

from torch.utils import data
from data import utils as du
from openfold.data import data_transforms
from openfold.np import residue_constants
from openfold.utils import rigid_utils

class PdbDataset(data.Dataset):
    def __init__(
            self,
            *,
            data_conf,
            diffuser,
            is_training,
        ):
        self._log = logging.getLogger(__name__)
        self._is_training = is_training
        self._data_conf = data_conf
        self._init_metadata()
        self._diffuser = diffuser

    @property
    def is_training(self):
        return self._is_training

    @property
    def diffuser(self):
        return self._diffuser

    @property
    def data_conf(self):
        return self._data_conf

    def _init_metadata(self):
        """Initialize metadata."""

        # Process CSV with different filtering criterions.
        filter_conf = self.data_conf.filtering
        pdb_csv = pd.read_csv(self.data_conf.csv_path)
        self.raw_csv = pdb_csv
        if filter_conf.allowed_oligomer is not None and len(filter_conf.allowed_oligomer) > 0:
            pdb_csv = pdb_csv[pdb_csv.oligomeric_detail.isin(
                filter_conf.allowed_oligomer)]
        if filter_conf.max_len is not None:
            pdb_csv = pdb_csv[pdb_csv.modeled_seq_len <= filter_conf.max_len]
        if filter_conf.min_len is not None:
            pdb_csv = pdb_csv[pdb_csv.modeled_seq_len >= filter_conf.min_len]
        if filter_conf.max_helix_percent is not None:
            pdb_csv = pdb_csv[
                pdb_csv.helix_percent < filter_conf.max_helix_percent]
        if filter_conf.max_loop_percent is not None:
            pdb_csv = pdb_csv[
                pdb_csv.coil_percent < filter_conf.max_loop_percent]
        if filter_conf.min_beta_percent is not None:
            pdb_csv = pdb_csv[
                pdb_csv.strand_percent > filter_conf.min_beta_percent]
        if filter_conf.subset is not None:
            pdb_csv = pdb_csv[:filter_conf.subset]
        pdb_csv = pdb_csv.sort_values('modeled_seq_len', ascending=False)
        self._create_split(pdb_csv)

    def _create_split(self, pdb_csv):
        # Training or validation specific logic.
        if self.is_training:
            self.csv = pdb_csv
            self._log.info(
                f'Training: {len(self.csv)} examples')
        else:
            all_lengths = np.sort(pdb_csv.modeled_seq_len.unique())
            length_indices = (len(all_lengths) - 1) * np.linspace(
                0.0, 1.0, self._data_conf.num_eval_lengths)
            length_indices = length_indices.astype(int)
            eval_lengths = all_lengths[length_indices]
            eval_csv = pdb_csv[pdb_csv.modeled_seq_len.isin(eval_lengths)]
            # Fix a random seed to get the same split each time.
            eval_csv = eval_csv.groupby('modeled_seq_len').sample(
                self._data_conf.samples_per_eval_length, replace=True, random_state=123)
            eval_csv = eval_csv.sort_values('modeled_seq_len', ascending=False)
            self.csv = eval_csv
            self._log.info(
                f'Validation: {len(self.csv)} examples with lengths {eval_lengths}')

    @fn.lru_cache(maxsize=50000)
    def _process_csv_row(self, processed_file_path):
        processed_feats = du.read_pkl(processed_file_path)
        processed_feats = du.parse_chain_feats(processed_feats)

        # Only take modeled residues.
        modeled_idx = processed_feats['modeled_idx']
        min_idx = np.min(modeled_idx)
        max_idx = np.max(modeled_idx)
        del processed_feats['modeled_idx']
        processed_feats = tree.map_structure(
            lambda x: x[min_idx:(max_idx+1)], processed_feats)

        # Run through OpenFold data transforms.
        chain_feats = {
            'aatype': torch.tensor(processed_feats['aatype']).long(),
            'all_atom_positions': torch.tensor(processed_feats['atom_positions']).double(),
            'all_atom_mask': torch.tensor(processed_feats['atom_mask']).double()
        }
        chain_feats = data_transforms.atom37_to_frames(chain_feats)
        chain_feats = data_transforms.make_atom14_masks(chain_feats)
        chain_feats = data_transforms.make_atom14_positions(chain_feats)
        chain_feats = data_transforms.atom37_to_torsion_angles()(chain_feats)

        # Re-number residue indices for each chain such that it starts from 1.
        # Randomize chain indices.
        chain_idx = processed_feats["chain_index"]
        res_idx = processed_feats['residue_index']
        new_res_idx = np.zeros_like(res_idx)
        new_chain_idx = np.zeros_like(res_idx)
        all_chain_idx = np.unique(chain_idx).tolist()
        shuffled_chain_idx = np.array(
            random.sample(all_chain_idx, len(all_chain_idx))) - np.min(all_chain_idx) + 1
        for i,chain_id in enumerate(all_chain_idx):
            chain_mask = (chain_idx == chain_id).astype(np.int)
            chain_min_idx = np.min(res_idx + (1 - chain_mask) * 1e3).astype(np.int)
            new_res_idx = new_res_idx + (res_idx - chain_min_idx + 1) * chain_mask

            # Shuffle chain_index
            replacement_chain_id = shuffled_chain_idx[i]
            new_chain_idx = new_chain_idx + replacement_chain_id * chain_mask

        # To speed up processing, only take necessary features
        final_feats = {
            'aatype': chain_feats['aatype'],
            'seq_idx': new_res_idx,
            'chain_idx': chain_idx,
            'residx_atom14_to_atom37': chain_feats['residx_atom14_to_atom37'],
            'residue_index': processed_feats['residue_index'],
            'res_mask': processed_feats['bb_mask'],
            'atom37_pos': chain_feats['all_atom_positions'],
            'atom37_mask': chain_feats['all_atom_mask'],
            'atom14_pos': chain_feats['atom14_gt_positions'],
            'rigidgroups_0': chain_feats['rigidgroups_gt_frames'],
            'torsion_angles_sin_cos': chain_feats['torsion_angles_sin_cos'],
        }
        return final_feats

    def _create_diffused_masks(self, atom37_pos, rng, row):
        bb_pos = atom37_pos[:, residue_constants.atom_order['CA']]
        dist2d = np.linalg.norm(bb_pos[:, None, :] - bb_pos[None, :, :], axis=-1)

        # Randomly select residue then sample a distance cutoff
        # TODO: Use a more robust diffuse mask sampling method.
        diff_mask = np.zeros_like(bb_pos)
        attempts = 0
        while np.sum(diff_mask) < 1:
            crop_seed = rng.integers(dist2d.shape[0])
            seed_dists = dist2d[crop_seed]
            max_scaffold_size = min(
                self._data_conf.scaffold_size_max,
                seed_dists.shape[0] - self._data_conf.motif_size_min
            )
            scaffold_size = rng.integers(
                low=self._data_conf.scaffold_size_min,
                high=max_scaffold_size
            )
            dist_cutoff = np.sort(seed_dists)[scaffold_size]
            diff_mask = (seed_dists < dist_cutoff).astype(float)
            attempts += 1
            if attempts > 100:
                raise ValueError(
                    f'Unable to generate diffusion mask for {row}')
        return diff_mask

    def __len__(self):
        return len(self.csv)

    def __getitem__(self, idx):

        # Sample data example.
        example_idx = idx
        csv_row = self.csv.iloc[example_idx]
        if 'pdb_name' in csv_row:
            pdb_name = csv_row['pdb_name']
        elif 'chain_name' in csv_row:
            pdb_name = csv_row['chain_name']
        else:
            raise ValueError('Need chain identifier.')
        processed_file_path = csv_row['processed_path']
        chain_feats = self._process_csv_row(processed_file_path)

        # Use a fixed seed for evaluation.
        if self.is_training:
            rng = np.random.default_rng(None)
        else:
            rng = np.random.default_rng(idx)

        gt_bb_rigid = rigid_utils.Rigid.from_tensor_4x4(
            chain_feats['rigidgroups_0'])[:, 0]
        diffused_mask = np.ones_like(chain_feats['res_mask'])
        if np.sum(diffused_mask) < 1:
            raise ValueError('Must be diffused')
        fixed_mask = 1 - diffused_mask
        chain_feats['fixed_mask'] = fixed_mask
        chain_feats['rigids_0'] = gt_bb_rigid.to_tensor_7()
        chain_feats['sc_ca_t'] = torch.zeros_like(gt_bb_rigid.get_trans())

        # Sample t and diffuse.
        if self.is_training:
            t = rng.uniform(self._data_conf.min_t, 1.0)
            diff_feats_t = self._diffuser.forward_marginal(
                rigids_0=gt_bb_rigid,
                t=t,
                diffuse_mask=None
            )
        else:
            t = 1.0
            diff_feats_t = self.diffuser.sample_ref(
                n_samples=gt_bb_rigid.shape[0],
                impute=gt_bb_rigid,
                diffuse_mask=None,
                as_tensor_7=True,
            )
        chain_feats.update(diff_feats_t)
        chain_feats['t'] = t

        # Convert all features to tensors.
        final_feats = tree.map_structure(
            lambda x: x if torch.is_tensor(x) else torch.tensor(x), chain_feats)
        final_feats = du.pad_feats(final_feats, csv_row['modeled_seq_len'])
        if self.is_training:
            return final_feats
        else:
            return final_feats, pdb_name

class LengthSampler(data.Sampler):

    def __init__(
            self,
            *,
            data_conf,
            dataset,
        ):
        self._data_conf = data_conf
        self._dataset = dataset
        self._data_csv = self._dataset.csv

    def __iter__(self):
        return iter(range(len(self._data_csv)))

    def __len__(self):
        return len(self._data_csv)

class TrainSampler(data.Sampler):

    def __init__(
            self,
            *,
            data_conf,
            dataset,
            batch_size,
        ):
        self._data_conf = data_conf
        self._dataset = dataset
        self._data_csv = self._dataset.csv
        self._dataset_indices = list(range(len(self._data_csv)))
        self._data_csv['index'] = self._dataset_indices
        self._batch_size = batch_size
        self.epoch = 0

    def __iter__(self):
        random.shuffle(self._dataset_indices)
        repeated_indices = np.repeat(self._dataset_indices, self._batch_size)
        return iter(repeated_indices)

    def set_epoch(self, epoch):
        self.epoch = epoch

    def __len__(self):
        return len(self._dataset_indices) * self._batch_size


import datetime
from django.utils import timezone

from django.core.paginator import Paginator
from django.db import transaction
from django.db.models import Q

from the_mechanic_backend.apps.accounts.models import Store
from the_mechanic_backend.apps.stock.models import Brand, BrandModel, Spare, SpareCustomer, SpareOrder, SpareSold
from the_mechanic_backend.v0.stock import serializers
from the_mechanic_backend.v0.utils import Utils, CustomBaseClass, AppUtils

class BrandList(CustomBaseClass):
    """
        Brand List and create Endpoint
    """

    def get(self, request):
        """
        returns the list of brand
        :param request:
        :return:
        """
        try:
            search = request.GET.get('search', '')
            if search:
                brands = self.get_filter_objects(Brand, name__icontains=search)
            else:
                brands = self.get_all_objects(Brand)
            serializer = serializers.BrandSerializer(brands, many=True)
            return Utils.dispatch_success(request, serializer.data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def post(self, request):
        """
        Creates a new brand
        :param request:
        {
           "name" : "Honda"
        }
        :return:
        """
        try:
            serializer = serializers.BrandSerializer(data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Utils.dispatch_success(request, serializer.data)
            return Utils.dispatch_failure(request, 'VALIDATION_ERROR', serializer.errors)
        except Exception as e:
            return self.internal_server_error(request, e)

class BrandModelList(CustomBaseClass):
    """
    Brand Model List and create Endpoint
    """

    def get(self, request, brand_id):
        """
        Returnt the list of Models of particular brand
        :param request:
        :param brand_id:
        :return:
        """
        try:
            search = request.GET.get('search', '')
            if search:
                brands = self.get_filter_objects(BrandModel, brand=brand_id, model_name__icontains=search)
            else:
                brands = self.get_filter_objects(BrandModel, brand=brand_id)
            serializer = serializers.BrandModelSerializer(brands, many=True)
            return Utils.dispatch_success(request, serializer.data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def post(self, request, brand_id):
        """
        Creates a new brand model
        :param request:
          {
             "model_name" : "Unicorn"
          }
        :param brand_id:
        :return:
        """
        try:
            data = request.data
            data['brand'] = brand_id
            serializer = serializers.AddBrandModelSerializer(data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Utils.dispatch_success(request, serializer.data)
            return Utils.dispatch_failure(request, 'VALIDATION_ERROR', serializer.errors)
        except Exception as e:
            return self.internal_server_error(request, e)

class SpareList(CustomBaseClass):
    def get(self, request, store_id, brand_model_id):
        """
        return a list of spares for particular model
        :param request:
        @query_param
        search=search_text - to search the spares
        out_of_stock=true - to get only out of stock
        Note - we can use both at same time :)
        :param store_id
        :param brand_model_id:
        :return:
        """
        try:
            search = request.GET.get('search')
            out_of_stock = request.GET.get('out_of_stock')
            spare = self.get_filter_objects(Spare, brand_model=brand_model_id, store=store_id)

            if search:
                spare = spare.filter(Q(spare_id__icontains=search) | Q(spare_name__icontains=search))

            if out_of_stock:
                spare = spare.filter(quantity=0)

            serializer = serializers.SpareSerializer(spare, many=True)
            return Utils.dispatch_success(request, serializer.data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def post(self, request, store_id, brand_model_id):
        """
        Create a spare
        :param request:
        {
            "spare_name": "SIde Mirror",
            "spare_id": #34545435,
            "quantity": 10,
            "per_price": "500",
            "suppliers": "Glass India",
            "quality_class": "FIRST"
        }
        :param store_id:
        :param brand_model_id:
        :return:
        """
        try:
            data = request.data
            brand_model = self.get_object(BrandModel, brand_model_id)
            if not brand_model:
                return self.object_not_found(request)
            data['brand'] = brand_model.brand.id
            data['store'] = store_id
            data['brand_model'] = brand_model_id
            serializer = serializers.AddSpareSerializer(data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Utils.dispatch_success(request, serializer.data)
            return Utils.dispatch_failure(request, 'VALIDATION_ERROR', serializer.errors)
        except Exception as e:
            return self.internal_server_error(request, e)

class SpareDetails(CustomBaseClass):
    """
    particular spare details
    """

    def get(self, request, spare_id):
        """
        Return requested spare
        :param request:
        :param spare_id:
        :return:
        """
        try:
            spare = self.get_object(Spare, spare_id)
            if not spare:
                return self.object_not_found(request)
            serializer = serializers.SpareSerializer(spare)
            return Utils.dispatch_success(request, serializer.data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def put(self, request, spare_id):
        """
        Updates the requested spare
        :param request:
        # partial fields are also acceptable
        {
            "spare_name": "SIde Mirror",
            "spare_id": #34545435,
            "quantity": 10,
            "per_price": "500",
            "suppliers": "Glass India",
            "quality_class": "FIRST"
        }
        :param spare_id:
        :return:
        """
        try:
            spare = self.get_object(Spare, spare_id)
            if not spare:
                return self.object_not_found(request)
            serializer = serializers.AddSpareSerializer(spare, request.data, partial=True)
            if serializer.is_valid():
                serializer.save()
                return Utils.dispatch_success(request, serializer.data)
            return Utils.dispatch_failure(request, 'VALIDATION_ERROR', serializer.errors)
        except Exception as e:
            return self.internal_server_error(request, e)

    def delete(self, request, spare_id):
        """
        Delete the request spare
        :param request:
        :param spare_id:
        :return:
        """
        try:
            spare = self.get_object(Spare, spare_id)
            if not spare:
                return self.object_not_found(request)
            spare.delete()
            return Utils.dispatch_success(request, 'SUCCESS')
        except Exception as e:
            return self.internal_server_error(request, e)

class SpareSearchList(CustomBaseClass):
    def get(self, request, store_id):
        """
        return a list of spares for all models
        :param request:
        @query_param
        search=search_text - to search the spares
        out_of_stock=true - to get only out of stock
        Note - we can use both at same time :)
        :param store_id
        :param brand_model_id:
        :return:
        """
        try:
            search = request.GET.get('search')
            out_of_stock = request.GET.get('out_of_stock')
            spare = self.get_filter_objects(Spare, store=store_id)

            if search:
                spare = spare.filter(Q(spare_id__icontains=search) | Q(spare_name__icontains=search))

            if out_of_stock:
                spare = spare.filter(quantity=0)

            serializer = serializers.SpareSerializer(spare, many=True)
            return Utils.dispatch_success(request, serializer.data)
        except Exception as e:
            return self.internal_server_error(request, e)

class SpareOrderList(CustomBaseClass):
    def get(self, request, store_id, *args, **kwargs):
        """
        Returns the list of Spares based on Store
        :param request:
        # params
        start_date=2019-01-31&&
        end_date=2019-12-31&&
        page=1
        :param store_id:
        :param args:
        :param kwargs:
        :return:
        """
        try:
            start_date = request.GET.get('start_date')
            end_date = request.GET.get('end_date')
            page = request.GET.get('page', 1)
            search = request.GET.get('search', None)
            if search:
                qs = SpareOrder.objects.filter(store=store_id, order_id__icontains=search)
            else:
                if start_date:
                    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
                    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') + datetime.timedelta(days=1)
                qs = SpareOrder.objects.filter(store=store_id, order_date__range=[start_date, end_date])

            paginator = Paginator(qs, per_page=10)
            serializer = serializers.SpareOrderHistorySerializer(paginator.page(page), many=True)
            response_data = {
                "data": serializer.data,
                "page": int(page),
                "total_pages": paginator.num_pages
            }
            return Utils.dispatch_success(request, response_data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def post(self, request, store_id, *args, **kwargs):
        """
        Create a new Order
        :param request:
        {
                "customer_info": {
                    "name": "Muthu Kumar",
                    "email": "itmemk@gmail.com",
                    "phone_number": "9876543210",
                    "address": "ADDRESSS"
                },
                "order_type": "IN_SOURCE / OUT_SOURCE",
                "bike_number": "TN41Y5644",
                "labour_charge": 0.00,
                "out_source_charge": 0.00,
                "spares": [
                    {
                        "spare_id": 1,
                        "spare_price_type": 'MRP / MECHANIC / WHOLESALER / CUSTOMER',
                        "spare_count": 2
                    },
                    {
                        "spare_id": 1,
                        "spare_price_type": 'MRP / MECHANIC / WHOLESALER / CUSTOMER',
                        "spare_count": 2
                    },
                    {
                        "spare_id": 1,
                        "spare_price_type": 'MRP / MECHANIC / WHOLESALER / CUSTOMER',
                        "spare_count": 2
                    }
                ]

            }
        :param store_id:
        :param args:
        :param kwargs:
        :return:
        """
        try:
            data = request.data
            try:
                customer = SpareCustomer.objects.get(phone_number=data['customer_info']['phone_number'])
            except SpareCustomer.DoesNotExist:
                customer_serializer = serializers.SpareCustomerSerializer(data=data['customer_info'])
                if customer_serializer.is_valid():
                    customer_serializer.save()
                else:
                    return Utils.dispatch_failure(request, "VALIDATION_ERROR", customer_serializer.errors)
                customer = SpareCustomer.objects.get(id=customer_serializer.data['id'])

            today = datetime.date.today()
            today_order_count = SpareOrder.objects.filter(order_date__year=today.year,
                                                          order_date__month=today.month).count()

            order_id = 'SPOR{}{:05d}'.format(today.strftime("%Y%m"), today_order_count + 1)
            with transaction.atomic():
                store = self.get_object(Store, store_id)
                share_message = f"You're successfully purchased following items from {store.name}, {store.branch}.\n"
                order = SpareOrder(order_id=order_id,
                                   store=store,
                                   order_type=data['order_type'],
                                   customer=customer,
                                   total=0.0,
                                   sold_by=request.user)
                order.save()
                total = 0.0
                spares_to_be_created = []
                for _spare in data['spares']:
                    spare = self.get_object(Spare, _spare['spare_id'])
                    price_map = {
                        'MRP': spare.mrp_price,
                        'MECHANIC': spare.mechanic_price,
                        'WHOLESALER': spare.wholesaler_price,
                        'CUSTOMER': spare.customer_price,
                    }
                    sold_spare = SpareSold(order=order,
                                           spare=spare,
                                           spare_count=_spare['spare_count'],
                                           spare_name=spare.spare_name,
                                           spare_buying_price=spare.buying_price,
                                           spare_price=price_map[_spare['spare_price_type']],
                                           spare_price_type=_spare['spare_price_type'])
                    spares_to_be_created.append(sold_spare)
                    current_total = float(sold_spare.spare_count * sold_spare.spare_price)
                    total = total + current_total
                    spare.quantity = spare.quantity - sold_spare.spare_count
                    spare.save()
                    share_message += f"{sold_spare.spare_name} -- {sold_spare.spare_count} x {sold_spare.spare_price} = {current_total}\n"

                SpareSold.objects.bulk_create(spares_to_be_created)
                if order.order_type:
                    order.bike_number = data['bike_number']
                    order.labour_charge = data['labour_charge']
                    order.out_source_charge = data['out_source_charge']
                    total = total + order.labour_charge + order.out_source_charge
                    share_message += f"Labour Charge = {order.labour_charge}\n\n" \
                        f"Out Source Charge = {order.out_source_charge}\n\n"
                order.total = total
                order.save()
                share_message += f"Grand total = {total}.\n\n" \
                    f"Order ID: {order_id}\n\n" \
                    f"Date: {today.strftime('%d-%m-%Y')}\n\nThank you for purchasing with us!"
                return Utils.dispatch_success(request, {'order_id': order.id, 'spareorder_id':order_id, 'share_info': share_message})
        except Exception as e:
            return self.internal_server_error(request, e)

class SparesAccountingView(CustomBaseClass):
    sell_report_type = ['IN_SELL', 'OUT_SELL', 'TOTAL_SELL']
    profit_report_type = ['IN_PROFIT', 'OUT_PROFIT', 'TOTAL_PROFIT']
    IN_SOURCE = ['IN_SELL', 'IN_PROFIT']
    OUT_SOURCE = ['OUT_SELL', 'OUT_PROFIT']

    def get_total(self, qs, report_type):
        """
        parms ?start_date=2019-01-31&&end_date=2019-12-31&&stores=16&&report_type=TOTAL_SELL
        IN_SELL', 'OUT_SELL', 'TOTAL_SELL', 'IN_PROFIT', 'OUT_PROFIT', 'TOTAL_PROFIT
        :param qs:
        :param report_type:
        :return:
        """
        total = 0.00
        total_items = 0
        spares = []

        if report_type in self.IN_SOURCE:
            qs = qs.filter(order_type=SpareOrder.IN_SOURCE)

        if report_type in self.OUT_SOURCE:
            qs = qs.filter(order_type=SpareOrder.OUT_SOURCE)

        for order in qs:
            for spare in SpareSold.objects.filter(order=order):
                if report_type in self.sell_report_type:
                    total = total + float(spare.spare_count * spare.spare_price)

                if report_type in self.profit_report_type:
                    total = total + float(spare.spare_count * spare.spare_buying_price)
                spares.append(spare.spare)
                total_items += spare.spare_count
        return total, total_items, len(set(spares))

    def get(self, request):
        try:
            stores = [int(x) for x in request.GET.get('stores', '').split(',')]
            start_date = request.GET.get('start_date')
            end_date = request.GET.get('end_date')
            report_type = request.GET.get('report_type')
            if start_date:
                start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
                end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') + datetime.timedelta(days=1)
            profit_map = {
                "IN_PROFIT": "IN_SELL",
                "OUT_PROFIT": "OUT_SELL",
                "TOTAL_PROFIT": "TOTAL_SELL",
            }

            qs = self.get_filter_objects(SpareOrder, store__in=stores, order_date__range=[start_date, end_date])

            if not qs:
                return Utils.dispatch_success(request, "DATA_NOT_FOUND")
            if report_type in self.sell_report_type:
                selling_total, total_items, total_spares = self.get_total(qs, report_type)
                response_data = {"selling_total": selling_total,
                                 "total_items": total_items,
                                 "total_spares": total_spares}
                return Utils.dispatch_success(request, response_data)

            if report_type in self.profit_report_type:
                buying_total, total_items, total_spares = self.get_total(qs, report_type)
                selling_total, total_items, total_spares = self.get_total(qs, profit_map[report_type])
                difference = selling_total - buying_total
                response_data = {"selling_total": selling_total,
                                 "buying_total": buying_total,
                                 "profit_total": abs(difference),
                                 "status": "LOSS" if difference < 0 else "PROFIT",
                                 "total_items": total_items,
                                 "total_spares": total_spares}
                return Utils.dispatch_success(request, response_data)
            return self.object_not_found(request)
        except Exception as e:
            return self.internal_server_error(request, e)

class UrgentSpareList(CustomBaseClass):
    def get(self, request, store_id, *args, **kwargs):
        """
        return list of urgent stock with pagination
        :param request:
        :param store_id:
        :param args:
        :param kwargs:
        :return:
        """
        try:
            qs = self.get_filter_objects(Spare, store=store_id, is_urgent_spare=True)
            page = request.GET.get('page', 1)
            paginator = Paginator(qs, per_page=10)
            serializer = serializers.SpareSerializer(paginator.page(page), many=True)
            response_data = {
                "data": serializer.data,
                "page": int(page),
                "total_pages": paginator.num_pages
            }
            return Utils.dispatch_success(request, response_data)
        except Exception as e:
            return self.internal_server_error(request, e)

    def put(self, request, store_id, *args, **kwargs):
        """
        Updates list of urgent stock with pagination
        :param request:
        {
        "spares":[{"id":23, "quantity": 20}, {"id":30, "quantity": 2}, {"id":11, "quantity": 12}, ]
        }
        :param store_id:
        :param args:
        :param kwargs:
        :return:
        """
        try:
            spares_list = request.data["spares"]
            for _spare in spares_list:
                spare = self.get_object(Spare, _spare["id"])
                spare.is_urgent_spare = False
                spare.quantity += _spare["quantity"]
                spare.save()
            return Utils.dispatch_success(request, 'SUCCESS')
        except Exception as e:
            return self.internal_server_error(request, e)

class SpareOrderEmailPdf(CustomBaseClass):
    def get(self, request, order_id, *args, **kwargs):
        """
        Returns PDF of the invoice or email's user
        :param request:
        @param action=email # to send email to customer
        @param action=download # to Download the invoice copy
        :param order_id:
        :param args:
        :param kwargs:
        :return:
        """
        try:
            action = request.GET.get('action')
            order = self.get_object(SpareOrder, order_id)
            data = {}
            store = order.store
            data['store'] = {
                'store_name': store.name.upper(),
                'store_branch': store.branch.upper(),
                'store_type': store.branch,
                'store_address': store.address.replace(',', '\n'),
                'store_phone': store.phone,
                'store_email': store.email,
                'store_website': store.website,
            }
            customer = order.customer
            data['customer'] = {
                'name': customer.name,
                'email': customer.email,
                'phone_number': customer.phone_number,
                'address': customer.address.replace(',', '\n')
            }
            data['order_id'] = order.order_id
            data['date'] = order.order_date.strftime('%d-%m-%Y %H:%M:%S')
            data['total'] = order.total
            data['type'] = order.order_type
            data['bike_number'] = order.bike_number
            data['labour_charge'] = order.labour_charge
            data['out_source_charge'] = order.out_source_charge
            data['sold_by'] = order.sold_by.first_name
            data['sub_total'] = order.total - order.labour_charge - order.out_source_charge
            response = {
                'csv': Utils.generate_csv,
                'xls': Utils.generate_xls,
                'pdf': Utils.generate_pdf
            }
            data['order'] = []
            for i, order_spare in enumerate(SpareSold.objects.filter(order=order)):
                data['order'].append([i + 1, order_spare.spare_name, order_spare.spare_price, order_spare.spare_count,
                                      order_spare.spare_price * order_spare.spare_count])
            dynamic_data = {
                'pdf_template': 'spare_invoice.html',
                'filename': f'Invoice_{order.order_id}',
                'data': data,
                'action': action
            }

            if action == 'download':
                return response.get('pdf')(**dynamic_data)
            elif action == 'email':
                if customer.email:
                    AppUtils.send_inovice_email(response.get('pdf')(**dynamic_data), data, f'Invoice_{order.order_id}' )
                    return Utils.dispatch_success(request, 'SUCCESS')
            return self.object_not_found(request)
        except Exception as e:
            return self.internal_server_error(request, e)


__all__ = ["Echo"]

from textwrap import dedent
from typing import Any, Dict

from ..imagecrawler import BaseImageCrawler, Image, ImageCollection, ImageCrawlerConfig, ImageCrawlerInfo

class Echo(BaseImageCrawler):

    def __init__(self, *, image_uri: str) -> None:
        super().__init__(image_uri=image_uri)

    @classmethod
    def info(cls) -> ImageCrawlerInfo:
        return ImageCrawlerInfo(
            description='"Finds" the same image ... again ... and again.',
            long_description=dedent('''
                Not an actual crawler.
                More like an Parrot that is trained to repeat what you tell it to say.
                ''').strip(),
            config={
                'image_uri': 'the URI of the image to "find"',
            },
            # does not have an icon
        )

    @classmethod
    def check_config(cls, config: Dict[str, Any]) -> ImageCrawlerConfig:
        image_uri = config['image_uri']
        if type(image_uri) is not str:
            raise TypeError(f'image_uri {image_uri!r} is not str')
        if len(image_uri) == 0:
            raise ValueError(f'image_uri {image_uri!r} is empty')
        return ImageCrawlerConfig(
            image_uri=image_uri,
        )

    def is_exhausted(self) -> bool:
        # is generic -> never exhausts
        return False

    def _reset(self) -> None:  # pragma: no cover
        pass

    def _crawl(self) -> ImageCollection:
        images = ImageCollection()
        image_uri = self.get_config()["image_uri"]
        images.add(
            Image(
                uri=image_uri,
                source=image_uri,
                is_generic=True,
                this_is_a_dummy=True,
            )
        )
        return images


# A transformation sequence from word beginWord to word endWord using a dictionary wordList is a sequence of words such that:

# The first word in the sequence is beginWord.
# The last word in the sequence is endWord.
# Only one letter is different between each adjacent pair of words in the sequence.
# Every word in the sequence is in wordList.
# Given two words, beginWord and endWord, and a dictionary wordList, return the number of words in the shortest transformation sequence from beginWord to endWord, or 0 if no such sequence exists.

# Example 1:

# Input: beginWord = "hit", endWord = "cog", wordList = ["hot","dot","dog","lot","log","cog"]
# Output: 5
# Explanation: One shortest transformation is "hit" -> "hot" -> "dot" -> "dog" -> "cog" with 5 words.
# Example 2:

# Input: beginWord = "hit", endWord = "cog", wordList = ["hot","dot","dog","lot","log"]
# Output: 0
# Explanation: The endWord "cog" is not in wordList, therefore there is no possible transformation.

# Constraints:

# 1 <= beginWord.length <= 10
# endWord.length == beginWord.length
# 1 <= wordList.length <= 5000
# wordList[i].length == beginWord.length
# beginWord, endWord, and wordList[i] consist of lowercase English letters.
# beginWord != endWord
# All the strings in wordList are unique.
from collections import deque
class Solution:
    def ladderLength(self, beginWord: str, endWord: str, wordList: List[str]) -> int:
        graph = {}
        for word in wordList:
            for i in range(len(word)):
                graph.setdefault(word[:i]+'*'+word[i+1:],[]).append(word)
        #print(graph)
        queue = deque([(beginWord,1)])
        visited = set([beginWord])
        while queue:
            curword, length = queue.popleft()
            #print(curword,length)
            if curword == endWord:
                return length
            for i in range(len(curword)):
                pattern = curword[:i]+'*'+curword[i+1:]
                #print("pattern",pattern)
                for nextWord in graph.get(pattern,[]):
                    #print(nextWord)
                    if nextWord not in visited:
                        queue.append((nextWord,length+1))
                        visited.add(nextWord)
        return 0


import cv2
import numpy as np

img2 = cv2.imread("images.jpg")
img1 = cv2.imread("new quantum.PNG")

rows, cols, channels = img2.shape  # Reading image details
roi = img1[0:rows, 0:cols]

img2g = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)  # converting to grayscale
# defining mask , makes our logo black and background white
ret, mask = cv2.threshold(img2g, 220, 255, cv2.THRESH_BINARY_INV)

# cv2.imshow('mask',mask)

mask_inv = cv2.bitwise_not(mask)  # defining non_masked area
# adds our inv_mask to region of image of main image
img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
img_fg = cv2.bitwise_and(img2, img2, mask=mask)  # our logo after adding mask
dst = img_bg+img_fg  # we get our logo with our original image background
# swaps the region of image original image to logo with same backgound
img1[0:rows, 0:cols] = dst

cv2.imshow('res', img1)
cv2.imwrite('result.jpg', img1)
cv2.waitKey(0)
cv2.destroyAllWindows()


import re
from pprint import pprint
import csv
from decorator import to_log

if __name__ == '__main__':
    # читаем адресную книгу в формате CSV в список contacts_list
    with open("phonebook_raw.csv", encoding='utf-8') as f:
        rows = csv.reader(f, delimiter=",")
        contacts_list = list(rows)

    # rewriting phone numbers
    phone_num = r'(\+7|8)\s?\(?(\w{3})\)?\s?-?(\w{3})-?(\w{2})-?(\w{2})(\s?)(\(?((доб\.)\s(\w{4}))\)?)?'
    num_format = r'+7(\2)\3-\4-\5\6\9\10'

    i = 1
    form_list = contacts_list
    while i < len(contacts_list):
        form_list[i][5] = re.sub(phone_num, num_format, contacts_list[i][5])
        i += 1

    # pprint(form_list)

    # put names, surnames... as different elements

    @to_log(path=r"C:\Users\olesy\PycharmProjects\regex_decorated\log.txt")
    def arrange_names(some_list):
        new_list = some_list

        for n in range(0, 2):
            for i in range(1, 9):
                if re.search(r'\s', some_list[i][n]):
                    name = some_list[i][n].split(' ')
                    if n == 0:
                        if some_list[i][1] != '':
                            name.extend(some_list[i][3:])
                            new_list[i] = name
                        else:
                            name.extend(some_list[i][2:])
                            new_list[i] = name
                    else:
                        new_list[i][n] = name[0]
                        new_list[i][n + 1] = name[1]
                else:
                    pass

        return new_list

    arrange_names(form_list)


import pandas as pd
import xml.etree.ElementTree as ET

# Load the XML data
tree = ET.parse('statement_short.xml')
root = tree.getroot()

# Define a function to extract data from the XML elements
def extract_data(elem):
    data = {}
    for child in elem:
        if len(child) == 0:
            data[child.tag] = child.text
        else:
            data[child.tag] = extract_data(child)
    return data

# Extract the data from the XML and create a DataFrame
data = []
for elem in root:
    data.append(extract_data(elem))
df = pd.DataFrame(data)
df1 = pd.read_xml('statement_short.xml')

# Display the DataFrame
print(df1)


from setuptools import setup, find_packages
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='pysnippets',
    version='0.1.0',
    description='Scattered python snippets',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/gongzhitaao/snippets/pysnippets',
    author='gongzhitaao',
    author_email='zhitaao.gong@gmail.com',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3',
    ],
    keywords='snippets',
    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
    install_requires=[],
    extras_require={
        'test': ['pytest'],
    },
)


import os
import atexit
import secrets
import unittest
from functools import wraps
import augpathlib as aug
from sxpyr import sxpyr
from pyontutils.utils import Async, deferred  # TODO -> asyncd in future
from pyontutils.utils_fast import isoformat
from sparcur import exceptions as exc
from sparcur.utils import GetTimeNow, log
from sparcur.paths import PennsieveCache, LocalPath, Path
from sparcur.backends import PennsieveRemote
from .common import test_organization, test_dataset, _pid
from .common import skipif_ci, skipif_no_net
import pytest

class _TestOperation:

    _cache_class = PennsieveCache
    _remote_class = PennsieveRemote

    @classmethod
    def tearDownClass(cls):
        base = aug.AugmentedPath(__file__).parent / f'test-operation-{_pid}'
        if base.exists():
            base.popd()  # in case we were inside it pop back out first
            base.rmtree()

    def setUp(self):
        class Cache(self._cache_class):
            pass

        Cache._bind_flavours()

        base = aug.AugmentedPath(__file__).parent / f'test-operation-{_pid}'

        if base.exists():
            base.popd()  # in case we were inside it pop back out first
            base.rmtree()

        base.mkdir()
        base.pushd()

        self.Remote = self._remote_class._new(Cache._local_class, Cache)
        self.Remote.init(test_organization)
        self.anchor = self.Remote.dropAnchor(base)

        self.root = self.anchor.remote
        self.project_path = self.anchor.local
        list(self.root.children)  # populate datasets
        self.test_base = [
            p for p in self.project_path.children
            if p.cache.id == test_dataset][0]
        list(self.test_base.rchildren)  # populate test dataset

        asdf = self.root / 'lol' / 'lol' / 'lol (1)'

        class Fun(os.PathLike):
            name = 'hohohohohoho'

            def __fspath__(self):
                return ''

            @property
            def size(self):
                return aug.FileSize(len(b''.join(self.data)))

            @property
            def data(self):
                for i in range(100):
                    yield b'0' * 1000

        #wat = asdf.bfobject.upload(Fun(), use_agent=False)
        #breakpoint()

@skipif_ci
@skipif_no_net
class TestDelete(_TestOperation, unittest.TestCase):

    def test_0(self):
        assert True

    def test_1_case(self):
        # this is an old scenario that happens because of how the old system worked
        # local working directory | x
        # local cache directory | o
        # remote | o
        pass

def make_rand(n, width=80):
    lw = width + 1
    hex_lw = lw // 2
    n_lines = n // lw
    n_accounted = n_lines * lw
    h_lines = n // (hex_lw * 2)
    h_accounted = h_lines * (hex_lw * 2)
    ldiff = n_lines - h_lines
    adiff = n_accounted - h_accounted
    accounted = n_accounted
    missing = n - accounted
    hex_missing = (missing + 1) // 2
    diff = hex_missing * 2 - missing
    hexstart = width % 2  # almost there fails on 71
    # also fails len(make_rand(102, 101)) - 102

    log.debug((adiff, ldiff, missing, diff))
    string = '\n'.join([secrets.token_hex(hex_lw)[hexstart:]
                        for i in range(n_lines)]
                       + [secrets.token_hex(hex_missing)[diff:-1] + '\n'])
    return string.encode()

@skipif_ci
@skipif_no_net
@pytest.mark.skip('VERY SLOW')
class TestFilenames(_TestOperation, unittest.TestCase):

    _evil_names = (
        # '......................',  # this breaks the agent with infinite timeout
        '!@#$%^&*()[]{}`~;:,',
        '(╯°□°）╯︵ ┻━┻)',
        'הָיְתָהtestالصفحات التّحول',
        'הָיְתָהtestالصفحا تالتّحول',  # check bucket names
        '᚛ᚄᚓᚐᚋᚒᚄ ᚑᚄᚂᚑᚏᚅ᚜',
        'Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮',
        # '𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌',  # this breaks the agent with ERRORED
        'evil file space',
        'evil_file underscore',
        'evil-file dash',
        'evil%20file percent 20',
        'hello%20world%20%60~%21%40%23%24%25%5E%26%2A%28%29%5B%5D%7B%7D%27',
        # the problem is that we don't know whether we can actually
        # decode a file name, and wthe database stores the encoded filename
        'hello%20world',
        'hello%20world~',
    )

    @property
    def _more_evil_names(self):
        # class scope strikes back! LOL PYTHON
        return [
            name
          for char in
            ('\x07',
             #'/',  # have to do this in a different way on unix
             '\\',
             '|',
             '!',
             '@',
             '#',
             '$',
             '%',
             '^',
             '&',
             '*',
             '(',
             ')',
             '[',
             ']',
             '{',
             '}',
             "'",
             '`',
             '~',
             ';',
             ':',
             ',',
             '"',
             '?',
             '<',
             '>',
             )
          for name in
            (f'prefix{char}',
             f'prefix{char}suffix',
             f'{char}suffix',)]

    @staticmethod
    def _op(test_folder, fsize, name):
        test_file_a = test_folder / (name + '.ext')
        test_file_b = test_folder / (name + '.txe')
        test_folder_i = test_folder / name

        for _f in (test_file_a, test_file_b):
            if _f.exists() or _f.is_broken_symlink():
                msg = (f'bad test environment: file/link already exists: {_f}')
                raise FileExistsError(msg)

        # FIXME maybe don't straight fail here, but instead
        # don't upload and just compare the existing name?
        # the fact that we get an error is a sign that the
        # name matches actually ... so not getting an error
        # in subsequent runs is bad ... for test_base at least

        test_file_a.data = iter((make_rand(fsize),))
        test_file_b.data = iter((make_rand(fsize),))

        try:
            remote_a = test_file_a.upload()
            name_a = remote_a.bfobject.name
        except Exception as e:
            name_a = e

        try:
            remote_b = test_file_b.upload()
            name_b = remote_b.bfobject.name
        except Exception as e:
            name_b = e

        try:
            remote_i = test_folder_i.mkdir_remote()
            name_i = remote_i.bfobject.name
        except Exception as e:
            name_i = e

        return name_a, name_b, name_i

    def test_filenames_more_evil(self):
        return self.test_filenames_evil(self._more_evil_names)

    def test_filenames_evil(self, names=_evil_names):
        # XXX warning slow!
        now = GetTimeNow()
        local = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}'
        remote = local.mkdir_remote()

        try:
            # FIXME consider going back to self.test_base instead of local here
            test_folder = local / 'pandora'
            test_folder.mkdir_remote()
            test_folder.__class__.upload = Path.upload
            fsize = 1024  # needed for uniqueish hashes colloisions will still happen
            # FIXME this pretty clearly reveals a need for
            # batching to multiplex the fetch ... SIGH
            results = Async(rate=10)(deferred(self._op)(test_folder, fsize, name) for name in names)
            #results = []
            #for name in names:
            #    name_a, name_b, name_i = self._op(test_folder, fsize, name)
            #    results.append((name_a, name_b, name_i))
        finally:
            remote.rmdir(force=True)
            # FIXME crumple fails in refresh since we use rmdir
            # instead of rmtree (for safety)
            #remote.cache.refresh()  # FIXME

@skipif_ci
@skipif_no_net
class TestUpdate(_TestOperation, unittest.TestCase):

    @pytest.mark.skip('the question has been answered')
    def test_process_filesize_limit(self):
        # so the 1 mb size file works, eventually, something else is wrong
        test_folder = self.test_base / 'hrm'
        test_folder.mkdir_remote()
        test_folder.__class__.upload = Path.upload
        for i in range(1024 ** 2, 5 * 1024 ** 2, 1024 ** 2):
            test_file = test_folder / f'size-{i}'
            if test_file.is_broken_symlink():
                test_file.remote.bfobject.package.delete()
                test_file.unlink()
                test_file = test_folder / f'size-{i}'  # remove stale cache

            test_file.data = iter((make_rand(i),))
            remote = test_file.upload()

    def test_upload_noreplace(self):
        for i in range(2):
            test_file = self.test_base / 'dataset_description.csv'
            test_file.data = iter((make_rand(100),))
            # FIXME temp sandboxing for upload until naming gets sorted
            test_file.__class__.upload = Path.upload
            # create some noise
            remote = test_file.upload(replace=False)
            print(remote.bfobject.package.name)

    def test_upload_noreplace_fail(self):
        # some persistent state from other tests is causing this to fail
        test_file = self.test_base / 'dataset_description.csv'
        test_file.data = iter((make_rand(100),))
        # FIXME temp sandboxing for upload until naming gets sorted
        test_file.__class__.upload = Path.upload
        test_file.upload(replace=False)
        try:
            test_file.upload(replace=False)
            assert False, 'should have failed'
        except exc.FileHasNotChangedError:
            pass

    def test_upload_replace(self):
        test_file = self.test_base / 'dataset_description.csv'
        test_file.data = iter((make_rand(100),))
        # FIXME temp sandboxing for upload until naming gets sorted
        test_file.__class__.upload = Path.upload
        test_file.upload()

@skipif_ci
@skipif_no_net
class TestClone(_TestOperation, unittest.TestCase):
    # TODO test a variety of clone scenarios
    # and consider whether testing for and
    # existing root should be done in dropAnchor

    def setUp(self):
        super().setUp()
        self.alt_project_path = self.project_path.parent / 'alt' / self.project_path.name
        if self.alt_project_path.parent.exists():
            self.alt_project_path.parent.rmtree()

        self.alt_project_path.mkdir(parents=True)

    def _do_target(self, target, expect_error_type=None):
        class Cache(self._cache_class):
            pass

        Cache._bind_flavours()

        BFR = self._remote_class._new(LocalPath, Cache)
        BFR.init(test_organization)

        if expect_error_type:
            try:
                anchor = BFR.dropAnchor(target.parent)
                raise AssertionError(f'should have failed with a {expect_error_type}')
            except expect_error_type as e:
                pass

        else:
            anchor = BFR.dropAnchor(target.parent)

    def test_1_in_project(self):
        target = self.project_path / 'some-new-folder'
        target.mkdir()
        self._do_target(target)  # FIXME succeeds for now, but probably should not?

    def test_2_project_top_level(self):
        target = self.project_path
        self._do_target(target, exc.DirectoryNotEmptyError)

    def test_3_existing_empty(self):
        target = self.alt_project_path
        self._do_target(target)

    def test_4_existing_has_folder(self):
        target = self.alt_project_path
        child = target / 'a-folder'
        child.mkdir(parents=True)
        self._do_target(target, exc.DirectoryNotEmptyError)

    def test_5_existing_has_file(self):
        target = self.alt_project_path
        child = target / 'a-file'
        child.touch()
        self._do_target(target, exc.DirectoryNotEmptyError)

    def test_6_existing_has_local_data_dir(self):
        target = self.alt_project_path
        child = target / self.anchor._local_data_dir
        child.mkdir()
        self._do_target(target, exc.DirectoryNotEmptyError)

@skipif_ci
@skipif_no_net
class TestMkdirRemote(_TestOperation, unittest.TestCase):

    def test_mkdir_remote_parents_false(self):
        now = GetTimeNow()
        local = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}' / 'some-folder'
        try:
            remote = local.mkdir_remote()
            raise AssertionError('Should have failed since parents=False')
        except FileNotFoundError:
            pass

    def test_0_mkdir_remote_will_be_dataset(self):
        now = GetTimeNow()
        local = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}'
        remote = local.mkdir_remote()
        remote.rmdir()
        remote.cache.refresh()  # reminder that remotes are a snapshot in time, NOT dynamic
        assert not local.exists(), f'should have been deleted {remote}'

    def test_1_mkdir_remote_will_be_collection(self):
        now = GetTimeNow()
        local = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}' / 'some-folder'
        remote = local.mkdir_remote(parents=True)
        parent = remote.parent
        try:
            parent.rmdir()  # should fail here
            try:
                remote.rmdir()  # insurance
            except BaseException as e:
                log.exception(e)
            finally:
                raise AssertionError('remote parent should NOT have rmdired {parent}')
        except exc.PathNotEmptyError:
            pass

        try:
            remote.rmdir()
            remote.cache.refresh()
            assert not local.exists(), f'should have been deleted {remote}'
        finally:
            lparent = parent.local
            parent.cache.refresh()  # we just removed the child so the parent is stale
            parent.rmdir()
            parent.cache.refresh()
            assert not lparent.exists(), f'should have been deleted {parent}'

class TestMoveFolder(_TestOperation, unittest.TestCase):

    def test_reparent(self):
        # TODO this is nowhere near complete with respect to synchronization
        # but it is sufficient to test the components needed for sync
        now = GetTimeNow()
        local = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}'
        remote = local.mkdir_remote()

        try:
            # FIXME consider going back to self.test_base instead of local here
            test_folder_1 = local / 'dire-1'
            test_folder_1.mkdir_remote()
            test_folder_2 = local / 'dire-2'
            test_folder_2.mkdir_remote()
            list(remote.cache.children)  # XXX populate with remote data

            test_folder_2.remote.reparent(test_folder_1.cache_id)

            test_folder_1.__class__.upload = Path.upload
            fsize = 1024
            test_file_1 = test_folder_1 / 'file-1.ext'
            test_file_1.data = iter((make_rand(fsize),))
            test_file_1.upload()
            test_file_1.remote.reparent(test_folder_2.cache_id)

        finally:
            remote.rmdir(force=True)
            # FIXME crumple fails in refresh since we use rmdir
            # instead of rmtree (for safety)
            #remote.cache.refresh()  # FIXME

class _ChangesHelper:
    _local_only = True

    def _make_ops(self):
        ops = tuple()
        # dirs
        dataset = 'project/dataset'
        ops += (
        #(0, 'mkdir', 'project'),  # don't actually need this since we replace it when building paths
        (0, 'mkdir', dataset),
        (0, 'mkdir', 'project/dataset/dire-1'),
        (0, 'mkdir', 'project/dataset/dire-2'),
        (0, 'mkdir', 'project/dataset/dire-6'),
        )

        # sources
        d3_1 = 'project/dataset/dire-1/dire-3-1-rn'
        d3_2 = 'project/dataset/dire-1/dire-3-2-rp'
        d3_3 = 'project/dataset/dire-1/dire-3-3-np'

        f1_0 = 'project/dataset/dire-1/file-1-0.ext'
        f1_1 = 'project/dataset/dire-1/file-1-1-rn.ext'
        f1_2 = 'project/dataset/dire-1/file-1-2-rp.ext'
        f1_3 = 'project/dataset/dire-1/file-1-3-np.ext'

        l1_0 = 'project/dataset/dire-1/link-1-0.ext'
        l1_1 = 'project/dataset/dire-1/link-1-1-rn.ext'
        l1_2 = 'project/dataset/dire-1/link-1-2-rp.ext'
        l1_3 = 'project/dataset/dire-1/link-1-3-np.ext'

        # targets
        # TODO need variants of all of these where we lose the metadata probably?

        ops += (
        (0, 'mkdir', d3_1),
        (0, 'mkdir', d3_2),
        (0, 'mkdir', d3_3),

        (0, 'mkfile', f1_0),  # nochange
        (0, 'mkfile', f1_1),
        (0, 'mkfile', f1_2),
        (0, 'mkfile', f1_3),

        (0, 'mklink', l1_0),  # nochange
        (0, 'mklink', l1_1),
        (0, 'mklink', l1_2),
        (0, 'mklink', l1_3),

        # moves: renames, reparents, rename_reparent
        (1, 'rename', d3_1, 'project/dataset/dire-1/dire-3-1-rn-r'),  # rn
        (1, 'rename', d3_2, 'project/dataset/dire-2/dire-3-2-rp'),  # rp
        (1, 'rename', d3_3, 'project/dataset/dire-2/dire-3-3-np-r'),  # rnp

        (1, 'rename', f1_1, 'project/dataset/dire-1/file-1-1-rn-r.ext'),  # rn
        (1, 'rename', f1_2, 'project/dataset/dire-2/file-1-2-rp.ext'),  # rp
        (1, 'rename', f1_3, 'project/dataset/dire-2/file-1-3-np-r.ext'),  # rnp

        (1, 'rename', l1_1, 'project/dataset/dire-1/link-1-1-rn-r.ext'),  # rn
        (1, 'rename', l1_2, 'project/dataset/dire-2/link-1-2-rp.ext'),  # rp
        (1, 'rename', l1_3, 'project/dataset/dire-2/link-1-3-np-r.ext'),  # rnp

        # add
        (1, 'mkdir',  'project/dataset/dire-6/dire-7-add'),
        (1, 'mkfile', 'project/dataset/dire-6/file-4-add.ext'),
        (2, 'mklink', 'project/dataset/dire-6/link-4-add.ext'),  # XXX this causes an error because it looks like the index is out of synx
        )

        # change (only applies to files)
        f5_1 = 'project/dataset/dire-6/file-5-1-cd_.ext'
        f5_2 = 'project/dataset/dire-6/file-5-2-c_m.ext'
        f5_3 = 'project/dataset/dire-6/file-5-3-c_x.ext'
        f5_4 = 'project/dataset/dire-6/file-5-4-cdm.ext'
        f5_5 = 'project/dataset/dire-6/file-5-5-cdx.ext'
        # file_id change ? should be impossible ...
        ops += (
        (0, 'mkfile', f5_1),
        (0, 'mkfile', f5_2),
        (0, 'mkfile', f5_3),
        (0, 'mkfile', f5_4),
        (0, 'mkfile', f5_5),

        # TODO probably also change size
        (1, 'change', f5_1, True, False),  # data
        (1, 'change', f5_2, False, True),  # metadata
        (1, 'change', f5_3, False, None),  # no metadata # can handle this from objects cache
        (1, 'change', f5_4, True, True),  # data metadata
        (1, 'change', f5_5, True, None),  # data no metadata
        )

        # remove
        d9 = 'project/dataset/dire-6/dire-9-rem'
        f6 = 'project/dataset/dire-6/file-6-rem.ext'
        l6 = 'project/dataset/dire-6/link-6-rem.ext'
        ops += (
        (0, 'mkdir', d9),
        (0, 'mkfile', f6),
        (0, 'mklink', l6),

        (1, 'remove', d9),
        (1, 'remove', f6),
        (1, 'remove', l6),
        )

        # build the indexes so we can do the diff
        ops += (
            (0.5, 'index', dataset),
        )
        return ops

    def setUp(self):
        # TODO construct the template we need
        super().setUp()
        self.Path = self.Remote._local_class

        #sigh = list(self.project_path.remote.children)
        #[s.rmdir(force=True) for s in sigh if '2023' in s.name]
        #breakpoint()
        #raise ValueError('cleanup tearDown failure mess')

        # TODO expected outcome after stage probably

        if self._local_only:
            def populate_cache(p, change=False):
                # FIXME TODO change=True case may need special handling
                # and probably represents a strange case of some kind
                return p._cache_class.fromLocal(p)

            def norm_path(p):
                return self.project_path / p.replace('project/', '')

        else:
            # XXX WARNING extremely slow due to sequentially creating each remote file
            now = GetTimeNow()
            local_dataset = self.project_path / f'test-dataset-{now.START_TIMESTAMP_LOCAL_FRIENDLY}'
            remote_dataset = local_dataset.mkdir_remote()
            #self._test_dataset = remote_dataset
            # tearDown fails to trigger if failure happens in setUp which is useless
            # so use atexit instead
            atexit.register(lambda : remote_dataset.rmdir(force=True))
            def populate_cache(p, change=False):
                # FIXME TODO change=True case may need special handling
                # and probably represents a strange case of some kind
                if change:
                    return

                remote = p.create_remote()
                return remote.cache

            def norm_path(p):
                return local_dataset / p.replace('project/dataset', '').strip('/')

        def mkdir(d, add=False):
            if not self._local_only and d == local_dataset:
                # FIXME HACK
                return

            d.mkdir()
            if not add:
                cache = populate_cache(d)
                d._cache = cache
            #d._cache_class.fromLocal(d)

        def mkfile(f, add=False):
            f.data = iter((make_rand(100),))
            if not add:
                cache = populate_cache(f)
                f._cache = cache
            #f._cache_class.fromLocal(f)

        def mklink(l):
            try:
                l.data = iter((make_rand(100),))
                # issue with id and parent_id not being set so use fromLocal since it does it correctly
                #meta = f.meta  # TODO checksum probably?
                #symlink = meta.as_symlink(local_name=l.name)
                #cache = l._cache_class.fromLocal(l)
                cache = populate_cache(l)
                symlink = cache.meta.as_symlink(local_name=l.name)
            finally:
                l.unlink()

            l.symlink_to(symlink)

        def rename(path, target):
            path.rename(target)

        def change(f, data, metadata):
            if data:
                f.data = iter((make_rand(100),))

            if metadata is None or metadata:
                # must set xattrs to nothing if
                # we want to change metadata otherwise
                # PrimaryCache._meta_updater will go haywire and
                # ... try to delete the file ... and it will
                # actually delete it instead of crumpling it
                # so definitely a FIXME very dangerous lose your work
                # kind of scenario between that _meta_updater and
                # BFPNCache._actually_crumple and change of BFPNCache.crumple
                [f.delxattr(k) for k in f.xattrs()]
                if f.xattrs():
                    breakpoint()

            if metadata:
                if not f.exists():
                    raise FileNotFoundError(f)

                try:
                    populate_cache(f, change=True)
                    #f._cache_class.fromLocal(f)
                except Exception as e:
                    breakpoint()
                    raise e

        def remove(path):
            if path.is_dir():
                path.rmdir()
            else:
                path.unlink()

        def index(ds):
            if self._local_only:
                caches = [l.cache for l in ds.rchildren]  # XXX reminder, NEVER use ds.cache.rchildren that will pull
                class fakeremote:
                    def __init__(self, id, name, parent_id, file_id, updated, local):
                        self.id = id
                        self.name = name
                        self._name = name
                        self.parent_id = parent_id
                        self.updated = updated
                        self._lol_local = local

                        if file_id is not None:
                            self.file_id = file_id

                    def is_dir(self):
                        return self._lol_local.is_dir()

                for c in caches:
                    # FIXME causes other issues ... even while trying to avoid init issues
                    # we should not have to do this
                    cmeta = c.meta
                    c._remote = fakeremote(
                        cmeta.id, cmeta.name, cmeta.parent_id, cmeta.file_id,
                        cmeta.updated, c.local)
            else:
                # this is safe at this stage since everything should match upstream
                caches = [c.cache for c in local_dataset.rchildren]

            ds._generate_pull_index(ds, caches)

        fops = {
            'mkdir': mkdir,
            'mkfile': mkfile,
            'mklink': mklink,
            'rename': rename,
            'change': change,
            'remove': remove,
            'index': index,
        }

        def make_closure(stage, op, obj, args):
            f = fops[op]

            if stage > 0 and op in ('mkdir', 'mkfile'):
                kwargs=dict(add=True)
            else:
                kwargs = {}

            @wraps(f)
            def inner():
                f(path, *args, **kwargs)

            return inner

        def cargs(args):
            for a in args:
                if isinstance(a, str) and a.startswith('project/'):
                    yield norm_path(a)
                else:
                    yield a

        ops = self._make_ops()
        pops = [(stage, op, norm_path(s), *cargs(args)) for stage, op, s, *args in ops]
        init = set([path for stage, op, path, *args in pops if stage == 0])
        test = set([p for stage, op, path, *args in pops if stage >= 1 for p in (path, *args) if isinstance(p, self.project_path.__class__)])
        nochange = init - test
        add_rename_reparent = test - init
        change_remove = test - add_rename_reparent

        cs = [(stage, path, make_closure(stage, op, path, args)) for stage, op, path, *args in pops]
        scs = sorted(cs, key=(lambda abc: (abc[0], len(abc[1].parts))))
        will_fails = []
        for stage, path, fun in scs:
            if stage > 1:
                will_fails.append(fun)
            else:
                fun()

        self._will_fails = will_fails
        self.dataset = pops[0][-1]

class TestChanges(_ChangesHelper, _TestOperation, unittest.TestCase):

    def test_changes(self):
        from dateutil import parser as dateparser
        dataset = self.dataset
        dataset_id, id_name, parent_children, name_id, updated_transitive = dataset._read_indexes()
        # XXX updated_transitive from _read_indexes is a string because that is what
        # _transitive_changes needs internally and then transforms to a datetime object
        # when it returns, therefore we don't fiddle with the types here

        #tc = dataset._transitive_changes()
        # XXX see sparcur.simple.utils
        dataset_id, updated_cache_transitive, diff = dataset.diff()
        blob = {
            'dataset-id': dataset_id.id,
            'updated-transitive': updated_transitive,
            'diff': diff,
        }
        pl = sxpyr.python_to_sxpr(blob, str_as_string=True)
        sxpr = pl._print(sxpyr.configure_print_plist(newline_keyword=False))
        print(sxpr)
        #pl = sxpyr.python_to_sxpr(diff, str_as_string=True)
        #sxpr = pl._print(sxpyr.configure_print_plist(newline_keyword=False))
        breakpoint()

class _WorkflowHelper:

    def _do_workflow(self, paths_to_add):
        # push button, receive bacon

        # 0. asumme there are changes to a dataset

        # 1. click upload button in main window (python get diff)
        #    argv-simple-diff -> sparcur.simple.utils for-racket diff -> path_dataset.diff()

        # 2. select specific paths for upload (python nothing)
        #    racket side selects the list of files to push (push_list) that goes into paths.sxpr
        #    which the python side then reads in the next step

        # 3. click confirm selection checkbox (python generate manifest)
        #    (ensure-directory! (push-dir)) -> updated_transitive push_id
        #    write-push-paths -> {:user-cache-path}/{dataset-uuid}/{updated-transitive}/{push-id}/paths.sxpr -> push_list
        #    argv-simple-make-push-manifest -> sparcur.simple.utils for-racket make-push-manifest -> path_dataset.make_mush_manifest()

        # 4. click push selected to remote (python push from manifest)
        #    argv-simple-push -> sparcur.simple.utils for-racket push -> path_dataset.push_from_manifest()

        # 5. TODO I think that after remote changes are made we probably want to create
        #    a modified index file that notes the changes so that incremental changes
        #    do not have to be pulled again ... of course if upstream has changed we are
        #    back in the usual world of pain ...

        path_dataset = self.dataset  # given
        dataset_id = path_dataset.cache.identifier
        # 1
        __dataset_id, updated_transitive, diff = path_dataset.diff()
        # 2
        # write the push_list to paths.sxpr
        push_id = path_dataset._write_push_list(dataset_id, updated_transitive, diff, paths_to_add)
        # 3
        path_dataset.make_push_manifest(dataset_id, updated_transitive, push_id)
        # 4
        if not self._local_only:  # this fails without remote
            import pennsieve.api.agent
            try:
                path_dataset.push_from_manifest(dataset_id, updated_transitive, push_id)
            except pennsieve.api.agent.AgentError as e:
                log.exception(e)
                pytest.skip('pennsieve error')

class TestWorkflow(_ChangesHelper, _WorkflowHelper, _TestOperation, unittest.TestCase):

    def test_workflow(self):
        # splitting d r l lets us test incremental changes
        paths_to_add_d = [
            'dire-1/dire-3-1-rn-r',  # rn
            'dire-2/dire-3-2-rp',  # rp
            'dire-2/dire-3-3-np-r',  # rnp
        ]
        paths_to_add_r = [
            'dire-1/file-1-1-rn-r.ext',  # rn
            'dire-2/file-1-2-rp.ext',  # rp
            'dire-2/file-1-3-np-r.ext',  # rnp
        ]
        paths_to_add_l = [
            'dire-1/link-1-1-rn-r.ext',  # rn
            'dire-2/link-1-2-rp.ext',  # rp
            'dire-2/link-1-3-np-r.ext',  # rnp
        ]

        paths_to_add_2 = [
            'dire-6/file-4-add.ext',  # should error for now
        ]
        self._do_workflow(paths_to_add_d)
        self._do_workflow(paths_to_add_r)
        self._do_workflow(paths_to_add_l)
        try:
            self._do_workflow(paths_to_add_2)
            raise AssertionError('should have failed due to forbidden ops')
        except ValueError:
            pass

class TestWithRemoteWorkflow(TestWorkflow):

    _local_only = False

class TestRemote(_TestOperation, unittest.TestCase):
    def test_remote_path_does_not_exist(self):
        new_thing = self.root / 'does not exist'

    @pytest.mark.skip('Not ready.')
    def test_cache_path_does_not_exist(self):
        """ This should not produce an error.
            Path objects should be able to be instantiated without
            po.exists() -> True at a point in time prior to instantiation.
        """
        new_thing = self.anchor / 'does not exist'

    def __test_cache_path_fake_id(self):
        # FIXME the right way to do this is
        np = self.project_path / 'new-path'
        np.mkdir()
        npm = np.meta

        # bad way
        class FakeBase(aug.RemotePath):
            def __init__(self, id, name, cache=None):
                super().__init__(id, cache)
                self.name = name
                now = GetTimeNow()
                self.created = now._start_time
                self.updated = now._start_time
                self.checksum = 'lolnone'
                self.chunksize = 4096
                self.file_id = 'asdfasdfasdf'

            @property
            def meta(self):
                return PathMeta(size=self.size,
                                created=self.created,
                                updated=self.updated,
                                checksum=self.checksum,
                                chunksize=self.chunksize,
                                id=self.id,
                                file_id=self.file_id)

            @property
            def parent(self):
                return None

            def _parts_relative_to(self, remote, cache_parent=None):
                return [self.name]

        # This takes way too many steps :/
        Fake = FakeBase._new(LocalPath, aug.CachePath)

        fake = Fake('lol', 'double lol')

        self.anchor / fake


import pandas as pd
from textblob import TextBlob
import multiprocessing as mp
import time

def calc(review):
    review_blob = TextBlob(str(review))
    polarity = review_blob.sentiment.polarity
    if polarity > 0:
        return "Positive"
    elif polarity < 0:
        return "Negative"
    else:
        return "Neutral"

df = pd.read_csv('googleplaystore_user_reviews.csv')

reviews = df.Translated_Review

pool = mp.Pool(processes=8)

start = time.time()
new_labels = pool.map(calc, [rev for rev in reviews])
end = time.time()
print(end - start)
pool.close()

df['new labels'] = new_labels
df.to_csv("new_data.csv")


from __future__ import absolute_import, division, print_function

import os
from setuptools import setup

HERE = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(HERE, 'README.rst')) as f:
    README = f.read()

setup(name='marv',
      version='3.2.0',
      description='MARV framework',
      long_description=README,
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          "Framework :: Flask",
          'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
          'Operating System :: POSIX :: Linux',  # for now
          'Programming Language :: Python',
          'Programming Language :: Python :: 2.7',
          'Programming Language :: Python :: 2 :: Only',  # for now
          'Programming Language :: Python :: Implementation :: CPython',  # for now
          "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
          'Topic :: Scientific/Engineering',
      ],
      author='Ternaris',
      author_email='team@ternaris.com',
      url='https://github.com/ternaris/marv',
      license='AGPLv3+',
      packages=[
          'marv',
          'marv.app',
          'marv.tests',
          'marv_detail',
          'marv_node',
          'marv_node.testing',
          'marv_node.tests',
          'marv_nodes',
          'marv_pycapnp',
          'marv_pycapnp.tests',
          'marv_store',
          'marv_webapi',
      ],
      include_package_data=True,
      zip_safe=False,
      test_suite='nose.collector',
      tests_require=['nose'],
      install_requires=['Flask-Cors',
                        'Flask-SQLAlchemy',
                        'PyJWT',
                        'bcrypt',
                        'configparser',
                        'cython',
                        'Jinja2>=2.7.3',
                        'requests-oauthlib',
                        'pycapnp-for-marv',
                        'marv-cli'],
      extras_require={
          'testing': ['coverage',
                      'ipdb',
                      'ipdbplugin',
                      'ipython',
                      'mock',
                      'nose',
                      'testfixtures'],
      },
      entry_points={'marv_cli': ['marv = marv.cli']})


'''
This module provides management methods for the pygame screen
'''

import sys
import pygame

class MetaGame(type):
    '''
    the metaclass for the game class - this implements classproperties on Game
    '''
    @property
    def clock(cls):
        '''
        produce the game clock
        '''
        return cls._clock

    @property
    def screen(cls):
        '''
        get the game screen
        '''
        return cls._screen

    @screen.setter
    def screen(cls, value):
        '''
        set the game screen
        '''
        cls._screen = value

    @property
    def scenes(cls):
        '''
        get the game scene stack
        '''
        return cls._scenes

    @scenes.setter
    def scenes(cls, value):
        '''
        set the game scene stack
        '''
        cls._scenes = value

class Game(object, metaclass=MetaGame):
    '''
    manage the pygame screen
    '''
    _clock = None
    _screen = None
    _scenes = None

    _fps_unlocked = False
    _max_fps = 0

    @classmethod
    def init(cls, title='pygame', max_fps=0):
        '''
        initialize pygame and some other important things
        '''
        # initialize pygame
        pygame.mixer.pre_init(channels=1)
        pygame.init()

        # set window caption
        pygame.display.set_caption(title)

        # initialize game clock
        cls._clock = pygame.time.Clock()
        cls._max_fps = max_fps

    @classmethod
    def main(cls):
        '''
        start the main loop of the game
        '''
        while cls._scenes:
            # get the scene on top of the scene stack
            scene = cls._scenes.peek()

            try:
                # process events
                for event in pygame.event.get():
                    if event.type == pygame.QUIT:
                        return
                    elif event.type == pygame.VIDEORESIZE:
                        cls.screen.resize(event.dict['size'])
                    elif event.type == pygame.KEYUP and event.key == pygame.K_HASH:
                        cls.screen.debug.enabled = not cls.screen.debug.enabled
                    elif event.type == pygame.KEYUP and event.key == pygame.K_EXCLAIM:
                        cls._fps_unlocked = not cls._fps_unlocked
                    else:
                        scene.on_event(event)

                # update the scenegraph objects and redraw
                scene.update()

                # flip the buffers at the given maximum refresh rate
                cls.screen.flip()
                cls._clock.tick(0 if cls._fps_unlocked else cls._max_fps)
            except StopIteration:
                Game.scenes.pop()

    @classmethod
    def quit(cls):
        '''
        quit the game
        '''
        sys.exit(0)


from torch.utils.data import *
from imutils import paths
import numpy as np
import random
import cv2
import os

CHARS = ['京', '沪', '津', '渝', '冀', '晋', '蒙', '辽', '吉', '黑',
         '苏', '浙', '皖', '闽', '赣', '鲁', '豫', '鄂', '湘', '粤',
         '桂', '琼', '川', '贵', '云', '藏', '陕', '甘', '青', '宁',
         '新',
         '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
         'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K',
         'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
         'W', 'X', 'Y', 'Z', 'I', 'O', '-'
         ]

CHARS_DICT = {char:i for i, char in enumerate(CHARS)}

class LPRDataLoader(Dataset):
    def __init__(self, img_dir, imgSize, lpr_max_len, PreprocFun=None):
        self.img_dir = img_dir
        self.img_paths = []
        for i in range(len(img_dir)):
            self.img_paths += [el for el in paths.list_images(img_dir[i])]
        random.shuffle(self.img_paths)
        self.img_size = imgSize
        self.lpr_max_len = lpr_max_len
        if PreprocFun is not None:
            self.PreprocFun = PreprocFun
        else:
            self.PreprocFun = self.transform

    def __len__(self):
        return len(self.img_paths)

    def __getitem__(self, index):
        filename = self.img_paths[index]
        Image = cv2.imread(filename)
        height, width, _ = Image.shape
        if height != self.img_size[1] or width != self.img_size[0]:
            Image = cv2.resize(Image, self.img_size)
        Image = self.PreprocFun(Image)

        basename = os.path.basename(filename)
        imgname, suffix = os.path.splitext(basename)
        imgname = imgname.split("-")[0].split("_")[0]
        label = list()
        for c in imgname:
            # one_hot_base = np.zeros(len(CHARS))
            # one_hot_base[CHARS_DICT[c]] = 1
            label.append(CHARS_DICT[c])

        if len(label) == 8:
            if self.check(label) == False:
                print(imgname)
                assert 0, "Error label ^~^!!!"

        return Image, label, len(label)

    def transform(self, img):
        img = img.astype('float32')
        img -= 127.5
        img *= 0.0078125
        img = np.transpose(img, (2, 0, 1))

        return img

    def check(self, label):
        if label[2] != CHARS_DICT['D'] and label[2] != CHARS_DICT['F'] \
                and label[-1] != CHARS_DICT['D'] and label[-1] != CHARS_DICT['F']:
            print("Error label, Please check!")
            return False
        else:
            return True