import PAsearchSites
import PAgenres

def search(results,encodedTitle,title,searchTitle,siteNum,lang,searchDate):
    searchResults = HTML.ElementFromURL(PAsearchSites.getSearchSearchURL(siteNum) + encodedTitle)
    for searchResult in searchResults.xpath('//div[@class="updateItem"] | //div[@class="photo-thumb video-thumb"]'):
        titleNoFormatting = searchResult.xpath('.//h4//a | .//p[@class="thumb-title"]')[0].text_content().strip()
        curID = searchResult.xpath('.//a')[0].get('href').replace('/', '_').replace('?', '!')
        releaseDate = parse(searchResult.xpath('.//span[@class="update_thumb_date"] | .//span[@class="date"]')[0].text_content().strip()).strftime('%Y-%m-%d')

        actors = searchResult.xpath('.//span[@class="tour_update_models"]//a | .//p[@class="model-name"]//a')
        firstActor = actors[0].text_content().strip()
        numActors = len(actors) - 1

        if searchDate:
            score = 100 - Util.LevenshteinDistance(searchDate, releaseDate)
        else:
            score = 100 - Util.LevenshteinDistance(searchTitle.lower(), titleNoFormatting.lower())

        results.Append(MetadataSearchResult(id='%s|%d' % (curID, siteNum), name='%s + %d in %s [%s] %s' % (firstActor, numActors, titleNoFormatting, PAsearchSites.getSearchSiteName(siteNum), releaseDate), score = score, lang = lang))

    return results

def update(metadata,siteID,movieGenres,movieActors):
    metadata_id = str(metadata.id).split('|')
    url = metadata_id[0].replace('_', '/').replace('!', '?')
    detailsPageElements = HTML.ElementFromURL(url)

    # Title
    metadata.title = detailsPageElements.xpath('//span[@class="update_title"] | //p[@class="raiting-section__title"]')[0].text_content().strip()

    # Studio/Tagline/Collection
    metadata.collections.clear()
    metadata.studio = 'AllHerLuv/MissaX'
    siteName = PAsearchSites.getSearchSiteName(siteID)
    metadata.tagline = siteName
    metadata.collections.add(siteName)

    # Summary
    try:
        metadata.summary = detailsPageElements.xpath('//span[@class="latest_update_description"] | //p[contains(@class,"text")]')[0].text_content().replace("Includes:","").replace("Synopsis:","").strip()
    except:
        Log('No summary found')

    # Date
    try:
        date = detailsPageElements.xpath('//span[@class="update_date"]')[0].text_content().strip()
    except:
        date = detailsPageElements.xpath('//p[@class="dvd-scenes__data"]')[0].text_content().split('|')[1].replace('Added:', '').strip()
    date_object = parse(date)
    metadata.originally_available_at = date_object
    metadata.year = metadata.originally_available_at.year

    # Actors
    movieActors.clearActors()
    actors = detailsPageElements.xpath('//div[@class="update_block"]//span[@class="tour_update_models"]//a | //p[@class="dvd-scenes__data"][1]//a')
    for actorLink in actors:
        actorName = actorLink.text_content().strip()
        actorPageURL = actorLink.get('href')
        actorPageElements = HTML.ElementFromURL(actorPageURL)
        actorPhotoURL = actorPageElements.xpath('//img[contains(@class, "model_bio_thumb")]/@src0_1x')[0]
        movieActors.addActor(actorName, actorPhotoURL)

    # Genres
    movieGenres.clearGenres()
    genres = detailsPageElements.xpath('//span[@class="tour_update_tags"]//a | //p[@class="dvd-scenes__data"][2]//a')
    for genreLink in genres:
        genre = genreLink.text_content()
        movieGenres.addGenre(genre)

    # Posters/Background
    art = [
        detailsPageElements.xpath('//img[contains(@class, "update_thumb")]/@src0_1x')[0]
    ]

    Log('Artwork found: %d' % len(art))
    for idx, posterUrl in enumerate(art, 1):
        if not PAsearchSites.posterAlreadyExists(posterUrl, metadata):
            # Download image file for analysis
            try:
                img_file = urllib.urlopen(posterUrl)
                im = StringIO(img_file.read())
                resized_image = Image.open(im)
                width, height = resized_image.size
                # Add the image proxy items to the collection
                if width > 1:
                    # Item is a poster
                    metadata.posters[posterUrl] = Proxy.Media(HTTP.Request(posterUrl, headers={'Referer': 'http://www.google.com'}).content, sort_order=idx)
                if width > 100:
                    # Item is an art item
                    metadata.art[posterUrl] = Proxy.Media(HTTP.Request(posterUrl, headers={'Referer': 'http://www.google.com'}).content, sort_order=idx)
            except:
                pass

    return metadata


# -*- coding: utf-8 -*-
"""
Created on Sun Aug  6 12:43:47 2017

@author: gansc
"""

import numpy as np
import csv
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
import cv2

class ImageGenerator(object):
    ''' Fetches all images from the driving similar in path + set
        and returns them as batch for training after preprocessing.
        Images from left and right camera are returned with a
        corrected steering angle
    '''
    def __init__(self, path, sets, batch_size=32, train_val_split=0.2):
        ''' Initialize samples with all files from .csv.
        '''
        self._samples = []
        self._correction = 0.09
        self._batch_size = batch_size
        self._cameras = ['center']
        for set in sets:
            with open(path + '/' + set + '/driving_log.csv') as csvfile:
                reader = csv.reader(csvfile)
                for line in reader:
                    self._samples.append(line)

        self._train_samples, self._valid_samples = train_test_split(self._samples, test_size=train_val_split)

    def train_data_gen(self):
        yield from self._data_gen(self._train_samples)

    def valid_data_gen(self, batch_size=32):
        yield from self._data_gen(self._valid_samples)

    def _read_image(self, file):
        name = './data/' + '/'.join(file.replace('\\', '/').split('/')[-3:])
        image = cv2.imread(name)
        return cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    def _data_gen(self, samples):
        ''' Generator to return batches of training / validation data
        '''
        batch_size = self._batch_size
        num_samples = len(samples)
        while 1: # Loop forever so the generator never terminates
            shuffle(samples)
            images = []
            cmds = []
            for i in range(batch_size):
                image_center = self._read_image(samples[i][0])
                angle_center, throttle, brake = [float(s) for s in samples[i][3:6]]
                images.append(image_center)
                cmds.append([angle_center, throttle, brake])

                image_center_flipped = np.fliplr(image_center)
                angle_center_flipped = -angle_center
                images.append(image_center_flipped)
                cmds.append([angle_center_flipped, throttle, brake])

                image_left = self._read_image(samples[i][1])
                angle_left = angle_center + self._correction
                images.append(image_left)
                cmds.append([angle_left, throttle, brake])

                image_right = self._read_image(samples[i][2])
                angle_right = angle_center - self._correction
                images.append(image_right)
                cmds.append([angle_right, throttle, brake])

            X_train = np.array(images)
            y_train = np.array(cmds)

            yield shuffle(X_train, y_train)

    def train_data_len(self):
        return len(self._train_samples) / self._batch_size

    def valid_data_len(self):
        return len(self._valid_samples) / self._batch_size

#############################

if __name__ == '__main__':
    import matplotlib.pyplot as plt

    img_gen = ImageGenerator(r'.\data', ['set1'], batch_size=1)
    train_gen = img_gen.train_data_gen()
    img,angle = next(train_gen)

    for i in img:
        plt.imshow(i)
        plt.show()


#Read the numbers from randomnum.txt and output some information

def main():
    #Perform the main body of the function

    #Open the file and catch the error
    try:
        num_file = open('randomnum.txt', 'r')

        #Begin reading and outputting the numbers in the file
        print("List of random numbers in randomnum.txt:")

        #Declare a variable to keep count of the amount of numbers
        line_count = 0

        #Print the numbers
        for line in num_file:
            line.rstrip('\n')
            print(line)
            line_count += 1

        #Print the number count
        print('Random number count:', line_count)

        input()

    except IOError:
        print("The file does not exist. Run randomwrite.py to generate some random numbers.")

main()


import random as r
from time import sleep as s

dragonNames = ['Sobek', 'Viper', 'Glycon', 'Typhon', 'Errol', 'Ramoth', 'Thorn', 'Eldrax', 'Pyre', 'Alduim']
colors = ['red', 'orange', 'yellow', 'blue', 'green', 'violet', 'cyan', 'magenta']

class Base():
    htp = 100
    att = 10
    dfc = 5
    dog = 5
    sta = 10
    spd = 10
    hap = 100
    name = 'Base'
    color = 'Base'
    xp = 0
    lvl = 1

class Dragon(Base):
    pass

class Level(Base):
    htp = 100
    win = None
    def __init__(self, level):
        if level == 1:
            self.att = r.randint(10, 17)
            self.dfc = r.randint(5, 12)
            self.dog = r.randint(6, 11)
            self.spd = r.randint(9, 12)
            self.sta = r.randint(9, 12)
            self.name = r.choice(dragonNames)
            self.color = r.choice(colors)
        elif level == 2:
            self.att = r.randint(23, 32)
            self.dfc = r.randint(17, 25)
            self.dog = r.randint(16, 21)
            self.spd = r.randint(17, 19)
            self.sta = r.randint(17, 19)
            self.name = r.choice(dragonNames)
            self.color = r.choice(colors)
        elif level == 3:
            self.att = r.randint(36, 43)
            self.dfc = r.randint(26, 31)
            self.dog = r.randint(28, 34)
            self.spd = r.randint(26, 30)
            self.sta = r.randint(26, 30)
            self.name = r.choice(dragonNames)
            self.color = r.choice(colors)
        elif level == 'bronze':
            self.att = 18
            self.dfc = 7
            self.dog = 13
            self.spd = 19
            self.sta = 13
            self.name = 'Mushu'
            self.color = 'orange'
        elif level == 'sliver':
            self.att = 26
            self.dfc = 14
            self.dog = 21
            self.spd = 29
            self.sta = 24
            self.name = 'Saphira'
            self.color = 'blue'
        elif level == 'gold':
            self.att = 35
            self.dfc = 19
            self.dog = 27
            self.spd = 35
            self.sta = 30
            self.name = 'Jade'
            self.color = 'cyan'
        elif level == 'crystal':
            self.att = 50
            self.dfc = 25
            self.dog = 40
            self.spd = 50
            self.sta = 40
            self.name = 'Valyrym'
            self.color = 'red'

class highScores():
        attack = 10000
        defense = 10000
        dodge = 10000
        speed = 10000
        stamina = 10000


"""Test module for data_interface"""

import os
from dataclasses import dataclass
import unittest
import pandas as pd
from torch.utils.data import Dataset
from pymarlin.core.data_interface import DataInterface, DataProcessor

@dataclass
class MyArgs:
    filepath_train: str = os.path.join("outputs", "file1.csv")
    filepath_test: str = os.path.join("outputs", "file2.csv")
    text_field: str = "text"
    label_field: str = "label"

class MyDataset(Dataset):
    def __init__(self, df, text_field, label_field):
        self.df = df
        self.text_field = text_field
        self.label_field = label_field

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        row = self.df.iloc[idx]
        return (row[self.text_field], row[self.label_field])

class MyData(DataInterface):

    def __init__(self):
        self._train_ds = None
        self._val_ds = None

    def setup_datasets(self, train_ds, val_ds):
        self._train_ds = train_ds
        self._val_ds = val_ds

    def get_train_dataset(self):
        return self._train_ds

    def get_val_dataset(self):
        return self._val_ds

class MyDataProcessor(DataProcessor):

    def __init__(self, args):
        super().__init__()
        self.args = args

    def process(self):
        df = pd.read_csv(self.args.filepath_train)
        return MyDataset(df, self.args.text_field,
                         self.args.label_field)

class MyDataMultiProcessor(DataProcessor):

    def __init__(self, args):
        super().__init__()
        self.args = args

    def process(self, filename):
        df = pd.read_csv(filename)
        return MyDataset(df, self.args.text_field,
                         self.args.label_field)

class TestDataInterface(unittest.TestCase):

    def setUp(self):

        self.args = MyArgs()
        self.data_interface = MyData()
        self.data_processor = MyDataProcessor(self.args)
        self.data_multiprocessor = MyDataMultiProcessor(self.args)

        if not os.path.exists("outputs"):
            os.makedirs("outputs")

    def tearDown(self):
        for f in os.listdir("outputs"):
            if f.endswith(".csv"):
                os.remove(os.path.join("outputs", f))

    def test_process_data(self):
        df = pd.DataFrame({self.args.text_field: ['one', 'two'],
                           self.args.label_field: [1, 2]})
        df.to_csv(self.args.filepath_train)

        train_ds = self.data_processor.process_data()
        assert train_ds[0] == ('one', 1)
        assert len(train_ds) == 2

    def test_multi_process_data(self):
        df1 = pd.DataFrame({self.args.text_field: ['one', 'two'],
                            self.args.label_field: [1, 2]})
        df2 = pd.DataFrame({self.args.text_field: ['three', 'four', 'five'],
                            self.args.label_field: [3, 4, 5]})
        df1.to_csv(self.args.filepath_train)
        df2.to_csv(self.args.filepath_test)

        train_ds_list = self.data_multiprocessor.multi_process_data(
            [self.args.filepath_train, self.args.filepath_test],
            process_count=2)
        assert train_ds_list[0] == ('one', 1)
        assert train_ds_list[2] == ('three', 3)
        assert len(train_ds_list) == 5

    def test_collect_params(self):
        a_number = 1
        single_argument = 'single_argument'
        list_to_split = ['first', 'second', 'third']
        a_dict = {'test': 'dict'}
        list_not_to_split = [['this', 'lists', 'elements', 'dont', 'split']]
        self.data_processor._set_ranks()
        list_params = self.data_processor._collect_params(
            a_number,
            single_argument,
            list_to_split,
            a_dict,
            list_not_to_split
        )
        assert len(list_params) == 5
        assert list_params[2][0] == list_to_split[0]
        assert list_params[2][1] == list_to_split[1]
        assert list_params[2][2] == list_to_split[2]

        for param in list_params[1]:
            assert param == single_argument

        for param in list_params:
            assert len(param) == 3


"""----------------------------------------------------------------------------
Project Euler
Gregory Gundersen
2013-01-24

Problem:
We shall say that an n-digit number is pandigital if it makes use of all the
digits 1 to n exactly once; for example, the 5-digit number, 15234, is 1
through 5 pandigital. The product 7254 is unusual, as the identity, 39 * 186 =
7254, containing multiplicand, multiplier, and product is 1 through 9
pandigital. Find the sum of all products whose multiplicand/multiplier/product
identity can be written as a 1 through 9 pandigital.

HINT: Some products can be obtained in more than one way so be sure to only
include it once in your sum.
----------------------------------------------------------------------------"""

import gmath

def main():
    result = []
    for x in range(1, 60):
        for y in range(1, 2000):
            p = x*y
            z = str(x) + str(y) + str(p)
            if gmath.is_pandigital(z) and len(z) == 9 and p not in result:
                result.append(p)
    return sum(result)


#!/usr/bin/env python
#!-*-coding:utf-8 -*-
"""
@version: python3.7
@author: v-enshi
@license: Apache Licence
@contact: 123@qq.com
@site:
@software: PyCharm
@file: model_traditional_attn.py
@time: 2019/4/26 17:51
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.nn.utils.rnn as rnn_utils

import random
import numpy as np

torch.manual_seed(1)
use_gpu = False
#use_gpu = True

##1. parameters setting
if use_gpu:
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

## 3.1 LSTM component
class LSTM_component(nn.Module):

    def __init__(self, vocab_value_size, value_dim, vocab_type_size, type_dim,
                 hidden_dim, batch_size, context_window, dropout_p=0.5):
        super(LSTM_component, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        self.context_window = context_window
        self.value_embeddings = nn.Embedding(vocab_value_size, value_dim, padding_idx=0)

        self.type_embeddings = nn.Embedding(vocab_type_size, type_dim, padding_idx=0)
        self.dropout = nn.Dropout(dropout_p)
        self.lstm = nn.LSTM(value_dim + type_dim, hidden_dim, batch_first=True)

    def forward(self, sentence, setence_len, hc):
        embeds_type = self.type_embeddings(sentence[0])

        embeds_value = self.value_embeddings(sentence[1])
        embeds = torch.cat([embeds_value, embeds_type], 2)
        h0 = hc[0]
        c0 = hc[1]

        embeded = rnn_utils.pack_padded_sequence(embeds, setence_len, batch_first=True)

        lstm_out, (lstm_h, lstm_c) = self.lstm(embeded, (h0, c0))
        lstm_out, _ = torch.nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)

        return lstm_out, lstm_h, lstm_c

    def initHidden(self):
        return torch.randn(1, self.batch_size, self.hidden_dim, device=device), torch.randn(1, self.batch_size,
                                                                                self.hidden_dim,
                                                                                device=device)

'''

#model_LSTM = LSTM_component(20,EMBEDDING_value,10,EMBEDDING_type,HIDDEN_SIZE, BATCH_SIZE, CONTEXT_WINDOW)

model_LSTM = LSTM_component(20,2,20,3,5, 2, 3)
with torch.no_grad():
    inputs = [torch.tensor([[6, 6, 5, 3], [7, 4, 0, 0]]), torch.tensor([[3, 3, 2, 1], [4, 1, 0, 0]])]
    output, hn,cn = model_LSTM(inputs,[4,2],model_LSTM.initHidden())
    #print("tag_scorces", output)
'''

## 3.2 attention component
class Context_atten(nn.Module):
    def __init__(self, hidden_dim, context_window, dropout_p=0.25):
        super(Context_atten, self).__init__()
        self.hidden_dim = hidden_dim
        self.context_window = context_window

        self.attn = nn.Linear(self.hidden_dim * 2, hidden_dim)
        self.v = nn.Parameter(torch.rand(hidden_dim))
        self.linear1 = nn.Linear(hidden_dim, hidden_dim, bias=False)

    def forward(self, inputs, setence_len, hc, parent):
        batch_size = inputs.size(0)
        timestep = self.context_window

        Mt = inputs[0, setence_len[0] - timestep: setence_len[0], :].unsqueeze(0)
        par = inputs[0, setence_len[0] - parent[0] - 1, :].unsqueeze(0)

        for i in range(batch_size - 1):
            Mt = torch.cat([Mt, inputs[i + 1, setence_len[i + 1] - timestep: setence_len[i + 1], :].unsqueeze(0)],
                           0)  # [B,T,H]
            par = torch.cat([par, inputs[i + 1, setence_len[i + 1] - parent[i + 1] - 1, :].unsqueeze(0)], 0)  # [B,H]

        h = hc.repeat(timestep, 1, 1).transpose(0, 1)  # hc[1,B,H] -> [B,T,H]
        attn_weights = self.score(h, Mt).unsqueeze(1)  # [B*T*1]
        context = attn_weights.bmm(Mt)  # (B,1,H)
        return attn_weights, context, par

    def score(self, hidden, attn_outputs):
        energy = torch.tanh(self.attn(torch.cat([hidden, attn_outputs], 2)))  # [B*T*2H]->[B*T*H]
        energy = energy.transpose(1, 2)  # [B*H*T]
        v = self.v.repeat(attn_outputs.size(0), 1).unsqueeze(1)  # [B*1*H]
        energy = torch.bmm(v, energy)  # [B*1*T] a
        energy = F.softmax(energy, dim=2)
        return energy.squeeze(1)  # [B*T]

'''
    def score(self, hidden, attn_outputs):
        # [B*T*2H]->[B*T*H]
        energy = F.softmax(self.attn(torch.cat([hidden, attn_outputs], 2)),dim = 2)
        energy = energy.transpose(1, 2)  # [B*H*T]
        v = self.v.repeat(attn_outputs.size(0), 1).unsqueeze(1)  # [B*1*H]
        energy = torch.bmm(v, energy)  # [B*1*T] a
        return energy.squeeze(1)  # [B*T]

'''

'''
HIDDEN_SIZE = 5
CONTEXT_WINDOW = 2
model_catten = Context_atten(HIDDEN_SIZE, CONTEXT_WINDOW)

with torch.no_grad():
    alpha , out_ct,parent = model_catten (output,[3,2], hn,[0,1])
    #print(out_ct)

'''

## 3.3parent attention
class Parent_atten(nn.Module):
    def __init__(self, hidden_dim, context_window):
        super(Parent_atten, self).__init__()
        self.hidden_dim = hidden_dim
        self.context_window = context_window
        self.Wg_linear = nn.Linear(hidden_dim * 3, hidden_dim, bias=False)
        self.Wv_linear = nn.Linear(hidden_dim, self.context_window)

    def forward(self, ht, ct, pt):
        # ht[1,B,H] ct[B,1,H] pt[B,H]
        ct = ct.transpose(0, 1)  # (1,B,H)
        pt = pt.unsqueeze(0)

        Gt = torch.tanh(self.Wg_linear(torch.cat([ht, ct, pt], 2)))
        yt = F.log_softmax(self.Wv_linear(Gt), dim=1)
        return yt

'''
model_par_atten_type = Parent_atten(HIDDEN_SIZE, CONTEXT_WINDOW)
with torch.no_grad():
    Yt_type = model_par_atten_type  ( hn,out_ct,parent)
    print(Yt_type)

'''

## 3 main model
class main_model(nn.Module):
    def __init__(self, vocab_value_size, value_dim, vocab_type_size, type_dim,
                 hidden_dim, batch_size, context_window, dropout_p=0.25):
        super(main_model, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        self.context_window = context_window

        self.model_LSTM = LSTM_component(vocab_value_size, value_dim, vocab_type_size, type_dim, hidden_dim, batch_size,
                                         context_window).to(device)
        self.model_catten = Context_atten(hidden_dim, context_window).to(device)
        self.model_par_atten_value = Parent_atten(hidden_dim, vocab_value_size).to(device)

    def forward(self, inputs, input_len, hc, parent):
        output, hn, cn = self.model_LSTM(inputs, input_len, hc)
        alpha, out_ct, par = self.model_catten(output, input_len, hn, parent)
        Yt = self.model_par_atten_value(hn, out_ct, par)
        return Yt

    def initHidden(self):
        # return torch.zeros(1, self.batch_size, self.hidden_dim, device=device), torch.zeros(1, self.batch_size,self.hidden_dim, device=device)
        return torch.randn(1, self.batch_size, self.hidden_dim, device=device), torch.randn(1, self.batch_size,
                                                                                self.hidden_dim,
                                                                                device=device)


"""
Issues:
    * Delete con_info and con_transfers when con is closed
    * Should contract also be deleted when its transfered? Prob
    * To do: add a clean up routine based on old cons

    Todo: implement RST for -- will probably need to look at  transfer_request_handlers
"""

import pyp2p.unl
import pyp2p.net
import pyp2p.dht_msg
import logging
import storjnode.storage as storage
from collections import OrderedDict
from btctxstore import BtcTxStore
import tempfile
import time
import json
import hashlib
import sys
import os
import binascii
import struct
from threading import Lock
from twisted.internet import defer

mutex = Lock()

_log = logging.getLogger(__name__)

class RequestDenied(Exception):
    pass

class TransferError(Exception):
    pass

def process_transfers(client):
    _log.debug("In process transfers")

    # Process contract messages.
    try:
        if client.net.dht_node.has_messages():
            for msg in client.net.dht_node.get_messages():
                _log.debug("Processing: " + msg["message"])
                client.protocol(msg["message"])
    except Exception as e:
        _log.debug(e)
        pass

    # Process new connections.
    client.net.synchronize()

    # Raise appropriate async callbacks for errors.
    for con in list(client.con_info):
        if not con.connected:
            # Broken connections.
            for contract_id in list(client.con_info[con]):
                if contract_id in client.defers:
                    e = TransferError("Connection died.")
                    client.defers[contract_id].errback(e)
                    del client.defers[contract_id]

                    if contract_id in client.contracts:
                        del client.contracts[contract_id]

    # Expired handshakes.
    for contract_id in list(client.contracts):
        if contract_id in client.handshake:
            handshake = client.handshake[contract_id]
            elapsed = time.time() - handshake["timestamp"]
            if elapsed >= 30:
                if contract_id in client.defers:
                    e = RequestDenied("Handshake timed out.")
                    client.defers[contract_id].errback(e)
                    del client.defers[contract_id]

    # Todo: cleanup con info and other structures.

    # Process connections.
    for con in client.net:
        _log.debug("In con.")

        # This is an new (or old) connection.
        if con not in client.con_info:
            _log.debug("Con not in con_info")
            continue

        # Connection is not synchronized yet.
        if con.nonce == None:
            _log.debug("Not synced yet")
            continue

        # Wait until there's new transfers to process.
        if not client.is_queued(con):
            # Socket has hung ungracefully.
            duration = time.time() - con.alive
            if duration >= 15.0:
                _log.debug("Ungraceful socket close")
                con.close()
                break

            _log.debug("Not queued: skipping")
            continue

        # Get active contract ID.
        if con not in client.con_transfer:
            _log.debug("Con not in con_Transfer")
            continue

        contract_id = client.con_transfer[con]
        _log.debug("Contract id =")
        _log.debug(contract_id)
        if len(contract_id) < 64:
            remaining = 64 - len(contract_id)
            _log.debug("Blocking = " + str(con.blocking))
            partial = con.recv(remaining)
            _log.debug("Contract id chunk = " + partial)
            _log.debug("Connected = " + str(con.connected))
            if not len(partial):
                _log.debug("Did not receive contract id")
                continue

            client.con_transfer[con] += partial
            _log.debug("Skipping contract id.")
            continue

        # Reached end of transfer queue.
        if contract_id == u"0" * 64:
            _log.debug("Skippng end of queue")
            continue

        # Anything left to do?
        if contract_id not in client.con_info[con]:
            continue
        con_info = client.con_info[con][contract_id]
        if not con_info["remaining"]:
            _log.debug("Skipping remaining.")
            continue

        # Upload.
        contract = client.contracts[contract_id]
        transfer_complete = 0
        if client.net.unl == pyp2p.unl.UNL(value=contract["host_unl"]):
            _log.debug("Uploading: Found our UNL")

            # Send file size.
            if not con_info["file_size"]:
                # Get file size.
                path = storage.manager.find(
                    client.store_config,
                    contract["data_id"]
                )
                file_size = os.path.getsize(path)
                con_info["file_size"] = file_size
                con_info["remaining"] = file_size

                # Marshal file size for network.
                if sys.version_info >= (3, 0, 0):
                    net_file_size = struct.pack(
                        "<20s",
                        str(file_size).encode("ascii")
                    )
                else:
                    net_file_size = struct.pack(
                        "<20s",
                        str(file_size)
                    )

                # Send file size.
                con.send(net_file_size, send_all=1)

            # Get next chunk from file.
            position = con_info["file_size"] - con_info["remaining"]
            data_chunk = client.get_data_chunk(
                contract["data_id"],
                position
            )

            # Upload chunk binary to socket.
            bytes_sent = con.send(data_chunk)
            _log.debug(bytes_sent)
            if bytes_sent:
                con_info["remaining"] -= bytes_sent

            _log.debug("Remaining = ")
            _log.debug(con_info["remaining"])

            # Everything uploaded.
            if not con_info["remaining"]:
                transfer_complete = 1
        else:
            _log.debug("Attempting to download.")

            # Get file size.
            if not con_info["file_size"]:
                file_size_buf = con_info["file_size_buf"]
                if len(file_size_buf) < 20:
                    remaining = 20 - len(file_size_buf)
                    partial = con.recv(remaining)
                    if not len(partial):
                        continue

                    file_size_buf += partial
                    if len(file_size_buf) == 20:
                        file_size, = struct.unpack("<20s", file_size_buf)
                        file_size = int(file_size_buf.rstrip(b"\0"))
                        con_info["file_size"] = file_size
                        con_info["remaining"] = file_size
                    else:
                        continue

            # Download.
            data = con.recv(
                con_info["remaining"],
                encoding="ascii"
            )
            _log.debug(con.connected)

            if len(data):
                con_info["remaining"] -= len(data)
                client.save_data_chunk(contract["data_id"], data)

            _log.debug("Remaining = ")
            _log.debug(con_info["remaining"])

            # When done downloading close con.
            if not con_info["remaining"]:
                # Check download.
                data_id = contract["data_id"]
                temp_path = client.downloading[data_id]
                with open(temp_path, "rw") as shard:
                    # Delete file if it doesn't hash right!
                    found_hash = storage.shard.get_id(shard)
                    if found_hash != data_id:
                        _log.debug(found_hash)
                        _log.debug(data_id)
                        _log.debug("Error: downloaded file doesn't hash right!")
                        os.remove(temp_path)
                        continue

                    # Move shard to storage.
                    storage.manager.add(
                        client.store_config,
                        shard
                    )

                # Remove that we're downloading this.
                del client.downloading[data_id]

                # Ready for a new transfer (if there are any.)
                transfer_complete = 1

        their_unl = client.get_their_unl(contract)
        is_master = client.net.unl.is_master(their_unl)
        _log.debug("Is master = " + str(is_master))
        if transfer_complete:
            # Return async success.
            if contract_id in client.defers:
                # Call any callbacks registered with this defer.
                client.defers[contract_id].callback(client.success_value)
                del client.defers[contract_id]

            if is_master:
                client.queue_next_transfer(con)
            else:
                client.con_transfer[con] = u""

class FileTransfer:
    def __init__(self, net, wif=None, store_config=None, handlers=None):
        # Accept direct connections.
        self.net = net

        # Returned by callbacks.
        self.success_value = ("127.0.0.1", 7777)

        # Used for signing messages.
        self.wallet = BtcTxStore(testnet=True, dryrun=True)
        self.wif = wif or self.wallet.create_key()

        # Where will the data be stored?
        self.store_config = store_config
        assert(len(list(store_config)))

        # Handlers for certain events.
        self.handlers = handlers

        # Start networking.
        if not self.net.is_net_started:
            self.net.start()

        # Dict of data requests.
        self.contracts = {}

        # Dict of defers for contracts.
        self.defers = {}

        # Three-way handshake status for contracts.
        self.handshake = {}

        # All contracts associated with this connection.
        self.con_info = {}

        # File transfer currently active on connection.
        self.con_transfer = {}

        # List of active downloads.
        # (Never try to download multiple copies of the same thing at once.)
        self.downloading = {}

    def get_their_unl(self, contract):
        if self.net.unl == pyp2p.unl.UNL(value=contract["dest_unl"]):
            their_unl = contract["src_unl"]
        else:
            their_unl = contract["dest_unl"]

        return their_unl

    def is_queued(self, con=None):
        if con is not None:
            if con not in self.con_info:
                return 0

        if con is None:
            con_list = list(self.con_info)
        else:
            con_list = [con]

        for con in con_list:
            for contract_id in list(self.con_info[con]):
                con_info = self.con_info[con][contract_id]
                if con_info["remaining"]:
                    return 1

        return 0

    def cleanup_transfers(self, con):
        # Close con - there's nothing left to download.
        if not self.is_queued(con):
            # Cleanup con transfers.
            if con in self.con_transfer:
                del self.con_transfer[con]

            # Cleanup con_info.
            if con in self.con_info:
                del self.con_info[con]

            # Todo: cleanup contract + handshake state.

    def queue_next_transfer(self, con):
        _log.debug("Queing next transfer")
        for contract_id in list(self.con_info[con]):
            con_info = self.con_info[con][contract_id]
            if con_info["remaining"]:
                self.con_transfer[con] = contract_id
                con.send(contract_id, send_all=1)
                return

        # Mark end of transfers.
        self.con_transfer[con] = u"0" * 64

    def is_valid_syn(self, msg):
        # List of expected fields.
        syn_schema = (
            u"status",
            u"direction",
            u"data_id",
            u"file_size",
            u"host_unl",
            u"dest_unl",
            u"src_unl",
            u"signature"
        )

        # Check all fields exist.
        if not all(key in msg for key in syn_schema):
            _log.debug("Missing required key.")
            return 0

        # Check SYN size.
        if len(msg) > 5242880: # 5 MB.
            _log.debug("SYN is too big")
            return 0

        # Check direction is valid.
        direction_tuple = (u"send", u"receive")
        if msg[u"direction"] not in direction_tuple:
            _log.debug("Missing required direction tuple.")
            return 0

        # Check the UNLs are valid.
        unl_tuple = (u"host_unl", u"dest_unl", u"src_unl")
        for unl_key in unl_tuple:
            if not pyp2p.unl.is_valid_unl(msg[unl_key]):
                _log.debug("Invalid UNL for " + unl_key)
                _log.debug(msg[unl_key])
                return 0

        # Check file size.
        file_size_type = type(msg[u"file_size"])
        if sys.version_info >= (3, 0, 0):
            expr = file_size_type != int
        else:
            expr = file_size_type != int and file_size_type != long
        if expr:
            _log.debug("File size validation failed")
            _log.debug(type(msg[u"file_size"]))
            return 0

        # Are we the host?
        if self.net.unl == pyp2p.unl.UNL(value=msg[u"host_unl"]):
            # Then check we have this file.
            path = storage.manager.find(self.store_config, msg[u"data_id"])
            if path is None:
                _log.debug("Failed to find file we're uploading")
                return 0
        else:
            # Do we already have this file?
            path = storage.manager.find(self.store_config, msg[u"data_id"])
            if path is not None:
                _log.debug("Attempting to download file we already have")
                return 0

            # Are we already trying to download this?
            if msg[u"data_id"] in self.downloading:
                _log.debug("We're already trying to download this")
                return 0

        return 1

    def protocol(self, msg):
        msg = json.loads(msg, object_pairs_hook=OrderedDict)

        # Associate TCP con with contract.
        def success_wrapper(self, contract_id, host_unl):
            def success(con):
                with mutex:
                    _log.debug("IN SUCCESS CALLBACK")
                    _log.debug("Success() contract_id = " + str(contract_id))

                    # Associate TCP con with contract.
                    contract = self.contracts[contract_id]
                    file_size = contract["file_size"]

                    # Store con association.
                    if con not in self.con_info:
                        self.con_info[con] = {}

                    # Associate contract with con.
                    if contract_id not in self.con_info[con]:
                        self.con_info[con][contract_id] = {
                            "contract_id": contract_id,
                            "remaining": 350, # Tree fiddy.
                            "file_size": file_size,
                            "file_size_buf": b""
                        }

                    # Record download state.
                    data_id = contract["data_id"]
                    if self.net.unl != pyp2p.unl.UNL(value=host_unl):
                        _log.debug("Success: download")
                        fp, self.downloading[data_id] = tempfile.mkstemp()
                    else:
                        # Set initial upload for this con.
                        _log.debug("Success: upload")

                    # Queue first transfer.
                    their_unl = self.get_their_unl(contract)
                    is_master = self.net.unl.is_master(their_unl)
                    _log.debug("Is master = " + str(is_master))
                    if con not in self.con_transfer:
                        if is_master:
                            # A transfer to queue processing.
                            self.queue_next_transfer(con)
                        else:
                            # A transfer to receive (unknown.)
                            self.con_transfer[con] = u""
                    else:
                        if self.con_transfer[con] == u"0" * 64:
                            if is_master:
                                self.queue_next_transfer(con)
                            else:
                                self.con_transfer[con] = u""
            return success

        # Sanity checking.
        if u"status" not in msg:
            return

        # Accept data request.
        if msg[u"status"] == u"SYN":
            # Check syn is valid.
            if not self.is_valid_syn(msg):
                _log.debug("SYN: invalid syn.")
                return

            # Save contract.
            contract_id = self.contract_id(msg)
            self.save_contract(msg)
            self.handshake[contract_id] = {
                "state": u"SYN-ACK",
                "timestamp": time.time()
            }

            # Create reply.
            reply = OrderedDict({
                u"status": u"SYN-ACK",
                u"syn": msg,
            })

            # Sign reply.
            reply = self.sign_contract(reply)

            # Save reply.
            self.send_msg(reply, msg[u"src_unl"])
            _log.debug("SYN")

        # Confirm accept and make connection if needed.
        if msg[u"status"] == u"SYN-ACK":
            # Valid syn-ack?
            if u"syn" not in msg:
                _log.debug("SYN-ACK: syn not in msg.")
                return

            # Is this a reply to our SYN?
            contract_id = self.contract_id(msg[u"syn"])
            if contract_id not in self.contracts:
                _log.debug("--------------")
                _log.debug(msg)
                _log.debug("--------------")
                _log.debug(self.contracts)
                _log.debug("--------------")
                _log.debug("SYN-ACK: contract not found.")
                return

            # Check syn is valid.
            if not self.is_valid_syn(msg[u"syn"]):
                _log.debug("SYN-ACK: invalid syn.")
                return

            # Did I sign this?
            if not self.is_valid_contract_sig(msg[u"syn"]):
                _log.debug("SYN-ACK: sig is invalid.")
                return

            # Update handshake.
            contract = self.contracts[contract_id]
            self.handshake[contract_id] = {
                "state": u"ACK",
                "timestamp": time.time()
            }

            # Create reply contract.
            reply = OrderedDict({
                u"status": u"ACK",
                u"syn_ack": msg
            })

            # Sign reply.
            reply = self.sign_contract(reply)

            # Try make TCP con.
            self.net.unl.connect(
                contract["dest_unl"],
                {
                    "success": success_wrapper(
                        self,
                        contract_id,
                        contract["host_unl"]
                    )
                },
                force_master=0,
                nonce=contract_id
            )

            # Send reply.
            self.send_msg(reply, msg[u"syn"][u"dest_unl"])
            _log.debug("SYN-ACK")

        if msg[u"status"] == u"ACK":
            # Valid ack.
            if u"syn_ack" not in msg:
                _log.debug("ACK: syn_ack not in msg.")
                return
            if u"syn" not in msg[u"syn_ack"]:
                _log.debug("ACK: syn not in msg.")
                return

            # Is this a reply to our SYN-ACK?
            contract_id = self.contract_id(msg[u"syn_ack"][u"syn"])
            if contract_id not in self.contracts:
                _log.debug("ACK: contract not found.")
                return

            # Did I sign this?
            if not self.is_valid_contract_sig(msg[u"syn_ack"]):
                _log.debug("--------------")
                _log.debug(msg)
                _log.debug("--------------")
                _log.debug(self.contracts)
                _log.debug("--------------")
                _log.debug("ACK: sig is invalid.")
                return

            # Is the syn valid?
            if not self.is_valid_syn(msg[u"syn_ack"][u"syn"]):
                _log.debug("ACK: syn is invalid.")
                return

            # Update handshake.
            contract = self.contracts[contract_id]
            self.handshake[contract_id] = {
                "state": u"ACK",
                "timestamp": time.time()
            }

            # Try make TCP con.
            self.net.unl.connect(
                contract["src_unl"],
                {
                    "success": success_wrapper(
                        self,
                        contract_id,
                        contract["host_unl"]
                    )
                },
                force_master=0,
                nonce=contract_id
            )

            _log.debug("ACK")

    def save_contract(self, contract):
        # Record contract details.
        contract_id = self.contract_id(contract)
        self.contracts[contract_id] = contract

        return contract_id

    def send_msg(self, dict_obj, unl):
        node_id = self.net.unl.deconstruct(unl)["node_id"]
        msg = json.dumps(dict_obj, ensure_ascii=True)
        self.net.dht_node.direct_message(
            node_id,
            msg
        )

    def contract_id(self, contract):
        if sys.version_info >= (3, 0, 0):
            contract = str(contract).encode("ascii")
        else:
            contract = str(contract)

        return hashlib.sha256(contract).hexdigest()

    def sign_contract(self, contract):
        if sys.version_info >= (3, 0, 0):
            msg = str(contract).encode("ascii")
        else:
            msg = str(contract)

        msg = binascii.hexlify(msg).decode("utf-8")
        sig = self.wallet.sign_data(self.wif, msg)

        if sys.version_info >= (3, 0, 0):
            contract[u"signature"] = sig.decode("utf-8")
        else:
            contract[u"signature"] = unicode(sig)

        return contract

    def is_valid_contract_sig(self, contract):
        sig = contract[u"signature"][:]
        del contract[u"signature"]

        if sys.version_info >= (3, 0, 0):
            msg = str(contract).encode("ascii")
        else:
            msg = str(contract)

        msg = binascii.hexlify(msg).decode("utf-8")
        address = self.wallet.get_address(self.wif)

        ret = self.wallet.verify_signature(address, sig, msg)
        contract[u"signature"] = sig[:]

        return ret

    def simple_data_request(self, data_id, node_unl, direction):
        file_size = 0
        if direction == u"send":
            action = u"upload"
        else:
            action = u"download"

        return self.data_request(action, data_id, file_size, node_unl)

    def data_request(self, action, data_id, file_size, node_unl):
        """
        Action = put (upload), get (download.)
        """
        _log.debug("In data request function")

        # Who is hosting this data?
        if action == "upload":
            # We store this data.
            direction = u"send"
            host_unl = self.net.unl.value
            assert(storage.manager.find(self.store_config, data_id) is not None)
        else:
            # They store the data.
            direction = u"receive"
            host_unl = node_unl
            if data_id in self.downloading:
                raise Exception("Already trying to download this.")

        # Encoding.
        if sys.version_info >= (3, 0, 0):
            if type(data_id) == bytes:
                data_id = data_id.decode("utf-8")

            if type(host_unl) == bytes:
                host_unl = host_unl.decode("utf-8")

            if type(node_unl) == bytes:
                node_unl = node_unl.decode("utf-8")
        else:
            if type(data_id) == str:
                data_id = unicode(data_id)

            if type(host_unl) == str:
                host_unl = unicode(host_unl)

            if type(node_unl) == str:
                node_unl = unicode(node_unl)

        # Create contract.
        contract = OrderedDict({
            u"status": u"SYN",
            u"direction": direction,
            u"data_id": data_id,
            u"file_size": file_size,
            u"host_unl": host_unl,
            u"dest_unl": node_unl,
            u"src_unl": self.net.unl.value
        })

        # Sign contract.
        contract = self.sign_contract(contract)

        # Route contract.
        contract_id = self.save_contract(contract)
        self.send_msg(contract, node_unl)
        _log.debug("Sending data request")

        # Update handshake.
        self.handshake[contract_id] = {
            "state": "SYN",
            "timestamp": time.time()
        }

        # For async code.
        d = defer.Deferred()
        self.defers[contract_id] = d

        # Return defer for async code.
        return d

    def remove_file_from_storage(self, data_id):
        storage.manager.remove(self.store_config, data_id)

    def move_file_to_storage(self, path):
        with open(path, "rb") as shard:
            storage.manager.add(self.store_config, shard)
            return {
                "file_size": storage.shard.get_size(shard),
                "data_id": storage.shard.get_id(shard)
            }

    def get_data_chunk(self, data_id, position, chunk_size=1048576):
        path = storage.manager.find(self.store_config, data_id)
        buf = b""
        with open(path, "rb") as fp:
            fp.seek(position, 0)
            buf = fp.read(chunk_size)

            return buf

    def save_data_chunk(self, data_id, chunk):
        _log.debug("Saving data chunk for " + str(data_id))
        _log.debug("of size + " + str(len(chunk)))
        assert(data_id in self.downloading)

        # Find temp file path.
        path = self.downloading[data_id]

        _log.debug(path)
        with open(path, "ab") as fp:
            fp.write(chunk)

if __name__ == "__main__":

    # Alice sample node.
    alice_wallet = BtcTxStore(testnet=True, dryrun=True)
    alice = FileTransfer(
        pyp2p.net.Net(
            net_type="direct",
            node_type="passive",
            nat_type="preserving",
            passive_port=60400,
            dht_node=pyp2p.dht_msg.DHT(),
        ),
        wif=alice_wallet.create_key(),
        store_config={"/home/laurence/Storj/Alice": None}  # FIXME temppath
    )

    # ed980e5ef780d5b9ca1a6200a03302f2a91223044bc63dacc6d9f07eead663ab
    # _log.debug(_log.debug(alice.move_file_to_storage("/home/laurence/Firefox_wallpaper.png")))

    # exit()

    # Bob sample node.
    bob_wallet = BtcTxStore(testnet=True, dryrun=True)
    bob = FileTransfer(
        pyp2p.net.Net(
            net_type="direct",
            node_type="passive",
            nat_type="preserving",
            passive_port=60401,
            dht_node=pyp2p.dht_msg.DHT(),
        ),
        wif=bob_wallet.create_key(),
        store_config={"/home/laurence/Storj/Bob": None}  # FIXME use temppath
    )

    _log.debug(alice.net.unl.deconstruct())
    _log.debug(bob.net.unl.deconstruct())

    # alice.net.unl == bob.net.unl

    _log.debug(type(alice.net.unl))
    _log.debug(type(pyp2p.unl.UNL(value=bob.net.unl.value)))

    # exit()

    # Alice wants data from Bob.
    d = alice.data_request(
        "download",
        "ed980e5ef780d5b9ca1a6200a03302f2a91223044bc63dacc6d9f07eead663ab",
        0,
        bob.net.unl.value
    )

    def do_beep(ret):
        if ret != None:
            print("\a")

    d.addCallback(do_beep)

    """
    alice.data_request(
        "upload",
        "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2",
        5,
        bob.net.unl.value
    )
    """

    # Main event loop.
    while 1:
        for client in [alice, bob]:
            if client == alice:
                _log.debug("Alice")
            else:
                _log.debug("Bob")
            process_transfers(client)

        time.sleep(0.002)


# -*- coding: utf-8 -*-

from datetime import datetime

from alascrapy.spiders.base_spiders.ala_spider import AlaSpider
import alascrapy.lib.dao.incremental_scraping as incremental_utils

class Whathifi_comSpider(AlaSpider):
    name = 'whathifi_com'
    allowed_domains = ['www.whathifi.com']
    start_urls = ['https://www.whathifi.com/products/headphones',
                  'https://www.whathifi.com/products/tvs',
                  'https://www.whathifi.com/products/tablets-and-smartphones',
                  'https://www.whathifi.com/products/home-cinema',
                  'https://www.whathifi.com/products/portable',
                  'https://www.whathifi.com/products/digital-tv-boxes',
                  'https://www.whathifi.com/products/all-in-one-systems',
                  'https://www.whathifi.com/products/games-consoles',
                  'https://www.whathifi.com/products/streaming',
                  'https://www.whathifi.com/products/hi-fi', ]

    def __init__(self, *args, **kwargs):
        super(Whathifi_comSpider, self).__init__(self, *args, **kwargs)
        self.stored_last_date = incremental_utils.get_latest_pro_review_date(
            self.mysql_manager, self.spider_conf["source_id"])

        if not self.stored_last_date:
            self.stored_last_date = datetime(1970, 1, 1)

    def parse(self, response):
        ocn_xpath = '//h1/text()'
        ocn = self.extract(response.xpath(ocn_xpath))

        category = ocn.split(' reviews')[0]

        urls_xpath = '//div[@data-page="1"]/a/@href'
        urls = self.extract_list(response.xpath(urls_xpath))

        for url in urls:
            yield response.follow(url, callback=self.parse_page,
                                  meta={'category': category})

    def parse_page(self, response):
        product_xpaths = {
            'PicURL': '//meta[@property="og:image"]/@content',
            'TestUrl': '//meta[@property="og:url"]/@content'
        }

        review_xpaths = {
            'TestSummary': '//meta[@property="og:description"]/@content',

            'Author': '//span[@class="no-wrap by-author"]//text()',

            'TestDateText': 'substring-before(//time/@datetime, "T")',

            'TestUrl': '//meta[@property="og:url"]/@content',
            'TestVerdict': '//div[@class="sub-box full"]/p/text()',
            'TestPros': '//div[@class="sub-box"][1]/ul/li/text()',
            'TestCons': '//div[@class="sub-box"][2]/ul/li/text()',
        }

        product = self.init_item_by_xpaths(response, 'product', product_xpaths)
        review = self.init_item_by_xpaths(response, 'review', review_xpaths)

        title_xpath = '//h1/text()'
        title = self.extract(response.xpath(title_xpath))
        productname = title.split(' review')[0]
        if 'Hands on:' in productname:
            productname = productname.split(': ')[1]
        else:
            productname = productname

        category = response.meta.get('category')
        product['OriginalCategoryName'] = category

        review['TestTitle'] = title
        review['ProductName'] = productname
        product['ProductName'] = productname

        rating_xpath = '//meta[@itemprop="ratingValue"]/@content'
        rating = self.extract(response.xpath(rating_xpath))

        SCALE = 5

        if rating:
            review['SourceTestRating'] = rating
            review['SourceTestScale'] = SCALE

        review['DBaseCategoryName'] = 'PRO'
        source_int_id = response.url
        source_internal_id = source_int_id.split('/')[-1]

        review['source_internal_id'] = source_internal_id
        product['source_internal_id'] = source_internal_id

        review_date = datetime.strptime(review['TestDateText'], "%Y-%m-%d")

        if review_date > self.stored_last_date:
            yield review
            yield product


# Matteo Golin
# Tic Tac Toe Game with AI

# Imports
import random
import time

# Difficulty easy by default
difficulty = "1"

# Global variables
pOneScore = 0
pTwoScore = 0
lastMove = (0, 0)
lastCompMove = (0, 0)
roundCount = 3
scoreCap = 2

# PvP or PvC selection
while True:

    PvP = input("Will you play against another player using the keyboard (1) or against the computer (2): ")

    if PvP == "1":
        PvP = True  # PvP is selected
        break

    elif PvP == "2":
        PvP = False  # PvC is selected

        # Difficulty selection
        while True:

            difficulty = input("Select easy (1), medium (2), hard (3) or impossible (4) mode: ")

            if difficulty not in ["1", "2", "3", "4"]:  # Must be valid option
                print("You must select option 1, 2, 3 or 4.")

            else:
                break

        break

    else:
        print("You must select option 1 or option 2.")  # Wrong input, try again

# Initial instructions
if PvP:  # Changing p1 and p2 names based on who is the opponent
    pOne = "Player 1"
    pTwo = "Player 2"
else:
    pOne = "You"
    pTwo = "The Computer"

print(f"""
{pOne} will use the letter X.
{pTwo} will use the letter O.

Each row and column corresponds to a number.
Input your numbers in the format [row][column] together with no spaces (12, 32, 23, etc.)

                                           1   2   3

                                     1   | X | O | X |
                                         -------------
                                     2   | O | X | O |
                                         -------------
                                     3   | X | O | X |

For example, there is an X at spot 11, 13, 22, 31 and 33.

The default rules use a game of {roundCount} rounds and {scoreCap} victories to win.
""")

# Rule selection
while True:

    changeRules = input("Use the default rules (1) or set custom rules (2): ")

    if changeRules not in ["1", "2"]:  # Invalid selection
        print("Pick option 1 or 2.")

    else:  # Valid selection

        if changeRules == "1":  # Keep the default rules
            break

        else:  # Change rules
            while True:  # Looping for input

                roundCount = input("Number of rounds (must be odd): ")

                try:
                    roundCount = int(roundCount)  # Attempt to convert input to integer

                    if roundCount <= 0:  # Zero or less
                        print("Number of rounds must be above 0!")

                    if roundCount % 2 == 0:  # Even
                        print("Number of rounds must be odd!")

                    else:  # Odd
                        scoreCap = round(roundCount / 2 + 0.5)

                        # Singular or plural depending on score cap
                        if scoreCap > 1:
                            victory = "victories"
                        else:
                            victory = "victory"

                        # Printing user set rules
                        print(f"Your game will have {roundCount} rounds "
                              f"and require {scoreCap} {victory} to win.")
                        break

                except ValueError:  # User didn't give an integer
                    print("You must input an integer!")

            break

# Function for printing board
def board_printer(board):

    print("-------------")  # Initial row cap

    # Printing the values for each spot of every row
    for row in range(3):
        for column in board[row]:
            print(f"| {column} ", end="")
        print("|\n-------------")  # Ending row with the final vertical line and printing the last horizontal bar

    print("")  # Ending with a line break

# Function for animation
def play_animation(frames):

    # Animation while player registers the computer's winning move
    index = 0  # Indexing variable for tracking frames
    animation = ["| Loading...", "/ Loading...", "- Loading...", "\\ Loading..."]  # Actual animation

    while index < frames:  # 0.1s x frames = # of seconds
        print(animation[index % len(animation)], end="\r")  # Printing next animation frame based on cycle
        index += 1  # Increasing index
        time.sleep(0.1)  # Waiting 0.1 seconds before next frame

# Function for making moves
def make_move(player, board):

    # Getting player name for customization
    if player:
        name = "Player 1"
    else:
        name = "Player 2"

    # Getting coordinates
    while True:

        coOrds = input(f"Coordinates for {name}'s move: ")

        if coOrds not in ['11', '12', '13',  # Coordinates must be one of these values
                          '21', '22', '23',
                          '31', '32', '33']:
            print("You did not input a valid set of coordinates!")
        else:
            # Formatting coordinates
            row = int(coOrds[0]) - 1  # Makes this start at index 0
            column = int(coOrds[1]) - 1  # Makes this start at index 0
            global lastMove
            lastMove = (row, column)  # Recording last move

            # Checking if the spot is already full
            if board[row][column] != " ":
                print("Oops, this space is already filled! Try again!")
            else:
                break

    # Adding move to board
    if player:  # Player one
        board[row][column] = "X"

    else:  # Player two
        board[row][column] = "O"

    # Printing board
    board_printer(board)

# Function for AI moves
def computer_move(diff, board):

    # Finding Xs and Os in a line function, for blocking or getting a winning line
    def block_and_win():

        # Arrays for O count
        oCountRows = [0, 0, 0]
        oCountColumns = [0, 0, 0]
        oCountDiagonals = [0, 0]

        # Arrays for X count
        xCountRows = [0, 0, 0]
        xCountColumns = [0, 0, 0]
        xCountDiagonals = [0, 0]

        # Diagonal coordinates
        leftDiagonal = [(0, 0), (1, 1), (2, 2)]
        rightDiagonal = [(0, 2), (1, 1), (2, 0)]
        diagonals = [leftDiagonal, rightDiagonal]

        # Actually counting Xs/Os

        for _ in range(2):  # Starting to check for wins with Os, and then block the Xs as a second priority

            if _ == 0:  # For Os
                rowCount = oCountRows
                columnCount = oCountColumns
                diagonalCount = oCountDiagonals
                character = "O"
            else:  # For Xs
                rowCount = xCountRows
                columnCount = xCountColumns
                diagonalCount = xCountDiagonals
                character = "X"

            # Counting characters in rows
            for row in range(3):
                for column in board[row]:
                    if column == character:
                        rowCount[row] += 1

            # Counting characters in columns
            for row in range(3):
                for column in range(3):
                    if board[column][row] == character:
                        columnCount[row] += 1

            # Counting characters in diagonals
            for _ in range(2):  # 0 = left diagonal, 1 = right diagonal | loops through both
                for coSet in diagonals[_]:
                    if board[coSet[0]][coSet[1]] == character:
                        diagonalCount[_] += 1

            # Filling spaces

            # Filling row if there's 2 Os
            for row in range(3):  # Each row
                if rowCount[row] == 2:  # If there's 2 Xs/Os and no spot has been filled yet
                    for column in range(3):  # Each column of the row

                        if board[row][column] == " ":  # If there's an empty space
                            board[row][column] = "O"  # Occupy it
                            coOrds = (row, column)  # Record move
                            return True, coOrds  # Spot found

            # Filling column if there's 2 Xs
            for column in range(3):  # Each column
                if columnCount[column] == 2:  # If there's 2 Xs/Os and no spot has been filled yet
                    for row in range(3):  # Each row of the column

                        if board[row][column] == " ":  # If there's an empty spot
                            board[row][column] = "O"  # Occupy it
                            coOrds = (row, column)  # Record move
                            return True, coOrds  # Spot found

            # Filling diagonal if there's 2 Os
            for index in range(2):  # Selecting left diagonal (0) or right diagonal (1)

                if diagonalCount[index] == 2:  # If there's 2 Xs/Os and we haven't found a spot
                    for coSet in diagonals[index]:  # For coordinates in the selected diagonal

                        if board[coSet[0]][coSet[1]] == " ":  # If there's an empty space
                            board[coSet[0]][coSet[1]] = "O"  # Occupy it
                            coOrds = (coSet[0], coSet[1])  # Record move
                            return True, coOrds  # Spot found

        return False, (0, 0)

    # Last computer move variable
    global lastCompMove

    # Easy
    if diff == "1":

        # Looping until computer selects a valid move
        while True:
            # Randomly selecting a coordinate
            coOrds = random.choice([[0, 0], [0, 1], [0, 2],
                                    [1, 0], [1, 1], [1, 2],
                                    [2, 0], [2, 1], [2, 2]])

            # Checking if space is filled
            if board[coOrds[0]][coOrds[1]] != " ":
                pass
            else:
                board[coOrds[0]][coOrds[1]] = "O"
                break

        # Printing results of move
        print(f"The Computer selected the spot {coOrds[0] + 1}{coOrds[1] + 1}.")
        lastCompMove = coOrds  # Saving last computer move
        board_printer(board)

    # Medium
    elif diff == "2":

        # Assuming no spot was selected
        spotFound = False

        if not spotFound:
            (spotFound, coOrds) = block_and_win()  # Checking for 2 Xs/Os in a row to block or win

        # Trying to find an open spot in the diagonal
        leftDiagonal = [(0, 0), (1, 1), (2, 2)]  # Left diagonal coordinates
        rightDiagonal = [(0, 2), (1, 1), (2, 0)]  # Right diagonal coordinates
        diagonals = [leftDiagonal, rightDiagonal]

        for _ in range(2):

            if lastMove in diagonals[_] and not spotFound:  # If the last move looks like a diagonal play

                for spot in leftDiagonal:  # Occupying a remaining diagonal space
                    if board[spot[0]][spot[1]] == " ":  # Checking to make sure space is empty
                        board[spot[0]][spot[1]] = "O"  # Filling space
                        coOrds = spot  # Recording the final spot
                        spotFound = True  # Recording that a spot was chosen
                        break
                    else:
                        spotFound = False

        # Diagonal spot not possible
        if not spotFound:

            # Selecting whether to occupy a space in the same row or column
            rowOrColumn = random.randint(1, 2)

            for index in range(3):  # Looping through indexes from 0-2

                if rowOrColumn == 1:  # Row
                    row = lastMove[0]
                    column = lastMove[1] + index

                else:  # Column
                    row = lastMove[0] + index
                    column = lastMove[1]

                # Trying to find an open spot in the same line
                try:
                    if board[row][column] == " ":
                        board[row][column] = "O"
                        spotFound = True
                        coOrds = (row, column)
                        break
                    else:
                        pass
                except IndexError:
                    pass

        # Neither diagonal nor row/column worked
        if not spotFound:
            computer_move("1", board)
        else:
            # Printing results
            print(f"The Computer selected the spot {coOrds[0] + 1}{coOrds[1] + 1}.")
            lastCompMove = coOrds  # Saving last computer move
            board_printer(board)

    # Hard
    elif diff == "3":

        (spotFound, coOrds) = block_and_win()  # Check if 2 Xs/Os in a row to block or win

        if not spotFound:  # If a move still isn't made, it checks to do a corner block

            corners = [(0, 0), (0, 2), (2, 0), (2, 2)]  # Corner coordinates

            for spot in corners:  # All corner spaces

                xFoundRow = False  # X in row boolean reset to false for every iteration
                xFoundColumn = False  # X in column boolean reset to false for every iteration

                for _ in range(3):  # Coordinates from 0-2 for iterating

                    if board[spot[0]][_] == "X" and spot[0] != _:  # X found in the row of corner space
                        xFoundRow = True

                    if board[_][spot[0]] == "X" and spot[0] != _:  # X found in the column of corner space
                        xFoundColumn = True

                # Tracking Os in the center row and column
                oCountCenterRow = 0
                oCountCenterColumn = 0

                # Don't skip the corner block
                skip = False

                # Counting Os in the center rows and columns
                for _ in range(3):  # Iterating coordinates

                    if board[1][_] == "O":  # Rows
                        oCountCenterRow += 1

                    if board[_][1] == "O":  # Columns
                        oCountCenterColumn += 1

                if oCountCenterColumn == 2 or oCountCenterRow == 2:  # If win scenario possible, skip corner block
                    skip = True  # This sends us to the next part of the algorithm which creates an inevitable win

                if lastMove in corners:  # This avoids the triple corner trap
                    skip = True

                # Condition to save space for next if statement
                condition = xFoundRow and xFoundColumn and not skip and board[spot[0]][spot[1]] == " "

                if condition:  # X found in lines that intersect at specified corner, and win scenario set-up impossible
                    board[spot[0]][spot[1]] = "O"  # Place an O
                    spotFound = True  # Spot has been found
                    coOrds = (spot[0], spot[1])  # Record coordinates
                    break  # Stop searching

        if not spotFound:  # If move still not made, it checks to set up a win scenario

            for _ in range(3):

                if board[lastCompMove[0]][_] == " ":  # If there's a space in the row
                    board[lastCompMove[0]][_] = "O"
                    spotFound = True
                    coOrds = (lastCompMove[0], _)
                    break

                if board[_][lastCompMove[1]] == " ":  # If there's a space in the column
                    board[_][lastCompMove[1]] = "O"
                    spotFound = True
                    coOrds = (_, lastCompMove[1])
                    break

        if not spotFound:  # If none of the above works, use the medium difficulty algorithm
            computer_move("2", board)

        else:  # Printing the results only if the hard mode HAS found a spot in order to avoid recursive printing
            # Printing results
            print(f"The Computer selected the spot {coOrds[0] + 1}{coOrds[1] + 1}.")
            lastCompMove = coOrds  # Saving last computer move
            board_printer(board)

    # Impossible
    else:

        # The crucial first move
        if maxMoves == 8:

            if lastMove == (1, 1):  # If center space is chosen, pick a corner
                coOrds = (0, 2)
                board[0][2] = "O"

            else:  # If any other space is chosen, pick center
                coOrds = (1, 1)
                board[1][1] = "O"

            # Printing results
            print(f"The Computer selected the spot {coOrds[0] + 1}{coOrds[1] + 1}.")
            lastCompMove = coOrds  # Saving last computer move
            board_printer(board)

        # After the first move
        else:
            computer_move("3", board)  # Calling the hard difficulty

# Logic for detecting win
def win_detect(board):

    # Assume no win
    win = False
    winner = None

    # Checking straight wins
    for index in range(3):

        # Checking for horizontal win
        if board[index][0] == board[index][1] == board[index][2] != " ":
            win = True
            winner = board[index][0]
            break

        # Checking for vertical win
        elif board[0][index] == board[1][index] == board[2][index] != " ":
            win = True
            winner = board[0][index]
            break

    # Checking for diagonal win
    if board[0][0] == board[1][1] == board[2][2] != " ":
        win = True
        winner = board[0][0]
    elif board[0][2] == board[1][1] == board[2][0] != " ":
        win = True
        winner = board[1][1]

    # Returning the win and winner

    if win:
        # Customizing output to match the player who one
        if winner == "X":
            print(f"The winner was {pOne}!\n")
            global pOneScore
            pOneScore += 1
        else:
            print(f"The winner was {pTwo}!\n")
            global pTwoScore
            pTwoScore += 1

    return win

# Setting originalRoundCount equal to the original roundCount selected by user
originalRoundCount = roundCount

# While loop to keep the game going for the number of rounds specified
while roundCount > 0 and pOneScore < scoreCap and pTwoScore < scoreCap:

    # Initializing empty board for each new round
    gameBoard = [[" ", " ", " "],
                 [" ", " ", " "],
                 [" ", " ", " "]]

    # Dividing line
    for _ in range(100):
        print("-", end="")
    print("\n")  # 2 new lines

    # Printing the start board
    print(f"Game {originalRoundCount + 1 - roundCount} has started!")  # Round number
    board_printer(gameBoard)

    # Max moves variable to determine when game must stop
    maxMoves = 9

    # Getting moves
    while maxMoves > 0:

        # Player two is second, so they will always play when the number of moves left is even
        if maxMoves % 2 == 0:
            if PvP:  # PvP logic
                make_move(False, gameBoard)

            else:  # PvC logic
                computer_move(difficulty, gameBoard)

        else:  # Player one is first, so they will always play when the number of moves left is odd
            make_move(True, gameBoard)

        # Reducing the number of remaining moves
        maxMoves -= 1

        # Checking for winner
        if win_detect(gameBoard):
            play_animation(30)
            print("Score Update")
            print(f"{pOne}: {pOneScore}")  # Printing p1 score
            print(f"{pTwo}: {pTwoScore}\n")  # Printing p2 score
            break  # Starts next round
        else:
            pass  # Continues game with no win

    # If game ends as a tie
    if maxMoves == 0:
        play_animation(30)
        print("This game was a tie!\n")
        print("Score Update")
        print(f"{pOne}: {pOneScore}")  # Printing p1 score
        print(f"{pTwo}: {pTwoScore}\n")  # Printing p2 score

    # Reducing round count after game finishes
    roundCount -= 1

# Displaying final scores
print("Final Scores")
print(f"{pOne}: {pOneScore}")
print(f"{pTwo}: {pTwoScore}\n")

# Checking for winner overall
if pOneScore > pTwoScore:
    finalWinner = pOne  # p1 wins
elif pOneScore == pTwoScore:
    finalWinner = "no one because the game ended in a draw"  # Draw
else:
    finalWinner = pTwo  # p2 wins

# Printing overall winner
print(f"Final winner is {finalWinner}!")


# 0501.py
import cv2
import numpy as np
src = cv2.imread('./heart10.jpg', cv2.IMREAD_GRAYSCALE)
cv2.imshow('src',  src)

#ret, dst = cv2.threshold(src, 120, 255, cv2.THRESH_BINARY) # thresholding 값: 120, 기준이 올라가면 올라갈 수록, 검은색 픽셀이 많아진다
ret, dst = cv2.threshold(src, 200, 255, cv2.THRESH_BINARY_INV)   # THRESH_BINARY_INV : 색 반전

print('ret=', ret)
cv2.imshow('dst',  dst)

ret2, dst2 = cv2.threshold(src, 200, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
#THRESH_OTSU : 제시된 thresholding 값을 쓰지 않고, 자동으로 계산한 뒤 Binary에서 이진화 계산

print('ret2=', ret2)
cv2.imshow('dst2',  dst2)

cv2.waitKey()
cv2.destroyAllWindows()


'''
Data Mining Project

Print topics found in Yelp business reviews

If training is specified, dataset will be read from disk and model will be trained.
Otherwise, pickled model will be read from disk.
'''

import argparse
from collections import defaultdict
import json
import pickle
import os
import sys
from time import time

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
import nltk
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.stem.porter import *
import numpy as np
np.random.seed(400)

PICKLED_MODEL_FILENAME = 'lda_model.p'
PICKLED_RESTAURANT_REVIEW_DATA_FILENAME = 'restaurants.p'

default_dataset_path = os.path.join(
	os.path.dirname(__file__),
	'dataset',
)

parser = argparse.ArgumentParser()

parser.add_argument(
	'-t', '--train',
	action='store_true',
	help='train new model (Yelp dataset must also be present on disk))',
)

parser.add_argument(
	'-d', '--dataset',
	default=default_dataset_path,
	help='specify absolute path to dataset',
)

parser.add_argument(
	'-n', '--num_topics',
	default=14,
	help='number of topics to train LDA on',
)

# Write functions to perform the preprocessing
stemmer = SnowballStemmer('english')
def lemmatize_stemming(text):
    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))

# Tokenize and lemmatize
def preprocess(text):
    result=[]
    for token in simple_preprocess(text, deacc=True, min_len=4):
        if token not in STOPWORDS:
            result.append(lemmatize_stemming(token))
    return result

if __name__ == '__main__':
	args = parser.parse_args()
	arg_dict = vars(args)

	if arg_dict['train']:

		# try to grab pickled version of dataset, if possible
		if not os.path.exists(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME):
			print('importing dataset...')

			dataset_path = arg_dict['dataset']

			try:
				datasets = {
					f.split('.')[0]: dataset_path + os.sep + f for f in filter(lambda x: x.endswith('.json'), os.listdir(dataset_path))
				}
			except:
				print('Could not find dataset at {}'.format(dataset_path))
				sys.exit(0)

			# organize data to be used in training model
			restaurants = defaultdict(dict)

			with open(datasets['business']) as f:
				for line in f:
					contents = json.loads(line)
					if contents['categories'] and 'Restaurants' in contents['categories']:
						restaurants[contents['business_id']]['categories'] = contents['categories']
						restaurants[contents['business_id']]['reviews'] = []

			with open(datasets['review']) as f:
				for line in f:
					contents = json.loads(line)
					if contents['business_id'] and contents['business_id'] in restaurants:
						restaurants[contents['business_id']]['reviews'].append(contents['text'])

			all_the_reviews = []
			for v in restaurants.values():
				all_the_reviews += v['reviews']

			print('Saving pickled dataset to disk as {}'.format(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME))
			pickle.dump(all_the_reviews, open(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME, 'wb'))

		else:
			print('Using pickled dataset file found on disk as {}'.format(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME))
			all_the_reviews = pickle.load(open(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME, 'rb'))

		# grab wordnet if not present for preprocessing data
		nltk.download('wordnet')

		print('Preview of document after preprocessing')

		doc_sample = all_the_reviews[0]

		print("Original document: ")
		words = [word for word in doc_sample.split(' ')]
		print("\nBefore preprocessing: ")
		print(words)
		print("\nTokenized and lemmatized document: ")
		print(preprocess(doc_sample))

		# Need to split reviews into training and testing
		training_reviews = all_the_reviews[:50000]
		processed_reviews = [preprocess(review) for review in training_reviews]

		'''
		Bag of words on the dataset
		Create a dictionary from 'processed_docs' containing the number of times a word appears
		in the training set. To do that, we pass processed_docs to gensim.corpora.Dictionary()
		'''
		dictionary = gensim.corpora.Dictionary(processed_reviews)

		'''
		Filter out tokens that appear only a few times or in more than 10% of documents
		'''
		dictionary.filter_extremes(
			no_below=0.001*len(processed_reviews),
			no_above=0.1,
		)

		'''
		Create the Bag-of-words model for each document i.e for each document we create a dictionary reporting how many
		words and how many times those words appear. Save this to 'bow_corpus'
		'''
		bow_corpus = [dictionary.doc2bow(review) for review in processed_reviews]

		'''
		Preview BOW for our sample preprocessed document
		'''
		print('\nBag of Words Preview on example document:')
		print(training_reviews[25])

		for doc in bow_corpus[25]:
		    print("Word {} (\"{}\") appears {} time.".format(doc[0], dictionary[doc[0]], doc[1]))

		'''
		Running LDA using Bag of Words
		We are going for 10 topics in the document corpus.

		We will be running LDA using 8 CPU cores to parallelize and speed up model training.
		'''
		num_topics = int(arg_dict['num_topics'])
		print('\nBuilding LDA model with {} topics...'.format(num_topics))
		lda_model =  gensim.models.LdaMulticore(
			bow_corpus,
			num_topics=num_topics,
			id2word=dictionary,
			passes=2,
			workers=8,
		)

		# serialize and save LDA model object to disk
		if not os.path.exists(PICKLED_MODEL_FILENAME):
			print('\nsaving pickled model to disk {}'.format(PICKLED_MODEL_FILENAME))
			pickle.dump(lda_model, open(PICKLED_MODEL_FILENAME, 'wb'))

	else:
		try:
			lda_model = pickle.load(open(PICKLED_MODEL_FILENAME, 'rb'))
		except:
			print('\nPickled LDA model not found on disk as {}... try running program with -t option to train new model!'.format(PICKLED_MODEL_FILENAME))
			sys.exit(0)

	# print discovered topics to screen
	for i, topic in lda_model.print_topics(
		num_topics=-1,
		num_words=20,
	):
	    print("Topic: {} \nWords: {}".format(i, topic))


import numpy as np
from pygef import geo
import pygef.utils as utils

def type_index_to_gamma(ic):
    gamma = None
    if ic > 3.22:
        gamma = 11
    elif 2.76 < ic <= 3.22:
        gamma = 16
    elif 2.40 < ic <= 2.76:
        gamma = 18
    elif 1.80 < ic <= 2.40:
        gamma = 18
    elif 1.25 < ic <= 1.80:
        gamma = 18
    elif ic <= 1.25:
        gamma = 18
    return gamma

def type_index_to_gamma_sat(ic):
    gamma_sat = None
    if ic > 3.22:
        gamma_sat = 11
    elif 2.76 < ic <= 3.22:
        gamma_sat = 16
    elif 2.40 < ic <= 2.76:
        gamma_sat = 18
    elif 1.80 < ic <= 2.40:
        gamma_sat = 19
    elif 1.25 < ic <= 1.80:
        gamma_sat = 20
    elif ic <= 1.25:
        gamma_sat = 20
    return gamma_sat

def type_index_to_soil_type(ic):
    soil_type = None
    if ic > 3.22:
        soil_type = 'Peat'
    elif 2.76 < ic <= 3.22:
        soil_type = 'Clays'
    elif 2.40 < ic <= 2.76:
        soil_type = 'Clayey silt to silty clay'
    elif 1.80 < ic <= 2.40:
        soil_type = 'Silty sand to sandy silt'
    elif 1.25 < ic <= 1.80:
        soil_type = 'Sands: clean sand to silty'
    elif ic <= 1.25:
        soil_type = 'Gravelly sands'
    return soil_type

def excess_pore_pressure_ratio(df):
    try:
        u2 = df['u2']
    except KeyError:
        raise SystemExit("ERROR: u2 not defined in .gef file, change classifier")
    return df.assign(excess_pore_pressure_ratio=(u2 - df['water_pressure']) / (df['qt'] - df['soil_pressure']))

def ic_to_gamma(df, water_level):
    return df.assign(gamma_predict=df.apply(
        lambda row: type_index_to_gamma(row['type_index']) if row['depth'] > water_level
        else type_index_to_gamma_sat(row['type_index']), axis=1))

def ic_to_soil_type(df):
    return df.assign(soil_type=df.apply(
        lambda row: type_index_to_soil_type(row['type_index']), axis=1))

def type_index(df):
    return df.assign(type_index=(((3 - np.log10(df['normalized_cone_resistance'] *
                                                (1 - df['excess_pore_pressure_ratio']) + 1)) ** 2 +
                                  (1.5 + 1.3 * np.log10(df['normalized_friction_ratio'])) ** 2) ** 0.5))

def iterate_been_jeffrey(original_df, water_level, area_quotient_cone_tip=None, pre_excavated_depth=None):
    gamma = np.ones(original_df.shape[0]) * 18

    c = 0
    while True:
        c += 1
        df = original_df.assign(gamma=gamma)

        def condition(x):
            return np.all(x['gamma_predict'] == gamma)
        df = been_jeffrey(df, water_level, area_quotient_cone_tip=area_quotient_cone_tip,
                          pre_excavated_depth=pre_excavated_depth)
        df = df.assign(gamma_predict=np.nan_to_num(df['gamma_predict']))
        if condition(df):
            break
        elif c > 10:
            break
        else:
            gamma = df['gamma_predict']

    return df.pipe(ic_to_soil_type)

def been_jeffrey(df, water_level, area_quotient_cone_tip=None, pre_excavated_depth=None):
    df = (df
          .pipe(geo.delta_depth, pre_excavated_depth)
          .pipe(geo.soil_pressure)
          .pipe(geo.qt, area_quotient_cone_tip)
          .pipe(geo.water_pressure, water_level)
          .pipe(geo.effective_soil_pressure)
          .pipe(utils.kpa_to_mpa, ['soil_pressure', 'effective_soil_pressure', 'water_pressure'])
          .pipe(excess_pore_pressure_ratio)
          .pipe(geo.normalized_cone_resistance)
          .pipe(geo.normalized_friction_ratio)
          .pipe(utils.nan_to_zero)
          .pipe(type_index)
          .pipe(ic_to_gamma, water_level)
          )
    return df