#!/usr/bin/env python
# coding: utf-8

#
COVID-19: ULAs

#
#

#

# Rodrigo Carrillo-Larco, Manuel Castillo-Cara

# Universidad Peruana Cayetano Heredia

# Universidad Nacional de Ingeniería
#
#

# ---
#
#
Índice

#
#
# * [1. Introducción](#section1)
# * [2. Análisis Exploratorio de Datos](#section2)
#     * [2.1. Limpieza de datos](#section21)
#     * [2.2. Boxplots](#section22)
#     * [2.3. Histogramas](#section23)
#     * [2.4. Matriz de correlación](#section24)
#     * [2.5. Matriz de dispersión](#section25)
# * [3. Transformaciones categórico a dummy](#section3)
# * [4. PCA + k-means](#section4)
# * [5. Análisis clustering](#section5)
#     * [5.1. Tamaño/Densidad de clústers](#section51)
#     * [5.2. Centroides de clústers (Formas)](#section52)

# ---
#
# #  1. Introducción

# In[2]:

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pylab as pl
import pandas as pd
import os

# ### Lectura de datos

# In[3]:

df_original = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df_original.head(1)

# In[5]:

df = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df.drop(['pop_men'], axis=1, inplace=True)
df.drop(['pop_women'], axis=1, inplace=True)
#df.drop(['p.db'], axis=1, inplace=True)
#df.drop(['p.copd'], axis=1, inplace=True)
#df.drop(['p.hiv'], axis=1, inplace=True)
#df.drop(['p.tbc'], axis=1, inplace=True)
df.drop(['ylds.db'], axis=1, inplace=True)
df.drop(['ylds.copd'], axis=1, inplace=True)
df.drop(['ylds.hiv'], axis=1, inplace=True)
df.drop(['ylds.tbc'], axis=1, inplace=True)
df.drop(['Country'], axis=1, inplace=True)
df.drop(['Speed.Number'], axis=1, inplace=True)
df.drop(['first_date'], axis=1, inplace=True)
df.drop(['order'], axis=1, inplace=True)
df.drop(['total_deaths'], axis=1, inplace=True)

# ---
#

#
#

# ---
#
# #  2. Análisis Exploratorio de Datos

# In[6]:

df.dtypes

# In[7]:

df.describe().transpose()

# In[8]:

df.shape

# In[9]:

list_columns = df.columns

# ---
#
# ##  2.1. Data Cleaning: Handling missing data

# In[10]:

df.isnull().any().any(), df.shape

# In[11]:

df.isnull().sum(axis=0)

# ---
#
# ##  2.2. Boxplots

# In[15]:

pl.figure(1)
plt.subplots(figsize=(20,20))
#pl.subplot(431)
#sns.boxplot(df["Speed.Number"])
#pl.grid(True)
pl.subplot(432)
sns.boxplot(df["p.db"])
pl.grid(True)
#pl.subplot(433)
#sns.boxplot(df["ylds.db"])
#pl.grid(True)
pl.subplot(434)
sns.boxplot(df["p.copd"])
pl.grid(True)
#pl.subplot(435)
#sns.boxplot(df["ylds.copd"])
#pl.grid(True)
pl.subplot(436)
sns.boxplot(df["p.hiv"])
pl.grid(True)
#pl.subplot(437)
#sns.boxplot(df["ylds.hiv"])
#pl.grid(True)
pl.subplot(438)
sns.boxplot(df["p.tbc"])
pl.grid(True)
pl.subplot(439)
sns.boxplot(df["GDP_2017"])
pl.grid(True)
#=======================
pl.figure(2)
plt.subplots(figsize=(20,20))
#pl.subplot(431)
#sns.boxplot(df["pop_men"])
pl.grid(True)
#pl.subplot(432)
#sns.boxplot(df["pop_women"])
#pl.grid(True)
pl.subplot(433)
sns.boxplot(df["prop_men"])
pl.grid(True)
pl.subplot(434)
sns.boxplot(df["pm2.5"])
pl.grid(True)
pl.subplot(435)
sns.boxplot(df["UHC_index_2017"])
pl.grid(True)

pl.show()

# ---
#
# ##  2.3. Histogramas

# In[17]:

get_ipython().run_line_magic('matplotlib', 'inline')
# Univariate Histograms
f, axes = plt.subplots(4, 4, figsize=(14, 14))
#sns.distplot(df["Speed.Number"], rug=True, ax=axes[0, 0])
sns.distplot(df["p.db"], rug=True, ax=axes[0, 1])
#sns.distplot(df["ylds.db"], rug=True, ax=axes[0, 2])
sns.distplot(df["p.copd"], rug=True, ax=axes[0, 3])
#sns.distplot(df["ylds.copd"], rug=True, ax=axes[1, 0])
sns.distplot(df["p.hiv"], rug=True, ax=axes[1, 1])
#sns.distplot(df["ylds.hiv"], rug=True, ax=axes[1, 2])
sns.distplot(df["p.tbc"], rug=True, ax=axes[2, 0])
sns.distplot(df["GDP_2017"], rug=True, ax=axes[2, 1])
#sns.distplot(df["pop_men"], rug=True, ax=axes[2, 2])
#sns.distplot(df["pop_women"], rug=True, ax=axes[2, 3])
sns.distplot(df["prop_men"], rug=True, ax=axes[3, 0])
sns.distplot(df["pm2.5"], rug=True, ax=axes[3, 1])
sns.distplot(df["UHC_index_2017"], ax=axes[3, 2])

# ---
#
# ##  2.4. Matriz de correlación

# In[18]:

correlation = df.corr()
plt.figure(figsize=(10,10))
ax = sns.heatmap(correlation, vmax=1, square=True, annot = True, cmap = 'viridis')
# Esto se ponde debido al bug de Matplotlib 3.1.1 (quitarlo en versiones diferentes)
bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
# ----------
plt.title('Correlación entre variables')
plt.show()

# ---
#
# ##  2.5. Matriz de dispersión

# Al igual que el diagrama de matriz de correlación anterior, la matriz de diagrama de dispersión es simétrica. Esto es útil para mirar las relaciones por pares desde diferentes perspectivas. Debido a que no tiene mucho sentido dibujar un diagrama de dispersión de cada variable consigo mismo, la diagonal muestra histogramas de cada atributo.
#
# **Aquí es importante ver el comportamiento entre variables, es decir, si tienen un comportamiento clusterizado entre ambas o tiene un comportamiento lineal**

# In[19]:

sns.pairplot(df);

# ---
#

#
#

# ---
#
# # 3. Transformaciones categórico a dummy

# In[12]:

cat_features = ['Country', 'first_date']

#num_features = ['Speed.Number', 'order', 'total_deaths', 'p.db', 'ylds.db', 'p.copd', 'ylds.copd', 'p.hiv', 'ylds.hiv', 'p.tbc', 'ylds.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']
num_features = ['Speed.Number', 'order', 'total_deaths', 'p.db', 'p.copd', 'p.hiv', 'p.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']
#num_features = ['Speed.Number', 'order', 'total_deaths', 'ylds.db', 'ylds.copd', 'ylds.hiv', 'ylds.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']

def generateonlyDummies2(X):
    outd_X = pd.DataFrame()
    categories = {}
    for c in cat_features:
        categories[c] = X[c].dropna().unique().tolist()

    for c in cat_features:
            # Cambiamos las variables object a categorical para que las categorias tengan siempre el mismo orden
            c_df = X[c].astype(pd.api.types.CategoricalDtype(categories = categories[c]))
            #c_df.reset_index(drop=True, inplace=True)
            # One-hot encoding. Utilizamos get_dummies de pandas!
            c_dummies = pd.get_dummies(c_df, prefix=c)
            outd_X = pd.concat([outd_X, c_dummies], axis=1)
    outc_X = pd.DataFrame(X[num_features],columns=num_features)

    for c in num_features:
        outd_X[c] = outc_X[c].values
    return outd_X

# ---
#

#
#

#
# ---

#
# # 4.PCA + k-Means

# Se generan las variables dummies correspondientes y con ello ya se tiene listo el dataset para aplicar k-means.
# El código en comentario es dependiendo las variables a analizar, también si hay que pasar a dummy o no.
#
# Cambiar los componentes de PCA
#
# O si no se utiliza PCA comentar el bloque correspondiente

# In[20]:

df_original = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df_original.head(1)

# In[21]:

df = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df.drop(['pop_men'], axis=1, inplace=True)
df.drop(['pop_women'], axis=1, inplace=True)
#df.drop(['p.db'], axis=1, inplace=True)
#df.drop(['p.copd'], axis=1, inplace=True)
#df.drop(['p.hiv'], axis=1, inplace=True)
#df.drop(['p.tbc'], axis=1, inplace=True)
df.drop(['ylds.db'], axis=1, inplace=True)
df.drop(['ylds.copd'], axis=1, inplace=True)
df.drop(['ylds.hiv'], axis=1, inplace=True)
df.drop(['ylds.tbc'], axis=1, inplace=True)
df.drop(['Country'], axis=1, inplace=True)
df.drop(['Speed.Number'], axis=1, inplace=True)
df.drop(['first_date'], axis=1, inplace=True)
df.drop(['order'], axis=1, inplace=True)
df.drop(['total_deaths'], axis=1, inplace=True)

# In[22]:

df.head(1)

# In[23]:

df_dummy = df.copy()
#df_dummy= generateonlyDummies2(df)

# In[24]:

df_dummy.head(1)

# In[25]:

from sklearn.decomposition import PCA
n_components = 3 # reducir de 67 a 40 variables
pca = PCA(n_components=n_components, iterated_power='auto', svd_solver='auto', whiten=True).fit(df_dummy)

# In[26]:

pca_trans = pca.transform(df_dummy)

# In[27]:

total_variance = np.cumsum(pca.explained_variance_ratio_)
total_variance[:5] # Cinco primeros...

# In[28]:

df_pca = pd.DataFrame(pca_trans)

# In[29]:

df_dummyo = df_pca.copy()
#df_dummyo = df_dummy.copy()

# In[30]:

df_dummyo.head(1)

# In[31]:

from sklearn.metrics import pairwise_distances

def cost(data, k, centroids, clusters):
    cost = 0.0
    for c in range(k):
        points_cluster = data[clusters==c, :]
        if points_cluster.shape[0] > 0:
            distances = pairwise_distances(points_cluster, [centroids[c]], metric='euclidean')
            cost += np.sum(distances**2)
    return cost/len(clusters)

# In[32]:

from sklearn.cluster import MiniBatchKMeans

# Arrat que guardará los costes
costsD = np.zeros(10,)
# Valores de k de 1 a 64
ks = 1+np.arange(10)
# Guarda los costes
#costs = np.zeros(20,)
# Valores de k de 5 a 125
#ks = np.linspace(1,20,20).astype(int)

# Ejecuta k-means para cada valor de k, y guarda el coste asociado
for i,k in enumerate(ks):
    kmeans = MiniBatchKMeans(n_clusters=k, init='k-means++', max_iter=500)
    kmeans.fit(df_dummyo.values)
    centroidsD = kmeans.cluster_centers_
    clustersD = kmeans.labels_
    costsD[i] = cost(df_dummyo.values, k, centroidsD, clustersD)
    print(i,end=',')

plt.xlabel('No. Clusters')
plt.ylabel('Cost')
plt.title('Cost Function')
plt.plot(ks,costsD, 'bo-');
plt.savefig('coste.eps', format='eps', dpi=400)

# In[33]:

for i in range(10):
    print(costsD[i])

# In[34]:

np.mean(costsD)

# La gráfica muestra claramente que existe un punto a partir del cual la disminución del coste es significativamente más lenta. En concreto, hasta 2-4 clusters, la disminución del coste es considerable. A partir de ese punto, es mucho más lenta.

# ### Con 2 clústers

# In[35]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_2 = KMeans(n_clusters=2, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_2.fit(df_dummy)
df_dummy_2 = df_dummy.copy()
df_dummy_2['label'] = km_2.labels_
df_dummy_2['Country'] = df_original['Country']
df_dummy_2.to_csv(r'Dataset_2_clusters.csv', index = False)
#df_dummy.drop(['Country'], axis=1, inplace=True)
#df_dummy.drop(['label'], axis=1, inplace=True)

# In[36]:

df_dummy_2.head(1)

# ### Con 3 clústers

# In[37]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_3 = KMeans(n_clusters=3, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_3.fit(df_dummy)
df_dummy_3 = df_dummy.copy()
df_dummy_3['label'] = km_3.labels_
df_dummy_3['Country'] = df_original['Country']
df_dummy_3.to_csv(r'Dataset_3_clusters.csv', index = False)
#df_dummy.drop(['Country'], axis=1, inplace=True)
#df_dummy.drop(['label'], axis=1, inplace=True)

# ### Con 4 clústers

# In[38]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_4 = KMeans(n_clusters=4, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_4.fit(df_dummy)
df_dummy_4 = df_dummy.copy()
df_dummy_4['label'] = km_4.labels_
df_dummy_4['Country'] = df_original['Country']
df_dummy_4.to_csv(r'Dataset_4_clusters.csv', index = False)

# ### Con 5 clústers

# In[39]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_5 = KMeans(n_clusters=5, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_5.fit(df_dummy)
df_dummy_5 = df_dummy.copy()
df_dummy_5['label'] = km_5.labels_
df_dummy_5['Country'] = df_original['Country']
df_dummy_5.to_csv(r'Dataset_5_clusters.csv', index = False)

# ### Con 6 clústers

# In[40]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_6 = KMeans(n_clusters=6, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_6.fit(df_dummy)
df_dummy_6 = df_dummy.copy()
df_dummy_6['label'] = km_6.labels_
df_dummy_6['Country'] = df_original['Country']
df_dummy_6.to_csv(r'Dataset_6_clusters.csv', index = False)

# ### Con 7 clústers

# In[41]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_7 = KMeans(n_clusters=7, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_7.fit(df_dummy)
df_dummy_7 = df_dummy.copy()
df_dummy_7['label'] = km_7.labels_
df_dummy_7['Country'] = df_original['Country']
df_dummy_7.to_csv(r'Dataset_7_clusters.csv', index = False)

# ### Con 8 clústers

# In[42]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_8 = KMeans(n_clusters=8, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_8.fit(df_dummy)
df_dummy_8 = df_dummy.copy()
df_dummy_8['label'] = km_8.labels_
df_dummy_8['Country'] = df_original['Country']
df_dummy_8.to_csv(r'Dataset_8_clusters.csv', index = False)

# In[59]:

df_dummy

# ---
#

#
#

#
# ---

#
# # 5. Análisis clustering

#
# ## 5.1. Tamaño/Densidad de clústers

# In[46]:

print(90*'_')
print("\nNumero de paises en cada cluster 2")
print(90*'_')

pd.value_counts(km_2.labels_, sort=False)

# In[47]:

print(90*'_')
print("\nNumero de paises en cada cluster 3")
print(90*'_')

pd.value_counts(km_3.labels_, sort=False)

# In[48]:

print(90*'_')
print("\nNumero de paises en cada cluster 4")
print(90*'_')

pd.value_counts(km_4.labels_, sort=False)

# In[49]:

print(90*'_')
print("\nNumero de paises en cada cluster 5")
print(90*'_')

pd.value_counts(km_5.labels_, sort=False)

# In[50]:

print(90*'_')
print("\nNumero de paises en cada cluster 6")
print(90*'_')

pd.value_counts(km_6.labels_, sort=False)

# In[51]:

print(90*'_')
print("\nNumero de paises en cada cluster 7")
print(90*'_')

pd.value_counts(km_7.labels_, sort=False)

# In[52]:

print(90*'_')
print("\nNumero de paises en cada cluster 8")
print(90*'_')

pd.value_counts(km_8.labels_, sort=False)

#
# ## 5.2. Centroides de clústers (Formas)

# In[54]:

def pd_centers(featuresUsed, centers):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt
	import pandas as pd
	import numpy as np

	colNames = list(featuresUsed)
	colNames.append('prediction')

	# Zip with a column called 'prediction' (index)
	Z = [np.append(A, index) for index, A in enumerate(centers)]

	# Convert to pandas for plotting
	P = pd.DataFrame(Z, columns=colNames)
	P['prediction'] = P['prediction'].astype(int)
	return P

def parallel_plot(data):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt

	my_colors = list(islice(cycle(['b', 'r', 'g', 'y', 'k']), None, len(data)))
	plt.figure(figsize=(15,8)).gca().axes.set_ylim([-2.5,+2.5])
	parallel_coordinates(data, 'prediction', color = my_colors, marker='o')

#
# First, let us create some utility functions for Plotting
#

def pd_centers(featuresUsed, centers):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt
	import pandas as pd
	import numpy as np

	colNames = list(featuresUsed)
	colNames.append('prediction')

	# Zip with a column called 'prediction' (index)
	Z = [np.append(A, index) for index, A in enumerate(centers)]

	# Convert to pandas for plotting
	P = pd.DataFrame(Z, columns=colNames)
	P['prediction'] = P['prediction'].astype(int)
	return P

def parallel_plot(data):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt

	my_colors = list(islice(cycle(['b', 'r', 'g', 'y', 'k']), None, len(data)))
	plt.figure(figsize=(15,8)).gca().axes.set_ylim([-2.5,+2.5])
	parallel_coordinates(data, 'prediction', color = my_colors, marker='o')

# ### 2 Clusters

# In[60]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_2.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 3 Clusters

# In[61]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_3.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 4 Clusters

# In[62]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_4.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 5 Clusters

# In[63]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_5.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 6 Clusters

# In[64]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_6.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 7 Clusters

# In[65]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_7.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 8 Clusters

# In[66]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_8.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ---
#

#
#

#
# ---


import sys
sys.path.insert(0, '.')
from operator import add

import toolz
from toolz import curry as c
import pytest

from bramin.subp import subp
from bramin import *

inc = c(map, lambda x: x + 1)

@pytest.fixture
def tmp_f():
    return "/tmp/bramin_test.txt"

class TestPipe(object):
    def test_simple_pipe(self):
        ans = range(10) | P | inc | list | END
        assert ans == list(range(1, 11))

    def test_pipe_as_func(self):
        pipe = P | inc | list
        ans = pipe(range(10))
        assert ans == list(range(1, 11))

    def test_concat_pipe(self):
        p1 = P | inc | inc
        p2 = P | list
        pipe = p1 | p2
        ans = pipe(range(10))
        assert ans == list(range(2, 12))

    def test_placeholder_in_func(self):
        pipe = P | c(map, lambda x: x+1, it) | list
        ans = pipe(range(10))
        assert ans == list(range(1, 11))
        ans = range(5) | P | c(toolz.accumulate, add, it) | list | END
        assert ans == [0, 1, 3, 6, 10]
        pipe = P | it + it
        assert pipe(1) == 2
        pipe = P | it + it + it - 1
        assert pipe(1) == 2
        assert pipe(1) == 2

    def test_placeholder_in_pipe(self):
        id_f = P | it
        assert id_f([1, 2, 3]) == [1, 2, 3]
        assert 1 | P | it + 1 | it // 2 | END == 1
        #g = range(10) | P | (i for i in _x_) | END
        #assert list(g) == list(range(10))

    def test_with_pandas(self):
        import pandas as pd
        df = pd.DataFrame([[1, 2, 3],
                           [2, 3, 4],
                           [4, 3, 1]], columns=['a', 'b', 'c'])
        pipe = P | it[it['a'] > 2].shape[0]
        assert pipe(df) == 1
        assert df | P | pipe | END == 1
        assert df | P | it[it['a'] > 2].shape[0] | END == 1

    def test_with_numpy(self):
        import numpy as np
        a = np.arange(10)
        pipe = P | np.sum
        assert pipe(a) == 45
        pipe = P | np.sin | it.shape[0]
        assert pipe(a) == 10
        pipe = P | np.sin(it) | it.shape[0]
        assert pipe(a) == 10
        assert a | P | pipe | END == 10

        b = np.linspace(0, 2*np.pi, 100)
        pipe = P | c(add, it, np.sin(it))
        assert type(pipe(b)) is np.ndarray
        assert pipe(b).shape == b.shape
        pipe = P | it[:10]
        assert (b | P | pipe | END).shape[0] == 10
        pipe = P | (it > np.pi)
        assert sum(pipe(b)) == 50
        pipe = P | it[it > np.pi]
        assert pipe(b).min() > np.pi

    def test_write_text(self, tmp_f):
        def gen_lines():
            for i in range(10):
                yield str(i)*i + "\n"
        gen_lines() | P | c(filter, lambda l: len(l.strip()) > 0) > tmp_f | END
        with open(tmp_f) as f:
            lines = f.readlines()
            assert lines == list(gen_lines())[1:]
        (gen_lines() | P | c(filter, lambda l: len(l.strip()) > 0)) >> tmp_f | END
        with open(tmp_f) as f:
            lines = f.readlines()
            assert lines == list(gen_lines())[1:] + list(gen_lines())[1:]

    def test_read_text(self, tmp_f):
        with open(tmp_f, 'w') as f:
            for i in range(10, 0, -1):
                f.write(str(i)+"\n")
        to_int = c(map, lambda l: int(l.strip()))
        grep_odd = c(filter, lambda i: i % 2 == 0)
        ans = tmp_f >> P | to_int | grep_odd | list | END
        assert ans == list(grep_odd(range(10, 0, -1)))

    def test_source_subp(self, tmp_f):
        with open(tmp_f, 'w') as f:
            for i in range(11):
                f.write(str(i)+"\n")
        filter_1 = c(filter, lambda l: "1" in l)
        pipe = P | subp(f"cat {tmp_f}") | filter_1 | list
        assert pipe() == ["1\n", "10\n"]

    def test_mid_subp(self):
        def gen_lines(n):
            for i in range(n):
                yield str(i) + "\n"
        pipe = P | gen_lines | subp("grep 1") | list
        assert pipe(11) == ["1\n", "10\n"]

    def test_compose_subp(self, tmp_f):
        with open(tmp_f, 'w') as f:
            for i in range(111111):
                f.write(str(i)+"\n")
        pipe = P | subp(f"cat {tmp_f}") | subp("grep 1") | list
        assert pipe._chain[0].cmd == f"cat {tmp_f} | grep 1"
        assert pipe()[:2] == ["1\n", "10\n"]


import numpy as np

def PCA(data: np.array, threshold: float):

    print("================", "Start PCA:", "================", sep='\n')
    print("Original Data:", data, sep='\n')
    n = data.shape[0]
    S_matrix = np.dot(data.transpose(), data) / n
    print("S:", S_matrix, sep='\n')

    eig_tuple = np.linalg.eig(S_matrix)
    print('eigenvalues and eigenvectors:\n', eig_tuple)
    sum_eigval = np.sum(eig_tuple[0])

    # Select first m eigenvalues
    sum_now = 0
    for m in range(len(eig_tuple[0])):
        sum_now += eig_tuple[0][m]
        if sum_now / sum_eigval > threshold:
            m -= 1
            break

    if m < 0:
        m = 0

    print("m:", m)
    P_matrix = eig_tuple[1][:m]
    print("P:", P_matrix.transpose(), sep='\n')

    new_data = np.dot(P_matrix, data.transpose()).transpose()
    print("\nNew data with %d dimension(s):" % m, new_data, sep='\n')
    return new_data


from project.player import Player

class Guild:
    list_of_players = []

    def __init__(self, name):
        self.name = name

    def assign_player(self, player: Player):
        if player in self.list_of_players:
            return f"Player {player.name} is already in the guild."
        elif not player.guild == "Unaffiliated":
            return f"Player {player.name} is in another guild."
        self.list_of_players.append(player)
        player.guild = self.name
        return f"Welcome player {player.name} to the guild {self.name}"

    def kick_player(self, player_name):
        for player in self.list_of_players:
            if player.name == player_name:
                self.list_of_players.remove(player)
                player.guild = "Unaffiliated"
                return f"Player {player_name} has been removed from the guild."
        return f"Player {player_name} is not in the guild."

    def guild_info(self):
        players_info = ""
        for player in self.list_of_players:
            players_info += player.player_info()

        return f"Guild: {self.name} \n" \
               f"{players_info}"


# https://onlinejudge.u-aizu.ac.jp/courses/lesson/1/ALDS1/6/ALDS1_6_D
# 最小コストソート

def get_groups(raw_nums, sorted_nums):
  allocated_nums = []
  groups = []
  for i in range(len(raw_nums)):
    num = raw_nums[i]
    if num in allocated_nums:
      continue
    group = []
    while True:
      if num in group:
        break
      group.append(num)
      allocated_nums.append(num)
      num = raw_nums[sorted_nums.index(num)]
    groups.append(group)
  return groups

def calc_min_cost(groups, min_val):
  cost = 0
  for group in groups:
    cost += min(
      sum(group)+(len(group)-2)*min(group),
      sum(group)+min(group)+(len(group)+1)*min_val
    )
  return cost

# main
qty = input()
nums = list(map(int, input().split(' ')))
sorted_nums = sorted(nums)
groups = get_groups(nums, sorted_nums)
print(calc_min_cost(groups, min(nums)))

"""input
7
4 3 2 7 1 6 5
"""
"""output
24
"""


import cv2 as cv
import matplotlib.pyplot as plt
import numpy as np

def read_data(imgpath='./datasets/library/lib_photo.png', labeltxt='./datasets/wenben/data1.txt'):
    lib_pic = cv.imread(imgpath)
    data = open(labeltxt, 'r')
    lines = data.readline().split(',')

    label_names = ['no', 'yes']
    labels = []
    positions = []
    x_starts = []
    y_starts = []
    x_ends = []
    y_ends = []
    for line in lines:
        words = line.split(' ')
        if len(words) < 6:
            break
        labels.append(int(words[0]))
        positions.append(int(words[1]))
        x_starts.append(int(words[2]))
        y_starts.append(int(words[3]))
        x_ends.append(int(words[4]))
        y_ends.append(int(words[5]))

    def get_labels():
        return labels

    def get_sub_images(photo=lib_pic):
        sub_images = []
        for index in range(0, len(labels)):
            img = photo[y_starts[index]: y_ends[index], x_starts[index]: x_ends[index]]
            img = cv.resize(img, (64, 64), interpolation=cv.INTER_CUBIC)
            b, g, r = cv.split(img)

            img2 = cv.merge([r, g, b])
            img2 = img2 / 255.0
            sub_images.append(img2)
        return sub_images

    return get_sub_images(), get_labels()

"""
plt.figure(figsize=(10, 10))
subs = get_sub_images()

for index in range(len(subs)):
    plt.subplot(6, 6, index + 1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)

    plt.imshow(subs[index])
    plt.xlabel(label_names[labels[index]])
print(labels[index], x_starts[index], y_starts[index], x_ends[index], y_ends[index])

img = lib_pic[y_starts[index]: y_ends[index], x_starts[index]: x_ends[index]]
print(img.shape)
img = cv.resize(img, (64, 64), interpolation=cv.INTER_LINEAR)
print(img.shape)

plt.imshow(img)
plt.show()
"""


from collections import defaultdict
import glob, json
import numpy as np
from IPython.display import clear_output
from bisect import bisect_left
from itertools import chain
import os.path
import argparse

# Data found here https://nvd.nist.gov/vuln/data-feeds#JSON_FEED

class Vulndb:
    def __init__(self, loadfile_path='data/*.json',
                 save_sdb='searchdb.json', rebuild=False):
        self.data = []
        self.searchdb = None
        self.save_sdb=save_sdb
        if not rebuild or os.path.isfile(self.save_sdb):
            self.load_saved_dbs()
        else:
            self.load_data(loadfile_path)
            self.build_searchdb()
            self.save_dbs()

    def load_data(self, loadfile_path):
        dataset = []
        for idx, file in enumerate(glob.glob(loadfile_path)):
            clear_output(wait=True)
            print("Processing data {}%".format(np.round(idx/len(glob.glob(loadfile_path)))*100))
            with open(file) as json_file:
                dataset[0:0] = json.load(json_file)['CVE_Items']
        print("load finished")
        self.data = dataset

    def build_searchdb(self):
        nested_dict = lambda:defaultdict(nested_dict)
        searchdb = nested_dict()
        dtlength = len(self.data)
        for idx, item in enumerate(self.data):
            if idx%10 == 0:
                clear_output(wait=True)
                print("Processing search graph {}%".format(np.round(idx/dtlength)*100))
            for vendor in item['cve']['affects']['vendor']['vendor_data']:
                for product in vendor['product']['product_data']:
                    for version in product['version']['version_data']:
                        if not searchdb[product['product_name']]['{}_{}'.format(
                            version['version_affected'], version['version_value'])]:
                            searchdb[product['product_name']]['{}_{}'.format(
                            version['version_affected'], version['version_value'])] = [item['cve']['CVE_data_meta']['ID']]
                        else:
                            searchdb[product['product_name']]['{}_{}'.format(
                            version['version_affected'], version['version_value'])].append(
                                item['cve']['CVE_data_meta']['ID'])
        self.searchdb = searchdb

    def save_dbs(self):
        with open(self.save_sdb, 'w+') as f:
            print("Writing Search DB")
            f.write(json.dumps(self.searchdb))

    def load_saved_dbs(self):
        print("Loading data")
        with open(self.save_sdb, 'r') as f:
            self.searchdb = json.load(f)

    def version_lt_search(self, target, versions):
        ltversions = [vs[3:] for vs in versions if vs.startswith("<=_")]
        ltversions.append(target)
        svers = sorted(ltversions, key=lambda v: [int(i) for i in v.rstrip('@').split('.')])
        return ["<=_{}".format(s) for s in svers[svers.index(target)+1:]]

    def search_vuln(self, software, version=None):
        matches = []
        software = software.lower()
        if software in self.searchdb:
            versions = self.searchdb[software].keys()
            # match any
            if "=_*" in versions:
                matches.append(self.searchdb[software]["=_*"])
            # match exact
            if version and "=_{}".format(version) in versions:
                matches.append(self.searchdb[software]["=_{}".format(version)])
            # match less than
            if version:
                try:
                    ltmatches = self.version_lt_search(version, versions)
                    for m in ltmatches:
                        matches.append(self.searchdb[software][m])
                except ValueError:
                    pass
        return set(chain.from_iterable(matches))

    def python_requirements_parse(self, filename):
        reponame=re.compile('^[\w]+')
        version=re.compile('(\d+\.)?(\d+\.)?(\*|\d+)$')
        with open(filename, 'r') as f:
            reqs=f.readlines()
        for line in reqs:
            if len(line) > 1:
                repo = re.search(reponame, line)
                vs = re.search(version, line)
                if repo and vs:
                    vulnlist = self.search_vuln(repo.group(0), vs.group(0))
                    if not vulnlist:
                        vulnlist = "✓"
                    print("    {} {} : {}".format(repo.group(0), str(vs.group(0)), vulnlist))

if __name__ == "__main__":
    vdb = Vulndb()

parser = argparse.ArgumentParser(description='Vuln Scanner.')
parser.add_argument('-n', '--name')
parser.add_argument('-v', '--version')
parser.add_argument('-f', '--file')
args = parser.parse_args()

if args.file:
    vdb.python_requirements.parse(args.file)
else:
    vdb.search_vuln(args.name, args.version)


######################################################
#  Ben Palmer University of Birmingham 2020
#  Free to use
######################################################

import numpy
import os
from labels import labels
from pwscf_output import pwscf_output
from potential_functions import potential_functions
from f2py_lib.f_interp import interp

class configs:

  @staticmethod
  def load():
    try:
      configs_dir = g.inp['configs']['dir']
    except:
      return False
    if(not os.path.isdir(configs_dir)):
      return False

    # Make file list
    g.configs['config_files'] = []
    g.configs['config_files'] = configs.load_files(configs_dir, g.configs['config_files'])

    # Read files in
    configs.read()

    #print(g.configs)

    return True

  @staticmethod
  def load_files(path_in, files):
    for path in os.listdir(path_in):
      path_new = path_in + "/" + path
      if(os.path.isdir(path_new)):
        files = configs.load_files(path_new, files)
      else:
        type = configs.file_type(path_new)
        files.append([path_new,type])
    return files

  @staticmethod
  def read():
    # Loop through files and read in
    for file in g.configs['config_files']:
      if(file[1] == 'std'):
        configs.std(file[0])
      elif(file[1] == 'qe'):
        configs.qe(file[0])

  @staticmethod
  def std(file_path):

    # Read content from file
    content = std.file_to_list(file_path)
    content = std.prep_data(content)
    content = std.remove_comments(content)

    # Split up into individual configs
    config_list = []
    temp = []
    last_line = None
    for line in content:
      if(last_line != None and line != '' and last_line != '' and last_line[0] != "#" and line[0] == "#"):
        config_list.append(temp)
        temp = []
      if(line != ''):
        temp.append(line)
        last_line = line
    if(len(temp)>0):
      config_list.append(temp)

    # Read each config
    for i in range(len(config_list)):
      configs.add_config(config_list[i], file_path, i)

  @staticmethod
  def make_config():
    return {
    'file_type': '',
    'file_path': '',
    'file_part': 0,
    'alat': 0.0,
    'uv_prim': numpy.zeros((3,3,),),
    'uv': numpy.zeros((3,3,),),
    'stress': numpy.zeros((3,3,),),
    'c': numpy.zeros((3,),dtype=numpy.int32,),
    'rcut': 0.0,
    'rverlet': 0.0,
    'mtemp': {},
    'm': {},
    'coord_count_prim': 0,
    'coords_label_prim': None,
    'coords_label_id_prim': None,
    'coords_prim': None,
    'forces_prim': None,
    'coord_count': 0,
    'coords_label': None,
    'coords_label_id': None,
    'coords': None,
    'forces': None,
    'energy_per_atom': None,
    'energy': None,
    'e': 0,
    'f': 0,
    's': 0,
    'n_atoms_prim': 0,
    'n_atoms': 0,
    'l_units': 'ANG',
    'e_units': 'EV',
    'f_units': 'EV/ANG',
    's_units': 'EV/ANG3',
    }

  @staticmethod
  def add_config(content, file_path, i, config_type='STANDARD'):
    fd = configs.make_config()
    fd['file_type'] = config_type
    fd['file_path'] = file_path
    fd['file_part'] = i

    # FIRST READ
    for line in content:
      line = line.strip()
      f = std.to_fields(line, ' ')
      epa_set = False

      if(len(f) > 1 and f[0].upper() == "#ALAT"):
        fd['alat'] = float(f[1])
      if(len(f) > 3 and f[0].upper() == "#X"):
        fd['uv_prim'][0,0] = float(f[1])
        fd['uv_prim'][0,1] = float(f[2])
        fd['uv_prim'][0,2] = float(f[3])
      if(len(f) > 3 and f[0].upper() == "#Y"):
        fd['uv_prim'][1,0] = float(f[1])
        fd['uv_prim'][1,1] = float(f[2])
        fd['uv_prim'][1,2] = float(f[3])
      if(len(f) > 3 and f[0].upper() == "#Z"):
        fd['uv_prim'][2,0] = float(f[1])
        fd['uv_prim'][2,1] = float(f[2])
        fd['uv_prim'][2,2] = float(f[3])
      if(len(f) > 3 and f[0].upper() == "#SX"):
        fd['s'] = 1
        fd['stress'][0,0] = float(f[1])
        fd['stress'][0,1] = float(f[2])
        fd['stress'][0,2] = float(f[3])
      if(len(f) > 3 and f[0].upper() == "#SY"):
        fd['s'] = 1
        fd['stress'][1,0] = float(f[1])
        fd['stress'][1,1] = float(f[2])
        fd['stress'][1,2] = float(f[3])
      if(len(f) > 3 and f[0].upper() == "#SZ"):
        fd['s'] = 1
        fd['stress'][2,0] = float(f[1])
        fd['stress'][2,1] = float(f[2])
        fd['stress'][2,2] = float(f[3])
      if(len(f) > 2 and f[0].upper() == "#M"):
        fd['mtemp'][f[1]] = f[2]
      if(len(f) > 3 and f[0].upper() == "#C"):
        fd['c'][0] = int(f[1])
        fd['c'][1] = int(f[2])
        fd['c'][2] = int(f[3])
      if(len(f) > 1 and f[0].upper() == "#E"):
        fd['energy'] = float(f[1])
        fd['e'] = 1
      if(len(f) > 1 and f[0].upper() == "#EPA"):
        fd['energy_per_atom'] = float(f[1])
        fd['e'] = 1
      if(len(f) > 1 and f[0].upper() == "#RCUT"):
        fd['rcut'] = f[1]
        if(fd['rverlet'] == 0):
          fd['rverlet'] = f[1]
      if(len(f) > 1 and f[0].upper() == "#RVERLET"):
        fd['rverlet'] = f[1]
      if(len(f) > 1 and f[0].upper() == "#L_UNITS"):
        fd['l_units'] = f[1]
      if(len(f) > 1 and f[0].upper() == "#E_UNITS"):
        fd['e_units'] = f[1]
      if(len(f) > 1 and f[0].upper() == "#F_UNITS"):
        fd['f_units'] = f[1]
      if(len(f) > 1 and f[0].upper() == "#S_UNITS"):
        fd['s_units'] = f[1]
      if(len(f) >= 4 and f[0][0] != "#"):
        count_coord = False
        if(len(f) >= 4):
          count_coord = True
        if(len(f) >= 7):
          fd['f'] = 1
        if(count_coord):
          fd['coord_count_prim'] = fd['coord_count_prim'] + 1

    # COORD SIZE
    c = numpy.identity(3)

    # EXPAND
    c[0,0] = float(fd['c'][0])
    c[1,1] = float(fd['c'][1])
    c[2,2] = float(fd['c'][2])
    fd['coord_count'] = (fd['c'][0] * fd['c'][1] * fd['c'][2]) * fd['coord_count_prim']
    fd['uv'] = numpy.matmul(c, fd['uv_prim'])

    # MAKE ARRAYS
    fd['coords_label_prim'] = [None] * fd['coord_count_prim']
    fd['coords_label_id_prim'] = numpy.zeros((fd['coord_count_prim'],), dtype=numpy.int32,)
    fd['coords_prim'] = numpy.zeros((fd['coord_count_prim'],3,),)
    if(fd['f'] == 1):
      fd['forces_prim'] = numpy.zeros((fd['coord_count_prim'],3,),)

    fd['coords_label'] = [None] * fd['coord_count']
    fd['coords_label_id'] = numpy.zeros((fd['coord_count'],), dtype=numpy.int32,)
    fd['coords'] = numpy.zeros((fd['coord_count'],3,),)
    if(fd['f'] == 1):
      fd['forces'] = numpy.zeros((fd['coord_count'],3,),)

    fd['n_atoms_prim'] = fd['coord_count_prim']
    fd['n_atoms'] = fd['coord_count']

    # READ COORDS
    n = 0
    for line in content:
      line = line.strip()
      f = std.to_fields(line, ' ')
      if(len(f) >= 4 and f[0][0] != "#"):
        if(len(f) >= 4):
          label_str, label_id = labels.add(f[0])
          fd['coords_label_prim'][n] = label_str
          fd['coords_label_id_prim'][n] = label_id
          fd['coords_prim'][n,0] = float(f[1])
          fd['coords_prim'][n,1] = float(f[2])
          fd['coords_prim'][n,2] = float(f[3])

        if(len(f) >= 7):
          fd['forces_prim'][n,0] = float(f[4])
          fd['forces_prim'][n,1] = float(f[5])
          fd['forces_prim'][n,2] = float(f[6])
        n = n + 1

    # EXPAND COORDS
    m = 0
    for i in range(fd['c'][0]):
      for j in range(fd['c'][1]):
        for k in range(fd['c'][2]):
          for n in range(fd['coord_count_prim']):
            fd['coords_label'][m] = fd['coords_label_prim'][n]
            fd['coords_label_id'][m] = fd['coords_label_id_prim'][n]
            fd['coords'][m,0] = (i + fd['coords_prim'][n,0]) / fd['c'][0]
            fd['coords'][m,1] = (j + fd['coords_prim'][n,1]) / fd['c'][1]
            fd['coords'][m,2] = (k + fd['coords_prim'][n,2]) / fd['c'][2]
            if(fd['f'] == 1):
              fd['forces'][m,0] = fd['forces_prim'][n,0]
              fd['forces'][m,1] = fd['forces_prim'][n,1]
              fd['forces'][m,2] = fd['forces_prim'][n,2]
            m = m + 1

    # SORT ENERGY
    if(fd['e'] == 1 and fd['energy_per_atom'] != None and fd['energy'] == None):
      try:
        fd['energy'] = fd['energy_per_atom'] * fd['n_atoms']
      except:
        pass
    if(fd['e'] == 1 and fd['energy'] != None and fd['energy_per_atom'] == None):
      try:
        fd['energy_per_atom'] = fd['energy'] / fd['n_atoms_prim']
        fd['energy'] = fd['energy_per_atom'] * fd['n_atoms']
      except:
        pass

    # APPEND
    g.configs['configs'].append(fd)

  #
  #  FILE TYPE (standard, quantum espresso)
  #

  @staticmethod
  def file_type(file_path):
    content = std.file_to_list(file_path)

    # Check if standard file
    count = 0
    for line in content:
      fields = line.split(" ")
      if(fields[0].upper() == "#ALAT"):
        count = count + 1
      if(fields[0].upper() == "#X"):
        count = count + 1
      if(fields[0].upper() == "#Y"):
        count = count + 1
      if(fields[0].upper() == "#Z"):
        count = count + 1
    if(count >= 4):
      return 'std'

    # Check if pwscf/qe file
    count = 0
    for line in content:
      if(line.strip()[0:13] == "Program PWSCF"):
        count = count + 1
      if(line.strip()[0:27] == "bravais-lattice index     ="):
        count = count + 1
      if(line.strip()[0:27] == "kinetic-energy cutoff     ="):
        count = count + 1
      if(line.strip()[0:27] == "mixing beta               ="):
        count = count + 1
      if(line.strip()[0:27] == "Exchange-correlation      ="):
        count = count + 1
      if(line.strip()[0:9] == "JOB DONE."):
        count = count + 1
    if(count >= 4):
      return 'qe'

  # Espresso files

  @staticmethod
  def qe(file_path):

    qe = pwscf_output(file_path)
    xyz = qe.make_xyz()
    n = 0
    for xyz_inner in xyz:
      n = n + 1
      configs.add_config(xyz_inner, file_path, n, config_type='QE')

  @staticmethod
  def output():
    if(g.outputs):
      fh = open(g.dirs['output'] + '/' + 'configs.dat', 'w')

      n = 0
      for c in g.configs['configs']:

        n = n + 1
        fh.write('#############################################################\n')
        fh.write('#############################################################\n')
        fh.write('CONFIG ' + str(n) + '\n')
        fh.write('#############################################################\n')
        fh.write('#############################################################\n')
        fh.write('file_type  ' + str(c['file_type']) + ' \n')
        fh.write('file_path  ' + str(c['file_path']) + ' \n')
        fh.write('file_part  ' + str(c['file_part']) + ' \n')
        fh.write('alat  ' + str(c['alat']) + ' \n')
        fh.write('uv_prim   ' + str(c['uv_prim'][0,0]) + ' ' + str(c['uv_prim'][0,1]) + ' ' + str(c['uv_prim'][0,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['uv_prim'][1,0]) + ' ' + str(c['uv_prim'][1,1]) + ' ' + str(c['uv_prim'][1,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['uv_prim'][2,0]) + ' ' + str(c['uv_prim'][2,1]) + ' ' + str(c['uv_prim'][2,2]) + ' ' + ' \n')
        fh.write('uv        ' + str(c['uv'][0,0]) + ' ' + str(c['uv'][0,1]) + ' ' + str(c['uv'][0,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['uv'][1,0]) + ' ' + str(c['uv'][1,1]) + ' ' + str(c['uv'][1,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['uv'][2,0]) + ' ' + str(c['uv'][2,1]) + ' ' + str(c['uv'][2,2]) + ' ' + ' \n')
        fh.write('stress    ' + str(c['stress'][0,0]) + ' ' + str(c['stress'][0,1]) + ' ' + str(c['stress'][0,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['stress'][1,0]) + ' ' + str(c['stress'][1,1]) + ' ' + str(c['stress'][1,2]) + ' ' + ' \n')
        fh.write('          ' + str(c['stress'][2,0]) + ' ' + str(c['stress'][2,1]) + ' ' + str(c['stress'][2,2]) + ' ' + ' \n')
        fh.write('c         ' + str(c['c'][0])  + ' ' + str(c['c'][1]) + ' ' + str(c['c'][2]) + ' ' + ' \n')
        fh.write('rcut      ' + str(c['rcut']) + ' \n')
        fh.write('rverlet   ' + str(c['rverlet']) + ' \n')
        fh.write('e         ' + str(c['e']) + ' \n')
        fh.write('f         ' + str(c['f']) + ' \n')
        fh.write('s         ' + str(c['s']) + ' \n')

        fh.write('\n')
        fh.write('COORDS PRIM (' + str(c['coord_count_prim']) + ')\n')
        for k in range(c['coord_count_prim']):
          fh.write(str(c['coords_label_prim'][k]) + '  ')
          fh.write(str(c['coords_prim'][k,0]) + '  ')
          fh.write(str(c['coords_prim'][k,1]) + '  ')
          fh.write(str(c['coords_prim'][k,2]) + '  ')
          if(c['f'] == 1):
            fh.write(str(c['forces_prim'][k,0]) + '  ')
            fh.write(str(c['forces_prim'][k,1]) + '  ')
            fh.write(str(c['forces_prim'][k,2]) + '  ')
          fh.write('\n')
        fh.write('\n')

        fh.write('\n')
        fh.write('COORDS (' + str(c['coord_count']) + ')\n')
        for k in range(c['coord_count']):
          fh.write(str(c['coords_label'][k]) + '  ')
          fh.write(str(c['coords'][k,0]) + '  ')
          fh.write(str(c['coords'][k,1]) + '  ')
          fh.write(str(c['coords'][k,2]) + '  ')
          if(c['f'] == 1):
            fh.write(str(c['forces'][k,0]) + '  ')
            fh.write(str(c['forces'][k,1]) + '  ')
            fh.write(str(c['forces'][k,2]) + '  ')
          fh.write('\n')
        fh.write('\n')

        fh.write('\n')
        fh.write('\n')

    fh.close()

  # Run after configs read in
  # After dft_energy_adjustments

  @staticmethod
  def complete():
    #print("Complete Configs")
    #print(g.configs['configs'])

    # CONVERT
    for i in range(len(g.configs['configs'])):

      g.configs['configs'][i]['alat'] = units.convert(g.configs['configs'][i]['l_units'], 'ang', g.configs['configs'][i]['alat'])
      g.configs['configs'][i]['l_units'] = 'ANG'

      #print(g.configs['configs'][i]['energy'], g.configs['configs'][i]['energy_per_atom'])

      g.configs['configs'][i]['energy_per_atom'] = units.convert(g.configs['configs'][i]['e_units'], 'ev', g.configs['configs'][i]['energy_per_atom'])
      g.configs['configs'][i]['energy'] = units.convert(g.configs['configs'][i]['e_units'], 'ev', g.configs['configs'][i]['energy'])
      g.configs['configs'][i]['e_units'] = 'EV'
      #print(g.configs['configs'][i]['energy'], g.configs['configs'][i]['energy_per_atom'])

      # print(g.configs['configs'][i]['f_units'])
      if(g.configs['configs'][i]['f'] == 1):
        for na in range(len(g.configs['configs'][i]['forces_prim'])):
          for nb in range(len(g.configs['configs'][i]['forces_prim'][na])):
            g.configs['configs'][i]['forces_prim'][na,nb] = units.convert(g.configs['configs'][i]['f_units'], 'EV/ANG', g.configs['configs'][i]['forces_prim'][na,nb])

      if(g.configs['configs'][i]['f'] == 1):
        for na in range(len(g.configs['configs'][i]['forces'])):
          for nb in range(len(g.configs['configs'][i]['forces'][na])):
            g.configs['configs'][i]['forces'][na,nb] = units.convert(g.configs['configs'][i]['f_units'], 'EV/ANG', g.configs['configs'][i]['forces'][na,nb])

      if(g.configs['configs'][i]['f'] == 1):
        for na in range(len(g.configs['configs'][i]['stress'])):
          for nb in range(len(g.configs['configs'][i]['stress'][na])):
            g.configs['configs'][i]['stress'][na,nb] = units.convert(g.configs['configs'][i]['s_units'], 'EV/ANG3', g.configs['configs'][i]['stress'][na,nb])

    # ADJUST ENERGY QE
    for i in range(len(g.configs['configs'])):
      if(g.configs['configs'][i]['file_type'] == 'QE'):
        #print(g.configs['configs'][i]['coord_count'], g.configs['configs'][i]['coord_count_prim'])
        for k in range(g.configs['configs'][i]['coord_count']):
          e_adj = g.dft_energy_adjustments[g.configs['configs'][i]['coords_label_id'][k]]
          g.configs['configs'][i]['energy'] = g.configs['configs'][i]['energy'] + e_adj['calc_apaev']
        g.configs['configs'][i]['energy_per_atom'] = g.configs['configs'][i]['energy'] / g.configs['configs'][i]['coord_count']
        #print(g.configs['configs'][i]['energy'], g.configs['configs'][i]['energy_per_atom'])

          #print(e_adj['calc_apaev'])

        #c['coords_label'][k]
        #print('QE')

    #  g.dft_energy_adjustments[label_id]

###########################################################
# F2PY functions
###########################################################

  @staticmethod
  def efs_add_config():
    for c in g.configs['configs']:
      efs.add_config(
                     6.5,
                     c['alat'],
                     c['uv'],
                     c['coords_label_id'],
                     c['coords'],
                     c['energy'],
                     c['forces'],
                     c['stress'],
                     c['f'],
                     c['s']
                    )
    efs.make_nl()


from math import pi
import random
import os
import sys
import math
assert sys.version_info >= (2, 5)
try:

  import pygame
except ImportError:
  print("Pygame not installed. Please install before running program.")
  exit()
import particle

(width, height) = (624,468)
screen = pygame.display.set_mode((width,height))
pygame.font.init()
myfont = pygame.font.SysFont('Sans Serif', 30)

pygame.display.set_caption('Springs')
paused = False
running = True
uni = particle.envi((width, height))
uni.color = (255,255,255)
uni.addFunctions(['move', 'bounce', 'collide', 'drag', 'accelerate'])
mypar = []
for p in range(9):
  particle = uni.addParticle(mass=100, size=16, speed=2, elasticity=1, color=(20,40,200))
  mypar.append(particle)
uni.addSpring(0,1, length=100, strength=0.5)
uni.addSpring(1,2, length=100, strength=0.1)
uni.addSpring(2,0, length=80, strength=0.05)
#
uni.addSpring(5,6,length=100,strength=0.4)
uni.addSpring(6,7,length=100,strength=0.4)
uni.addSpring(7,8,length=100,strength=0.4)
uni.addSpring(8,5,length=100,strength=0.4)
uni.addSpring(7,5,length=100,strength=0.4)
uni.addSpring(8,6,length=100,strength=0.4)
#uni.addSpring(9,5,length=100,strength=0.4)
selected_particle = None
clock = pygame.time.Clock()

while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_SPACE:
                paused = (True, False)[paused]
        elif event.type == pygame.MOUSEBUTTONDOWN:
          (mouseX, mouseY) = pygame.mouse.get_pos()
          selected_particle = uni.findPar(mypar,mouseX,mouseY)
        elif event.type == pygame.MOUSEBUTTONUP:
          selected_particle = None
    if selected_particle:

          (mouseX, mouseY) = pygame.mouse.get_pos()
          dx = mouseX - selected_particle.x
          dy = mouseY - selected_particle.y
          selected_particle.angle = 0.5 * math.pi + math.atan2(dy, dx)
          selected_particle.speed = math.hypot(dx, dy) * 0.1
    if not paused:
      uni.update()
    screen.fill(uni.color)
    for p in uni.particles:
      pygame.draw.circle(screen, p.color, (int(p.x), int(p.y)), p.size, 0)
    for s in uni.springs:
      pygame.draw.aaline(screen,(0,0,0), (int(s.p1.x), int(s.p1.y)), (int(s.p2.x), int(s.p2.y)))
    clock.tick()
    text = "FPS: " + str(clock.get_fps())
    textsurface = myfont.render(text, False, (0, 20, 0))
    screen.blit(textsurface, (0, 20))
    pygame.display.flip()


import os
import json
from kafka import KafkaConsumer
import boto3

# Simple Kafka on docker compose so same host for all brokers
KAFKA_IP = os.environ.get('KAFKA_IP', 'localhost')

consumer = KafkaConsumer(
    bootstrap_servers=[f'{KAFKA_IP}:9093', f'{KAFKA_IP}:9094', f'{KAFKA_IP}:9095'],
    auto_offset_reset='latest', enable_auto_commit=True,
    auto_commit_interval_ms=1000)

consumer.subscribe(topics=['raw_table_acct','raw_table_cust'])

lambda_client = boto3.client('lambda')

for message in consumer:
    value = json.loads(message.value)

    payload = {"topic": message.topic, "partition": message.partition, "offset": message.offset, "key": message.key, "value": value }

    response = lambda_client.invoke(
        FunctionName='test_kafka_s3',
        Payload=json.dumps(payload),
    )

    print( response['Payload'].read().decode("utf-8") )


import sqlite3
from datetime import date

class database:
    def __init__(self) -> None:
        self.create_tables_if_not_exists()
        self.print_fts_table = False

    def search_items(self, query) -> list:
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        cursor.execute("SELECT item.*, item_fts.rank FROM item JOIN item_fts ON item.id = item_fts.id WHERE item_fts.name MATCH ? ORDER BY item_fts.rank", (query,))
        results = cursor.fetchall()

        print(f"Search results for '{query}':")
        for result in results:
            print(f"ID: {result[0]}, Name: {result[1]}, Row: {result[3]}, Col: {result[4]} Relevance Score: {result[-1]}")

        conn.close()

        return results

    def add_or_update_item(self, item_name: str, additional_quantity: int=1):
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        try:
            # Check if the item already exists
            existing_items = self.search_items(item_name)

            if existing_items:
                # Items exists, return found items
                return existing_items

            else:
                # Item does not exist, find the next available row and column
                # Query all rows and columns in the item table
                cursor.execute("SELECT row, col FROM item")
                occupied_positions = cursor.fetchall()

                # Determine the minimum available row and column
                occupied_positions_set = set(occupied_positions)
                if len(occupied_positions_set) > 0:
                    max_row = max([pos[0] for pos in occupied_positions])
                    max_col = max([pos[1] for pos in occupied_positions])
                else:
                    max_row, max_col = 0, 0

                next_row, next_column = None, None
                for row in range(1, min(max_row + 2, 8)):
                    for col in range(1, min(max_col + 2, 16)):
                        if (row, col) not in occupied_positions_set:
                            next_row, next_column = row, col
                            break
                    if next_row is not None:
                        break

                if next_row == None or next_column == None:
                    raise Exception("There is no remaining space in the organizer!")

                print(f"Inserting item '{item_name}' at [{next_row}, {next_column}]")

                # Insert the new item with the given quantity at the next available row and column
                cursor.execute("""
                    INSERT INTO item (name, quantity, row, col, date_created, date_updated)
                    VALUES (?, ?, ?, ?, ?, ?)
                    """, (item_name, additional_quantity, next_row, next_column, date.today(), date.today()))

                cursor.execute("""
                    INSERT INTO item_fts (id, name)
                    VALUES (?, ?)
                    """, (cursor.lastrowid, item_name))

                # Commit the changes before querying the db for the newly inserted item
                conn.commit()
                added_item = self.search_items(item_name)

                return added_item #[item_name, additional_quantity, next_row, next_column, date.today(), date.today()]
        finally:
            # Close the connection
            conn.close()

    def delete_items(self, items) -> None:
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        agg_results = []

        for item in items:
            cursor.execute("""
                SELECT item.*, item_fts.rank FROM item JOIN item_fts
                ON item.id = item_fts.id WHERE item_fts.name
                MATCH ? ORDER BY item_fts.rank""", (item,))

            results = cursor.fetchall()
            agg_results.append(*results)

            cursor.execute(f"DELETE FROM item WHERE name=?", (item,))

            if cursor.rowcount > 0:
                print(f"'{item}' deleted from the item table.")
            else:
                print(f"'{item}' not found in the item table.")

            cursor.execute(f"DELETE FROM item_fts WHERE name=?", (item,))

            if cursor.rowcount > 0:
                print(f"'{item}' deleted from the item_fts table.")
            else:
                print(f"'{item}' not found in the item_fts table.")

        conn.commit()
        conn.close()

        return agg_results

    def delete_items2(self, items) -> None:
        # Connect to the database
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        agg_deleted_items = []

        for item in items:

            cursor.execute("""
                SELECT item.*, item_fts.rank FROM item JOIN item_fts
                ON item.id = item_fts.id WHERE item_fts.name MATCH ?
                ORDER BY item_fts.rank""", (item,))

            #cursor.execute("SELECT * FROM item_fts WHERE name MATCH ?", (item,))

            # Fetch the matching items
            matching_items = cursor.fetchall()

            if len(matching_items) > 1:
                print(f"More than one item found for {item}. Skipping to avoid unintentional deletions.")
                print(matching_items)

            print(f"Deleting:")
            print(matching_items)

            agg_deleted_items.append(*matching_items)

            # Delete the matching items from the item table and the item_fts table using the ids
            for item_id in matching_items:
                cursor.execute("DELETE FROM item WHERE id=?", (item_id[0],))
                cursor.execute("DELETE FROM item_fts WHERE id=?", (item_id[0],))

        # Commit the changes and close the connection
        conn.commit()
        conn.close()

        return agg_deleted_items

    def print_tables(self) -> None:
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        cursor.execute("SELECT * FROM item")
        items = cursor.fetchall()

        print("Item table:")
        print(f"{'ID':<4} | {'Name':<15} | {'Quantity':<8} | {'Row':<4} | {'Col':<6} | {'Date Created':<12} | {'Date Updated':<16}")
        print("-----------------------------------------------------------------------------")

        for item in items:
            print(f"{item[0]:<4} | {item[1]:<15} | {item[2]:<8} | {item[3]:<4} | {item[4]:<6} | {item[5]:<12} | {item[6]:<16}")

        if self.print_fts_table:
            print()
            print("Item FTS table:")
            cursor.execute("SELECT * FROM item_fts")
            rows = cursor.fetchall()

            if not rows:
                print("The item_fts table is empty.")
            else:
                print(f"{'ID':<4} {'Name':<30}")
                print("-" * 40)
                for row in rows:
                    print(f"{row[0]:<4} {row[1]:<30}")

        conn.close()

    def create_tables_if_not_exists(self) -> None:
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        cursor.execute("""
            CREATE TABLE IF NOT EXISTS item (
                id INTEGER PRIMARY KEY,
                name TEXT UNIQUE NOT NULL,
                quantity INTEGER NOT NULL DEFAULT 1,
                row INTEGER NOT NULL,
                col INTEGER NOT NULL,
                date_created DATE NOT NULL,
                date_updated DATE NOT NULL
            )
        """)

        cursor.execute("""
            CREATE VIRTUAL TABLE IF NOT EXISTS item_fts USING fts5(id, name, tokenize='porter');
        """)

        conn.commit()
        conn.close()

    def delete_tables(self) -> None:
        conn = sqlite3.connect("inventory.db")
        cursor = conn.cursor()

        cursor.execute("DROP TABLE IF EXISTS item")
        cursor.execute("DROP TABLE IF EXISTS item_fts")

        conn.commit()
        conn.close()


from aiogram.types import ReplyKeyboardMarkup
from data.config import BASE_URL
import requests

rs = requests.get(url=f"{BASE_URL}main/schedule/create/")
data = rs.json()

available_time = ReplyKeyboardMarkup(row_width=3, resize_keyboard=True)
for i in data:
    if i['user_id'] is None:
        available_time.add(f"{i['month']} {i['day']} {i['time']}")
available_time.add('Bekor qilish')


import cv2 as cv
import numpy as np
# from get_lines import *
# from square_check import *
# from image_transform import *
from model.get_surface import *
from model.surface_deal import *

colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (100, 100, 0)]

def parse_args():
    '''Parse input arguments
    '''
    parser = argparse.ArgumentParser(description="Create dataset with different augmentations")
    # parser.add_argument("-ff","--foreground_files",
    #   help="The root directory which contains the images ,mask files and annotations of the foreground,"
    #        "image file path : */img/*.jpg;mask  file path : */mask/*/*.pbm;json  file path : */anno/*.json")
    parser.add_argument("-bf","--background_files",
      help="The root directory which contains the images,mask files and annotations of the background."
           "image file path : */img/*.jpg;mask  file path : */mask/*/*.pbm;json  file path : */anno/*.json")
    parser.add_argument("-sf","--save_files",
      help="The root directory which contains the images and annotations. ")
    # parser.add_argument("-NO","--noise_object",
    #   help="The image types will  generate. :Ture is positive images ,False is negative images")
    args = parser.parse_args()
    return args

def get_construction_result(file,save_file):
    img_files = get_list_of_images(file)
    for file_path in img_files:
        img = cv2.imread(file_path)
        print('file  path is :', file_path)
        h, w, c = img.shape
        anno_path = get_list_of_labels(file_path)
        print('json path is :', anno_path)
        json_data = json.load(open(anno_path))

        all_data = json_data['shapes']
        k = 0
        for data in all_data:
            label = data['label']
            flag = label.split('-')[-1]
            points = data['points']

            print('-----------calculate surface START---------')
            points = calc_surface(points)
            state_flage = False
            for num_surface in range(len(points)):
                if len(points[num_surface]) < 4:
                    state_flage = True
                    break
            if state_flage:
                continue
            print('-----------construct square  END---------')
            # surface=get_square(points,flag)
            hello = construction_surface(points, flag)
            surface = hello.surface
            # print('surface is :',surface)
            print('')
            if surface == -1:
                print('error of surface :')
                continue
            j = 0
            for point_list in surface:
                for i in range(len(point_list)):
                    if point_list[i][0] < 0:
                        point_list[i][0] = 0
                    if point_list[i][1] < 0:
                        point_list[i][1] = 0
                    point_list[i][0] = int(point_list[i][0])
                    point_list[i][1] = int(point_list[i][1])
                    if point_list[i][0] > w:
                        point_list[i][0] = w
                    if point_list[i][1] > h:
                        point_list[i][1] = h
                img = polygons_to_mask_(img, point_list, colors[j])
                j = j + 1
        file_name = file_path.split('/')[-1]
        print('file name is :',file_name)
        img_save=os.path.join(save_file)
        if not os.path.exists(img_save):
            os.makedirs(img_save)
        img_save = os.path.join(img_save, file_name)
        cv2.imwrite(img_save, img)

if __name__ == "__main__":
    arg=parse_args()
    get_construction_result(arg.background_files,arg.save_files)

# python test_construction_surface.py -bf ./data/back -sf ./data/cons


#!/usr/bin/env python
""" Index IntEnz xml files, with Elasticsearch, MongoDB, or Neo4j """
from __future__ import print_function

import argparse
import os
import traceback

import xmltodict
from six import string_types

from nosqlbiosets.dbutils import DBconnection
from nosqlbiosets.objutils import *

DOCTYPE = 'intenz'     # Default document-type or collection name

class Indexer(DBconnection):

    def __init__(self, db, index, host=None, port=None, doctype=DOCTYPE):
        self.doctype = doctype
        self.index = index
        self.db = db
        super(Indexer, self).__init__(db, index, host, port,
                                      mdbcollection=doctype, recreateindex=True)
        if db == "MongoDB":
            self.mcl = self.mdbi[doctype]
        elif db == "Neo4j":
            self.reactions = dict()
            self.reactants = set()
            self.products = set()
            self.edges = set()

    # Parse IntEnz xml file, call index function after each entry is parsed
    def parse_intenz_xmlfiles(self, infile):
        infile = str(infile)
        print("Reading/indexing %s " % infile)
        if not infile.endswith(".xml"):
            print("Input file should be IntEnz xml file")
        else:
            with open(infile, 'rb', buffering=1000) as inf:
                namespaces = {
                    'http://www.xml-cml.org/schema/cml2/react': None,
                    'http://www.ebi.ac.uk/intenz': None
                }
                xmltodict.parse(inf, item_depth=5,
                                item_callback=self.index_intenz_entry,
                                process_namespaces=True,
                                namespaces=namespaces,
                                attr_prefix='')
        print("\nCompleted")
        if self.db == "Neo4j":
            self.indexwithneo4j()

    def index_intenz_entry(self, _, entry):
        slim = False  # TODO: option to select indexing selected fields only
        if not isinstance(entry, string_types):
            docid = entry['ec'][3:]
            list_attrs = ["reactions", "cofactors"]
            if not slim:
                list_attrs += ["synonyms", "comments", "links"]
            unifylistattributes(entry, list_attrs)
            if slim:
                for attr in ['map', 'comments', 'links', 'references',
                             'synonyms']:
                    if attr in entry:
                        del entry[attr]
            if 'reactions' in entry:
                for r in entry['reactions']:
                    unifylistattribute(r, 'reactantList', 'reactant',
                                       'reactants')
                    unifylistattribute(r, 'productList', 'product', 'products')
            # TODO: make accepted_name list
            try:
                if self.db == "Elasticsearch":
                    self.es.index(index=self.index,
                                  op_type='create', ignore=409,
                                  filter_path=['hits.hits._id'],
                                  id=docid, body=entry)
                elif self.db == "MongoDB":
                    entry["_id"] = docid
                    self.mcl.insert_one(entry)
                else:  # Neo4j
                    self.updatereactionsandelements_sets(entry)
            except Exception as e:
                print("ERROR: %s" % e)
                print(traceback.format_exc())
                exit(-1)
            self.reportprogress(40)
        return True

    def indexwithneo4j(self):
        print("Indexing collected data with Neo4j")
        with self.neo4jc.begin_transaction() as tx:
            tx.run("match ()-[a:Produces]-() delete a")
            tx.run("match ()-[a:Reactant_in]-() delete a")
            tx.run("match (a:Substrate) delete a")
            tx.run("match (a:Product) delete a")
            tx.run("match (a:Reaction) delete a")
            tx.sync()
            for r in self.reactants:
                c = "CREATE (a:Substrate {id:{id}})"
                tx.run(c, {"id": r})
            for r in self.products:
                c = "CREATE (a:Product {id:{id}})"
                tx.run(c, {"id": r})
        with self.neo4jc.begin_transaction() as tx:
            for r in self.reactions:
                r = self.reactions[r]
                if 'id' in r:
                    rid = r['id']  # Rhea reaction id
                    tx.run("CREATE (a:Reaction {id:{rid}, name:{name}})",
                           rid=rid, name=r['name'])
                    for re in r['reactants']:
                        if isinstance(re, dict):
                            substrate = re['title']
                        else:
                            substrate = re
                        if (substrate, rid) in self.edges:
                            c = "MATCH (r:Reaction), (s:Substrate)" \
                                " WHERE r.id = {rid}" \
                                " AND s.id = {substrate} " \
                                "CREATE (s)-[:Reactant_in {r:{rid}}]->(r)"
                            tx.run(c, rid=rid,
                                   substrate=substrate)
                            self.edges.remove((substrate, rid))
                    for pr in r['products']:
                        if isinstance(pr, dict):
                            product = pr['title']
                        else:
                            product = pr
                        if (rid, product) in self.edges:
                            c = "MATCH (r:Reaction), (t:Product) " \
                                " WHERE  r.id = {rid} " \
                                " AND t.id = {productid} " \
                                "CREATE (r)-[:Produces {r:{rid}}]->(t)"
                            tx.run(c, rid=rid,
                                   productid=product)
                            self.edges.remove((rid, product))
                    tx.sync()

    def updatereactionsandelements_sets(self, e):
        if 'reactions' not in e:
            return
        for r in e['reactions']:
            if 'id' in r:
                rid = r['id']
                if rid not in self.reactions:
                    self.reactions[rid] = r
                rproducts = set()
                for pr in r['products']:
                    if isinstance(pr, dict):
                        product = pr['title']
                    else:
                        product = pr
                    rproducts.add(product)
                    self.products.add(product)
                    self.edges.add((rid, product))
                for re in r['reactants']:
                    if isinstance(re, dict):
                        substrate = re['title']
                    else:
                        substrate = re
                    self.reactants.add(substrate)
                    self.edges.add((substrate, rid))

def mongodb_textindex(mdb):
    index = [
        ("accepted_name.#text", "text"),
        ("reactions.name", "text"),
        ("reactions.reactants.title", "text"),
        ("reactions.products.title", "text"),
        ("comments.#text", "text"), ("synonyms.#text", "text")
    ]
    mdb.create_index(index, name="text fields")

def main(infile, index, doctype, db, host=None, port=None):
    indxr = Indexer(db, index, host, port, doctype)
    indxr.parse_intenz_xmlfiles(infile)
    if db == 'Elasticsearch':
        indxr.es.indices.refresh(index=index)
    elif db == 'MongoDB':
        mongodb_textindex(indxr.mcl)

if __name__ == '__main__':
    d = os.path.dirname(os.path.abspath(__file__))
    parser = argparse.ArgumentParser(
        description='Index IntEnz xml files,'
                    ' with Elasticsearch, MongoDB or Neo4j')
    parser.add_argument('-infile', '--infile',
                        help='Input file name (intenz/ASCII/intenz.xml)')
    parser.add_argument('--index',
                        default="biosets",
                        help='Name of the Elasticsearch index'
                             ' or MongoDB database')
    parser.add_argument('--doctype', default=DOCTYPE,
                        help='Document type name for Elasticsearch, '
                             'collection name for MongoDB')
    parser.add_argument('--host',
                        help='Elasticsearch, MongoDB or Neo4j server hostname')
    parser.add_argument('--port', type=int,
                        help="Elasticsearch, MongoDB or Neo4j server port")
    parser.add_argument('--db', default='MongoDB',
                        help="Database: 'Elasticsearch', 'MongoDB' or 'Neo4j'")
    args = parser.parse_args()
    main(args.infile, args.index, args.doctype, args.db, args.host, args.port)


from sys import stdin
input = stdin.readline

n = int(input())
piles = list(map(int, input().split()))
left = n - piles.count(0)
one_cnt = piles.count(1)
turn = 0
while True:
    if one_cnt == left:
        turn += one_cnt
        break

    turn += 1
    if left % 2:
        for i in range(n):
            if piles[i] > 1:
                piles[i] = 1
                one_cnt += 1
                break
    else:
        for i in range(n):
            if piles[i] > 1:
                piles[i] = 0
                left -= 1
                break
if turn % 2:
    print('cubelover')
else:
    print('koosaga')


# import the Flask class from the flask module
from flask import Flask, render_template, redirect, url_for, request
import requests
import json
from stats import Stats
from bs4 import BeautifulSoup
import base64
import os

# create the application object
app = Flask(__name__)

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/callback/instagram')
def on_callback():
    error = request.args.get('error')
    if error:
        return 'Error: ' + error
    code = request.args.get("code")
    if not code:
        return 'Instagram not allowed'
    try:
        response = requests.post('https://api.instagram.com/oauth/' +
                                 'access_token',
                                 data={'client_id':
                                       'f3ef7014dfdc4a64a832af7d487d787a',
                                       'client_secret':
                                       'b32ac1a8ad6b47a5bf5e5ed3548cf675',
                                       'grant_type': 'authorization_code',
                                       'redirect_uri':
                                       'http://localhost:5000/callback/' +
                                       'instagram', 'code': code})
        json_obj = json.loads(response.text)
        access_token = json_obj['access_token']
        if not access_token:
            return 'Could not get access token'

        statsObj = Stats(access_token)
        formatted_time = statsObj.compute_optimal_time()
        statsObj.create_frequently_used_tags_wordcloud()
        statsObj.create_popular_tags_wordcloud()
        statsObj.create_histogram_likes_time()
        best_filter = statsObj.get_best_filter()

        f_read = open('templates/results.html', 'r')
        soup = BeautifulSoup(f_read, 'html.parser')
        new_tag1 = soup.new_tag('h1')
        opt_msg = 'Optimal time to Post: '
        new_tag1.string = opt_msg + '{} (Hours after midnight)'.format(
            formatted_time)
        original_tag1 = soup.div
        if original_tag1.h1 is not None:
            original_tag1.h1.replace_with(new_tag1)
        else:
            original_tag1.append(new_tag1)
        new_tag2 = soup.new_tag('h2')
        new_tag2.string = 'Best Filter to Use: {}'.format(best_filter)
        original_tag2 = soup.div
        if original_tag2.h2 is not None:
            original_tag2.h2.replace_with(new_tag2)
        else:
            original_tag2.append(new_tag2)
        colors = statsObj.get_dominant_colors()
        head = soup.head
        new_tag3 = soup.new_tag('style', type='text/css')
        new_tag3.string = ("body{margin:0;padding:0;font:100%/1.3 arial," +
                           "helvetica,sans-serif;background:#FFFFFF;}" +
                           "#wrap{width:900px;margin:0 auto;text-align:" +
                           "center;font-weight:bold;color:#FFF;}" +
                           "#nav{min-height:50px;" +
                           "background:{};}}".format(colors[4]) +
                           "#header{min-height:200px;" +
                           "background:{};}}".format(colors[0]) +
                           "#main{width:100%;overflow:hidden;}#content{" +
                           "float:left;width:600px;min-height:200px;" +
                           "background:{};}}".format(colors[1]) +
                           "#links{float:left;width:300px;min-height:200px;" +
                           "background:{};}}".format(colors[2]) +
                           "#footer{min-height:50px;" +
                           "background:{};}}".format(colors[3]))
        if head.style is not None:
            head.style.replace_with(new_tag3)
        else:
            head.append(new_tag3)
        new_tag6 = soup.new_tag('h3')
        new_tag6['id'] = 'frequent_tags_title'
        new_tag6.string = 'Most Frequently Used Tags'

        src_bytes = None
        with open('frequent_wordcloud.png', 'rb') as image_file:
            src_bytes = base64.b64encode(image_file.read())
        src_text = str(src_bytes)
        src_text = src_text[2:len(src_text) - 1]
        src_text = 'data:image/png;base64,' + src_text
        new_tag4 = soup.new_tag('img', src=src_text)
        new_tag4['id'] = 'frequent_tags'
        body = soup.body
        t = body.find('h3', {'id': 'frequent_tags_title'})
        if t is not None:
            t.replace_with(new_tag6)
        else:
            body.append(new_tag6)
        body = soup.body
        t = body.find('img', {'id': 'frequent_tags'})
        if t is not None:
            t.replace_with(new_tag4)
        else:
            body.append(new_tag4)
        body = soup.body
        new_tag7 = soup.new_tag('h3')
        new_tag7['id'] = 'popular_tags_title'
        new_tag7.string = 'Most Liked Tags'
        src_bytes = None
        with open('popular_wordcloud.png', 'rb') as image_file:
            src_bytes = base64.b64encode(image_file.read())
        src_text = str(src_bytes)
        src_text = src_text[2:len(src_text) - 1]
        src_text = 'data:image/png;base64,' + src_text
        new_tag5 = soup.new_tag('img', src=src_text)
        new_tag5['id'] = 'popular_tags'
        t = body.find('h3', {'id': 'popular_tags_title'})
        if t is not None:
            t.replace_with(new_tag7)
        else:
            body.append(new_tag7)
        t = body.find('img', {'id': 'popular_tags'})
        if t is not None:
            t.replace_with(new_tag5)
        else:
            body.append(new_tag5)

        body = soup.body
        new_tag8 = soup.new_tag('h3')
        new_tag8['id'] = 'histogram_title'
        new_tag8.string = 'Histogram'
        src_bytes = None
        with open('hist.png', 'rb') as image_file:
            src_bytes = base64.b64encode(image_file.read())
        src_text = str(src_bytes)
        src_text = src_text[2:len(src_text) - 1]
        src_text = 'data:image/png;base64,' + src_text
        new_tag9 = soup.new_tag('img', src=src_text)
        new_tag9['id'] = 'histogram'
        t = body.find('h3', {'id': 'histogram_title'})
        if t is not None:
            t.replace_with(new_tag8)
        else:
            body.append(new_tag8)
        t = body.find('img', {'id': 'histogram'})
        if t is not None:
            t.replace_with(new_tag9)
        else:
            body.append(new_tag9)
        html_str = soup.prettify(formatter='html')
        f_read.close()
        f_write = open('templates/results.html', 'w')
        f_write.write(html_str)
        f_write.close()
        os.remove('popular_wordcloud.png')
        os.remove('frequent_wordcloud.png')
    except Exception as e:
        print(e)
    return render_template('results.html')

# start the server with the 'run()' method
if __name__ == '__main__':
    app.run(debug=True)


import re
from pathlib import Path

import pandas as pd
from joblib import load

import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

PATH_DATA = Path(__file__).parent / 'data'

trained_models = Path(__file__).parent / 'trained_models/'
tfidf_vect = load(trained_models  / 'tfidf.joblib')
model = load(trained_models / 'model.joblib')

REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z +_-]')
STOPWORDS = stopwords.words('english')
RANDOM_STATE = 17

def text_prepare(text):
    """
        text: a string

        return: modified initial string
    """
    if pd.isna(text):
        return ''

    text = text.strip().lower()
    text = re.sub(REPLACE_BY_SPACE_RE, ' ', text)  # replace REPLACE_BY_SPACE_RE symbols by space in text
    text = re.sub(BAD_SYMBOLS_RE, '', text)  # delete symbols which are in BAD_SYMBOLS_RE from text
    text = re.sub('-', ' ', text)  # if there is defise split the words
    text = ' '.join([s for s in text.split() if s not in STOPWORDS])  # delete stopwords from text
    text = [token for token in text.split() if not token.isdigit()]  # remove digits
    text = ' '.join([s for s in text])

    return text

def prepare_input(multiplier=1):
    data = pd.read_csv(PATH_DATA / 'train.csv')
    data['text_prepared'] = data['excerpt'].apply(text_prepare)

    x = [t for t in data['text_prepared']] * multiplier
    return x

def predict(data):
    x = tfidf_vect.transform(data)
    pred = model.predict(x)

    return list(zip(data, pred))

if __name__ == '__main__':
    a = predict(prepare_input())
    print(a[:10])


# Relative Sort Array
from typing import List

class Solution:
    def relativeSortArray(self, arr1: List[int], arr2: List[int]) -> List[int]:
        if not arr2:
            arr1.sort()
            return
        d = {x: 0 for x in arr2}
        s = set(arr2)
        notIn = []
        for n in arr1:
            if n in s:
                d[n] = d.get(n) + 1
            else:
                notIn.append(n)
        print(d)
        notIn.sort()
        ans = []
        for x in arr2:
            for _ in range(d.get(x)):
                ans.append(x)
        return ans + notIn

s = Solution()
print(s.relativeSortArray([2,3,1,3,2,4,6,7,9,2,19], [2,1,4,3,9,6]))


#-*- coding: utf-8 -*-
import requests
import json
import configparser
import xlrd
from vendoasg.vendoasg import Vendo
config = configparser.ConfigParser()
config.read('vendo.ini')

# połączenie z bazą vendo
vendoApi = Vendo(config.get('vendo','vendo_API_port'))
vendoApi.logInApi(config.get('vendo','logInApi_user'),config.get('vendo','logInApi_pass'))
vendoApi.loginUser(config.get('vendo','loginUser_user'),config.get('vendo','loginUser_pass'))

###################################################################
def addTag(numerID,nazwa_wd,nowa_lista):
    response_data = vendoApi.getJson ('/json/reply/Magazyn_Towary_Aktualizuj', {"Token":vendoApi.USER_TOKEN,"Model":{
        "ID":numerID,
        "PolaUzytkownika":{
            "NazwaWewnetrzna":nazwa_wd ,
            "Wartosc":nowa_lista
            }
        }})
###################################################################
def checkWD():
    plik = xlrd.open_workbook("tagi.xls")
    strona = plik.sheet_by_index(0)
    total_cols = strona.ncols
    total_rows = strona.nrows
    jezyki = ['pl','de','fr','en','cz']
    for jezyk in jezyki:
        i = 1
        nazwa_wd = f'tagi_{jezyk}'
        for e in range(total_rows-1):
            kod_towaru = strona.cell(i,0).value
            nowy_tag = str(strona.cell(i,1).value)
            if kod_towaru != "":
                if isinstance(kod_towaru, float):
                    kod_towaru = str(int(kod_towaru))
                if len(kod_towaru) == 4:
                    kod_towaru = "0" + kod_towaru
                print(kod_towaru)
                try:
                    kod_query = vendoApi.getJson ('/Magazyn/Towary/TowarRozszerzony', {"Token":vendoApi.USER_TOKEN,"Model":{
                        "ZwrocWartosciDowolne":True,
                        "Towar":{"Kod":kod_towaru}
                        }})
                    kod_query = kod_query["Wynik"]

                    numerID = kod_query["KartaTowaru"]["Towar"]["ID"]
                    wd = kod_query["WartosciDowolne"]
                    lista_wd = []
                    for w in wd:
                        lista_wd.append(w['Nazwa'])

                        if w['Nazwa'] == nazwa_wd:
                            lista_tagow = w['Wartosci'][0].replace(" ", "").split(",")
                            dodac = True
                            for tag in lista_tagow:
                                if nowy_tag in tag:
                                    dodac = False
                                    break
                            if dodac == True:
                                lista_tagow = (";").join(lista_tagow)

                                lista_tagow = lista_tagow +";"+ nowy_tag

                                addTag(numerID,nazwa_wd,lista_tagow)
                            else:
                                print("TAG już jest")
                    if nazwa_wd not in lista_wd:
                        addTag(numerID,nazwa_wd,nowy_tag)
                        print("BYŁO PUSTE")
                    i +=1
                except:
                    i +=1
                    continue

if __name__ == '__main__':
    checkWD()
# zakończenie pracy z bazą vendo
    vendoApi.logOutApi()


import os

ims_pths = []
for root, dirs, files in os.walk(".."):
   for name in files:
       if name.lower().endswith(".ims") or name.lower().endswith(".ims6"):
           pth = os.path.join(root, name)
           ims_pths.append(pth)

replace_dict = {
    "inner_hclose": "inner_dvclose",
    "outer_hclose": "outer_dvclose",

}

for idx, pth in enumerate(ims_pths):
    print("{:3d}: {}".format(idx + 1, pth))
    with open(pth) as f:
        lines = f.read().splitlines()

        f = open(pth, "w")
        for line in lines:
            for text0, text1 in replace_dict.items():
                line = line.lower().replace(text0, text1)
            f.write("{}\n".format(line))
        f.close()


age = 0
rem = 0  # remaining candles
boxes = 0

while age <= 100:
    i = 0

    if rem < age:
        while rem < age:
            i += 1
            rem += 24

        boxes += i

        if i > 0:
            print("Before birthday", str(age) + ",", "buy", i, "box(es)")

    rem -= age
    age += 1

print("\nTotal number of boxes:", str(boxes) + ",", "Remaining candles:", rem)


class Solution:
    def solve(self, board: List[List[str]]) -> None:
        """
        Do not return anything, modify board in-place instead.
        """
        # use dfs on the O tiles near the border and hange all tiles connected to it to T
        # those are tiles which are not surrounded by water so they won't be converted to X
        # change the ones that are still O to X
        m=len(board)
        n=len(board[0])
        dirs=[(0,1),(0,-1),(1,0),(-1,0)]

        def dfs(i,j):
            board[i][j]='T'
            for dx,dy in dirs:
                nx=dx+i
                ny=dy+j
                if 0<=nx<m and 0<=ny<n and board[nx][ny]=='O':
                    dfs(nx,ny)

        for i in range(n):
            if board[0][i]=='O':
                dfs(0,i)
            if board[m-1][i]=='O':
                dfs(m-1,i)

        for j in range(m):
            if board[j][0]=='O':
                dfs(j,0)
            if board[j][n-1]=='O':
                dfs(j,n-1)

        for i in range(m):
            for j in range(n):
                if board[i][j]=='T':
                    board[i][j]='O'
                elif board[i][j]=='O':
                    board[i][j]='X'


from days import parser

sample_data = ["00100",
                "11110",
                "10110",
                "10111",
                "10101",
                "01111",
                "00111",
                "11100",
                "10000",
                "11001",
                "00010",
                "01010"]

def get_zeros_number(data):
    total_zeros = [0 for _ in data[0]]
    for row in data:
        for i, bit in enumerate(row):
            total_zeros[i] += bit == "0"
    return total_zeros

def task_01(data):
    total_zeros = get_zeros_number(data)
    total_length = len(data)
    half_total_length = total_length/2
    print(total_zeros)
    total_ones = [total_length - zero for zero in total_zeros]
    print(total_ones)
    gamma = "".join(["0" if zero > half_total_length else "1" for zero in total_zeros])
    print(gamma)
    epsilon = "".join(["1" if zero == "0" else "0" for zero in gamma])
    print(epsilon)
    return int(gamma, 2) * int(epsilon, 2)

def get_zeros_number_on_index(data, index):
    return sum([x[index] == "0" for x in data])

def task_02(data):
    tmp = data

    idx = 0
    while len(tmp) > 1:
        total_zeros = get_zeros_number_on_index(tmp, idx)
        half_total_length = len(tmp) / 2
        if total_zeros - half_total_length > 0:
            new_data = filter(lambda x: x[idx] == "0", tmp)
        else:
            new_data = filter(lambda x: x[idx] == "1", tmp)
        tmp = list(new_data)
        idx += 1
    oxygen = int(tmp[0], 2)
    print(tmp, oxygen)

    tmp = data
    idx = 0
    while len(tmp) > 1:
        total_zeros = get_zeros_number_on_index(tmp, idx)
        half_total_length = len(tmp) / 2
        if total_zeros - half_total_length <= 0:
            new_data = filter(lambda x: x[idx] == "0", tmp)
        else:
            new_data = filter(lambda x: x[idx] == "1", tmp)
        tmp = list(new_data)
        idx += 1
    co2 = int(tmp[0], 2)
    print(tmp, co2)
    return oxygen * co2

if __name__ == "__main__":
    data = parser.load_rows_to_list("day03.txt", parser.string_without_newline)
    # print(data)
    # data = sample_data
    print(task_01(data))
    print(task_02(data))


#!/usr/bin/env python

import re
import platform
import socket
import time
from subprocess import Popen, PIPE, CalledProcessError
from uuid import getnode as get_mac

import config

# Base class for tasks that can be accomplished on both Linux and Windows.
class CrossPlatform:
    def get_name(self):
        try:
            name = socket.gethostname().lower()
        except BaseException:
            name = socket.gethostname().lower()

        return name

    def get_os(self):
        sysos = platform.system().lower()

        return sysos

    # Get Operating System type and version, then format for SQL.
    def get_os_version(self):
        osversion = platform.platform()
        if 'centos-7.' in osversion:
            return "cent7_64"
        elif 'redhat-7' in osversion:
            return "cent7_64"
        elif 'centos-6.' in osversion:
            return "cent6_64"
        elif 'redhat-6' in osversion:
            return "cent6_64"
        elif 'XP' in osversion:
            return "xp_32"
        elif 'Windows-7' in osversion:
            return "win7_64"
        elif 'Windows-10' in osversion:
            return "win10_64"
        elif 'Windows-post2008Server' in osversion:
            return "win7_64"
        else:
            return "unknown"

    # Get the IPv4 address.
    def get_ip(self):
        try:
            ipaddr = socket.gethostbyname(socket.gethostname())
            return ipaddr
        except:
            ipaddr = '0.0.0.0'
            return ipaddr

    # Get the MAC address.
    def get_mac(self):
        macaddr = ':'.join(("%012X" % get_mac())[i:i + 2] for i in range(0, 12, 2))

        return macaddr

    # Get the current time in YYYY-mm-dd HH:MM:SS format.
    def get_current_time(self):
        lastupdate = time.strftime("%Y-%m-%d %H:%M:%S")

        return lastupdate

    # Get NVIDIA GPU information.
    def get_gpu(self, smi):
        try:
            cmdgpu = [smi, '-L']
            rungpu = Popen(cmdgpu, stdout=PIPE)
            gpu = rungpu.communicate()[0].strip()
            gpu = re.sub(r'\([^)]*\)', '', gpu)
            gpu = re.sub(r'GPU.+: ', '', gpu).strip()
            gpu = gpu.replace('\r', '')
            if '\n' in gpu:
                gpu = gpu.replace(' \n', ',')
            if 'NVIDIA-SMI' in gpu:
                gpu = ''
            if 'Failed' in gpu:
                gpu = ''
            if 'Unable' in gpu:
                gpu = ''
        except BaseException:
            try:
                gpu = ''
                cmdgpu = [
                    'powershell',
                    '(Get-WMIObject Win32_VideoController).Name |  Where-Object {$_ -notmatch "Remote"}']
                rungpu = Popen(cmdgpu, stdout=PIPE)
                gpu = rungpu.communicate()[0].strip(b'\r\n')
                gpu = gpu.decode("utf-8")
                if '\n' in gpu:
                    gpu = gpu.replace(' \n', ',')
            except BaseException:
                gpu = ''

        return gpu

    # Get the Serial number of the NVIDIA GPU.
    def get_gpu_serial(self, smi):
        try:
            cmdgpuserial = [smi, '--query-gpu=serial', '--format=csv,noheader']
            rungpuserial = Popen(
                cmdgpuserial,
                stdout=PIPE,
                stderr=subprocess.STDOUT)
            gpuserial = rungpuserial.communicate()[0].strip()
            gpuserial = gpuserial.replace('\r', '')
            if '\n' in gpuserial:
                gpuserial = gpuserial.replace('\n', ',')
            if 'NVIDIA-SMI' in gpuserial:
                gpuserial = ''
            if 'Invalid' in gpuserial:
                gpuserial = ''
            if 'Failed' in gpuserial:
                gpuserial = ''
            if 'Unable' in gpuserial:
                gpuserial = ''
            if 'Not Supported' in gpuserial:
                gpuserial = ''
        except BaseException:
            gpuserial = ''

        return gpuserial

    # Get the amount of VRAM on the NVIDIA GPU.
    def get_gpu_ram(self, smi):
        try:
            cmdgpuram = [
            	smi,
            	'--query-gpu=memory.total',
            	'--format=csv,noheader']
            rungpuram = Popen(cmdgpuram, stdout=PIPE)
            gpuram = rungpuram.communicate()[0].strip()
            gpuram = gpuram.decode("utf-8")
            gpuram = re.sub("[^0-9\n]", "", gpuram)
            if '\n' in gpuram:
                gpuram = gpuram.replace('\n', ',')
            if 'NVIDIA-SMI' in gpuram:
                gpuram = ''
            if 'Invalid' in gpuram:
                gpuram = ''
            if 'Failed' in gpuram:
                gpuram = ''
            if 'Unable' in gpuram:
                gpuram = ''
        except BaseException:
            gpuram = ''

        return gpuram

    # Get the microarchitecture of the NVIDIA GPU.
    # These values are embedded in config.
    def get_gpu_arch(self, gpu):
        gpuarch = ''
        try:
            if ',' in gpu:
                gpuarch = []
                for x in gpu.split(b','):
                    for k in list(config.gpu_arch_dict.keys()):
                        if re.search(k, gpu):
                            gpuarch.append(config.gpu_arch_dict[k])
                gpuarch = ','.join(gpuarch)
            else:
                for k in list(config.gpu_arch_dict.keys()):
                    if re.search(k, gpu):
                        gpuarch = config.gpu_arch_dict[k]
        except BaseException:
            gpuarch = ''

        return gpuarch


def s(liczba: int):  # funkcja s() opisana w zadaniu
    suma = 0
    for cyfra in str(liczba):
        suma += int(cyfra)
    return suma

def m(liczba: int):  # funkcja m() opisana w zadaniu
    if liczba <= 9:
        return 1
    else:
        return m(s(liczba)) + 1

print(m(578))  # zadanie opisane w zadaniu


# 방문할 수 있는 정점이 여러 개인 경우 정점번호가 작은 것 먼저
# 더 이상 방문 할 점이 없을 경우 종료

# n = 정점 수, m = 간선의 수 , v = 시작할 번호

import sys
sys.setrecursionlimit(10**6)
n, m, v = map(int, sys.stdin.readline().split())

#https://duwjdtn11.tistory.com/515 참고
# 4 * 4 만듬
adj = [ [] for _ in range(n+1)]

for _ in range(m):
    x, y = map(int, sys.stdin.readline().split())
    adj[x].append(y)
    adj[y].append(x)

sort_adj = []
for i in adj:
    i.sort()
    sort_adj.append(i)

# dfs 재귀함수
visited_dfs =[]
def dfs_recursive(sort_adj, cur_position):

    # 만약 방문했으면 돌아감
    if cur_position in visited_dfs:
        return

    # v먼저 추가
    visited_dfs.append(cur_position)
    print(cur_position, end =" ")
    for node in sort_adj[cur_position]:
        dfs_recursive(sort_adj, node)

dfs_recursive(sort_adj, v)
print()

from collections import deque

visited_bfs = [False]*(n+1)

def bfs(sort_adj, start, visited):
    queue = deque([start])

    visited[start] = True

    while queue:
        v = queue.popleft()
        print(v, end =" ")

        for i in sort_adj[v]:
            if visited[i] == False:
                queue.append(i)
                visited[i] = True
    return 0

bfs(sort_adj, v, visited_bfs)


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 12 11:19:41 2019

@author: user
"""

import numpy as np
a = np.array([[1,2,3]])
arr2D = np.array([[1,2,3],[4,5,6]])
zeroes = np.zeros((2,4))
z_1 = np.zeros_like(arr2D)
ones = np.ones((3,3))
full = np.full((3,3),8) # 3X3 all  8's
f_1 = np.full_like(arr2D,90)
identity = np.eye(5)
trans = arr2D.T

lins = np.linspace(0,100,5)

arange = np.arange(0,12,3)

# 3X4 array of random data distribution
# b/w 0 and 1

rand = np.random.rand(3,4)

# b/w 0 and 100

rand1 = (np.random.rand(3,4)*100)

#randints

randint = np.random.randint(10,50,(3,4))

# normal data distribution
# Gaussian bell curve

normal = np.random.normal(10,2,10)

#uniform

uniform = np.random.uniform(-5,5,100)


#encoding=utf-8
from RankingList import RankingList
import requests
import json
import xml.etree.ElementTree as ET
from AnonymousIP import AnonymousIP

class Barrage:
    def __init__(self):
        self.rankingList = RankingList().getRankingList()
        self.list = []
        self.anony = AnonymousIP(1)

    def getCid(self,aid):
        header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)\
             AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"}
        res = requests.get("https://api.bilibili.com/x/player/pagelist?aid=" + aid + "&jsonp=jsonp",headers=header,proxies=self.anony.getProxy())
        return res.json()["data"][0]['cid']

    def getBarrages(self,aid,aid_title):
        oid = self.getCid(aid)
        # url = "https://api.bilibili.com/x/v2/dm/history?type=1&oid="+ str(oid) +"&date=2019-06-09"
        url = "https://api.bilibili.com/x/v1/dm/list.so?oid=" + str(oid)
        header = {  "Origin": "https://www.bilibili.com",
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)\
                    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"}
        res = requests.get(url,headers=header,proxies=self.anony.getProxy()).content.decode()
        root = tree = ET.fromstring(res)
        #root = ET.parse(path).getroot()

        with open('barrages.txt','a+',encoding="utf-8") as f:
            for e in root.iter('d'):
                attr = e.attrib['p'].split(',')
                # 弹幕出现的时间以秒为单位
                second = attr[0]
                #弹幕类型（1-跑马灯，4-底部）
                bar_type = attr[1]
                #字体
                size = attr[2]
                #颜色
                color = attr[3]
                #Unix时间戳基准时间为 1970.01.01，日期=（Unix时间戳+83600）/86400+70365+闰天
                timestamp = attr[4]
                #弹幕池（0-普通池 1-字幕池 2-特殊池）
                bar_pool = attr[5]
                #用户id
                user_id = attr[6]
                #唯一标识
                token = attr[7]

                text = e.text

                content = str(aid) + "\t" + str(aid_title) + "\t" +  str(second) + "\t" + str(bar_type) + "\t" + str(size) + "\t" + str(color) + "\t" + str(timestamp) + "\t" \
                    + str(bar_pool) + "\t" + str(user_id) + '\t' +str(token) + '\t' + text + '\n'

                #print("content::" + content)
                f.write(content)

    def save(self):
        # for movie in self.rankingList[0:3]:
        for movie in self.rankingList:
            aid = movie['aid']
            aid_title = movie['title']
            self.getBarrages(aid,aid_title)

if __name__ == "__main__":
    Barrage().save()