import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from keras.utils.np_utils import to_categorical
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import LabelBinarizer
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.linear_model import LinearRegression
import math
import random
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import scale
from sklearn.ensemble import ExtraTreesClassifier

counter = 0

#NOTE:- cars_price_clean.csv is the csv file which doesn't include rows with missing features.

cars_price = pd.read_csv('cars_price_clean.csv')

all_cat = ['make', 'model', 'condition', 'fuel_type', 'transmission', 'year', 'mileage(kilometers)', 'volume(cm3)', 'color', 'drive_unit', 'segment']
all_cat = np.array(all_cat)
all_cat = all_cat.reshape(11, 1)

#print(cars_price.dtypes)
#cars_price.dropna().to_csv('cars_price_clean.csv')
y = cars_price['make']

#storing the 'make' classes in 'cat_make' variable
cat_make = np.unique(y)
#print(cat_make)
y = cars_price['model']

#storing the 'model' classes in 'cat_model' variable
cat_model = np.unique(y)
#print(cat_model)

y = cars_price['condition']

#storing the 'condition' classes in 'cat_condition' variable
cat_condition = np.unique(y)
#print(cat_condition)

y = cars_price['fuel_type']

#storing the 'fuel_type' classes in 'cat_fuel_type' variable
cat_fuel_type = np.unique(y)
#print(cat_fuel_type)

y = cars_price['transmission']

#storing the 'transmission' classes in 'cat_transmission' variable
cat_transmission = np.unique(y)
#print(cat_transmission)

#y = cars_price['segment']

#storing the 'segment' classes in 'cat_segment' variable
#cat_segment = np.unique(y)
#print(cat_segment)

y = cars_price['year']

cat_year = np.unique(y)

#storing classes for color
y = cars_price['color']

cat_color = np.unique(y)

#storing classes for drive_unit
y = cars_price['drive_unit']

cat_drive_unit = np.unique(y)

#storing classes for segment
y = cars_price['segment']

cat_segment = np.unique(y)

#print(cars_price.dtypes)
cars_price['priceUSD'] = cars_price['priceUSD'].astype('float64')
#print(cars_price['priceUSD'])

#to understand relation between 'mileage' and 'price'

#cars_price = cars_price.dropna(axis = 0, how = 'any', inplace = True)

#print(cars_price)

new_col = cars_price['segment']
#print(new_col)

new_col = np.array(new_col)
print(new_col)
nan_counter = 0

#preparing the training data:-
number = LabelEncoder()

#encoding the str column data into numerical data
num_make = number.fit_transform(cars_price['make'])
#max_make = max(num_make)
#num_make = num_make / max_make
print("the shape of num_make is {}".format(num_make.shape))

num_model = number.fit_transform(cars_price['model'])
#max_model = max(num_model)
#num_model = num_model / max_model
print("the shape of num_model is {}".format(num_model.shape))

num_condition = number.fit_transform(cars_price['condition'])
print("the shape of num_condition is {}".format(num_condition.shape))

num_fuel_type = number.fit_transform(cars_price['fuel_type'])
print("the shape of num_fuel_type is {}".format(num_fuel_type.shape))

num_year = number.fit_transform(cars_price['year'])
print("the shape of num_year is {}".format(num_year.shape))

num_mil = cars_price['mileage(kilometers)']
print(num_mil.shape)
#num_mil = num_mil * 1000.0
#max_mil = max(num_mil)
#num_mil = num_mil / float(max_mil)

num_vol = cars_price['volume(cm3)']
print(num_vol.shape)
#max_vol = max(num_vol)
#num_vol = num_vol / float(max_vol)
#num_vol = num_vol / 1000000.0

num_col = number.fit_transform(cars_price['color'])
#max_col = max(num_col)
#num_col = num_col / max_col
print("The shape of num_col is {}".format(num_col.shape))

num_trans = number.fit_transform(cars_price['transmission'])
#max_trans = max(num_trans)
#num_trans = num_trans / max_trans
print("The shape of num_trans is {}".format(num_trans.shape))

num_drive_unit = number.fit_transform(cars_price['drive_unit'])
#max_drive = max(num_drive_unit)
#num_drive_unit = num_drive_unit / max_drive
print("The shape of num_drive_unit is {}".format(num_drive_unit.shape))

num_segment = number.fit_transform(cars_price['segment'])
#max_seg = max(num_segment)
#num_segment = num_segment / max_seg

print("The shape of num_segment is {}".format(num_segment.shape))

target_price = cars_price['priceUSD']
print("The size of The traget_price to predict is   {}".format(target_price.shape))

print(num_make[0])
print(num_model[0])
print(num_condition[0])
print(num_fuel_type[0])
print(num_year[0])

training_data = []
target = []

#Logic for appending the features using training data.
for i in range(0, len(num_year)):
    sample_data = []
    sample_data.append(num_make[i])
    sample_data.append(num_model[i])
    sample_data.append(num_condition[i])
    sample_data.append(num_fuel_type[i])
    sample_data.append(num_year[i])
    sample_data.append(num_mil[i])
    sample_data.append(num_vol[i])
    sample_data.append(num_col[i])
    sample_data.append(num_trans[i])
    sample_data.append(num_drive_unit[i])
    sample_data.append(num_segment[i])

    target_data = []
    target_data.append(target_price[i])

    training_data.append([sample_data, target_data])

#shuffling the training data
random.shuffle(training_data)

X = []
y = []

for features, label in training_data:
    X.append(features)
    y.append(label)

X = np.array(X)
y = np.array(y)

#Scaling the feature data for better computation and increased accuracy
Xs = scale(X)
print(Xs)

y = y.flatten()

print(X.shape)

print(y.shape)

  #Using chi-squared to understand the impact of most important feature on our target_price
  #Infered from chi-squared:- Segment feature of the data is really impactful on the price followed by 'color'
bestfeatures = SelectKBest(score_func=chi2, k=10)
fit = bestfeatures.fit(X,y)
dfscores = pd.DataFrame(fit.scores_)
print("here are scores: ")
print(dfscores.shape)
dfcolumns = pd.DataFrame(all_cat)
#concat two dataframes for better visualization
featureScores = pd.concat([dfcolumns,dfscores],axis=1)
featureScores.columns = ['Specs','Score']  #naming the dataframe columns
print(featureScores.nlargest(10,'Score'))  #print 10 best features

#Splitting the data into training and testing(20%)
X_train, X_test, y_train, y_test = train_test_split(Xs, y, test_size = 0.2, random_state = 0)

#using sklearn's regression model
regressor = LinearRegression()

regressor.fit(X_train, y_train)

y_pred = regressor.predict(X_test)

print("Accuracy using regression model is:   {}%".format(regressor.score(X_test, y_test) * 100))

print("Here are the actual and predicted values using regression:--------")

#displaying the actual and predicted values for better understanding
result = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
print(result.head())

y_test_new = []
y_test_orig = []

#Second model using neural networks
model = Sequential()
model.add(Dense(20, input_dim = X_train.shape[1], activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))

model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))
model.add(Dense(32, kernel_initializer='normal', activation='relu'))

model.add(Dense(1, activation='linear'))

model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_squared_error'])

hist = model.fit(X_train, y_train, epochs = 20, validation_split = 0.2, shuffle=True)

model.save('train_cars.h5', hist)

y_new_pred = model.predict(X_test)

print(X_test.shape)

print(y_new_pred.shape)

y_new_pred = y_new_pred.flatten()

#printing the actual and predicted values for comparison
print("Here are the actual and predicted values using neural nets:--------")
result = pd.DataFrame({'Actual': y_test, 'Predicted': y_new_pred})
print(result.head())

#Logic for calculating accuracy on test_data
new_count = 0
for i in range(0, 7004):
    accuracy_pos = y_test[i] + 1000
    accuracy_neg = y_test[i] - 1000
    if(y_new_pred[i] < y_test[i] and y_new_pred[i] > accuracy_neg):
        new_count = new_count + 1

    if((y_new_pred[i] > y_test[i]) and (y_new_pred[i] < accuracy_pos)):
        new_count = new_count + 1

print("Right predictions are out of 7004 are:  {}".format(new_count))

percent = (float(new_count) * 100) / 7004

print("accuracy using neural nets is:   {}  %".format(percent))

print("Accuracy using regression model is:   {}%".format(regressor.score(X_test, y_test) * 100))

print("plot showing test and training loss:---")

plt.title('Loss / Mean Squared Error')
plt.plot(hist.history['loss'], label='train')
plt.plot(hist.history['val_loss'], label='test')
plt.legend()
plt.show()

#Note:- Used two models for predictions. Both reached the max accuracy of prediction upto 55%. Accuracy can further be increased after monitoring the relationship between data or adding more relevant features


if not __debug__:
    raise RuntimeError(
        'DataLad cannot run in "optimized" mode, i.e. python -O')

import atexit
import os

# this is not to be modified. for querying use get_apimode()
__api = 'python'

def get_apimode():
    """Returns the API mode label for the current session.

    The API mode label indicates whether DataLad is running in "normal"
    mode in a Python session, or whether it is used via the command line
    interface.

    This function is a utility for optimizing behavior and messaging to the
    particular API (Python vs command line) in use in a given process.

    Returns
    {'python', 'cmdline'}
      The API mode is 'python' by default, unless the main command line
      entrypoint set it to 'cmdline'.
    """
    return __api

def in_librarymode():
    """Returns whether DataLad is requested to run in "library mode"

    In this mode DataLad aims to behave without the assumption that it is
    itself the front-end of a process and in full control over messaging
    and parameters.

    Returns
    -------
    bool
    """
    return __runtime_mode == 'library'

def enable_librarymode():
    """Request DataLad to operate in library mode.

    This function should be executed immediately after importing the `datalad`
    package, when DataLad is not used as an application, or in interactive
    scenarios, but as a utility library inside other applications. Enabling
    this mode will turn off some convenience feature that are irrelevant in
    such use cases (with performance benefits), and alters it messaging
    behavior to better interoperate with 3rd-party front-ends.

    Library mode can only be enabled once. Switching it on and off within
    the runtime of a process is not supported.

    Example::

        >>> import datalad
        >>> datalad.enable_librarymode()
    """
    global __runtime_mode
    __runtime_mode = 'library'
    # export into the environment for child processes to inherit
    os.environ['DATALAD_RUNTIME_LIBRARYMODE'] = '1'

# For reproducible demos/tests
_seed = os.environ.get('DATALAD_SEED', None)
if _seed is not None:
    import random
    _seed = int(_seed)
    random.seed(_seed)

# Colorama (for Windows terminal colors) must be imported before we use/bind
# any sys.stdout
try:
    # this will fix the rendering of ANSI escape sequences
    # for colored terminal output on windows
    # it will do nothing on any other platform, hence it
    # is safe to call unconditionally
    import colorama
    colorama.init()
    atexit.register(colorama.deinit)
except ImportError as e:
    pass

# Other imports are interspersed with lgr.debug to ease troubleshooting startup
# delays etc.

from .config import (
    ConfigManager,
    warn_on_undefined_git_identity,
)
cfg = ConfigManager()
warn_on_undefined_git_identity(cfg)

# must come after config manager
# this is not to be modified. see enable/in_librarymode()
# by default, we are in application-mode, simply because most of
# datalad was originally implemented with this scenario assumption
__runtime_mode = 'library' \
    if cfg.getbool('datalad.runtime', 'librarymode', False) \
    else 'application'

from datalad.utils import (
    get_encoding_info,
    get_envvars_info,
    getpwd,
)

from .log import lgr

def setup_package():
    import warnings

    warnings.warn(
        "setup_package() and testing with nose are deprecated."
        "  Switch to using pytest instead.",
        DeprecationWarning,
    )
    from datalad.tests.utils import setup_package
    return setup_package()

def teardown_package():
    import warnings
    warnings.warn(
        "teardown_package() and testing with nose are deprecated."
        "  Switch to using pytest instead.",
        DeprecationWarning,
    )
    from datalad.tests.utils import teardown_package
    return teardown_package()

# To analyze/initiate our decision making on what current directory to return
getpwd()

lgr.log(5, "Instantiating ssh manager")
from .support.sshconnector import SSHManager

ssh_manager = SSHManager()
atexit.register(ssh_manager.close, allow_fail=False)
atexit.register(lgr.log, 5, "Exiting")

from ._version import get_versions

__version__ = get_versions()['version']
del get_versions

if str(__version__) == '0' or __version__.startswith('0+'):
    lgr.warning(
        "DataLad was not installed 'properly' so its version is an uninformative %r.\n"
        "It can happen e.g. if datalad was installed via\n"
        "  pip install https://github.com/.../archive/{commitish}.zip\n"
        "instead of\n"
        "  pip install git+https://github.com/...@{commitish} .\n"
        "We advise to re-install datalad or downstream projects might not operate correctly.",
        __version__
    )

lgr.log(5, "Done importing main __init__")


from math import sqrt
from numpy import asarray
import operator
import sys
from warnings import warn

# relax module imports.
from lib.errors import RelaxError, RelaxImplementError, RelaxNoSpectraError
from lib.io import sort_filenames, write_data
from lib.text.sectioning import section, subsection
from lib.spectrum.peak_list import read_peak_list
from lib.statistics import std
from lib.warnings import RelaxWarning, RelaxNoSpinWarning
from pipe_control.mol_res_spin import check_mol_res_spin_data, create_spin, generate_spin_id_unique, return_spin, spin_loop
from pipe_control.pipes import check_pipe
from pipe_control.selection import desel_spin, sel_spin

def __errors_height_no_repl():
    """Calculate the errors for peak heights when no spectra are replicated."""

    # Loop over the spins and set the error to the RMSD of the base plane noise.
    for spin, spin_id in spin_loop(return_id=True):
        # Skip deselected spins.
        if not spin.select:
            continue

        # Skip spins missing intensity data.
        if not hasattr(spin, 'peak_intensity'):
            continue

        # Test if the RMSD has been set.
        if not hasattr(spin, 'baseplane_rmsd'):
            raise RelaxError("The RMSD of the base plane noise for spin '%s' has not been set." % spin_id)

        # Set the error to the RMSD.
        spin.peak_intensity_err = spin.baseplane_rmsd

def __errors_repl(subset=None, verbosity=0):
    """Calculate the errors for peak intensities from replicated spectra.

    @keyword subset:    The list of spectrum ID strings to restrict the analysis to.
    @type subset:       list of str
    @keyword verbosity: The amount of information to print.  The higher the value, the greater the verbosity.
    @type verbosity:    int
    """

    # Replicated spectra.
    repl = replicated_flags()

    # Are all spectra replicated?
    if False in list(repl.values()):
        all_repl = False
        print("All spectra replicated:  No.")
    else:
        all_repl = True
        print("All spectra replicated:  Yes.")

    # Initialise.
    if not hasattr(cdp, 'sigma_I'):
        cdp.sigma_I = {}
    if not hasattr(cdp, 'var_I'):
        cdp.var_I = {}

    # The subset.
    subset_flag = False
    if not subset:
        subset_flag = True
        subset = cdp.spectrum_ids

    # Loop over the spectra.
    for id in subset:
        # Skip non-replicated spectra.
        if not repl[id]:
            continue

        # Skip replicated spectra which already have been used.
        if id in cdp.var_I and cdp.var_I[id] != 0.0:
            continue

        # The replicated spectra.
        for j in range(len(cdp.replicates)):
            if id in cdp.replicates[j]:
                spectra = cdp.replicates[j]

        # Number of spectra.
        num_spectra = len(spectra)

        # Printout.
        print("\nReplicated spectra:  " + repr(spectra))
        if verbosity:
            print("%-20s%-20s" % ("Spin_ID", "SD"))

        # Calculate the mean value.
        count = 0
        for spin, spin_id in spin_loop(return_id=True):
            # Skip deselected spins.
            if not spin.select:
                continue

            # Skip and deselect spins which have no data.
            if not hasattr(spin, 'peak_intensity'):
                spin.select = False
                continue

            # Missing data.
            missing = False
            for j in range(num_spectra):
                if not spectra[j] in spin.peak_intensity:
                    missing = True
            if missing:
                continue

            # The peak intensities.
            values = []
            for j in range(num_spectra):
                values.append(spin.peak_intensity[spectra[j]])

            # The standard deviation.
            sd = std(values=values, dof=1)

            # Printout.
            if verbosity:
                print("%-20s%-20s" % (spin_id, sd))

            # Sum of variances (for average).
            if not id in cdp.var_I:
                cdp.var_I[id] = 0.0
            cdp.var_I[id] = cdp.var_I[id] + sd**2
            count = count + 1

        # No data catch.
        if not count:
            raise RelaxError("No data is present, unable to calculate errors from replicated spectra.")

        # Average variance.
        cdp.var_I[id] = cdp.var_I[id] / float(count)

        # Set all spectra variances.
        for j in range(num_spectra):
            cdp.var_I[spectra[j]] = cdp.var_I[id]

        # Print out.
        print("Standard deviation:  %s" % sqrt(cdp.var_I[id]))

    # Average across all spectra if there are time points with a single spectrum.
    if not all_repl:
        # Print out.
        if subset_flag:
            print("\nVariance averaging over the spectra subset.")
        else:
            print("\nVariance averaging over all spectra.")

        # Initialise.
        var_I = 0.0
        num_dups = 0

        # Loop over all time points.
        for id in cdp.var_I:
            # Only use id's defined in subset
            if id not in subset:
                continue

            # Single spectrum (or extraordinarily accurate NMR spectra!).
            if cdp.var_I[id] == 0.0:
                continue

            # Sum and count.
            var_I = var_I + cdp.var_I[id]
            num_dups = num_dups + 1

        # Average value.
        var_I = var_I / float(num_dups)

        # Assign the average value to all time points.
        for id in subset:
            cdp.var_I[id] = var_I

        # Print out.
        print("Standard deviation for all spins:  " + repr(sqrt(var_I)))

    # Loop over the spectra.
    for id in cdp.var_I:
        # Create the standard deviation data structure.
        cdp.sigma_I[id] = sqrt(cdp.var_I[id])

    # Set the spin specific errors.
    for spin in spin_loop():
        # Skip deselected spins.
        if not spin.select:
            continue

        # Set the error.
        spin.peak_intensity_err = cdp.sigma_I

def __errors_volume_no_repl(subset=None):
    """Calculate the errors for peak volumes when no spectra are replicated."""

    # Loop over the spins and set the error to the RMSD of the base plane noise.
    for spin, spin_id in spin_loop(return_id=True):
        # Skip deselected spins.
        if not spin.select:
            continue

        # Skip spins missing intensity data.
        if not hasattr(spin, 'peak_intensity'):
            continue

        # Test if the RMSD has been set.
        if not hasattr(spin, 'baseplane_rmsd'):
            raise RelaxError("The RMSD of the base plane noise for spin '%s' has not been set." % spin_id)

        # Test that the total number of points have been set.
        if not hasattr(spin, 'N'):
            raise RelaxError("The total number of points used in the volume integration has not been specified for spin '%s'." % spin_id)

        # Set the error to the RMSD multiplied by the square root of the total number of points.
        for key in spin.peak_intensity:
            spin.peak_intensity_err[key] = spin.baseplane_rmsd[key] * sqrt(spin.N)

def add_spectrum_id(spectrum_id=None):
    """Add the spectrum ID to the data store.

    @keyword spectrum_id:   The spectrum ID string.
    @type spectrum_id:      str
    """

    # Initialise the structure, if needed.
    if not hasattr(cdp, 'spectrum_ids'):
        cdp.spectrum_ids = []

    # The ID already exists.
    if spectrum_id in cdp.spectrum_ids:
        return

    # Add the ID.
    cdp.spectrum_ids.append(spectrum_id)

def baseplane_rmsd(error=0.0, spectrum_id=None, spin_id=None):
    """Set the peak intensity errors, as defined as the baseplane RMSD.

    @param error:           The peak intensity error value defined as the RMSD of the base plane
                            noise.
    @type error:            float
    @keyword spectrum_id:   The spectrum id.
    @type spectrum_id:      str
    @param spin_id:         The spin identification string.
    @type spin_id:          str
    """

    # Data checks.
    check_pipe()
    check_mol_res_spin_data()
    check_spectrum_id(spectrum_id)

    # The scaling by NC_proc.
    if hasattr(cdp, 'ncproc') and spectrum_id in cdp.ncproc:
        scale = 1.0 / 2**cdp.ncproc[spectrum_id]
    else:
        scale = 1.0

    # Loop over the spins.
    for spin in spin_loop(spin_id):
        # Skip deselected spins.
        if not spin.select:
            continue

        # Initialise or update the baseplane_rmsd data structure as necessary.
        if not hasattr(spin, 'baseplane_rmsd'):
            spin.baseplane_rmsd = {}

        # Set the error.
        spin.baseplane_rmsd[spectrum_id] = float(error) * scale

def check_spectrum_id(id):
    """Check that the give spectrum ID exists.

    @param id:                      The spectrum ID to check for.
    @type id:                       str
    @raises RelaxNoSpectraError:    If the ID does not exist.
    """

    # Check that the spectrum ID structure exists.
    if not hasattr(cdp, 'spectrum_ids'):
        raise RelaxNoSpectraError(id)

    # Test if the spectrum ID exists.
    if id not in cdp.spectrum_ids:
        raise RelaxNoSpectraError(id)

def delete(spectrum_id=None):
    """Delete spectral data corresponding to the spectrum ID.

    @keyword spectrum_id:   The spectrum ID string.
    @type spectrum_id:      str
    """

    # Data checks.
    check_pipe()
    check_mol_res_spin_data()
    check_spectrum_id(spectrum_id)

    # Remove the ID.
    cdp.spectrum_ids.pop(cdp.spectrum_ids.index(spectrum_id))

    # The ncproc parameter.
    if hasattr(cdp, 'ncproc') and spectrum_id in cdp.ncproc:
        del cdp.ncproc[spectrum_id]

    # Replicates.
    if hasattr(cdp, 'replicates'):
        # Loop over the replicates.
        for i in range(len(cdp.replicates)):
            # The spectrum is replicated.
            if spectrum_id in cdp.replicates[i]:
                # Duplicate.
                if len(cdp.replicates[i]) == 2:
                    cdp.replicates.pop(i)

                # More than two spectra:
                else:
                    cdp.replicates[i].pop(cdp.replicates[i].index(spectrum_id))

                # No need to check further.
                break

    # Errors.
    if hasattr(cdp, 'sigma_I') and spectrum_id in cdp.sigma_I:
        del cdp.sigma_I[spectrum_id]
    if hasattr(cdp, 'var_I') and spectrum_id in cdp.var_I:
        del cdp.var_I[spectrum_id]

    # Loop over the spins.
    for spin in spin_loop():
        # Intensity data.
        if hasattr(spin, 'peak_intensity') and spectrum_id in spin.peak_intensity:
            del spin.peak_intensity[spectrum_id]

def error_analysis(subset=None):
    """Determine the peak intensity standard deviation.

    @keyword subset:    The list of spectrum ID strings to restrict the analysis to.
    @type subset:       list of str
    """

    # Tests.
    check_pipe()
    check_mol_res_spin_data()

    # Test if spectra have been loaded.
    if not hasattr(cdp, 'spectrum_ids'):
        raise RelaxError("Error analysis is not possible, no spectra have been loaded.")

    # Check the IDs.
    if subset:
        for id in subset:
            if id not in cdp.spectrum_ids:
                raise RelaxError("The spectrum ID '%s' has not been loaded into relax." % id)

    # Peak height category.
    if cdp.int_method == 'height':
        # Print out.
        print("Intensity measure:  Peak heights.")

        # Do we have replicated spectra?
        if hasattr(cdp, 'replicates'):
            # Print out.
            print("Replicated spectra:  Yes.")

            # Set the errors.
            __errors_repl(subset=subset)

        # No replicated spectra.
        else:
            # Print out.
            print("Replicated spectra:  No.")
            if subset:
                print("Spectra ID subset ignored.")

            # Set the errors.
            __errors_height_no_repl()

    # Peak volume category.
    if cdp.int_method == 'point sum':
        # Print out.
        print("Intensity measure:  Peak volumes.")

        # Do we have replicated spectra?
        if hasattr(cdp, 'replicates'):
            # Print out.
            print("Replicated spectra:  Yes.")

            # Set the errors.
            __errors_repl(subset=subset)

        # No replicated spectra.
        else:
            # Print out.
            print("Replicated spectra:  No.")

            # No implemented.
            raise RelaxImplementError

            # Set the errors.
            __errors_vol_no_repl()

def error_analysis_per_field():
    """Perform an error analysis of the peak intensities for each field strength separately."""

    # Printout.
    section(file=sys.stdout, text="Automatic Error analysis per field strength", prespace=2)

    # Handle missing frequency data.
    frqs = [None]
    if hasattr(cdp, 'spectrometer_frq_list'):
        frqs = cdp.spectrometer_frq_list

    # Loop over the spectrometer frequencies.
    for frq in frqs:
        # Generate a list of spectrum IDs matching the frequency.
        ids = []
        for id in cdp.spectrum_ids:
            # Check that the spectrometer frequency matches.
            match_frq = True
            if frq != None and cdp.spectrometer_frq[id] != frq:
                match_frq = False

            # Add the ID.
            if match_frq:
                ids.append(id)

        # Run the error analysis on the subset.
        print("For field strength %.8f MHz, subset ids for spectrum.error_analysis is: %s" % (frq/1e6, ids))
        error_analysis(subset=ids)

def get_ids():
    """Return a list of all spectrum IDs.

    @return:    The list of spectrum IDs.
    @rtype:     list of str
    """

    # No IDs.
    if not hasattr(cdp, 'spectrum_ids'):
        return []

    # Return the IDs.
    return cdp.spectrum_ids

def integration_points(N=0, spectrum_id=None, spin_id=None):
    """Set the number of integration points for the given spectrum.

    @param N:               The number of integration points.
    @type N:                int
    @keyword spectrum_id:   The spectrum ID string.
    @type spectrum_id:      str
    @keyword spin_id:       The spin ID string used to restrict the value to.
    @type spin_id:          None or str
    """

    raise RelaxImplementError

def read(file=None, dir=None, spectrum_id=None, dim=1, int_col=None, int_method=None, spin_id_col=None, mol_name_col=None, res_num_col=None, res_name_col=None, spin_num_col=None, spin_name_col=None, sep=None, spin_id=None, ncproc=None, verbose=True):
    """Read the peak intensity data.

    @keyword file:          The name of the file(s) containing the peak intensities.
    @type file:             str or list of str
    @keyword dir:           The directory where the file is located.
    @type dir:              str
    @keyword spectrum_id:   The spectrum identification string.
    @type spectrum_id:      str or list of str
    @keyword dim:           The dimension of the peak list to associate the data with.
    @type dim:              int
    @keyword int_col:       The column containing the peak intensity data (used by the generic intensity file format).
    @type int_col:          int or list of int
    @keyword int_method:    The integration method, one of 'height', 'point sum' or 'other'.
    @type int_method:       str
    @keyword spin_id_col:   The column containing the spin ID strings (used by the generic intensity file format).  If supplied, the mol_name_col, res_name_col, res_num_col, spin_name_col, and spin_num_col arguments must be none.
    @type spin_id_col:      int or None
    @keyword mol_name_col:  The column containing the molecule name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type mol_name_col:     int or None
    @keyword res_name_col:  The column containing the residue name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type res_name_col:     int or None
    @keyword res_num_col:   The column containing the residue number information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type res_num_col:      int or None
    @keyword spin_name_col: The column containing the spin name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type spin_name_col:    int or None
    @keyword spin_num_col:  The column containing the spin number information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type spin_num_col:     int or None
    @keyword sep:           The column separator which, if None, defaults to whitespace.
    @type sep:              str or None
    @keyword spin_id:       The spin ID string used to restrict data loading to a subset of all spins.  If 'auto' is provided for a NMRPipe seriesTab formatted file, the ID's are auto generated in form of Z_Ai.
    @type spin_id:          None or str
    @keyword ncproc:        The Bruker ncproc binary intensity scaling factor.
    @type ncproc:           int or None
    @keyword verbose:       A flag which if True will cause all relaxation data loaded to be printed out.
    @type verbose:          bool
    """

    # Data checks.
    check_pipe()
    check_mol_res_spin_data()

    # Check the file name.
    if file == None:
        raise RelaxError("The file name must be supplied.")

    # Test that the intensity measures are identical.
    if hasattr(cdp, 'int_method') and cdp.int_method != int_method:
        raise RelaxError("The '%s' measure of peak intensities does not match '%s' of the previously loaded spectra." % (int_method, cdp.int_method))

    # Multiple ID flags.
    flag_multi = False
    flag_multi_file = False
    flag_multi_col = False
    if isinstance(spectrum_id, list) or spectrum_id == 'auto':
        flag_multi = True
    if isinstance(file, list):
        flag_multi_file = True
    if isinstance(int_col, list) or spectrum_id == 'auto':
        flag_multi_col = True

    # List argument checks.
    if flag_multi:
        # Too many lists.
        if flag_multi_file and flag_multi_col:
            raise RelaxError("If a list of spectrum IDs is supplied, the file names and intensity column arguments cannot both be lists.")

        # Not enough lists.
        if not flag_multi_file and not flag_multi_col:
            raise RelaxError("If a list of spectrum IDs is supplied, either the file name or intensity column arguments must be a list of equal length.")

        # List lengths for multiple files.
        if flag_multi_file and len(spectrum_id) != len(file):
                raise RelaxError("The file list %s and spectrum ID list %s do not have the same number of elements." % (file, spectrum_id))

        # List lengths for multiple intensity columns.
        if flag_multi_col and spectrum_id != 'auto' and len(spectrum_id) != len(int_col):
            raise RelaxError("The spectrum ID list %s and intensity column list %s do not have the same number of elements." % (spectrum_id, int_col))

    # More list argument checks (when only one spectrum ID is supplied).
    else:
        # Multiple files.
        if flag_multi_file:
            raise RelaxError("If multiple files are supplied, then multiple spectrum IDs must also be supplied.")

        # Multiple intensity columns.
        if flag_multi_col:
            raise RelaxError("If multiple intensity columns are supplied, then multiple spectrum IDs must also be supplied.")

    # Intensity column checks.
    if spectrum_id != 'auto' and not flag_multi and flag_multi_col:
        raise RelaxError("If a list of intensity columns is supplied, the spectrum ID argument must also be a list of equal length.")

    # Check the intensity measure.
    if not int_method in ['height', 'point sum', 'other']:
        raise RelaxError("The intensity measure '%s' is not one of 'height', 'point sum', 'other'." % int_method)

    # Set the peak intensity measure.
    cdp.int_method = int_method

    # Convert the file argument to a list if necessary.
    if not isinstance(file, list):
        file = [file]

    # Loop over all files.
    for file_index in range(len(file)):
        # Read the peak list data.
        peak_list = read_peak_list(file=file[file_index], dir=dir, int_col=int_col, spin_id_col=spin_id_col, mol_name_col=mol_name_col, res_num_col=res_num_col, res_name_col=res_name_col, spin_num_col=spin_num_col, spin_name_col=spin_name_col, sep=sep, spin_id=spin_id)

        # Automatic spectrum IDs.
        if spectrum_id == 'auto':
            spectrum_id = peak_list[0].intensity_name

        # Loop over the assignments.
        data = []
        data_flag = False
        for assign in peak_list:
            # Generate the spin_id.
            spin_id = generate_spin_id_unique(res_num=assign.res_nums[dim-1], spin_name=assign.spin_names[dim-1])

            # Convert the intensity data to a list if needed.
            intensity = assign.intensity
            if not isinstance(intensity, list):
                intensity = [intensity]

            # Loop over the intensity data.
            for int_index in range(len(intensity)):
                # Sanity check.
                if intensity[int_index] == 0.0:
                    warn(RelaxWarning("A peak intensity of zero has been encountered for the spin '%s' - this could be fatal later on." % spin_id))

                # Get the spin container.
                spin = return_spin(spin_id=spin_id)
                if not spin:
                    warn(RelaxNoSpinWarning(spin_id))
                    continue

                # Skip deselected spins.
                if not spin.select:
                    continue

                # Initialise.
                if not hasattr(spin, 'peak_intensity'):
                    spin.peak_intensity = {}

                # Intensity scaling.
                if ncproc != None:
                    intensity[int_index] = intensity[int_index] / float(2**ncproc)

                # Add the data.
                if flag_multi_file:
                    id = spectrum_id[file_index]
                elif flag_multi_col:
                    id = spectrum_id[int_index]
                else:
                    id = spectrum_id
                spin.peak_intensity[id] = intensity[int_index]

                # Switch the flag.
                data_flag = True

                # Append the data for printing out.
                data.append([spin_id, repr(intensity[int_index])])

        # Add the spectrum id (and ncproc) to the relax data store.
        spectrum_ids = spectrum_id
        if isinstance(spectrum_id, str):
            spectrum_ids = [spectrum_id]
        if ncproc != None and not hasattr(cdp, 'ncproc'):
            cdp.ncproc = {}
        for i in range(len(spectrum_ids)):
            add_spectrum_id(spectrum_ids[i])
            if ncproc != None:
                cdp.ncproc[spectrum_ids[i]] = ncproc

        # No data.
        if not data_flag:
            # Delete all the data.
            delete(spectrum_id)

            # Raise the error.
            raise RelaxError("No data could be loaded from the peak list")

        # Printout.
        if verbose:
            print("\nThe following intensities have been loaded into the relax data store:\n")
            write_data(out=sys.stdout, headings=["Spin_ID", "Intensity"], data=data)
        print('')

def read_spins(file=None, dir=None, dim=1, spin_id_col=None, mol_name_col=None, res_num_col=None, res_name_col=None, spin_num_col=None, spin_name_col=None, sep=None, spin_id=None, verbose=True):
    """Read the peak intensity data.

    @keyword file:          The name of the file containing the peak intensities.
    @type file:             str
    @keyword dir:           The directory where the file is located.
    @type dir:              str
    @keyword dim:           The dimension of the peak list to associate the data with.
    @type dim:              int
    @keyword spin_id_col:   The column containing the spin ID strings (used by the generic intensity file format).  If supplied, the mol_name_col, res_name_col, res_num_col, spin_name_col, and spin_num_col arguments must be none.
    @type spin_id_col:      int or None
    @keyword mol_name_col:  The column containing the molecule name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type mol_name_col:     int or None
    @keyword res_name_col:  The column containing the residue name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type res_name_col:     int or None
    @keyword res_num_col:   The column containing the residue number information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type res_num_col:      int or None
    @keyword spin_name_col: The column containing the spin name information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type spin_name_col:    int or None
    @keyword spin_num_col:  The column containing the spin number information (used by the generic intensity file format).  If supplied, spin_id_col must be None.
    @type spin_num_col:     int or None
    @keyword sep:           The column separator which, if None, defaults to whitespace.
    @type sep:              str or None
    @keyword spin_id:       The spin ID string used to restrict data loading to a subset of all spins.  If 'auto' is provided for a NMRPipe seriesTab formatted file, the ID's are auto generated in form of Z_Ai.
    @type spin_id:          None or str
    @keyword verbose:       A flag which if True will cause all relaxation data loaded to be printed out.
    @type verbose:          bool
    """

    # Data checks.
    check_pipe()

    # Check the file name.
    if file == None:
        raise RelaxError("The file name must be supplied.")

    # Read the peak list data.
    peak_list = read_peak_list(file=file, dir=dir, spin_id_col=spin_id_col, mol_name_col=mol_name_col, res_num_col=res_num_col, res_name_col=res_name_col, spin_num_col=spin_num_col, spin_name_col=spin_name_col, sep=sep, spin_id=spin_id)

    # Loop over the peak_list.
    created_spins = []
    for assign in peak_list:
        mol_name = assign.mol_names[dim-1]
        res_num = assign.res_nums[dim-1]
        res_name = assign.res_names[dim-1]
        spin_num = assign.spin_nums[dim-1]
        spin_name = assign.spin_names[dim-1]

        # Generate the spin_id.
        spin_id = generate_spin_id_unique(mol_name=mol_name, res_num=res_num, res_name=res_name, spin_name=spin_name)

        # Check if the spin already exist.
        if return_spin(spin_id=spin_id) == None:
            # Create the spin if not exist.
            create_spin(spin_num=spin_num, spin_name=spin_name, res_num=res_num, res_name=res_name, mol_name=mol_name)

    # Test that data exists.
    check_mol_res_spin_data()

def replicated(spectrum_ids=None):
    """Set which spectra are replicates.

    @keyword spectrum_ids:  A list of spectrum ids corresponding to replicated spectra.
    @type spectrum_ids:     list of str
    """

    # Test if the current pipe exists.
    check_pipe()

    # Test for None.
    if spectrum_ids == None:
        warn(RelaxWarning("The spectrum ID list cannot be None."))
        return

    # Test if spectra have been loaded.
    if not hasattr(cdp, 'spectrum_ids'):
        raise RelaxError("No spectra have been loaded therefore replicates cannot be specified.")

    # Test the spectrum id strings.
    for spectrum_id in spectrum_ids:
        check_spectrum_id(spectrum_id)

    # Test for more than one element.
    if len(spectrum_ids) == 1:
        warn(RelaxWarning("The number of spectrum IDs in the list %s must be greater than one." % spectrum_ids))
        return

    # Initialise.
    if not hasattr(cdp, 'replicates'):
        cdp.replicates = []

    # Check if the spectrum IDs are already in the list.
    found = False
    for i in range(len(cdp.replicates)):
        # Loop over all elements of the first.
        for j in range(len(spectrum_ids)):
            if spectrum_ids[j] in cdp.replicates[i]:
                found = True

        # One of the spectrum IDs already have a replicate specified.
        if found:
            # Add the remaining replicates to the list and quit this function.
            for j in range(len(spectrum_ids)):
                if spectrum_ids[j] not in cdp.replicates[i]:
                    cdp.replicates[i].append(spectrum_ids[j])

            # Nothing more to do.
            return

    # A new set of replicates.
    cdp.replicates.append(spectrum_ids)

def replicated_flags():
    """Create and return a dictionary of flags of whether the spectrum is replicated or not.

    @return:    The dictionary of flags of whether the spectrum is replicated or not.
    @rtype:     dict of bool
    """

    # Initialise all IDs to false.
    repl = {}
    for id in cdp.spectrum_ids:
        repl[id] = False

    # Loop over the replicates.
    for i in range(len(cdp.replicates)):
        for j in range(len(cdp.replicates[i])):
            repl[cdp.replicates[i][j]] = True

    # Return the dictionary.
    return repl

def replicated_ids(spectrum_id):
    """Create and return a list of spectra ID which are replicates of the given ID.

    @param spectrum_id: The spectrum ID to find all the replicates of.
    @type spectrum_id:  str
    @return:            The list of spectrum IDs which are replicates of spectrum_id.
    @rtype:             list of str
    """

    # No replicates present.
    if not hasattr(cdp, 'replicates'):
        return []

    # Initialise the ID list.
    repl = []

    # Loop over the replicate lists.
    for i in range(len(cdp.replicates)):
        # The spectrum ID is in the list.
        if spectrum_id in cdp.replicates[i]:
            # Loop over the inner list.
            for j in range(len(cdp.replicates[i])):
                # Spectrum ID match.
                if spectrum_id == cdp.replicates[i][j]:
                    continue

                # Append the replicated ID.
                repl.append(cdp.replicates[i][j])

    # Nothing.
    if repl == []:
        return repl

    # Sort the list.
    repl.sort()

    # Remove duplicates (backward).
    id = repl[-1]
    for i in range(len(repl)-2, -1, -1):
        # Duplicate.
        if id == repl[i]:
            del repl[i]

        # Unique.
        else:
            id = repl[i]

    # Return the list.
    return repl

def signal_noise_ratio(verbose=True):
    """Calculate the signal to noise ratio per spin.

    @keyword verbose:       A flag which if True will print additional information out.
    @type verbose:          bool
    """

    # Tests.
    check_pipe()
    check_mol_res_spin_data()

    # Test if spectra have been loaded.
    if not hasattr(cdp, 'spectrum_ids'):
        raise RelaxError("No spectra have been loaded.")

    # Possible print.
    if verbose:
        print("\nThe following signal to noise ratios has been calculated:\n")

    # Set the spin specific signal to noise ratio.
    for spin, spin_id in spin_loop(return_id=True):
        # Skip deselected spins.
        if not spin.select:
            continue

        # Skip spins missing intensity data.
        if not hasattr(spin, 'peak_intensity'):
            continue

        # Test if error analysis has been performed.
        if not hasattr(spin, 'peak_intensity_err'):
            raise RelaxError("Intensity error analysis has not been performed.  Please see spectrum.error_analysis().")

        # If necessary, create the dictionary.
        if not hasattr(spin, 'sn_ratio'):
            spin.sn_ratio = {}

        # Loop over the ID.
        ids = []
        for id in spin.peak_intensity:
            # Append the ID to the list.
            ids.append(id)

            # Calculate the sn_ratio.
            pint = float(spin.peak_intensity[id])
            pint_err = float(spin.peak_intensity_err[id])
            sn_ratio = pint / pint_err

            # Assign the sn_ratio.
            spin.sn_ratio[id] = sn_ratio

        # Sort the ids alphanumeric.
        ids = sort_filenames(filenames=ids, rev=False)

        # Collect the data under sorted ids.
        data_i = []
        for id in ids:
            # Get the values.
            pint = spin.peak_intensity[id]
            pint_err = spin.peak_intensity_err[id]
            sn_ratio = spin.sn_ratio[id]

            # Store the data.
            data_i.append([id, repr(pint), repr(pint_err), repr(sn_ratio)])

        if verbose:
            section(file=sys.stdout, text="Signal to noise ratio for spin ID '%s'"%spin_id, prespace=1)
            write_data(out=sys.stdout, headings=["Spectrum ID", "Signal", "Noise", "S/N"], data=data_i)

def sn_ratio_deselection(ratio=10.0, operation='<', all_sn=False, select=False, verbose=True):
    """Use user function deselect.spin on spins with signal to noise ratio higher or lower than ratio.  The operation determines the selection operation.

    @keyword ratio:         The ratio to compare to.
    @type ratio:            float
    @keyword operation:     The comparison operation by which to select the spins.  Of the operation(sn_ratio, ratio), where operation can either be:  '<', '<=', '>', '>=', '==', '!='.
    @type operation:        str
    @keyword all_sn:        A flag specifying if all the signal to noise ratios per spin should match the comparison operator, of if just a single comparison match is enough.
    @type all_sn:           bool
    @keyword select:        A flag specifying if the user function select.spin should be used instead.
    @type select:           bool
    @keyword verbose:       A flag which if True will print additional information out.
    @type verbose:          bool
    """

    # Tests.
    check_pipe()
    check_mol_res_spin_data()

    # Test if spectra have been loaded.
    if not hasattr(cdp, 'spectrum_ids'):
        raise RelaxError("No spectra have been loaded.")

    # Assign the comparison operator.
    # "'<' : strictly less than"
    if operation == '<':
        op = operator.lt

    # "'<=' : less than or equal"
    elif operation == '<=':
        op = operator.le

    # "'>' : strictly greater than"
    elif operation == '>':
        op = operator.gt

    # "'>=' : greater than or equal"
    elif operation == '>=':
        op = operator.ge

    # "'==' : equal"
    elif operation == '==':
        op = operator.eq

    # "'!=' : not equal",
    elif operation == '!=':
        op = operator.ne

    # If not assigned, raise error.
    else:
        raise RelaxError("The compare operation does not belong to the allowed list of methods: ['<', '<=', '>', '>=', '==', '!=']")

    # Assign text for print out.
    if all_sn:
        text_all_sn = "all"
    else:
        text_all_sn = "any"

    if select:
        text_sel = "selected"
        sel_func = sel_spin
    else:
        text_sel = "deselected"
        sel_func = desel_spin

    # Print
    section(file=sys.stdout, text="Signal to noise ratio comparison selection", prespace=1, postspace=0)
    print("For the comparion test: S/N %s %1.1f"%(operation, ratio))

    # Loop over the spins.
    spin_ids = []
    for spin, spin_id in spin_loop(return_id=True):
        # Skip spins missing sn_ratio.
        if not hasattr(spin, 'sn_ratio'):
            # Skip warning for deselected spins.
            if spin.select:
                warn(RelaxWarning("Spin '%s' does not contain Signal to Noise calculations. Perform the user function 'spectrum.sn_ratio'. This spin is skipped." % spin_id))
            continue

        # Loop over the ID, collect and sort.
        ids = []
        for id in spin.peak_intensity:
            # Append the ID to the list.
            ids.append(id)

        # Sort the ids alphanumeric.
        ids = sort_filenames(filenames=ids, rev=False)

        # Loop over the sorted ids.
        sn_val = []
        for id in ids:
            # Append the Signal to Noise in the list.
            sn_val.append(spin.sn_ratio[id])

        # Convert the list to array.
        sn_val = asarray(sn_val)

        # Make the comparison for the whole array.
        test_arr = op(sn_val, ratio)

        # Determine how the test should evaluate.
        if all_sn:
            test = test_arr.all()
        else:
            test = test_arr.any()

        # Make an numpy array for the ids, an extract id which failed the test.
        ids_arr = asarray(ids)
        ids_test_arr = ids_arr[test_arr]

        # Make inversion of bool
        test_arr_inv = test_arr == False
        ids_test_arr_inv = ids_arr[test_arr_inv]

        # print
        if verbose:
            subsection(file=sys.stdout, text="Signal to noise ratio comparison for spin ID '%s'"%spin_id, prespace=1, postspace=0)
            print("Following spectra ID evaluated to True: %s"%ids_test_arr)
            print("Following spectra ID evaluated to False: %s"%ids_test_arr_inv)
            print("'%s' comparisons have been used for evaluation, which evaluated to: %s"%(text_all_sn, test))
            if test:
                print("The spin ID '%s' is %s"%(spin_id, text_sel))
            else:
                print("The spin ID '%s' is skipped"%spin_id)

        # If the test evaluates to True, then do selection action.
        if test:
            # Select/Deselect the spin.
            sel_func(spin_id=spin_id)

            # Assign spin_id to list, for printing.
            spin_ids.append(spin_id)

    # Make summary
    if verbose:
        if len(spin_ids) > 0:
            subsection(file=sys.stdout, text="For all of the S/N comparion test, the following spin ID's was %s"%text_sel, prespace=1, postspace=0)
            print(spin_ids)

def sn_ratio_selection(ratio=10.0, operation='>', all_sn=True, verbose=True):
    """Use user function select.spin on spins with signal to noise ratio higher or lower than ratio.  The operation determines the selection operation.

    @keyword ratio:         The ratio to compare to.
    @type ratio:            float
    @keyword operation:     The comparison operation by which to select the spins.  Of the operation(sn_ratio, ratio), where operation can either be:  '<', '<=', '>', '>=', '==', '!='.
    @type operation:        str
    @keyword all_sn:        A flag specifying if all the signal to noise ratios per spin should match the comparison operator, of if just a single comparison match is enough.
    @type all_sn:           bool
    @keyword verbose:       A flag which if True will print additional information out.
    @type verbose:          bool
    """

    # Forward function.
    sn_ratio_deselection(ratio=ratio, operation=operation, all_sn=all_sn, select=True, verbose=verbose)


# -*- coding:utf-8 -*-
import logging

from flask import (
    render_template, request, url_for, redirect, abort, current_app
)
from sqlalchemy import extract, func, desc
from flask_login import current_user

from ..models import Blog, User, Tag, Comment
from . import main
from .. import db

@main.route('/')
def index():
    page = request.args.get('page', 1, type=int)
    pagination = Blog.query.filter_by(type='blog').order_by(Blog.createtime.desc()).paginate(
        page, per_page=current_app.config['BLOGS_PER_PAGE'], error_out=False)
    blogs = pagination.items
    tags = Tag.query.order_by(Tag.id).all()
    return render_template('main/index.html', blogs=blogs, tags=tags, pagination=pagination)

@main.route('/login/')
def login_view():
    if current_user.is_authenticated:
        return redirect(url_for('admin.index'))
    return render_template('admin/login.html')

@main.route('/blogs/<blog_id>')
def blog_view(blog_id):
    blog = Blog.query.get_or_404(blog_id)
    comments = blog.comments.filter_by(is_child=False).order_by(Comment.createtime)
    blog.click()
    return render_template('main/blog.html', blog=blog, comments=comments)

@main.route('/about/')
def about_view():
    blog = Blog.query.filter_by(type='about').first_or_404()
    comments = blog.comments.filter_by(is_child=False).order_by(Comment.createtime)
    blog.click()
    return render_template('main/blog.html', blog=blog, comments=comments)

@main.route('/project/')
def project_view():
    blog = Blog.query.filter_by(type='project').first_or_404()
    comments = blog.comments.filter_by(is_child=False).order_by(Comment.createtime)
    blog.click()
    return render_template('main/blog.html', blog=blog, comments=comments)

@main.route('/archives/')
def archives_view():
    tags = Tag.query.order_by(Tag.id).all()
    blogs_group = []
    archives = db.session.query(extract('year', Blog.createtime).label('year'),
                                func.count('*').label('count')).group_by('year').order_by(desc('year')).all()
    for archive in archives:
        blogs_group.append((archive[0], db.session.query(Blog).filter(
            extract('year', Blog.createtime) == archive[0], Blog.type == 'blog').order_by(desc('createtime')).all()))
    return render_template('main/archives.html', blogs_group=blogs_group, tags=tags)

@main.route('/tags/<tag_id>')
def tag_view(tag_id):
    tag = Tag.query.get_or_404(tag_id)
    tags = Tag.query.all()
    page = request.args.get('page', 1, type=int)
    pagination = tag.blogs.order_by(Blog.createtime.desc()).paginate(
        page, per_page=current_app.config['BLOGS_PER_PAGE'], error_out=False)
    blogs = pagination.items
    return render_template('main/index.html', blogs=blogs, tags=tags, pagination=pagination)


from keras.layers.convolutional import Conv2D
from keras.legacy import interfaces
import keras.backend as K
from keras.engine.base_layer import InputSpec
from cnns.nnlib.utils.general_utils import ConvType
from cnns.tensorflow.utils import tensor_shape
from cnns.nnlib.utils.general_utils import next_power2
import tensorflow as tf

def forward_padding(x, kernel, padding, H, W, HH, WW, args=None):
    # Prepare for rfft2d.
    # The 2 most inner dimensions are H, W (channels first).
    # from N, H, W, C -> N, C, H, W
    x = tf.transpose(x, perm=[0, 3, 1, 2])
    # from H, W, C, F -> F, C, H, W
    kernel = tf.transpose(kernel, perm=[3, 2, 0, 1])

    if padding == "same":
        out_H = H
        out_W = W
    elif padding == "valid":
        # The size of the output of the convolution.
        out_H = H - HH + 1
        out_W = W - WW + 1

    HHH = max(out_H, HH)
    WWW = max(out_W, WW)

    # Padding of x to prevent the effects of wrapped-around filter/kernel
    # data.
    init_H_fft = H + (HHH - 1)
    init_W_fft = W + (WWW - 1)

    if args is None or args.next_power2:
        init_H_fft = next_power2(init_H_fft)
        init_W_fft = next_power2(init_W_fft)

    x_pad_H = init_H_fft - H
    x_pad_W = init_W_fft - W

    y_pad_H = init_H_fft - HH
    y_pad_W = init_W_fft - WW

    x = tf.pad(x, [[0, 0], [0, 0], [0, x_pad_H], [0, x_pad_W]])
    kernel = tf.pad(kernel, [[0, 0], [0, 0], [0, y_pad_H], [0, y_pad_W]])

    return x, kernel, out_H, out_W, init_H_fft, init_W_fft

@tf.custom_gradient
def conv_fft_forward(x, kernel, strides, padding, data_format, dilation_rate,
                     args=None):
    print("conv_fft_forward")
    N, H, W, C = tensor_shape(x)
    HH, WW, CC, F = tensor_shape(kernel)
    assert C == CC
    assert H >= HH
    assert W >= WW
    assert strides == (1, 1)
    assert data_format == "channels_last"
    assert dilation_rate == (1, 1)

    x, kernel, out_H, out_W, init_H_fft, init_W_fft = forward_padding(
        x=x, kernel=kernel, padding=padding, H=H, W=W, HH=HH, WW=WW, args=args)

    xfft = tf.signal.rfft2d(x)
    yfft = tf.signal.rfft2d(kernel)

    # from N, C, H, W -> H, W, N, C
    xfft = tf.transpose(xfft, perm=[2, 3, 0, 1])
    # from F, C, H, W -> H, W, F, C
    yfft = tf.transpose(yfft, perm=[2, 3, 1, 0], conjugate=True)

    outfft = tf.linalg.matmul(xfft, yfft)

    # from H, W, N, F -> N, F, H, W
    outfft = tf.transpose(outfft, perm=[2, 3, 0, 1])

    out = tf.signal.irfft2d(outfft)
    out = out[..., :out_H, :out_W]

    # from N, F, H, W -> N, H, W, F (new channel last).
    out = tf.transpose(out, perm=[0, 2, 3, 1])

    def grad(dout):
        return conv_fft_backward(dout)

    return out, grad

def conv_fft_backward(dout):
    print("conv_fft_backward")
    return dout + 2

class Conv2D_fft(tf.keras.layers.Conv2D):
    """2D convolution layer via FFT.

    This layer creates a convolution kernel that is convolved
    with the layer input to produce a tensor of
    outputs. If `use_bias` is True,
    a bias vector is created and added to the outputs. Finally, if
    `activation` is not `None`, it is applied to the outputs as well.

    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(128, 128, 3)` for 128x128 RGB pictures
    in `data_format="channels_last"`.

    # Arguments
        filters: Integer, the dimensionality of the output space
            (i.e. the number of output filters in the convolution).
        kernel_size: An integer or tuple/list of 2 integers, specifying the
            height and width of the 2D convolution window.
            Can be a single integer to specify the same value for
            all spatial dimensions.
        strides: An integer or tuple/list of 2 integers,
            specifying the strides of the convolution
            along the height and width.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Specifying any stride value != 1 is incompatible with specifying
            any `dilation_rate` value != 1.
        padding: one of `"valid"` or `"same"` (case-insensitive).
            Note that `"same"` is slightly inconsistent across backends with
            `strides` != 1, as described
            [here](https://github.com/keras-team/keras/pull/9473#issuecomment-372166860)
        data_format: A string,
            one of `"channels_last"` or `"channels_first"`.
            The ordering of the dimensions in the inputs.
            `"channels_last"` corresponds to inputs with shape
            `(batch, height, width, channels)` while `"channels_first"`
            corresponds to inputs with shape
            `(batch, channels, height, width)`.
            It defaults to the `image_data_format` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "channels_last".
        dilation_rate: an integer or tuple/list of 2 integers, specifying
            the dilation rate to use for dilated convolution.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Currently, specifying any `dilation_rate` value != 1 is
            incompatible with specifying any stride value != 1.
        activation: Activation function to use
            (see [activations](../activations.md)).
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: `a(x) = x`).
        use_bias: Boolean, whether the layer uses a bias vector.
        kernel_initializer: Initializer for the `kernel` weights matrix
            (see [initializers](../initializers.md)).
        bias_initializer: Initializer for the bias vector
            (see [initializers](../initializers.md)).
        kernel_regularizer: Regularizer function applied to
            the `kernel` weights matrix
            (see [regularizer](../regularizers.md)).
        bias_regularizer: Regularizer function applied to the bias vector
            (see [regularizer](../regularizers.md)).
        activity_regularizer: Regularizer function applied to
            the output of the layer (its "activation").
            (see [regularizer](../regularizers.md)).
        kernel_constraint: Constraint function applied to the kernel matrix
            (see [constraints](../constraints.md)).
        bias_constraint: Constraint function applied to the bias vector
            (see [constraints](../constraints.md)).

    # Input shape
        4D tensor with shape:
        `(batch, channels, rows, cols)`
        if `data_format` is `"channels_first"`
        or 4D tensor with shape:
        `(batch, rows, cols, channels)`
        if `data_format` is `"channels_last"`.

    # Output shape
        4D tensor with shape:
        `(batch, filters, new_rows, new_cols)`
        if `data_format` is `"channels_first"`
        or 4D tensor with shape:
        `(batch, new_rows, new_cols, filters)`
        if `data_format` is `"channels_last"`.
        `rows` and `cols` values might have changed due to padding.
    """

    @interfaces.legacy_conv2d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=(1, 1),
                 padding='valid',
                 data_format=None,
                 dilation_rate=(1, 1),
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 args=None,  # the global arguments for the code
                 **kwargs):
        super(Conv2D_fft, self).__init__(
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format=data_format,
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.args = args

    @interfaces.legacy_add_weight_support
    def add_weight_custom(self,
                          name,
                          value,
                          dtype=None,
                          regularizer=None,
                          trainable=True,
                          constraint=None):
        """Adds a weight variable to the layer.

        # Arguments
            name: String, the name for the weight variable.
            shape: The shape tuple of the weight.
            dtype: The dtype of the weight.
            initializer: An Initializer instance (callable).
            regularizer: An optional Regularizer instance.
            trainable: A boolean, whether the weight should
                be trained via backprop or not (assuming
                that the layer itself is also trainable).
            constraint: An optional Constraint instance.

        # Returns
            The created weight variable.
        """
        if dtype is None:
            dtype = K.floatx()
        weight = K.variable(value,
                            dtype=dtype,
                            name=name,
                            constraint=constraint)
        if regularizer is not None:
            with K.name_scope('weight_regularizer'):
                self.add_loss(regularizer(weight))
        if trainable:
            self._trainable_weights.append(weight)
        else:
            self._non_trainable_weights.append(weight)
        return weight

    def build_custom(self, input_shape, kernel, bias=None):
        if self.data_format == 'channels_first':
            channel_axis = 1
        else:
            channel_axis = -1
        if input_shape[channel_axis] is None:
            raise ValueError('The channel dimension of the inputs '
                             'should be defined. Found `None`.')
        input_dim = input_shape[channel_axis]
        # kernel_shape = self.kernel_size + (input_dim, self.filters)
        self.kernel = self.add_weight_custom(value=kernel,
                                             name='kernel',
                                             regularizer=self.kernel_regularizer,
                                             constraint=self.kernel_constraint)
        if self.use_bias:
            self.bias = self.add_weight_custom(value=bias,
                                               name='bias',
                                               regularizer=self.bias_regularizer,
                                               constraint=self.bias_constraint)
        else:
            self.bias = None
        # Set input spec.
        self.input_spec = InputSpec(ndim=self.rank + 2,
                                    axes={channel_axis: input_dim})
        self.built = True

    def get_config(self):
        config = super(Conv2D_fft, self).get_config()
        config.pop('rank')
        return config

    def exec(self, x, kernel, strides, padding, data_format, dilation_rate):
        if self.args is None or self.args.conv_type == ConvType.STANDARD2D:
            outputs = K.conv2d(
                x=x,
                kernel=kernel,
                strides=strides,
                padding=padding,
                data_format=data_format,
                dilation_rate=dilation_rate)
        elif self.args.conv_type == ConvType.FFT2D:
            # raise Exception("Not implemented yet")
            print("Execute convolution via FFT")
            outputs = conv_fft_forward(x=x,
                                       kernel=kernel,
                                       strides=strides,
                                       padding=padding,
                                       data_format=data_format,
                                       dilation_rate=dilation_rate,
                                       args=self.args)
        else:
            raise Exception(f"Unknown convolution version: {self.version.name}")
        return outputs

    def call(self, inputs):
        outputs = self.exec(x=inputs,
                            kernel=self.kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate)

        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs


# -*- coding: utf-8 -*-
"""
Compare the set3 sims

@author: Tom Williams
"""

import os

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

from vars import phangs_folder, output_folder, plot_folder

matplotlib.rcParams['mathtext.fontset'] = 'stix'
matplotlib.rcParams['font.family'] = 'STIXGeneral'
matplotlib.rcParams['font.size'] = 14

os.chdir(phangs_folder)

plot_filename = plot_folder + 'mock_set3/'

plot_filename += 'mock_set3_pattern_speeds_comparison'

vrots = ['9.8', '10.2', '10.6']
bar_angs = ['60.0', '210.0']
bar_ellips = ['0.25', '0.5']
bar_lens = ['10.0', '30.0', '50.0']
bar_len = '50.0'
spiral_arm_types = ['DW5', 'DW15', 'MAT']

colours = ['r', 'g']

bar_angs_label = [float(bar_ang) for bar_ang in bar_angs]

plt.figure(figsize=(8, 6))

for i, vrot in enumerate(vrots):

    plt.subplot(len(vrots), 1, i + 1)

    x_labels = []
    omega_bars = []
    omega_bars_err_up = []
    omega_bars_err_down = []

    for spiral_arm_type in spiral_arm_types:

        for bar_ang in bar_angs:

            for bar_ellip in bar_ellips:

                file_name = output_folder + 'mock_set3/' + vrot + bar_ang + bar_ellip + bar_len + spiral_arm_type + \
                            '_pattern_speed.txt'

                try:
                    omega_bar, omega_bar_err_up, omega_bar_err_down = np.loadtxt(file_name)
                except OSError:
                    omega_bar, omega_bar_err_up, omega_bar_err_down = np.nan, np.nan, np.nan

                x_labels.append(", ".join([spiral_arm_type, bar_ang, bar_ellip]))

                omega_bars.append(omega_bar)
                omega_bars_err_up.append(omega_bar_err_up)
                omega_bars_err_down.append(omega_bar_err_down)

    plt.errorbar(np.arange(len(omega_bars)), omega_bars, yerr=[omega_bars_err_down, omega_bars_err_up],
                 c='k', marker='o', ls='none')
    plt.title(vrot)

    plt.xticks([])

plt.xticks(np.arange(len(omega_bars)), x_labels, rotation=90)
plt.xlabel('spiral arm type, bar angle, bar ellipticity')

# plt.ylim([0, 30])

plt.tight_layout()
# plt.subplots_adjust(hspace=0)
plt.savefig(plot_filename + '.png', bbox_inches='tight')
plt.savefig(plot_filename + '.pdf', bbox_inches='tight')
plt.close()

print('Complete!')


# You are free to use, modify, copy, distribute the code.
# Please give a clap on medium, star on github, or share the article if you
# like.
# Created by Jonas, github.com/jkoendev

# Double pendulum on a cart (dpc) simulation code
#
# This file runs the simulation using python initial value problem solver.
# You need to generate the dynamics before running this file!
#
# run `dpc_lagrange.py` first!

import numpy as np
import math
from scipy.integrate import solve_ivp

def main():
  # get a random starting state between min state and max state
  x_min = np.array([-1, -np.pi, -np.pi, -.05, -1, -1]);
  x_max = -x_min;
  x0 = np.random.uniform(low=0.0, high=1.0, size=6) * \
       (x_max-x_min)+x_min;

  # simulate
  t_eval = np.arange(0, 8, 0.025);
  sol = solve_ivp(dpc_ode, [0,8], x0, method="RK45", t_eval=t_eval);

def dpc_ode(t, x):
  q_0 = x[0];
  q_1 = x[1];
  q_2 = x[2];

  qdot_0 = x[3];
  qdot_1 = x[4];
  qdot_2 = x[5];
  f = 0;

  qddot_0 = (-4.0*f*math.cos(2.0*q_2)+6.0*f+
             4.0*qdot_1**2*math.cos(q_1)+
             qdot_1**2*math.cos(q_1-q_2)-
             qdot_1**2*math.cos(q_1+2.0*q_2)+
             2.0*qdot_1*qdot_2*math.cos(q_1-q_2)+
             qdot_2**2*math.cos(q_1-q_2)-
             29.43*math.sin(2.0*q_1)+
             9.81*math.sin(2.0*q_1+2.0*q_2)) / \
            (3.0*math.cos(2.0*q_1)-22.0*math.cos(2.0*q_2)-
             math.cos(2.0*q_1+2.0*q_2)+34.0)
  qddot_1 = (8.0*f*math.sin(q_1)-4.0*f*math.sin(q_1+2.0*q_2)+
             3.0*qdot_1**2*math.sin(2.0*q_1)+
             23.0*qdot_1**2*math.sin(q_2)+
             22.0*qdot_1**2*math.sin(2.0*q_2)+
             qdot_1**2*math.sin(2.0*q_1+q_2)+
             46.0*qdot_1*qdot_2*math.sin(q_2)+
             2.0*qdot_1*qdot_2*math.sin(2.0*q_1+q_2)+
             23.0*qdot_2**2*math.sin(q_2)+
             qdot_2**2*math.sin(2.0*q_1+q_2)-
             490.5*math.cos(q_1)+
             215.82*math.cos(q_1+2.0*q_2)) / \
            (3.0*math.cos(2.0*q_1)-
             22.0*math.cos(2.0*q_2)-
             math.cos(2.0*q_1+2.0*q_2)+34.0)
  qddot_2 = -((100.0*qdot_1**2*math.sin(q_2)+
               981.0*math.cos(q_1+q_2)) *
              (-(3.0*math.sin(q_1)+
                 math.sin(q_1+q_2))**2+
               28.0*math.cos(q_2)+42.0)+
              0.5*(200.0*qdot_1*qdot_2*math.sin(q_2)+
                   100.0*qdot_2**2*math.sin(q_2)-
                   2943.0*math.cos(q_1)-
                   981.0*math.cos(q_1+q_2))*
              (25.0*math.cos(q_2)+3.0*math.cos(2.0*q_1+q_2)+
               math.cos(2.0*q_1+2.0*q_2)+13.0)+
               50.0*(2.0*math.sin(q_1)+3.0*math.sin(q_1-q_2)-
               2.0*math.sin(q_1+q_2)-math.sin(q_1+2.0*q_2))*
              (2.0*f+3.0*qdot_1**2*math.cos(q_1)+
               qdot_1**2*math.cos(q_1+q_2)+
               2.0*qdot_1*qdot_2*math.cos(q_1+q_2)+
               qdot_2**2*math.cos(q_1+q_2))) / \
             (75.0*math.cos(2.0*q_1)-550.0*math.cos(2.0*q_2)-
              25.0*math.cos(2.0*q_1+2.0*q_2)+850.0)

  return [qdot_0, qdot_1, qdot_2, qddot_0, qddot_1, qddot_2]

if __name__ == '__main__':
  main()


import logging
from typing import Dict, List

from yarl import URL

logger = logging.getLogger(__name__)

def get_manifest(
    image_url: URL, image_hash: str, protocols: List[str], outbound_urls: List[str]
) -> Dict:
    manifest = {
        "version": "0.1.0",
        "createdAt": "2023-06-26T00:00:00.000000Z",
        "expiresAt": "2100-01-01T00:00:00.000000Z",
        "metadata": {
            "name": "Golem Ray",
            "description": "Golem ray webserver",
            "version": "0.0.1",
        },
        "payload": [
            {
                "urls": [str(image_url)],
                "hash": f"sha3:{image_hash}",
            }
        ],
        "compManifest": {
            "version": "0.1.0",
            "script": {
                "commands": [
                    "run /bin/sh -c *",
                ],
                "match": "regex",
            },
            "net": {
                "inet": {
                    "out": {
                        "protocols": protocols,
                        "urls": outbound_urls,
                    }
                }
            },
        },
    }
    logger.debug(f"Manifest generated: {manifest}")
    return manifest


# coding=utf-8
import time

from six import StringIO

from stash.tests.stashtest import StashTestCase

class ThreadsTests(StashTestCase):

    setup_commands = ['BIN_PATH=$STASH_ROOT/tests/system/data:$BIN_PATH']

    def test_101(self):
        """
        background thread clears properly
        """
        self.stash('test_101_1.py &')
        time.sleep(4)
        cmp_str = r"""[stash]$ [stash]$ sleeping ... 0
sleeping ... 1
"""
        assert self.stash.main_screen.text == cmp_str, 'output not identical'

    def test_102(self):
        """
        Two parallel threads with same stdout should interleave
        """
        outs = StringIO()
        self.stash('test_102_1.py &', final_outs=outs)
        self.stash('test_102_2.py &', final_outs=outs)
        time.sleep(5)
        s = outs.getvalue()

        # Count the number of times the output switches between threads
        change_cnt = 0
        prev_line = None
        for cur_line in outs.getvalue().splitlines():
            if prev_line is None:
                prev_line = cur_line
            elif prev_line != cur_line:
                change_cnt += 1
                prev_line = cur_line

        self.assertTrue(change_cnt > 2, 'Output do not interleave')

    def test_103(self):
        """
        Two threads in parallel with different stdout do not interfere
        """
        outs1 = StringIO()

        self.stash('test_102_1.py &', final_outs=outs1)
        self.stash('test_102_2.py')
        time.sleep(1)

        cmp_str1 = r"""[stash]$ [stash]$ test_102_2.py
test_102_2.py
test_102_2.py
test_102_2.py
test_102_2.py
[stash]$ """
        assert self.stash.main_screen.text == cmp_str1, 'output not identical'

        cmp_str2 = r"""test_102_1.py
test_102_1.py
test_102_1.py
test_102_1.py
test_102_1.py
"""
        assert outs1.getvalue() == cmp_str2, 'output not identical'


from pyspark.sql import SparkSession

# 创建一个连接
spark = SparkSession.Builder().appName('sql_yang').master('local[*]').enableHiveSupport().getOrCreate()
spark.sparkContext.setLogLevel("ERROR")

# spark.sql执行的就直接是DataFrame类型了
df1 = spark.sql("show tables")
spark.sql("set  hive.exec.dynamic.partition.mode = nonstrict")
spark.sql("set  hive.exec.dynamic.partition = true")
# df2 = spark.sql(
#     "create table tab_test(name string,age int,num1 double,num2 bigint,msg varchar(80)) partitioned by (p_age int,p_name string) row format delimited fields terminated by ',' stored as textfile location '/tab/test/tab_test'; ")

df2 = spark.sql("CREATE TABLE IF NOT EXISTS src_yang (key_yang INT, value_yang STRING)")

df3 = spark.sql("show tables")
df4 = spark.sql("show databases;")
# 打印列头及字段类型
print("\n\n\n")
print(df1)
print(df2)
print(df3)
print(df4)

# 打印查询得到的数据
print("\n\n\n\n\n")
df1.show()
df2.show()
df3.show()
df4.show()

# 关闭spark回话
spark.stop()


#!/usr/bin/env python
# -*- encoding:utf-8 -*-

import redis
import sqlite3
import time
import commands

#productList = [
#                "hundred",
#                "hb",
#                "reward",
#                "notice",
#                "public",
#                "linkface",
#                "api",
#                "jiaoyi",
#                "friend",
#                "ermas",
#                "repayment",
#                "payui",
#                "coupon",
#                "feed",
#                "hadoop",
#                "huodong",
#                "inner",
#                "payaccount",
#                "cuiji",
#                "zhifu",
#              ]
productList = []
conn = sqlite3.connect("/home/work/CodisCenter/CodisCenter.db")
cursor = None
cursor = conn.execute('select ProductName from codisapp_codisinfo order by CodisID ASC')
for row in cursor:
  productList.append(row[0])
conn.close()
#print productList

command = "/usr/bin/python /home/work/CodisCenter/checkCodis.py %s"

conn = redis.Redis("127.0.0.1", 6000)

for product in productList:
  output = commands.getoutput(command % product)
  if output == "":
    ret = conn.hset("codis_monitor", product, "good")
  else:
    ret = conn.hset("codis_monitor", product, output)

t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
ret = conn.set("codis_monitor_time", t)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_bower import Bower
import os, sqlite3

db = SQLAlchemy()
bower = Bower()

def create_app(config=None):
    app = Flask(__name__)

    if config is not None:
        app.config.from_object(config)

    db.init_app(app)
    app.engine = db.create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
    Bower(app)

    from .features_bp.views import features_bp
    app.register_blueprint(features_bp, url_prefix='')

    @app.cli.command('initdb')
    def initdb_command():
        """Initializes the database."""
        db.create_all()
        print('Initialized the database.')

    @app.cli.command('secretkey')
    def secretkey_command():
        """Print the secret key."""
        print(app.config['SECRET_KEY'])

    return app

app = create_app('config.DevelopmentConfig')


from flask import Flask, request
from flask_restful import Resource, Api
import irohaSDK
import irohaBUY
import binascii
import os
from iroha import IrohaCrypto

app = Flask(__name__)
api = Api(app)

@app.route('/')
def hello_world():
    return 'Hello, World!'

class ValidateUser(Resource):
    def post(self):
        name = request.form['Name']+'@mchain'
        private_key = request.form['priv_key']
        global priv_key
        priv_key =  private_key.encode()
        D = irohaSDK.get_userone_details(name)
        if D.detail:
            return {'user':'exist'}
        else:
            return {'user': 'Not exist'}

class CreateUser(Resource):
    def post(self):
        signin_name = request.form['SigninName']
        priva_key = IrohaCrypto.private_key()
        public_key = IrohaCrypto.derive_public_key(priva_key)
        D = irohaSDK.create_account(signin_name,public_key)
        return {'priv_key': priva_key.decode("utf-8")}

class GetBalance(Resource):
    def post(self):
        acc_name = request.form['acc_name']
        bal = irohaSDK.get_account_assets(acc_name)
        if bal:
            for asset in bal:
                balance = asset.balance
            return {'ans': balance}
        else:
            return {'ans': 0 }

class GetLabels(Resource):
    def get(self):
        assets = irohaSDK.get_account_assets('admin')
        Data = []
        for asset in assets:
            Data.append(asset.asset_id)
        return Data

class BuyMusic(Resource):
    def post(self):
        d = irohaBUY.instantiate(request.form['src'], request.form['dest'], request.form['amt'], request.form['priv_key'])
        return {'bought': "success"}

class QueryAccount(Resource):
    def get(self):
        assets = irohaSDK.get_account_assets(request.form['acc_id'])
        Data = []
        for asset in assets:
            Data.append(asset.asset_id)
        return Data

class QueryMusic(Resource):
    def get(self):
        d = irohaSDK.get_asset_transaction(request.form['acc_id'], request.form['asset'])
        ans = []
        for i in d:
            for j in i.payload.reduced_payload.commands:
                ans.append(str(j.transfer_asset))
        return ans

class UploadMusic(Resource):
    def post(self):
        key = request.form['priv_key']
        d = irohaBUY.createAsset(request.form['asset'],'mchain',0,key,request.form['acc_name'])
        return {'upload': 'finished'}

class BuyCoin(Resource):
    def post(self):
        d = irohaSDK.transfer_coin_from_admin_to_user(request.form['acc_name'],request.form['amt'])
        return {'Bought': "successful"}

api.add_resource(ValidateUser, '/ValidateUser/')
api.add_resource(CreateUser, '/CreateUser/')
api.add_resource(GetBalance, '/GetBalance/')
api.add_resource(GetLabels, '/GetLabels/')
api.add_resource(BuyMusic, '/BuyMusic/')
api.add_resource(QueryAccount, '/QueryAccount/')
api.add_resource(QueryMusic, '/QueryMusic/')
api.add_resource(UploadMusic, '/UploadMusic/')
api.add_resource(BuyCoin, '/BuyCoin/')


import time, re, sys, os, tempfile, subprocess, threading, queue, json
from typing import Callable, Generator, Optional, Dict, Any, List, Union
from . import manager, domain, utils
import zmq
import ast

class DialogueSystem:

    def __init__(self, domain_file: Optional[str]=None):
        self.config = domain.DomainConfig(domain_file)
        self.manager = manager.DialogueManager(self.config)

        listen_thread = threading.Thread(target=listen_to_inputs,
                             args=(self.manager.input_queue, self.config))
        sendoff_thread = threading.Thread(target=send_outputs,
                             args=(self.manager.output_queue, self.config))
        listen_thread.start()
        sendoff_thread.start()

        self.manager.run()

def listen_to_inputs(input_queue: queue.Queue,
                     config: domain.DomainConfig):
    context = zmq.Context() # type: ignore
    socket = context.socket(zmq.PULL) # type: ignore
    socket.bind("tcp://*:%i"%config.params["zmq_input_port"])
    while True:
        message = socket.recv_json()
        if message.get("label", None) in config.inputs:

            if message["label"]=="HumanUtterance":
                converter = convert_asr_results
            else:
                converter = default_converter

            queries = converter(message)
            input_queue.put(queries)

def send_outputs(output_queue: queue.Queue,
                 config: domain.DomainConfig):

    context = zmq.Context() # type: ignore
    socket = context.socket(zmq.PUSH) # type: ignore
    socket.bind("tcp://*:%i"%(config.params["zmq_output_port"]))

    while True:
        message = output_queue.get()
        if any(l in config.outputs for l in message.get("labels", [])):
            socket.send_json(message)

def convert_asr_results(mess_dict: Dict[str,Any]) -> List[str]:

    queries = [ f"MERGE (u:HumanUtterance {{id:{mess_dict['id']}}}) "
               + f"SET u.start={mess_dict['start']}, u.end={mess_dict['end']};",

               f"MATCH (u:HumanUtterance {{id:{mess_dict['id']}}})-[:ALTERNATIVE]->(r_old:ASRHypothesis) "
               + "DETACH DELETE r_old ;"]

    for hypo in ast.literal_eval(mess_dict["hypotheses"]):
        if (not mess_dict["isFinal"]): hypo["confidence"] = 0.7
        if (mess_dict["isFinal"]): mess_dict["stability"] = 0.7
        properties = f"{{transcript:\"{hypo['transcript']}\", prob:{hypo['confidence']}, stability:{mess_dict['stability']}}}"

        queries.append(f"MATCH (u:HumanUtterance {{id:{mess_dict['id']}}}) "
                       + f"CREATE (r_new:ASRHypothesis {properties})<-[:ALTERNATIVE]-(u) ;")

    floor_status = "free" if mess_dict.get("isFinal", False) else "busy"
    queries.append(f"MERGE (f:Floor) SET f.status='{floor_status}';")

    queries = [utils.normalise_query(q) for q in queries]
    return queries

def default_converter(mess_dict: Dict[str,Any]) -> List[str]:

    props = (", ".join("%s:%s"%(k, '%s'%v if type(v)==str else str(v))
                       for k, v in mess_dict.items() if k != "label"))
    cypher_query = "CREATE {n:%%s {%s} ;"%(mess_dict["type"], props)
    return [cypher_query]


'''
Author: Elite_zhangjunjie
CreateDate:
LastEditors: Elite_zhangjunjie
LastEditTime: 2022-05-10 13:35:46
Description: 基于list实现的Pose类,主要实现打印和调试中角度和弧度的显示,以及求逆和乘
'''
import math
from typing import Any, SupportsIndex, overload

from others.ElitePoseCali import pose_inv, pose_mul

class Pose(list):

    MAX_BITS = 6

    __solts__ = ("x", "y", "z", "rx", "ry", "rz")

    @overload
    def __init__(self,_list: list, is_rad:bool=True): ...

    @overload
    def __init__(self, _x: float, _y: float, _z: float, _rx: float, _ry: float, _rz: float, is_rad:bool=True): ...

    def __init__(self, *args, **kwargs):
        is_rad = kwargs["is_rad"] if "is_rad" in kwargs else True
        if type(args[0]) == list:
            self.__refresh(args[0], is_rad)
        elif len(args) == 6:
            self.__refresh(args[:6], is_rad)
        super().__init__([self.x, self.y ,self.z, self.rx, self.ry, self.rz])

    def __refresh(self, _pose: list, is_rad: bool=True):
        self.x,  self.y,  self.z = _pose[0], _pose[1], _pose[2]
        if is_rad:
            self.rx, self.ry, self.rz = _pose[3], _pose[4], _pose[5]
        else:
            self.rx, self.ry, self.rz = math.radians(_pose[3]), math.radians(_pose[4]), math.radians(_pose[5])

    def __to_deg(self):
        return self.__round([self.x, self.y, self.z, math.degrees(self.rx), math.degrees(self.ry), math.degrees(self.rz)])

    def __round(self, pose: list=None):
        if pose is not None:
            return [round(i, self.MAX_BITS) for i in pose]
        else:
            return [round(i, self.MAX_BITS) for i in [self.x, self.y, self.z, self.rx, self.ry, self.rz]]

    def __str__(self) -> str:
        return f"Pose:{self.__to_deg()} in deg"

    def __repr__(self) -> str:
        return f"Pose:{self.__round()} in rad"

    def __mul__(self, __n: SupportsIndex) -> list:
        return Pose(pose_mul(self, __n))

    @property
    def inv(self) -> list:
        return Pose(pose_inv(self))

if __name__ == "__main__":
    x = Pose([30, 40, 50, 90.030030303, 180.030030303, 30.030030303], is_rad=False)
    y = Pose(1,2,3,4,5,6)

    a = Pose([1,2,3,1,1,1])
    b = Pose([1,2,3,1,1,1])
    c = a* b
    print(a.inv)
    print(a * b)
    print(x.x)
    print(x)
    print(x[-1])


from flask import Flask, render_template, request, jsonify, redirect, url_for
from flask_sqlalchemy import SQLAlchemy
from flask_bootstrap import Bootstrap

app = Flask(__name__)
app.secret_key = 'K705k705'
app.app_context().push()
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///vins.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
bootstrap = Bootstrap(app)

# Available options for equipment code
EQUIPMENT_OPTIONS = {
    '': '',
    '000': '000 - Base platform',
    '014': '014 - Bumper',
    '037': '037 - Drum Mulcher',
    '036': '036 - Side Trimmer',
    '038': '038 - Sprayer',
    '027': '027 - Lawn Mower'
}

# Available options for place of production
PLACE_OPTIONS = {
    '': '',
    '00': '00 - Slovenia',
    '01': '01 - Turkey'
}

class Vin(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    version = db.Column(db.String(3), nullable=False)
    equipment = db.Column(db.String(3), nullable=False)
    year = db.Column(db.Integer(), nullable=False)
    serial_number = db.Column(db.Integer(), nullable=False)
    place = db.Column(db.String(2), nullable=False)

db.create_all()

@app.route('/', methods=['GET', 'POST'])
def vin_generator():
    if request.method == 'POST':
        version = request.form['version']
        equipment = request.form['equipment']
        year = int(request.form['year'])
        place = request.form['place']
        serial_number = request.form['serialNumber']

        # Generate the VIN
        vin = f'{version}{equipment}{year:02d}{serial_number}{place}'

        return render_template('index.html', vin=vin, equipment_options=EQUIPMENT_OPTIONS, place_options=PLACE_OPTIONS)

    return render_template('index.html', equipment_options=EQUIPMENT_OPTIONS, place_options=PLACE_OPTIONS)

@app.route('/search', methods=['POST'])
def search_serial_number():
    data = request.get_json()
    version = data['version']
    equipment = data['equipment']
    year = data['year']
    place = data['place']

    serial_number = 1000000

    while db.session.query(Vin).filter_by(version=f"{version}", equipment=f"{equipment}", year=year, serial_number=serial_number, place=f"{place}").scalar() is not None:

        serial_number += 1
        print(serial_number)

    return jsonify({'serialNumber': serial_number})

@app.route("/add", methods=["GET", "POST"])
def add():
    vin = request.get_json()['vin']
    # Extract information from the VIN
    version = vin[:3]
    equipment = vin[3:6]
    year = int(vin[6:8])
    serial_number = vin[8:15]
    place = vin[-2:]

    # Create a new Vin object
    new_vin = Vin(
        version=f"{version}",
        equipment=f"{equipment}",
        year=year,
        serial_number=serial_number,
        place=f"{place}"
    )

    try:
        # Add the new VIN to the database
        db.session.add(new_vin)
        db.session.commit()

        return jsonify({'message': 'VIN number added successfully'})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/vins')
def vins():
    all_vins = db.session.query(Vin).all()
    db.session.commit()

    return render_template('vins.html', all_vins=all_vins)

@app.route('/delete', methods=["GET", "POST"])
def delete():
    if request.method == 'POST':
        vin_id = request.form["id"]
        vin_to_delete = Vin.query.filter_by(id=vin_id).first()

        if vin_to_delete:
            db.session.delete(vin_to_delete)
            db.session.commit()
            return redirect(url_for('vins', success='VIN deleted successfully.'))
        else:
            return redirect(url_for('vins', error='VIN not found.'))

    vin_id = request.args.get('id')
    vin_to_delete = Vin.query.get(vin_id)
    vin_selected = f'{vin_to_delete.version}{vin_to_delete.equipment}{vin_to_delete.year}{vin_to_delete.serial_number}{vin_to_delete.place}'

    return render_template('delete.html', vin=vin_selected, vin_to_delete=vin_to_delete)

if __name__ == '__main__':
    app.run(debug=True)


import time

import torch
import torch.nn.functional as F
from torch import tensor
from torch.optim import Adam

from utils.ppr_utils import select_from_csr

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def index_to_mask(index, size):
    mask = torch.zeros(size, dtype=torch.bool, device=index.device)
    mask[index] = 1
    return mask

def random_planetoid_splits(data, num_classes):
    # Set new random planetoid splits:
    # * 20 * num_classes labels for training
    # * 500 labels for validation
    # * 1000 labels for testing

    indices = []
    for i in range(num_classes):
        index = (data.y == i).nonzero().view(-1)
        index = index[torch.randperm(index.size(0))]
        indices.append(index)

    train_index = torch.cat([i[:20] for i in indices], dim=0)

    rest_index = torch.cat([i[20:] for i in indices], dim=0)
    rest_index = rest_index[torch.randperm(rest_index.size(0))]

    data.train_mask = index_to_mask(train_index, size=data.num_nodes)
    data.val_mask = index_to_mask(rest_index[:500], size=data.num_nodes)
    data.test_mask = index_to_mask(rest_index[500:1500], size=data.num_nodes)

    return data

def train_and_evaluate(
    dataset,
    model,
    wandb,
    permute_masks=None,
    logger=None,
    runs=3,
    epochs=100,
    lr=0.005,
    weight_decay=0.0005,
    early_stopping=100,
    ppr=True,
    **kwargs,
):
    data = dataset[0]
    param_forward = kwargs.copy()
    if "ppr_matrix" in kwargs:
        ppr_matrix = param_forward.pop("ppr_matrix")
        ppr_vals = select_from_csr(ppr_matrix, data.edge_index).to(device)
        param_forward["ppr_vals"] = ppr_vals

    val_losses, train_accs, val_accs, test_accs, durations = [], [], [], [], []
    for r in range(runs):
        if r % 10 == 0:
            print(f"\nRun {r+1}/{runs}")

        if permute_masks is not None:
            data = permute_masks(data, dataset.num_classes)
        data = data.to(device)

        model.to(device).reset_parameters()
        optimizer = Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

        if torch.cuda.is_available():
            torch.cuda.synchronize()

        t_start = time.perf_counter()

        best_val_loss = float("inf")
        train_acc, val_acc, test_acc = 0.0, 0.0, 0.0
        val_loss_history = []

        for epoch in range(1, epochs + 1):
            train(model, optimizer, data, ppr=ppr, **param_forward)
            eval_info = evaluate(model, data, ppr=ppr, **param_forward)
            eval_info["epoch"] = epoch

            if logger is not None:
                logger(eval_info)

            if eval_info["val_loss"] < best_val_loss:
                best_val_loss = eval_info["val_loss"]
                test_acc = eval_info["test_acc"]
                train_acc = eval_info["train_acc"]
                val_acc = eval_info["val_acc"]

            val_loss_history.append(eval_info["val_loss"])
            if early_stopping > 0 and epoch > epochs // 2:
                tmp = tensor(
                    val_loss_history[-(early_stopping + 1) : -1], dtype=torch.float64
                )
                if eval_info["val_loss"] > tmp.mean().item():
                    print(f"@@@@@@@ early_stopping: epoch {epoch}")
                    break
            if epoch % 10 == 0:
                wandb.log(eval_info)

        if torch.cuda.is_available():
            torch.cuda.synchronize()

        t_end = time.perf_counter()

        val_losses.append(best_val_loss)
        test_accs.append(test_acc)
        train_accs.append(train_acc)
        val_accs.append(val_acc)
        durations.append(t_end - t_start)

    val_loss, train_acc, val_acc, test_acc, duration = [
        tensor(i) for i in [val_losses, train_accs, val_accs, test_accs, durations]
    ]
    loss_mean, train_acc_mean, val_acc_mean, test_acc_mean, duration_mean = [
        i.mean().item() for i in [val_loss, train_acc, val_acc, test_acc, duration]
    ]
    test_acc_std = test_acc.std().item()
    wandb_log = {
        "val_loss": loss_mean,
        "train_acc": train_acc_mean,
        "val_acc": val_acc_mean,
        "test_acc": test_acc_mean,
        "test_acc_std": test_acc_std,
        "duration": duration_mean,
        "epoch_end": epoch,
    }
    print(
        "Val Loss: {:.4f}, Test Accuracy: {:.3f} ± {:.3f}, Duration: {:.3f}".format(
            val_loss.mean().item(),
            test_acc.mean().item(),
            test_acc.std().item(),
            duration.mean().item(),
        )
    )
    return wandb_log

def train(
    model,
    optimizer,
    data,
    lambda1=1e-4,
    lambda2=1e-4,
    p=1,
    fused=0,
    ppr=True,
    **kwargs,
):
    model.train()
    optimizer.zero_grad()
    if ppr:
        out, (edge_index1, alpha1), (edge_index2, alpha2) = model(
            data.x, return_attention_weights=True, **kwargs
        )
        loss0 = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
        reg_l1_1 = lambda1 * torch.norm(alpha1, int(p))
        reg_l1_2 = lambda2 * torch.norm(alpha2, int(p))
        loss = loss0 + reg_l1_1 + reg_l1_2
        if fused > 0:
            alpha1_norm = alpha1 / alpha1.sum()
            alpha2_norm = alpha2 / alpha2.sum()
            fused_l1 = fused * torch.norm(alpha1_norm - alpha2_norm, int(p))
            loss += fused_l1
    else:
        out = model(data.x, **kwargs)
        loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])

    loss.backward()
    optimizer.step()

def evaluate(model, data, ppr=True, **kwargs):
    model.eval()

    with torch.no_grad():
        if ppr:
            logits, (edge_index1, alpha1), (edge_index2, alpha2) = model(
                data.x, **kwargs
            )
        else:
            logits = model(data.x, **kwargs)

    outs = {}
    for key in ["train", "val", "test"]:
        mask = data["{}_mask".format(key)]
        loss = F.nll_loss(logits[mask], data.y[mask]).item()
        pred = logits[mask].max(1)[1]
        acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item()

        outs["{}_loss".format(key)] = loss
        outs["{}_acc".format(key)] = acc

    return outs


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import webapp2

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/html; charset=UTF-8'
        self.response.write('こんにちは！')

app = webapp2.WSGIApplication([
    ('/', MainPage),
], debug=True)