"""
Singly Linked List implementation in Python 3.10

Node: (the first node is called: head)
- data: data it stored in Node
- next_node: contains the reference to next node

LinkedLists are used as: queues/stacks/graphs

"""
from typing import Any

# Create a Node:
class Node:
    """
    Class that creates each node
    - data: data that is stored in Node
    - next_node: reference to the next node
    """
    def __init__(self, data: Any, next_node: Any = None):
        self.data = data
        self.next = next_node

    def __repr__(self):
        return self.data

# Create a LinkedList:
class LinkedList:
    """
    Implementing a Linked Lists
    - nodes: list of predefined values to be treated as Node
    """
    def __init__(self, nodes = None):
        self.head = None
        if nodes is not None:
            node = Node(data=nodes.pop(0))
            self.head = node
            for i in nodes:
                node.next = Node(data=i)
                node = node.next

    def add_left(self, node):
        node.next = self.head
        self.head = node

    def add_right(self, node):
        if self.head is None:
            self.head = node
            return
        for current_node in self:
            pass
        current_node.next = node

    def add_after(self, target, new_node):
        if self.head is None:
            raise Exception("LinkedList is empty.")

        for node in self:
            if node.data == target:
                new_node.next = node.next
                node.next = new_node
                return

        raise Exception(f"Node with data {target} not found")

    def add_before(self, target, new_node):
        if self.head is None:
            raise Exception("LinkedList is empty.")

        if self.head.data == target:
            return self.add_left(new_node)

        prev_node = self.head
        for node in self:
            if node.data == target:
                prev_node.next = new_node
                new_node.next = node
                return
            prev_node = node

        raise Exception(f"Node with data {target} not found")

    def remove(self, target):
        if self.head is None:
            raise Exception("LinkedList is empty.")

        if self.head.data == target:
            self.head = self.head.next
            return

        previous_node = self.head
        for node in self:
            if node.data == target:
                previous_node.next = node.next
                return
            previous_node = node

        raise Exception(f"Node with data {target} not found")

    def __iter__(self):
        node = self.head
        while node is not None:
            yield node
            node = node.next

    def __repr__(self) -> str:
        node = self.head
        nodes = []
        while node is not None:
            nodes.append(node.data)
            node = node.next

        nodes.append("None")
        return " -> ".join(nodes)


import wandb
import time
import numpy as np
NO_REWARD_VALUE = 1000
NO_POSITION_VALUE = 10000

def train(agent, env, config):
    print("Starting training...")
    update_timestep = config.update_multiplier * env.max_ep_length
    debug = False
    time_step = 0
    # training loop
    reward_times = np.ones(config.num_episodes) * NO_REWARD_VALUE
    log_running_reward = []
    steps_per_100_episodes = []
    trajectories = np.ones((config.num_episodes, env.max_ep_length + 1, 2)) * NO_POSITION_VALUE
    t0 = time.time()

    for episode in range(config.num_episodes):

        state = env.reset()
        if episode >= 100 and episode % 100 == 0:
            print(f"episode: {episode}")
            print(f"took {time.time() - t0} s")
            t0 = time.time()
            average_reward = np.mean(log_running_reward)
            average_episode_length = np.mean(steps_per_100_episodes)
            wandb.log({f"epoch n={env.n}": episode, f"average_reward n={env.n}": average_reward, f"average_episode_length n={env.n}": average_episode_length}, step=time_step)
            print(f"n: {env.n}")
            print(f"average reward: {average_reward}")
            print(f"average episode length: {average_episode_length}")
            log_running_reward = []
            steps_per_100_episodes = []

        trajectories[episode][0] = np.array(env.coordinates)

        for t in range(1, env.max_ep_length + 1):

            # select action with policy
            if agent.variable_action_space:
                action_mask = env.get_action_mask()
            else:
                action_mask = None
            action = agent.select_action(state, action_mask, debug)
            state, reward, done = env.step(action)
            trajectories[episode][t] = np.array(env.coordinates)

            # saving reward and is_terminals
            agent.buffer.rewards.append(reward)
            agent.buffer.is_terminals.append(done)

            time_step += 1

            # update weights of agent
            if time_step % update_timestep == 0:
                agent.update()

            # break; if the episode is over
            if done:
                # save time till reward
                if reward > 0:
                    reward_times[episode] = env.t
                else:
                    reward_times[episode] = NO_REWARD_VALUE

                log_running_reward.append(reward)
                steps_per_100_episodes.append(env.t)

                break

    return reward_times, trajectories


from django.views.decorators.http import require_http_methods
from django.http import JsonResponse
from common.json import ModelEncoder
import json
from .models import InvVO, Technician, ServiceAppointment

class InvVOEncoder(ModelEncoder):
    model = InvVO
    properties = [
        "VINVO", "VIP",
    ]

class TechnicianEncoder(ModelEncoder):
    model = Technician
    properties = [
        "name", "employee_number",
    ]

class ServiceAppointmentEncoder(ModelEncoder):
    model = ServiceAppointment
    properties = [
        "id", "date", "service_reason", "owner", "completed",
        "VIN", "technician",
    ]
    encoders = {
        "VIN": InvVOEncoder(),
        "technician": TechnicianEncoder()
    }

@require_http_methods(["GET", "POST"])
def api_list_service_appointments(request):
    if request.method == "GET":
        appointments = ServiceAppointment.objects.all()
        return JsonResponse({"appointments": appointments}, encoder=ServiceAppointmentEncoder, safe=False)
    else:
        content = json.loads(request.body)

        try:
            vinvo = content["VIN"]
            vin = InvVO.objects.get(VINVO=vinvo)
            content["VIN"] = vin
        except InvVO.DoesNotExist:
            vin = InvVO.objects.create(VINVO=content["VIN"])
            content["VIN"] = vin

        try:
            employee_number = content["technician"]
            technician = Technician.objects.get(employee_number=employee_number)
            content["technician"] = technician
        except Technician.DoesNotExist:
            return JsonResponse({"Message": "Invalid employee ID"})

        appointment = ServiceAppointment.objects.create(**content)
        return JsonResponse(appointment, encoder=ServiceAppointmentEncoder, safe=False)

@require_http_methods(["GET", "DELETE", "PUT"])
def api_detail_service_appointment(request, pk):
    if request.method == "GET":
        service = ServiceAppointment.objects.filter(id=pk)
        return JsonResponse({"appointment": service}, encoder=ServiceAppointmentEncoder, safe=False)
    elif request.method == "DELETE":
        count, _ = ServiceAppointment.objects.filter(id=pk).delete()
        return JsonResponse({"deleted": count > 0})
    else:
        content = json.loads(request.body)

        ServiceAppointment.objects.filter(id=pk).update(**content)
        appointment = ServiceAppointment.objects.get(id=pk)
        return JsonResponse({"appointment": appointment}, encoder=ServiceAppointmentEncoder, safe=False)

@require_http_methods(["GET", "POST"])
def api_list_techs(request):
    if request.method == "GET":
        technicians = Technician.objects.all()
        return JsonResponse({"technicians": technicians}, encoder=TechnicianEncoder, safe=False)
    else:
        content = json.loads(request.body)
        technician = Technician.objects.create(**content)
        return JsonResponse(technician, encoder=TechnicianEncoder, safe=False)

@require_http_methods(["DELETE"])
def api_detail_techs(request, num):
    count, _ = Technician.objects.filter(employee_number=num).delete()
    return JsonResponse({"Deleted": count > 0})


import logging
from django_gcp.tasks import OnDemandTask
from example.models import ERROR_STATUS, IN_PROGRESS_STATUS, FooFightingQuestion, FooFightingTest

logger = logging.getLogger(__name__)

class FooFightingTestTask(OnDemandTask):
    """An on-demand task to prepare and ask a question to foo-fighting-service

    This process is done in an async task, rather than immediately in a request,
    because it can take several seconds to create the resources required for the question
    """

    def run(self, foo_fighting_test_id=None, **__):
        """Run a series of FooFightingQuestions to load test the hardware

        Question results should contain:
         - The number of iterations of the mandelbrot calculation completed in the duration of the question
         - The duration of the questions, which will be equal to `max_duration` if `randomise_duration` is false, otherwise will be `0 <= duration <= max_duration`

        :param int foo_fighting_test_id: The id of the test run
        :param int number_of_questions: The number of questions that will get created and sent to the foo-fighting service.
        :param int max_duration: The duration of calculation in seconds that each service will run for.
        :param bool randomise_duration: Randomise durations of calculation if True. If False, all calculations will run for max_durations seconds.
        :param int mandelbrot_array_size: The dimension of the mandelbrot array that will be calculated.
        """

        test = FooFightingTest.objects.get(id=foo_fighting_test_id)

        for idx in range(test.number_of_questions):
            logger.info("Creating question %s of %s for FooFightingTest %s", idx + 1, test.number_of_questions, test.id)

            question = FooFightingQuestion(foo_fighting_test=test, calculation_status=IN_PROGRESS_STATUS)
            question.save()

            try:
                logger.info("Running FooFightingQuestionTask with question_id=%s", question.id)
                question.ask()
            except Exception as e:
                question.calculation_status = ERROR_STATUS
                question.save()
                raise e


import time
import pandas as pd
from datetime import datetime
import numpy as np

from sklearn import preprocessing
import matplotlib
import matplotlib.pyplot as plt

import seaborn as sns
from scipy.stats import pearsonr
import os
from matplotlib.font_manager import FontProperties
from statistics import stdev

import pwrcommon as pc

prec = 'fp64'
mode = 'load'
app = 'pymm'

def pltIndividual(output,fileID,size):

        file = output+fileID+size
        gpu0Pwr = pc.readMetrics(file)
        GPU0MeanPwr = gpu0Pwr['pwr'].mean()
        plt.figure()
        plt.plot(gpu0Pwr['time'], gpu0Pwr['pwr'],label = "Real-time")
        meanGPUPwr = [GPU0MeanPwr for x in range(len(gpu0Pwr))]
        plt.plot(gpu0Pwr['time'], meanGPUPwr, linestyle = '--',label = "Mean")
        plt.xlabel('Time')
        plt.ylabel('GPU Power Consumption (W)')
        plt.title('Matrix size: '+size+'x'+size)
        plt.legend()
        plt.grid()
        plt.ticklabel_format(useOffset=False)
        plt.savefig(output+fileID+size+'.png')

        meanGPUPwr.clear()

def getPwrPerfCounters(output,fileID,size,pwrThresh = 100):
        #os.chdir('../')
        #baseDir = os.getcwd()
        #file = baseDir+'/test/output/'+app+'-'+prec+'-'+mode+'-'+size
        file = output+fileID+size
        # striping non-computing power consumption

        data = pc.readMetrics(file)

        gdata, cpStartTime,cpEndTime = pc.getComputationStartEnd (data,pwrThresh)
        df = pc.getStablePwr(gdata,cpStartTime,cpEndTime)

        #gpu0Pwr['time'] =  (df['time'] - (df['time'].iloc[0]))

        tot_seconds = max(df['time']) - min(df['time'])
        #timedelta = tot_time - datetime(1900, 1, 1)

        #tot_seconds = timedelta.total_seconds()
        print ('\nTotal Seconds',tot_seconds)
        s=0.0
        if size == '16' or size == '32' or size == '48' or size == '64' or  size == '72':
                s = float(size) * 320
        else:
                s = float(size)

        gflop = 2*s*s*s/1e9
        print('\nGFlops:',gflop)
        if tot_seconds != 0:
                gflops = gflop/tot_seconds
        else:
                gflops = gflop

        return (df['pwr'].mean()),gflops

def pltPwrPerf(output,fileID,loadSize,thresh):
        flops = []
        meanPwr = []
        fltPerW = []
        normflops = []
        normMeanPwr = []
        for size in loadSize:
                pwr,gfs = getPwrPerfCounters(output,fileID,size,thresh)
                meanPwr.append(pwr)
                flops.append(gfs)
                fltPerW.append(gfs/pwr)

        # TDP
        tdpY = [250,250,250,250,250]
        print ('\nGfltPerW:',fltPerW)
        print ('\nGFlops:',flops)
        print ('\nMeanPwr:',meanPwr)
        # attainable GFLOPS: 7065
        pltGFLOPS(output,fileID,loadSize,meanPwr,tdpY,flops)
        pltGFLOPSWatt(output,fileID,loadSize, meanPwr,tdpY,fltPerW)

def pltGFLOPS(output,fileID,loadSize, meanPwr,tdpY,flops):
        fig = plt.figure()
        ax = fig.add_subplot(111)

        #tdp line
        lns1 = ax.plot(loadSize, meanPwr, 'H', color = 'b',label = 'MeanPower(W)')
        lns2 = ax.plot(loadSize, tdpY, '--', color = 'b',label = 'Thermal Design Power(TDP)')
        ax2 = ax.twinx()
        #attainable flops line
        attainableY = [7065,7065,7065,7065,7065]
        lns3 = ax2.plot(loadSize, attainableY, '--', color = 'r',label = 'Attainable GFLOPS')
        lns4 = ax2.plot(loadSize, flops, 'o', color = 'r',label = 'GFLOPS')

        # added these four lines
        lns = lns1+lns2+lns3+lns4
        labs = [l.get_label() for l in lns]
        ax.legend(lns, labs, loc=4)

        ax.grid()
        ax.set_xlabel("Data Points (FP64)")
        ax.set_ylabel("Mean Power Usage (W)")
        ax2.set_ylabel("GFLOPS")
        ax2.set_ylim(0, 4500)
        ax.set_ylim(0,275)
        plt.title('GFLOPS versus Power Usage')
        plt.savefig(output+fileID+'-gflops.png')

def pltGFLOPSWatt(output,fileID,loadSize, meanPwr,tdpY,fltPerW):
        #GFLOPS/WATT
        #attainable GFLOPS/WATT: 7065/250=28.26

        Gflopsperwatt = [28.26,28.26,28.26,28.26,28.26]
        fig = plt.figure()
        ax = fig.add_subplot(111)

        #mean power usage
        lns1 = ax.plot(loadSize, meanPwr, 'H', color = 'b',label = 'Power(W)')
        # tdp
        lns2 = ax.plot(loadSize, tdpY, '--', color = 'b',label = 'TDP')
        ax2 = ax.twinx()
        #attainable flops/watt line
        lns3 = ax2.plot(loadSize, fltPerW, 'o', color = 'r',label = 'GFLOPS/Watt')
        lns4 = ax2.plot(loadSize, Gflopsperwatt, '--', color = 'r',label = 'Attainable GFLOPS/Watt')
        # added these four lines
        lns = lns1+lns2+lns3+lns4
        labs = [l.get_label() for l in lns]
        ax.legend(lns, labs, loc=4)

        ax.grid()
        ax.set_xlabel("Data Points (FP64)")
        ax.set_ylabel("Mean Power")
        ax2.set_ylabel("GFLOPS/Watt")
        ax2.set_ylim(0, 22)
        ax.set_ylim(0,275)
        plt.title('GFLOPS/Watt vs Power Usage')

        plt.savefig(output+fileID+'-gflopswatt.png')


import os
import argparse
from render import *

def run_script(args):
    steps = args.num_steps
    env_name = args.env
    seed = args.seed

    if args.episodic:
        train_beh = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed) + " --episodic --train_behavioral"
        gen_buff = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed) + " --episodic --generate_buffer"
        train_bcq = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed) + " --episodic"
    else:
        train_beh = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed) + " --train_behavioral"
        gen_buff = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed) + " --generate_buffer"
        train_bcq = lambda x: "python3 ./BCQ/discrete_BCQ/main.py --env=" + str(env_name) + " --max_timesteps=" + str(x) + " --seed=" + str(seed)

    for command in [train_beh, gen_buff, train_bcq]:
        os.system(command(steps))

    for dir in ["./results/BCQ_", "./results/behavioral_", "./results/buffer_performance_"]:
        os.system("mv " + dir + str(env_name) + "_" + str(seed) + ".npy " + dir + str(env_name) + "_" + str(steps) + "_" + str(seed) + ".npy")

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--env", default="LunarLander-v2")                      # OpenAI gym environment name
    parser.add_argument("--goal", default=200)                                  # sucess score / goal
    parser.add_argument("--seed", default=0, type=int)                          # Sets Gym, PyTorch and Numpy seeds
    parser.add_argument("--num_steps", default=500000)                          # Max time steps to run environment or train for
    parser.add_argument("--render_only", default=0)                             # only render or run_script and render
    parser.add_argument("--include_baselines", default=0)                       # include baselines result while rendering
    parser.add_argument("--episodic", action="store_true")                      # If true, task is episodic
    args = parser.parse_args()

    if str(args.render_only) == '0':
        run_script(args)
        render_script(args)
    else:
        render_script(args)


#!/usr/bin/env python3
# По материалам интернета:
# https://habrahabr.ru/post/321834/
# https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html

# Отключение предупреждений и информационных сообщений по уровням.
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
# Отключение предупреждений по категориям.
import warnings
warnings.filterwarnings(action = 'ignore', category = FutureWarning)

from keras.preprocessing.image import ImageDataGenerator, img_to_array
from keras.models import Sequential, Model
from keras.callbacks import ModelCheckpoint
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D

import numpy as np
import h5py
import matplotlib.pyplot as plt

# Параметры.
train_data_size = 3000
validate_data_size = 210
batch_size = 16
epochs = 20
image_size = 35
# Включён режим тестирование без обучения.
test_only = True
# Включён режим пошагового обучения с загрузкой файла весов, полученных на предыдущем шаге.
next_training_phase = False
# Блокировка первого свёрточного слоя.
lock_convolution_layer1 = False
# Блокировка второго свёрточного слоя.
lock_convolution_layer2 = False
# Блокировка третьего свёрточного слоя.
lock_convolution_layer3 = False
# Блокировка четвёртого свёрточного слоя.
lock_convolution_layer4 = False
# Блокировка пятого свёрточного слоя.
lock_convolution_layer5 = False
# Блокировка шестого свёрточного слоя.
lock_convolution_layer6 = False
# Блокировка полносвязных слоёв.
lock_dense_part = False
tests_number = 20

# Модель.
model = Sequential()
# filters - сколько вариации пикселя будет из одного пикселя
# kernel - сколько обовляется пикселей одновременно с исходным
# padding - вид обработки краев
# input_shape (size, size, кол-во каналов (rgb))
model.add(Convolution2D(filters = 32, kernel_size = (3, 3), padding = 'valid',
    input_shape = (image_size, image_size, 3)))
# Вызов функции активации вида relu
model.add(Activation('relu'))
model.add(Convolution2D(filters = 32, kernel_size = (3, 3), padding = 'valid'))
model.add(Activation('relu'))
# Сужение изображения в 2 раза
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Convolution2D(filters = 32, kernel_size = (3, 3), padding = 'valid'))
model.add(Activation('relu'))
model.add(Convolution2D(filters = 32, kernel_size = (3, 3), padding = 'valid'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size = (2, 2)))
# model.add(Convolution2D(filters = 64, kernel_size = (3, 3), padding = 'valid'))
# model.add(Activation('relu'))
# model.add(Convolution2D(filters = 64, kernel_size = (3, 3), padding = 'valid'))
# model.add(Activation('relu'))
# model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Flatten())
# Кол-во нейронов на relu
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
# Кол-во нейронов на softmax (10 папок) - softmax для распознавания хорошо
model.add(Dense(10))
model.add(Activation('softmax'))
data_generator = ImageDataGenerator(rescale = 1.0 / 255)
if test_only:
    print('Testing only.')
else:
    if next_training_phase:
        print('Continued training.')
        model.load_weights('best_model.hdf5')
    else:
        print('The starting phase of training.')
    if lock_convolution_layer1:
        print('Locking of the first convolution layer.')
        for layer in model.layers[0:2]:
            layer.trainable = False
    if lock_convolution_layer2:
        print('Locking of the second convolution layer.')
        for layer in model.layers[2:5]:
            layer.trainable = False
    if lock_convolution_layer3:
        print('Locking of the third convolution layer.')
        for layer in model.layers[5:7]:
            layer.trainable = False
    if lock_convolution_layer4:
        print('Locking of the fourth convolutional layer.')
        for layer in model.layers[7:10]:
            layer.trainable = False
    if lock_convolution_layer5:
        print('Locking of the fifth convolutional layer.')
        for layer in model.layers[10:12]:
            layer.trainable = False
    if lock_convolution_layer6:
        print('Locking of the sixth convolutional layer.')
        for layer in model.layers[12:15]:
            layer.trainable = False
    if lock_dense_part:
        print('Locking dense layers.')
        for layer in model.layers[15:]:
            layer.trainable = False
model.compile(loss = 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy'])
if not test_only:
    # Обучение.
    train_generator = data_generator.flow_from_directory('data/train',
        target_size = (image_size, image_size), batch_size = batch_size, class_mode = 'categorical')
    validation_generator = data_generator.flow_from_directory('data/validation',
        target_size = (image_size, image_size), batch_size = batch_size, class_mode = 'categorical')
    model.fit_generator(train_generator, callbacks = [ModelCheckpoint('best_model.hdf5',
        monitor = 'val_acc', save_best_only = True, save_weights_only = False, mode = 'auto')],
        steps_per_epoch = train_data_size // batch_size, epochs = epochs,
        validation_data = validation_generator, validation_steps = validate_data_size // batch_size)
# Тестирование.
model.load_weights('best_model.hdf5')
test_generator = data_generator.flow_from_directory('data/test',
    target_size = (image_size, image_size), batch_size = tests_number, class_mode = 'categorical')
model.evaluate_generator(test_generator, steps = tests_number)
data, labels = test_generator.next()
images = np.asarray([img_to_array(item) for item in data])
predictions = np.asarray(model.predict(data))
samples = [sample for sample in zip(images, predictions, labels)]
plt.figure(figsize = (10, 8))
plt.subplots_adjust(left = 0.05, right = 0.95, bottom = 0.05, top = 0.95, wspace = 0.2, hspace = 0.2)
for index, sample in enumerate(samples[:tests_number]):
    plt.subplot(4, 5, index + 1)

    image = sample[0]

    max_value = 0.0
    prediction = 0
    for i in range(10):
        value = sample[1][i]
        if (value > max_value):
            prediction = i
            max_value = value

    max_value = 0.0
    max_label = 0
    for i in range(10):
        value = sample[2][i]
        if (value > max_value):
            max_label = i
            max_value = value

    if(prediction == max_label):
        plt.text(0, 0, max_label, fontsize=11, color='green')
        plt.text(30, 0, prediction, fontsize=11, color='green')
    else:
        plt.text(0, 0, max_label, fontsize=11, color='blue')
        plt.text(30, 0, prediction, fontsize=11, color='red')

    plt.imshow(image)
    plt.axis('off')
plt.gcf().canvas.set_window_title('Первые 20 результатов тестирования')
plt.show()


import matplotlib.pyplot as plt

## scatter plots
x = [2, 4, 6, 8, 10]
y = [6, 7, 4, 2, 9]

plt.scatter(x, y, label="Sc1", color="r", marker="^")

plt.xlabel("X-axis")
plt.ylabel("Y-axis")

plt.title("Simple graph")

plt.legend()

plt.show()


from model.afnonet import BaseModel
import torch.nn as nn
import torch
from einops.layers.torch import Rearrange

from networks.utils.utils import DropPath, PatchEmbed
from functools import partial
class PatchEmbedAlignModel(BaseModel):
    def __init__(self, low_level_img_size=None, low_level_patch_size=None,
                 high_level_img_size=None, high_level_patch_size=None,
                 **kargs):
        super().__init__()
        norm_layer=None
        in_chans = kargs.get('in_chans',68)
        embed_dim=kargs.get('embed_dim',512)
        self.low_level_patch_embed  = PatchEmbed(patch_size=low_level_patch_size, in_c=in_chans, embed_dim=embed_dim, norm_layer=norm_layer)
        self.high_level_patch_embed = PatchEmbed(
            patch_size=high_level_patch_size, in_c=in_chans, embed_dim=embed_dim, norm_layer=norm_layer)
    def forward(self, x1, x2):
        # x: [B, C, H, W]
        with torch.no_grad():
            x1, T, H, W = self.low_level_patch_embed(x1)  # x:[B, H*W, C]
        x2, T, H, W = self.high_level_patch_embed(x2)  # x:[B, H*W, C]
        return x1, x2

class PatchAlign_64_to_128(PatchEmbedAlignModel):
    def __init__(self, **kargs):
        super().__init__(low_level_img_size=(64, 128), low_level_patch_size=(1, 1),
                  high_level_img_size = (128, 256), high_level_patch_size = (2, 2), **kargs)
        pretrain_weight = torch.load(
        "/nvme/zhangtianning/checkpoints/wpredict/lgnet_finetune/world_size8-swinvrnn64x128_lgnet_possloss_finetune_norandom/checkpoint_best.pth", map_location='cpu')['model']['lgnet']
        low_level_patch_embed_weight = {'proj.weight':pretrain_weight['module.net.patch_embed.proj.weight'],
                         'proj.bias': pretrain_weight['module.net.patch_embed.proj.bias']}
        self.low_level_patch_embed.load_state_dict(low_level_patch_embed_weight)
        high_level_patch_embed_weight = {'proj.weight':pretrain_weight['module.net.patch_embed.proj.weight'].repeat(1,1,2,2)/4,
                          'proj.bias': pretrain_weight['module.net.patch_embed.proj.bias']}
        self.high_level_patch_embed.load_state_dict(high_level_patch_embed_weight)

    def set_epoch(self,epoch=None,epoch_total=None,eval_mode=False):
        for p in self.low_level_patch_embed.parameters():p.requires_grad=False


import torch

ALPHA = 0.8
GAMMA = 2

class DiceLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(DiceLoss, self).__init__()

    def forward(self, inputs, targets, smooth=1):
        inputs = inputs.view(-1)
        targets = targets.view(-1)

        intersection = (inputs * targets).sum()
        dice = (2.0 * intersection + smooth) / (inputs.sum() + targets.sum() + smooth)
        return 1 - dice

class IoULoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(IoULoss, self).__init__()

    def forward(self, inputs, targets, smooth=1):
        inputs = inputs.view(-1)
        targets = targets.view(-1)
        intersection = (inputs * targets).sum()
        total = (inputs + targets).sum()
        union = total - intersection

        IoU = (intersection + smooth) / (union + smooth)

        return 1 - IoU

class FocalLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(FocalLoss, self).__init__()

    def forward(self, inputs, targets, alpha=ALPHA, gamma=GAMMA, smooth=1):

        inputs = torch.sigmoid(inputs)

        inputs = inputs.view(-1)
        targets = targets.view(-1)

        BCE = F.binary_cross_entropy(inputs, targets, reduction="mean")
        BCE_EXP = torch.exp(-BCE)
        focal_loss = alpha * (1 - BCE_EXP) ** gamma * BCE

        return focal_loss


import matplotlib.pyplot as plt
from matplotlib.patches import Patch

color_evt = {
    'offer received': 'red',
    'offer viewed': 'blue',
    'transaction': 'black',
    'offer completed': 'yellow'
}

legend_elements = [
    Patch(facecolor='red', label='offer received'),
    Patch(facecolor='blue', label='offer viewed'),
    Patch(facecolor='black', label='transaction'),
    Patch(facecolor='yellow', label='offer completed')
]

def time_line(event, transactions=[], legend=False, xlabel=True, text=''):
    ax = plt.gca()
    ax.yaxis.set_visible(False)
    ymax = 1
    tag = 'VIEWED' if event.viewed > 0 else 'not viewed'
    Text = text + event.details['offer_type'] + ' / ' + tag
    # Text_code = text + event.details['code'] + ' / ' + tag
    evt_list = event.events['event'].tolist()
    color_list = [color_evt[k] for k in evt_list]

    evt_id = event.events['offer_id'].tolist()
    style_list = ['solid' if x == event.offer_id else 'dashed' for x in evt_id]

    if len(transactions) > 0:
        ymax = transactions.amount.max()+5
        ax.plot(
            transactions.time,
            transactions.amount,
            'ko',
            mfc='white',
            ms=10
        )
        ax.yaxis.set_visible(True)
        ax.set_ylabel('transaction $')

    ax.vlines(
        x=event.events['time'],
        ymax=ymax,
        ymin=0,
        color=color_list,
        ls=style_list,
        lw=3
    )
    ax.fill_between(
        x=[event.start, event.end],
        y1=[0, 0],
        y2=[ymax, ymax],
        alpha=.2
    )
    ax.text(
        0.01, 0.8,
        Text,
        weight='bold',
        ha='left',
        transform=ax.transAxes
    )
    if xlabel:
        ax.set_xlabel('time (hours)')

    if legend:
        ax.legend(handles=legend_elements, loc='best')

    plt.grid(False, axis='y')


from cmath import sqrt
import scipy.linalg as la

x,y,z = 30,10,20
volume = 30*20*10
CUBE = [x,y,z]
STEEL = {
    "yield": 350,
    "elastic": 210000,
    "shear":81000,
    "poisson":0.3
}
ALUMINIUM = {
    "yield": 270,
    "elastic": 70000,
    "shear":27000,
    "poisson":0.3
}
# data points:
points = [[0,0,20],[30,0,20],[30,10,20],[0,10,20],[0,0,0],[30,10,0],[0,10,0],[0,5,0],[0,5,20],[30,5,20],[30,5,0]]
# need to respect order (same index in points and displacements)
after_loading = [[0.001,0.002,20],[30.001,0,20.004],[29.997,10.003,19.996],[0.004,10.009,19.995],[0,0,0],[30.003,0.001,0.0026],[29.996,10.003,0],[0.001,9.996,0.0021],[0,4.995,0],[0.004,5.009,19.996],[29.997,5.003,19.996],[29.996,5.003,0]] #[u,v,w]
def strain_transform(point,afterLoad,MATERIAL) :
    displacement=[]
    for x in range(len(point)):
        displacement.append(afterLoad[x]-point[x])
    c = [0,0,0]
    eps = [0,0,0]
    for i in range(3):
        c[i] = (displacement[i] - point[i]) / volume
        eps[i] = displacement[i]/CUBE[i]
    e = sum(eps)
    gxy = c[0]*x*z + c[1]*y*z
    gxz = c[0]*x*y + c[2]*y*z
    gyz = c[2]*x*z + c[1]*x*y
    d =[0,0,0]
    for i in range(len(eps)):
        d[i] = 2*MATERIAL["shear"]*eps[i] + get_lambda(MATERIAL)*e
    txy = MATERIAL["shear"]*2*gxy
    txz = MATERIAL["shear"]*2*gxz
    tyz = MATERIAL["shear"]*2*gyz
    strain_matrix = [
        [eps[0],gxy,gxz],
        [gxy,eps[1],gyz],
        [gxz,gyz,eps[2]]
    ]
    stress_matrix = [
        [d[0],txy,txz],
        [txy,d[1],tyz],
        [txz,tyz,d[2]]
        ]
    oct_stress = (d[0] - d[1])**2 + (d[1]-d[2])**2 + 6*(txy**2 + txz**2 +tyz**2)
    oct_stress = sqrt(oct_stress) / 3
    uod = (3 * oct_stress**2) / (4*MATERIAL["shear"])
    u0 = sum([d[0]*eps[0], d[1] * eps[1], d[2] * eps[2],txy*gxy,txz*gxz,tyz*gyz]) /2
    v,d= la.eig(stress_matrix)
    return {
       "strain" : strain_matrix,
        "stess": stress_matrix,
        "v":v,
        "d":d,
        "octahedral_stress":oct_stress,
        "distortional_strain":uod,
        "energy_density":u0
    }

def get_lambda(MATERIAL):
    return (MATERIAL["poisson"]*MATERIAL["elastic"]/((1+MATERIAL["poisson"])*(1-2*MATERIAL["poisson"])))

def save(filename, results):
    print(filename)
    with open(filename, 'w',newline='') as file:
        file.write('-'*30 + '\n')
        file.write('{:<17}'.format("name") + " | " + "value\n")
        for (key,value) in results.items():
            file.write('\n{:<17}'.format(key) + " |\n")
            file.write(str(value))
            file.write('\n')
            file.write("-"*30)

if __name__ == "__main__":
    for x in range (len(points)):
        if x in (2,3,6):
            material = STEEL
        else :
            material = ALUMINIUM
        result = strain_transform(points[x],after_loading[x], material)
        save(f"point_{x}.txt", result)
    for x in range (len(points)):
        if x in (8,9,10,11,12,13):
            material = STEEL
        result = strain_transform(points[x],after_loading[x], material)
        save(f"point[mid]_{x}.txt", result)


from snowflake.connector import DictCursor

def databases_get(self,env_database_prefix:str)->dict:
    cur = self._conn.cursor(DictCursor)
    query = "SHOW DATABASES like '" + env_database_prefix + "%' in ACCOUNT;"

    data=[]
    try:
        cur.execute(query)
        for rec in cur:
            nw = {}
            nw['DATABASE_NAME'] = rec['name']
            nw['COMMENT'] = rec['comment']
            nw['DATA_RETENTION_TIME_IN_DAYS'] = int(rec['retention_time'])
            nw['OWNER'] = rec['owner']
            data.append(nw)
    except Exception as ex:
        msg = 'SQL Error:\n\nQuery: ' + query + '\n\nError Message:\n' + str(ex) + '\n\n'
        raise Exception(msg)
    finally:
        cur.close()
    return data


'''
Various utlity functions that are independant of any Django app or
model.
'''

from nturl2path import pathname2url as ntpathname2url
from nturl2path import url2pathname as url2ntpathname
import random
import re
import string
from unicodedata import normalize
from urllib.parse import urljoin, urlparse
from urllib.request import pathname2url, url2pathname

from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.utils.encoding import iri_to_uri, uri_to_iri

from .models import Setting

GROUP_NT_UNC = r'file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/'

GROUP_NT_DRIVE_LETTER = r'file:///[A-Za-z](?:\:|\|)/'

GROUP_NON_AUTH = r'file:///[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+'

FILE_IRI_PATTERN = (
    r'^(?P<unc>' +
    GROUP_NT_UNC +
    r')|(?P<driveletter>' +
    GROUP_NT_DRIVE_LETTER +
    r')|(?P<nonauth>' +
    GROUP_NON_AUTH +
    r')'
)

def generate_password(length=32):
    '''
    Quick and dirty random password generator.

    ***WARNING*** - Although this is likely "good enough" to create a secure
    password, there are no validations (suitible entropy, dictionary words,
    etc.) and should not be completely trusted. YOU HAVE BEEN WARNED.
    '''
    chars = string.ascii_letters + string.digits + string.punctuation
    rng = random.SystemRandom()
    return ''.join([rng.choice(chars) for i in range(length)])

def get_len(rawqueryset):
    '''
    Adds/Overrides a dynamic implementation of the length protocol to the
    definition of RawQuerySet.
    '''
    def __len__(self):
        params = ['{}'.format(p) for p in self.params]
        sql = ''.join(('SELECT COUNT(*) FROM (',
                       rawqueryset.raw_query.format(tuple(params)),
                       ') B;'))
        cursor = connection.cursor()
        cursor.execute(sql)
        row = cursor.fetchone()
        return row[0]
    return __len__

def get_setting(name):
    '''Helper function to get dynamic settings from the database.'''
    setting = Setting.objects.get(name=name)
    return setting.get()

def set_setting(name, value, setting_type=None):
    '''Helper function to set dynamic settings from the database.'''
    setting_types = {'Integer': 0, 'Float': 1, 'String': 2, 'Bool': 3}
    try:
        setting = Setting.objects.get(name=name)
        setting.data = str(value)
        if setting_type in setting_types:
            setting.setting_type = setting_types[setting_type]
        setting.save()
    except ObjectDoesNotExist:
        if setting_type in setting_types:
            Setting.objects.create(name=name,
                                   setting_type=setting_types[setting_type],
                                   data=str(value))
        else:
            error_msg = 'New settings need type (Integer, Float, String, Bool)'
            raise TypeError(error_msg)
    return

def naturalize(text):
    '''
    Return a normalized unicode string, with removed starting articles, for use
    in natural sorting.

    Code was inspired by 'django-naturalsortfield' from Nathan Reynolds:
    https://github.com/nathforge/django-naturalsortfield
    '''
    def naturalize_int_match(match):
        return '{:08d}'.format(int(match.group(0)))

    text = normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
    text = text.lower()
    punc = re.compile('[{}]'.format(re.escape(string.punctuation)))
    text = re.sub(punc, ' ', text)
    text = text.strip()
    text = re.sub(r'^(a|an|the)\s+', '', text)
    text = re.sub(r'\d+', naturalize_int_match, text)

    return text

def quantify(quantity, model):
    '''
    A message based on the quantity and singular/plural name of the model.
    '''
    if quantity == 1:
        message = '1 {}'.format(model._meta.verbose_name)
    else:
        message = '{} {}'.format(str(quantity),
                                 model._meta.verbose_name_plural)
    return message

def create_success_message(parent_model, parent_quantity, child_model,
                           child_quantity, remove=False):
    '''
    Creates a message for displaying the success of model modification.
    '''
    p_message = quantify(parent_quantity, parent_model)
    c_message = quantify(child_quantity, child_model)
    if remove:
        return '{} successfully removed from {}'.format(c_message, p_message)
    return '{} successfully added to {}.'.format(c_message, p_message)

def get_pretty_time(seconds):
    '''
    Displays a human-readable representation of time.
    '''
    if seconds > 0:
        periods = [
            ('year', 60*60*24*365.25),
            ('day', 60*60*24),
            ('hour', 60*60),
            ('minute', 60),
            ('second', 1)
        ]
        strings = []
        for period_name, period_seconds in periods:
            if seconds >= period_seconds:
                period_value, seconds = divmod(seconds, period_seconds)
                strings.append('{} {}{}'.format(period_value,
                                                period_name,
                                                ('s', '')[period_value == 1]))
        return ', '.join(strings)
    return 'Now'

def path_to_iri(path):
    '''
    OS-independant attempt at converting any OS absolute path to an
    RFC3987-defined IRI along with the file scheme from RFC8089.
    '''
    # Looking to see if the path starts with a drive letter or UNC path
    # (eg. 'D:\' or '\\')
    windows = re.match(r'^(?:[A-Za-z]:|\\)\\', path)
    if windows:
        return uri_to_iri(urljoin('file:', ntpathname2url(path)))
    return uri_to_iri(urljoin('file:', pathname2url(path)))

def iri_to_path(iri):
    '''
    OS-independant attempt at converting an RFC3987-defined IRI with a file
    scheme from RFC8089 to an OS-specific absolute path.
    '''
    # Drive letter IRI will have three slashes followed by the drive letter
    # UNC path IRI will have two slashes followed by the UNC path
    uri = iri_to_uri(iri)
    patt = r'^(?:' + GROUP_NT_DRIVE_LETTER + r'|' + GROUP_NT_UNC + r')'
    windows = re.match(patt, uri)
    if windows:
        parse = urlparse(uri)
        # UNC path URIs put the server name in the 'netloc' parameter.
        if parse.netloc:
            return '\\' + url2ntpathname('/' + parse.netloc + parse.path)
        return url2ntpathname(parse.path)
    return url2pathname(urlparse(uri).path)


import sys
import pygame
from settings import Settings
from ship import Ship

class AlienInvasion:
    """初始化游戏并创建游戏"""

    def __init__(self):
        pygame.init()

        self.settings = Settings()
        self.screen = pygame.display.set_mode((0, 0), pygame.FULLSCREEN)
        self.settings.screen_width = self.screen.get_rect().width
        self.settings.screen_height = self.screen.get_rect().height
        pygame.display.set_caption("Alien Invasion")

        self.ship = Ship(self)

    def _check_events(self):
        """响应键盘和鼠标事件"""
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            # 判断键盘按下
            elif event.type == pygame.KEYDOWN:
                self._check_keydown_events(event)
            # 判断键盘松开
            elif event.type == pygame.KEYUP:
                self._check_keyup_events(event)

    def _check_keydown_events(self, event):
        if event.key == pygame.K_RIGHT:
            self.ship.moving_right = True
        elif event.key == pygame.K_LEFT:
            self.ship.moving_left = True
        elif event.key == pygame.K_q:
            sys.exit()

    def _check_keyup_events(self, event):
        if event.key == pygame.K_RIGHT:
            self.ship.moving_right = False
        elif event.key == pygame.K_LEFT:
            self.ship.moving_left = False

    def _update_screen(self):
        """更新屏幕上的图像，并切换到新屏幕"""
        self.screen.fill(self.settings.bg_color)
        self.ship.blitme()
        pygame.display.flip()

    def run_game(self):
        """开始游戏的主循环"""
        while True:
            self._check_events()
            self._update_screen()
            self.ship.update()

if __name__ == '__main__':
    ai = AlienInvasion()
    ai.run_game()


# This Python 3 environment comes with many helpful analytics libraries installed

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import matplotlib.pyplot as plt

import seaborn as sns

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os

print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
train = pd.read_csv('../input/train.csv')

test = pd.read_csv('../input/test.csv')
train.head()
test.head()
print(train.isnull().sum().sort_values(ascending = False)) #as you can see there is no null value in the columns

print("**"*50)

print(test.isnull().sum().sort_values(ascending = False))
print(train.info())

print('**'*50)

print(test.info())
train.datetime = pd.to_datetime(train.datetime)

test.datetime = pd.to_datetime(test.datetime)
print(train.info())

print('**'*50)

print(test.info())
train['year'] = train['datetime'].dt.year

train['month'] = train['datetime'].dt.month

train['day'] = train['datetime'].dt.day

train['hour'] = train['datetime'].dt.hour

train['dayofweek'] = train['datetime'].dt.weekday_name

test['year'] = test['datetime'].dt.year

test['month'] = test['datetime'].dt.month

test['day'] = test['datetime'].dt.day

test['hour'] = test['datetime'].dt.hour

test['dayofweek'] = test['datetime'].dt.weekday_name
train.tail()
plt.figure(figsize=(16,8))

sns.heatmap(train.corr(), annot=True)

plt.show()
plt.figure(figsize=(16,8))

sns.distplot(train['count'])

plt.show()
plt.figure(figsize=(16,8))

plt.plot(train["count"][0:300])

plt.show()
# we need to convert categorical data to numeric data.

from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()

train['dayofweek'] = le.fit_transform(train['dayofweek'])

test['dayofweek'] = le.transform(test['dayofweek'])
plt.figure(figsize=(16,8))

sns.boxplot(x='season', y='count', data=train)
plt.figure(figsize=(16,8))

sns.boxplot(x='dayofweek',y='count', data=train)
plt.figure(figsize=(16,8))

sns.boxplot(x='hour',y='count', data=train)
plt.figure(figsize=(16,8))

sns.boxplot(x='year',y='count', data=train)
plt.figure(figsize=(16,8))

plt.hist(train['count'][train['year'] == 2011], alpha=0.5, label='2011')

plt.hist(train['count'][train['year'] == 2012], alpha=0.5, label='2012', color='red')
plt.scatter(train['hour'], train['count'])
train.head()
train.set_index('datetime', inplace=True)
train['2011-01-19 23:00:00':]
Q1 = train.quantile(0.25)

Q3 = train.quantile(0.75)

IQR = Q3 - Q1

print(IQR)
train_without_outliers =train[~((train < (Q1 - 1.5 * IQR)) |(train > (Q3 + 1.5 * IQR))).any(axis=1)]
train_without_outliers.dropna(inplace=True)
print(train.info())

print('*********************************************************************************')

print(train_without_outliers.info())
train_without_outliers.head(2)
plt.figure(figsize=(12, 7))

sns.boxplot(x='season',y='windspeed',data=train_without_outliers,palette='winter')
def wind(cols):

    windspeed = cols[0]

    season = cols[1]

    if windspeed==0:

        if season == 1 :

            return 14

        elif season == 2 :

            return 14

        else:

            return 13

    else:

        return windspeed
train_without_outliers['wind'] = train_without_outliers[['windspeed','season']].apply(wind,axis=1)

test['wind'] = test[['windspeed', 'season']].apply(wind, axis=1)
test.head()
train_without_outliers.head(5)
train_without_outliers[['season','holiday','workingday','weather', 'year','month','day','hour','dayofweek']] = train_without_outliers[['season','holiday','workingday','weather', 'year','month','day','hour','dayofweek']].astype('category')

test[['season','holiday','workingday','weather', 'year','month','day','hour','dayofweek']] = test[['season','holiday','workingday','weather', 'year','month','day','hour','dayofweek']].astype('category')
train_without_outliers.info()
from sklearn.model_selection import train_test_split
X = train_without_outliers[['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp','humidity', 'year', 'month', 'day', 'hour', 'dayofweek','wind']]

y = train_without_outliers['count']
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.1)
y_train
y_train= y_train.values.reshape(-1,1)

y_test= y_test.values.reshape(-1,1)

from sklearn.preprocessing import MinMaxScaler

sc_X = MinMaxScaler()

sc_y = MinMaxScaler()

X_train = sc_X.fit_transform(X_train)

X_test = sc_X.fit_transform(X_test)

y_train = sc_X.fit_transform(y_train)

y_test = sc_y.fit_transform(y_test)
from sklearn.ensemble import RandomForestRegressor

rf = RandomForestRegressor(n_estimators=100)

rf.fit(X_train, y_train)
rf_prediction = rf.predict(X_test)
from sklearn.metrics import mean_squared_error

from sklearn import metrics

print('MSE:', metrics.mean_squared_error(y_test, rf_prediction))
plt.scatter(y_test,rf_prediction)
plt.figure(figsize=(16,8))

plt.plot(rf_prediction[0:200],'r')

plt.plot(y_test[0:200])
from sklearn.tree import DecisionTreeRegressor

dt_reg = DecisionTreeRegressor()

dt_reg.fit(X_train, y_train)
dt_prediction = dt_reg.predict(X_test)
print('MSE:', metrics.mean_squared_error(y_test, dt_prediction))
plt.scatter(y_test,dt_prediction)
test.head()
test[['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp','humidity', 'year', 'month', 'day', 'hour', 'dayofweek','wind']] = sc_X.fit_transform(test[['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp','humidity', 'year', 'month', 'day', 'hour', 'dayofweek','wind']])
test_pred= rf.predict(test[['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp','humidity', 'year', 'month', 'day', 'hour', 'dayofweek','wind']])
test_pred
test_pred=test_pred.reshape(-1,1)
test_pred = sc_y.inverse_transform(test_pred)
test_pred = pd.DataFrame(test_pred, columns=['count'])
df = pd.concat([test['datetime'], test_pred],axis=1)
df.head()
df['count'] = df['count'].astype('int')
df.to_csv('submission1.csv' , index=False)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
to compile fortran code, mk_fortran.f90:
try first this
f2py -c mk_fortran.f90 -m mankendall_fortran_repeat_exp2 --fcompiler=gfortran

if not try this
LDFLAGS=-shared f2py -c mk_fortran.f90 -m mankendall_fortran_repeat_exp2 --fcompiler=gfortran

"""

import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from mpl_toolkits.axes_grid1 import make_axes_locatable
import numpy as np
import netCDF4
import h5py # import after netCDF4 otherwise 'RuntimeError: NetCDF: HDF error' on VITO VM
from datetime import datetime
import os,sys
import re
import fnmatch
import traceback
from timeit import default_timer as timer
import generic
import pathlib
from tools import SimpleTimer

class TimeSeriesExtractor():
    def __init__(self, product, chunks, config, b_delete):
        self.start = product.start_date
        self.end = product.end_date
        self.product = product.name
        self.sensor = product.sensor
        self.chunks = chunks
        self.hash = product.hash
        self.b_delete = b_delete

        self.config = config[self.product]

        self.output_path = pathlib.Path(config['output_path']['extract'])
        (self.output_path / self.product).mkdir(parents=True, exist_ok=True)

        self.mode = self.config['mode']

        if self.config['freq']=='10D':
            # Every 10D means 3 snapshots a month and 36 snapshots a year
            print('INFO: frequency = 10D so start and end dates are trimmed to start and end of month respectively')
            self.start_01 = self.start.replace(day=1)
            self.end_31 = self.end.replace(day=pd.Period(self.end, freq='D').days_in_month) # freq='xxx' is here just to avoid a bug
            self.dseries = pd.date_range(self.start_01, self.end_31, freq='D')
            self.dseries = self.dseries[[d in [1,2,3] for d in self.dseries.day]] # select only days 3, 13 and 23 by month
        else:
            self.dseries = pd.date_range(self.start, self.end, freq=self.config['freq'])

        self.source = self.config['source']

    def get_lw_mask(self):
        """ Get the MSG disk land mask (0: see, 1: land, 2: outside space, 3: river/lake)"""
        hlw = h5py.File('./input_ancillary/HDF5_LSASAF_USGS-IGBP_LWMASK_MSG-Disk_201610171300','r')
        self.lwmsk = hlw['LWMASK'][self.chunks.get_limits('global', 'slice')]
        hlw.close()

    def DEPRECATED_extract_albedo(self, chunk, date, dateindex):
        files = {}
        files['mdal_nrt'] = self.product_files['mdal_nrt'][dateindex]
        files['mdal'] = self.product_files['mdal'][dateindex]

        ## Reading MDAL albedo
        if date.year>=2016:
            file_name = files['mdal_nrt']
            if date.year<=2018:
                file_name += '.h5'
        else:
            file_name = files['mdal']
        print(file_name)

        ## Extract data zone
        try:
            fh5 = h5py.File(file_name,'r')
        except Exception:
            traceback.print_exc()
            print ('File not found, moving to next file, assigning NaN to extacted pixel.')
            return None

        albedo = fh5['AL-BB-DH'][chunk.get_limits('global', 'slice')] # array of int
        zage = fh5['Z_Age'][chunk.get_limits('global', 'slice')]
        fh5.close()

        ## debug
        if 1:
            #probe = (slice(400, 420, 10), slice(1900, 1920, 10))
            probe = (400, 1900)
            albedo = fh5['AL-BB-DH'][probe] # array of int
            zage = fh5['Z_Age'][probe]
            qf = fh5['Q-flag'][probe]
            return [albedo, zage, qf]

        ## remove invalid data
        albedo = np.where(albedo==-1, np.nan, albedo/10000.)
        albedo = np.where(zage>0, np.nan, albedo)

        return albedo

    def DEPRECATED_extract_evapo(self, chunk, date, dateindex):
        file_name = self.product_files['mdal'][dateindex]

        print(file_name)

        ## Extract data zone
        try:
            fh5 = h5py.File(file_name,'r')
        except Exception:
            traceback.print_exc()
            print ('File not found, moving to next file, assigning NaN to extacted pixel.')
            return None

        var = fh5['METREF'][chunk.get_limits('global', 'slice')] # array of int
        #zage = fh5['Z_Age'][*chunk.global_lim]
        fh5.close()

        ## remove invalid data
        var = np.where(var==-8000, np.nan, var/100.)
        #albedo = np.where(zage>0, np.nan, albedo)

        return var

    def plot_histogram(self, tseries):
        ## Initialize an array to store histogram stats
        nbins = 100
        res_h = np.zeros((tseries.shape[0], nbins))
        dmin, dmax = (np.nanmin(tseries), np.nanmax(tseries))
        bins = np.linspace(dmin, dmax, nbins+1)
        pct_nan = []

        for i,cut in enumerate(tseries):
            ## Fill histogram
            h = np.histogram(cut, bins=bins)[0]
            #res_h[i] = np.log10(h) # w/o scaling
            #res_h[i] = np.log10(h/(np.count_nonzero(~np.isnan(net_albedo)))) # scale by total nb of not nan
            res_h[i] = np.log10(h/h.max()) # scale by max

            ## Count nan
            pct_nan.append(100*np.count_nonzero(np.isnan(cut))/cut.size)

        ## Generate and save histogram
        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax2 = ax.twinx()
        #res_h[res_h==0.0] = np.nan

        # You can then convert these datetime.datetime objects to the correct
        # format for matplotlib to work with.
        xnum = mdates.date2num(self.dseries)
        xlims = [xnum[0], xnum[-1]]

        # plot histogram
        ax.imshow(res_h.T, aspect='auto', origin='lower', extent=(xlims[0], xlims[1], dmin, dmax))

        # plot nan count
        ax2.plot(xnum, pct_nan, c='k', lw=1)

        ax.grid()

        plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
        plt.gca().xaxis.set_major_locator(mdates.MonthLocator())
        plt.gcf().autofmt_xdate()

        ax.set_xlabel('date')
        ax.set_ylabel(self.product)
        ax2.set_ylabel('% NaN')
        ax2.set_ylim(0,100)
        plt.savefig('res_hist_{}_{}.png'.format(self.hash, self.product))

    def plot_image_series(self, tseries):
        dmin, dmax = (np.nanmin(tseries), np.nanmax(tseries))

        out_path = './output_imageseries/'
        pathlib.Path(out_path).mkdir(parents=True, exist_ok=True)

        ## Generate and save histogram
        fig = plt.figure()
        ax = fig.add_subplot(111)

        init = dmin+np.zeros_like(tseries[0])
        init[0,0] = dmax
        img = ax.imshow(init, aspect='auto')
        for i,cut in enumerate(tseries):
            img.set_data(cut)
            #plt.pause(0.1)

            ax.set_title('{} - {}'.format(self.product, self.dseries[i].isoformat()))
            plt.savefig(out_path+'/res{1}_{0:03d}_{2}.png'.format(i, self.hash, self.product))

    def get_product_files(self, mode='predict'):
        """
        Generator that yield recursively open H5 file between two dates

        Parameters
        ----------

        mode : string, 'predict' or 'walk'
            - 'predict' is used to load daily/hourly files with predictable filename
            - 'walk' is used to load not easily predictable names and so 'walk' in subdirs of roots
            to find relevant files.
        """
        if self.mode=='predict':
            for date in self.dseries:
                ## merge root dir with substituted template
                file_name = pathlib.Path(self.config['root']) / date.strftime(self.config['template'])
                print(file_name)
                # Note: the with/yield pattern should be checked to see if files are corectly closed
                try:
                    with h5py.File(file_name, 'r') as h5f:
                        yield date, h5f
                except Exception:
                    #traceback.print_exc()
                    print ('File not found, moving to next file, assigning NaN to extacted pixel.')
                    yield date, None

        elif self.mode=='walk':
            ## Get year min and year max from start and end
            ymin = self.start.year
            ymax = self.end.year

            #print(ymin, ymax)

            ## Find in the subdir all the *.nc files and save as [datetime, file path]
            print('Look for data files in:\n', self.config['root'])
            flist = []
            for y in range(ymin, ymax+1):
                for dp, dn, fn in os.walk((pathlib.Path(self.config['root'])/str(y)).as_posix()):
                    if len(fn)>0:
                        fl = fnmatch.filter(fn, '*.nc')
                        if len(fl)==1: # Normally there is is only one nc file in each folder
                            ncfile = fl[0]
                            # Check sensor type
                            if self.sensor not in ncfile:
                                continue
                        else:
                            print('WARNING: no .nc file found')
                            continue
                        # Use regex to catch the date in the filename:
                        # starts with '_' followed by a 1 or 2
                        # then between 7 and 13 digits and a final '_'
                        p = re.compile('_[12]\d{7,13}_')
                        m = p.search(ncfile)
                        date_str = m.group()[1:-1]
                        if len(date_str[8:])==6:
                            pattern = "%H%M%S"
                        elif len(date_str[8:])==4:
                            pattern = "%H%M"
                        date_obj = datetime.strptime(date_str, "%Y%m%d"+pattern)
                        flist.append([date_obj, pathlib.Path(dp) / ncfile])

            ## Process data get from walking with pandas

            df = pd.DataFrame(flist, columns=['datetime','path'])
            # Trim the dates using start and end in a DataFrame
            df = df.set_index('datetime')
            df = df.sort_index().loc[self.start:self.end]

            ## The time series may have missing data and we have to take them into account.
            ## self.dfseries is the list of all theoretical dates (ex d1 to d4) so we are going
            ## to construct a dataframe based on it and we are then join it with df that contains
            ## only valid data in order to pass from df =
            ##  d2 path2
            ##  d3 path3
            ## to self.df_full =
            ##  d1 nan
            ##  d2 path2
            ##  d3 path3
            ##  d4 nan

            def test(date):
                """
                Apply a formated index to the three month observations
                xx, yy and zz with xx < yy < zz become 1, 2 and 3
                We use this because day number are not always the same, ex:
                - c3s albedo : 10, 20 and 28|29|30|31
                - c3s LAI: 3, 13, 21|23|24
                """
                if date.day < 11:
                    return date.replace(day=1)
                elif date.day < 21:
                    return date.replace(day=2)
                else:
                    return date.replace(day=3)

            # Create fake formated date and set it as index
            df['global_id'] = df.index.to_series().map(test)
            df = df.reset_index().set_index('global_id')

            # debug
            #df = df.loc["2019-05-01":"2019-07-31"]
            #print(df)

            # Create a new theoretical df based only on dseries as index
            self.df_full = pd.DataFrame(index=self.dseries)
            self.df_full.index.name = 'global_id'

            # Join with valid data, empty date will automatically be filled with NaN or NaT
            self.df_full = self.df_full.join(df)

            # Show summary of valid data by year (or month)
            #df = df.groupby([df.index.year, df.index.month]).count()
            sum = self.df_full.groupby([self.df_full.index.year]).count()
            print(sum)
            print("Total path:", sum['path'].sum())

            # datetime index back to column to be easily selected
            self.df_full = self.df_full.reset_index()

            # Add helper columns
            # global_id is a timestamp like starting at 0 from 1970-01-01 on a 36 based year
            # useful to merge different cache file or with // or % operator to have year or position in year
            fd = self.df_full['global_id']
            fd = pd.to_datetime(fd).apply(lambda t: (t.year-1970)*36 + (t.month-1)*3 + t.day-1)
            self.df_full['global_id'] = fd
            #self.df_full = self.df_full.set_index('global_id')

            ## Debug option to output filepath list to csv
            #self.df_full.to_csv('path_to_file.csv', sep=';')
            #sys.exit()

            ## Loop on these files and yield

            #for row in df.to_dict(orient="records"):
            for t in self.df_full.itertuples():
                # Note: the with/yield pattern should be checked to see if files are corectly closed
                try:
                    self.b_h5py_r = 1
                    if self.b_h5py_r:
                        with h5py.File(t.path, 'r') as h5file:
                            self.h5f = h5file
                            yield t
                    else:
                        with netCDF4.Dataset(t.path, 'r') as h5file:
                            self.h5f = h5file
                            yield t
                except Exception:
                    #traceback.print_exc()
                    t2 = t._replace(path=None)
                    yield t2

    def _extract_points(self, var, dtype):
        """
        This version load all the dataset and extract location from memory

        Direct load may be faster until a certain number of point depending on the system.
        On VITO benchmark give a value of 150.

        Notes
        -----
        1/ Profiling
        1.1/ Timing is good if there are a lot of points and loading of data is not
        too long. Best way should be using hyperslabs in h5py low level api
        but there are some array manipulation do to after extraction: hyperslab
        blocks are sorted by lat then lon (to be efficiently read in the file)
        and so it may yield mixed data when regions share same lat or overlap.

        1.2/ On 1km dataset, profiling gives:
        h5py:
        data = self.h5f[var][:] --> ~2.4s
        netCDF4:
        data = self.h5f.variables[var][:] --> ~24s, ie 10x h5py, to be continued...

        2/ To plot netCDF file attributes
        #print(len(self.h5f.ncattrs()))
        try:
            print(self.h5f.getncattr('platform'))
        except:
            print('warning: no platform attribute')
        """
        chunk = self.chunk

        prod_chunk = np.zeros((chunk.dim[1], chunk.box_size, chunk.box_size), dtype=dtype)

        ti = SimpleTimer()
        # check the number of points to load
        if chunk.dim[1] > 150: # two-step load
            if self.b_h5py_r:
                data = self.h5f[var][:]
                ti('h5py_indirect load time for {}:'.format(var))
            else:
                data = self.h5f.variables[var][:]
                ti('netCDF4 load time for {}:'.format(var))

            for ii,s in enumerate(chunk.get_limits('global', 'slice')):
                prod_chunk[ii] = data[s]
            del(data)
            ti('memory load time for {}:'.format(var))

        else: # direct load
            for ii,s in enumerate(chunk.get_limits('global', 'slice')):
                prod_chunk[ii] = self.h5f[var][s]
            ti('h5py_direct load time for {}:'.format(var))

        return prod_chunk

    def _get_c3s_albedo_points(self):

        if self.chunk.sensor != 'SENTINEL3':
            age_chunk = self._extract_points('AGE', np.int16)
        q_chunk = self._extract_points('QFLAG', np.uint8)

        d = {}
        for v in self.config['var']:
            prod_chunk = self._extract_points(v, np.int16)
            error_chunk = self._extract_points(v+'_ERR', np.int16)

            # Discard pixels in the analysis when (fill value in AL_* is -32767 ):
            # - Outside valid range in AL_* ([0, 10000])
            # - QFLAG indicates ‘sea’ or ‘continental water’ (QFLAG bits 0-1) -> Fill value in product
            # - QFLAG indicates the algorithm failed (QFLAG bit 7) -> Fill value in product
            # Optionally, also the following thresholds can be considered:
            # - *_ERR > 0.2
            # - AGE > 30
            b_print_sel = 0
            # int value is used instead of np.nan because nan is a float
            discard = -32767
            prod_chunk[ (prod_chunk<0) | (prod_chunk>10000) ] = discard
            if b_print_sel: self._count_val('data range', prod_chunk, discard)
            if b_print_sel: print(np.vectorize(np.binary_repr)(q_chunk, width=8))
            prod_chunk[ ~((q_chunk & 0b11) == 0b01) ] = discard # if bit 1 and 0 are not land (= 01) set np.nan
            if b_print_sel: self._count_val('land', prod_chunk, discard)
            prod_chunk[ ((q_chunk >> 7) & 1) == 1 ] = discard # if bit 7 is set
            if b_print_sel: self._count_val('algo failed', prod_chunk, discard)
            prod_chunk[error_chunk > 2000] = discard
            if b_print_sel: self._count_val('error', prod_chunk, discard)
            if self.chunk.sensor != 'SENTINEL3':
                prod_chunk[age_chunk > 30] = discard
                if b_print_sel: self._count_val('age', prod_chunk, discard)
            #sys.exit()
            #prod_chunk = prod_chunk.

            ## Agregate data using quality flag
            # Count True in each window and keep only when 75% is ok
            # 1 for 1x1, 12 for 4x4 and 108 for 12x12
            if self.chunk.c3s_resol=='4km':
                threshold = 1
            if self.chunk.c3s_resol=='1km':
                threshold = 12
            if self.chunk.c3s_resol=='300m':
                threshold = 108
            q_mask = np.count_nonzero(~(prod_chunk==discard), axis=(1,2)) >= threshold
            print('valid : {0} / {1}'.format(np.count_nonzero(q_mask), self.chunk.dim[1]) )
            #print('invalid indices:')
            #print(np.where(q_mask==0)[0])
            d[v] = 1e-4 * prod_chunk
            d[v][ d[v]==discard*1e-4 ] = np.nan # replace discard value by nan
            d[v] = np.nanmean(d[v], axis=(1,2))
            d[v] = np.where(q_mask,  d[v], np.nan)
            d[v].reshape((1,-1)) # fake 2D array (1,nb_pts)

        return d

    def _get_c3s_lai_fapar_points(self):

        if 'QFLAG' in self.h5f.keys():
            print('---qflag')
            flag_layer = 'q'
        elif 'retrieval_flag' in self.h5f.keys():
            print('---retrieval_flag')
            flag_layer = 'r'
        else:
            print('ERROR: no known quality flag found in h5 file.')
            sys.exit()

        if flag_layer=='q':
            q_chunk = self._extract_points('QFLAG', np.uint8)
        elif flag_layer=='r':
            q_chunk = self._extract_points('retrieval_flag', np.uint32)

        d = {}
        for v in self.config['var']:
            prod_chunk = self._extract_points(v, np.uint16)
            #error_chunk = self._extract_points(v+'_ERR', np.int16)

            # Discard pixels (ie. Apply fill value 65535) in the analysis when:
            # - Outside valid range in LAI / fAPAR ([0, 65534])
            # EITHER:
            # - QFLAG indicates ‘sea’ or ‘continental water’ (QFLAG bits 0-1) -> Fill value in product
            # - QFLAG indicates the algorithm failed (QFLAG bit 7) -> Fill value in product
            # OR:
            # - retrieval_flag indicates ‘obs_is_fillvalue’ (QFLAG bit 0)
            # - retrieval_flag indicates ‘tip_untrusted’ (QFLAG bit 6)
            # - retrieval_flag indicates ‘obs unusable’ (QFLAG bit 7)

            b_print_sel = 0
            # int value is used instead of np.nan because nan is a float
            discard = 65535
            prod_chunk[ (prod_chunk<0) | (prod_chunk>65534) ] = discard
            if b_print_sel: self._count_val('data range', prod_chunk, discard)
            if b_print_sel: print(np.vectorize(np.binary_repr)(q_chunk, width=8))
            if flag_layer=='q': # if qflag
                prod_chunk[ ~((q_chunk & 0b11) == 0b01) ] = discard # if bit 1 and 0 are not land (= 01) set np.nan
                if b_print_sel: self._count_val('land', prod_chunk, discard)
                prod_chunk[ ((q_chunk >> 7) & 1) == 1 ] = discard # if bit 7 is set
                if b_print_sel: self._count_val('algo failed', prod_chunk, discard)
            elif flag_layer=='r': # if retrieval_flag
                prod_chunk[ ((q_chunk >> 0) & 1) == 1 ] = discard # if bit 0 is set
                if b_print_sel: self._count_val('obs_is_fillvalue', prod_chunk, discard)
                prod_chunk[ ((q_chunk >> 6) & 1) == 1 ] = discard # if bit 6 is set
                if b_print_sel: self._count_val('tip_untrusted', prod_chunk, discard)
                prod_chunk[ ((q_chunk >> 7) & 1) == 1 ] = discard # if bit 7 is set
                if b_print_sel: self._count_val('obs_unusable', prod_chunk, discard)
            #sys.exit()

            ## Agregate data using quality flag
            # Count True in each window and keep only when 75% is ok
            # 1 for 1x1, 12 for 4x4 and 108 for 12x12
            if self.chunk.c3s_resol=='4km':
                threshold = 1
            if self.chunk.c3s_resol=='1km':
                threshold = 12
            if self.chunk.c3s_resol=='300m':
                threshold = 108
            q_mask = np.count_nonzero(~(prod_chunk==discard), axis=(1,2)) >= threshold
            print('valid : {0} / {1}'.format(np.count_nonzero(q_mask), self.chunk.dim[1]) )
            #print('invalid indices:')
            #print(np.where(q_mask==0)[0])
            d[v] = prod_chunk/6553.
            #d[v] = prod_chunk*1.
            d[v][ d[v]==discard/6553. ] = np.nan # replace discard value by nan
            d[v] = np.nanmean(d[v], axis=(1,2))
            d[v] = np.where(q_mask,  d[v], np.nan)
            d[v].reshape((1,-1)) # fake 2D array (1,nb_pts)

        return d

    def _count_val(self, txt, a, val):
        print("{}: {}".format(txt, np.count_nonzero(a==val)) )

    def extract_product(self):
        chunk = self.chunk
        if self.source=='msg':
            prod_chunk = self.h5f[self.config['var']][chunk.get_limits('global', 'slice')] # array of int

        elif self.source=='c3s':

            if chunk.input=='box':
                chunk_range = (0, *chunk.get_limits('global', 'slice')) # array of int
                prod_chunk = self.h5f[self.config['var']][chunk_range] # array of int

            elif chunk.input=='points':
                # Save point name
                self.ascii_pointnames = [n.encode("ascii", "ignore") for n in chunk.site_coor['NAME']]

                ## DEBUG: Plot landval
                if 0:
                    print("DEBUG: print landval sites")
                    print('Load...')
                    data_in = self.h5f[self.config['var']][:]
                    #data = self.h5f[self.config['var']][0,::2,::2]
                    #data = h5f['retrieval_flag'][:]
                    data = data_in[0,::2,::2]
                    del(data_in)

                    print('Plot...')
                    ti = SimpleTimer()
                    ti('t01')
                    fig, ax = plt.subplots()
                    ti('t02')

                    data = 1e-4*data
                    ti('t1')
                    data[data<0] = 0.0
                    ti('t2')
                    print(data.shape)
                    print(data.min(), data.max())
                    #data = data & 0xFC1 # See Appendix A of D3.3.8-v2.1_PUGS_CDR-ICDR_LAI_FAPAR_PROBAV_v2.0_PRODUCTS_v1.1.pdf
                    print(data.min(), data.max())
                    ti('t3')

                    ptype = 2
                    # Add landval sites directly in array
                    if ptype==1:
                        for ii,s in enumerate(chunk.get_limits('global', 'slice')):
                            data[s] = 2e3
                    ti('t4')
                    if 1:
                        cm = 'jet'
                        cm = 'gist_ncar'
                        #mat = ax.imshow(data[0,::2,::2], cmap=cm)
                        mat = ax.imshow(data, cmap=cm)
                        # create an axes on the right side of ax. The width of cax will be 5%
                        # of ax and the padding between cax and ax will be fixed at 0.05 inch.
                        divider = make_axes_locatable(ax)
                        cax = divider.append_axes("right", size="5%", pad=0.05)
                        plt.colorbar(mat, cax=cax)

                    # Add landval sites with scatter
                    if ptype==2:
                        pts = (0.5*chunk.site_coor[['ilat', 'ilon']].values.T).astype(np.int32)
                        print(pts.shape)
                        ax.scatter(pts[1], pts[0])

                    #plt.savefig('res_c3s.png')
                    plt.show()

                    sys.exit()
                ## END DEBUG

                if ('LAI' in self.config['var']) or ('fAPAR' in self.config['var']):
                    prod_chunk = self._get_c3s_lai_fapar_points()
                else:
                    prod_chunk = self._get_c3s_albedo_points()

        return prod_chunk

    def rebin(self, a, shape=None, scale=None):
        """Downsample a 2D numpy array"""
        if shape is not None:
            sh = shape[0],a.shape[0]//shape[0],shape[1],a.shape[1]//shape[1]
        elif scale is not None:
            sh = a.shape[0]//scale,scale,a.shape[1]//scale,scale
        else:
            print('ERROR: need to provide either shape or scale')
            sys.exit()
        #return a.reshape(sh).mean(-1).mean(1)
        return a.reshape(sh).max(-1).max(1)

    def extract_snow1(self):
        #data = self.h5f[var][:]

        qdata = self.h5f['QFLAG']
        snow_mask = ((qdata[0]>>5)&1)==1
        snow_sel = self.area[snow_mask]

        d = {}
        for v in self.config['var']:
            #count = np.count_nonzero((qdata>>5)&1) #calculate mean over time
            #count = self.area[((qdata[0]>>5)&1)==1].sum() # calculate total surface of snow
            count = snow_sel.sum()
            d[v] = np.array([count])
            d[v].reshape((1,-1)) # fake 2D array (1,nb_pts)

        #self.snow_map.set_array(qdata[0])
        #print(self.snow_map.get_array().max())
        #self.snow_map = self.ax1.imshow(qdata[0])
        self.ax1.clear()
        #self.ax1.imshow(self.rebin(snow_mask, shape=self.avhrr_dim))
        self.ax1.imshow(self.rebin(snow_mask, scale=4))
        self.ax1.set_title("{} | {}".format(self.sensor, self.current.datetime.strftime('%Y-%m-%d')))

        plt.savefig("output_snow/snow_{}_{}.png".format(self.sensor, self.current.global_id))

        return d

    def _write_ts_chunk(self, out_var):
        """
        Write time series of the data
        Variable have to be given in a list as dict with the following keys:
        'name': string, name of the variable
        'data': numpy array, the actual data
        'type': string, representing the data type
        """
        ## New H5 simplified version
        if 1:
            hf = h5py.File(self.write_file, 'w')

            if len(out_var)>0:
                for v in out_var:
                    hf.create_dataset(v['name'], data=v['data'].astype(v['type']))
                    #hf.create_dataset(v['name'], data=v['data'].astype(v['type']), compression="gzip", compression_opts=9)

            hf.close()
            print(">>> Data chunk written to:", self.write_file)
            return self.write_file

        ## Original NETCDF4 version
        else:
            ds = netCDF4.Dataset(self.write_file, 'w', format='NETCDF4')

            if len(out_var)>0:
                for v in out_var:
                    tdim = []
                    for idd,d in enumerate(v['data'].shape):
                        dname = '{}_d{}'.format(v['name'], idd)
                        ds.createDimension(dname, d)
                        tdim.append(dname)
                    tdim = tuple(tdim)

                    var = ds.createVariable(v['name'], v['type'], tdim, zlib=True)
                    var_data = v['data']
                    var[:] = var_data

            ds.close()
            print(">>> Data chunk written to:", self.write_file)
            return self.write_file

    def _check_previous_cache(self):
        """
        Check if cache file already exists and must be overwritten
        """
        #write_file = self.output_path / self.product / (self.hash+'_timeseries_'+'_'.join(self.chunk.get_limits('global', 'str'))+'.nc')
        write_file = self.output_path / self.product / (self.hash+'_timeseries_'+'_'.join(self.chunk.get_limits('global', 'str'))+'.h5')

        self.write_file = write_file

        print ('INFO: Check for previous Cache file {} '.format(write_file))

        if not self.b_delete:
            if write_file.is_file():
                print ('INFO: Cache file {} already exists. Use -f option to overwrite it.'.format(write_file))
                return True

    def run(self):
        b_use_mask = 0

        if b_use_mask: self.get_lw_mask()
        write_files = []

        ## Loop on chunks
        for chunk in self.chunks.list:
            self.chunk = chunk

            ## Print some info
            if chunk.input=='box':
                print('***', 'SIZE (y,x)=(row,col)=({},{})'.format(*chunk.dim), 'GLOBAL_LOCATION=[{}:{},{}:{}]'.format(*chunk.get_limits('global', 'str')))
            elif chunk.input=='points':
                print('***', 'SIZE {} points'.format(chunk.dim[1]))

            ## Check if cache file already exists
            if self._check_previous_cache():
                write_files.append(self.write_file)
                continue

            ## Initialize time series with array of nan
            data_ts = {}
            for v in self.config['var']:
                data_ts[v] = np.full([len(self.dseries), *chunk.dim], np.nan)
            time_ts = []
            print('data shape:', data_ts[self.config['var'][0]].shape)

            ## Create the chunk mask
            if b_use_mask:
                lwmsk_chunk = self.lwmsk[chunk.get_limits('local', 'slice')]
                ocean = np.where(lwmsk_chunk==0)
                land = np.where(lwmsk_chunk==1)

            res = [] # debug

            t0 = datetime.now()

            ## Loop on files
            for f in self.get_product_files():
                print("####################################################")
                print("{}/{} - {}".format(f.Index, len(self.dseries), f.datetime))
                if f.path is not None:
                    print(f.path.name)
                    tmp = self.extract_product()
                    for v in self.config['var']:
                        data_ts[v][f.Index] = tmp[v]
                    time_ts.append(f.datetime)
                else:
                    print ('File not found, moving to next file, assigning NaN to extacted pixel.')
                    time_ts.append(datetime(1970,1,1)) # set timestamp to 0 (=1970-01-01) if no data

                now = datetime.now()
                print('{} | elapsed: {}'.format(now, now-t0), flush=True)

            time_ts = np.array([np.datetime64(d).astype('<M8[s]') for d in time_ts])
            time_ts = time_ts.astype(np.int64)
            #print(time_ts)
            ## To store dates
            #bla = np.datetime64(datetime.datetime(2018,1,1)).astype('<M8[s]')
            #np.int64(4290000000).astype('<M8[s]')
            #datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc).timestamp() == 0

            # debug
            if 0:
                import matplotlib.pyplot as plt
                res = np.array(res)
                print(res.shape)
                plt.plot(res)
                plt.savefig('comp_age_qflag.png')
                return

            ## Write output data
            out_var = []
            for name, data in data_ts.items():
                out_var.append({'type':np.float, 'name':'vars/'+name, 'data':data})
            out_var.append({'type':np.int64, 'name':'meta/ts_dates', 'data':time_ts})
            out_var.append({'type':'S', 'name':'meta/point_names', 'data':np.array(chunk.site_coor['NAME'])})
            out_var.append({'type':np.uint16, 'name':'meta/global_id', 'data':self.df_full['global_id'].values})
            self._write_ts_chunk(out_var)
            write_files.append(self.write_file)

            #self.plot_histogram(tseries)
            #self.plot_histogram(tseries)
            #self.plot_image_series(tseries)

            del data_ts

        return write_files

    def run_snow(self):

        ## Initialize time series with array of nan
        data_ts = {}
        for v in self.config['var']:
            data_ts[v] = np.full([len(self.dseries),1,1], np.nan) # Add fake dimensions to fit the pipeline
        time_ts = []
        print('data shape:', data_ts[self.config['var'][0]].shape)

        ## Import pixel area
        with h5py.File('output_snow/pixel_area_{}.h5'.format(self.sensor), 'r') as ahf:
            self.area = ahf['area'][:]
            data_dim = ahf['data_dim'][:]
            nlon = data_dim[1]
            self.area = np.tile(self.area, (nlon,1)).T
            #self.area = ahf['area'][:].reshape((1,-1,1)) # add fake dimension

        ## Initialize interactive plot
        self.ax1 = plt.subplot(111)
        self.avhrr_dim = (4200,10800)
        self.snow_map = self.ax1.imshow(np.zeros(self.avhrr_dim))

        res = [] # debug

        t0 = datetime.now()

        ## Loop on files
        for f in self.get_product_files():
            ts = datetime.now()
            print("####################################################")
            print("{}/{} - {}".format(f.Index, len(self.dseries), f.datetime))
            if f.path is not None:
                print(f.path.name)
                self.current = f
                tmp = self.extract_snow1()
                for v in self.config['var']:
                    data_ts[v][f.Index] = tmp[v]
                time_ts.append(f.datetime)
            else:
                print ('File not found, moving to next file, assigning NaN to extacted pixel.')
                time_ts.append(datetime(1970,1,1)) # set timestamp to 0 (=1970-01-01) if no data

            now = datetime.now()
            print('{} | iteration: {} | elapsed: {}'.format(now, now-ts, now-t0), flush=True)

        time_ts = np.array([np.datetime64(d).astype('<M8[s]') for d in time_ts])
        time_ts = time_ts.astype(np.int64)

        ## Write output data
        out_var = []
        gid0 = self.df_full['global_id'].values[0]
        gid0 = "{}{:02d}".format(1970+gid0//36, gid0%36)
        gid1 = self.df_full['global_id'].values[-1]
        gid1 = "{}{:02d}".format(1970+gid1//36, gid1%36)
        self.write_file = pathlib.Path('output_snow/test_snow_{}_{}_{}.h5'.format(gid0, gid1, self.sensor))
        for name, data in data_ts.items():
            out_var.append({'type':np.float, 'name':'vars/'+name, 'data':data})
        out_var.append({'type':np.int64, 'name':'meta/ts_dates', 'data':time_ts})
        out_var.append({'type':'S', 'name':'meta/point_names', 'data':np.array(['WORLD'])})
        out_var.append({'type':np.uint16, 'name':'meta/global_id', 'data':self.df_full['global_id'].values})
        self._write_ts_chunk(out_var)

        if 0:
            for name, data in data_ts.items():
                xnum = time_ts.view('datetime64[s]')

                plt.plot(xnum, data.ravel())
                plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
                #plt.gca().xaxis.set_major_locator(mdates.MonthLocator())
                plt.gcf().autofmt_xdate()
                plt.grid()
                plt.savefig('output_snow/test_snow_{}.png'.format(name))

        del data_ts

        return [self.write_file]


#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json as json

__author__ = 'Mx'

pinyin_analyzer={
    "index" : {
        "analysis" : {
            "tokenizer" : {
                "py" : {
                    "lowercase" : True,
                    "type" : "pinyin",
                    "keep_original" : False,
                    "keep_separate_first_letter" : False,
                    "keep_full_pinyin" : True,
                    "keep_first_letter": False,
                    "limit_first_letter_length" : 10,
                },
                "jp" : {
                    "lowercase" : True,
                    "type" : "pinyin",
                    "keep_original" : False,
                    "keep_separate_first_letter" : False,
                    "keep_full_pinyin" : False,
                    "keep_first_letter": True,
                }
            },
            "analyzer" : {
                "py" : {
                    "tokenizer" : "py"
                },
                "jp" : {
                    "tokenizer" : "jp"
                }
            }
        }
    }
}

mapping_all = {
    "analyzer": "ik_max_word",
    "search_analyzer": "ik_max_word",
    "term_vector": "yes",
    "store": "yes"
}
#"term_vector": "with_positions_offsets",
field_mapping = {
    "type": "text",
    "analyzer": "ik_max_word",
    "search_analyzer": "ik_max_word",
    "boost": 1.2,
    "fields": {
        "py": {
            "type": "text",
            "term_vector": "with_offsets",
            "analyzer": "py",
            "boost": 1.0
        },
        "jp": {
            "type": "text",
            "term_vector": "with_offsets",
            "analyzer": "jp",
            "boost": 1.0
        }
    }
}
def gen_create(idx_name):
    print("""
# Create a index with custom analyzer
curl -XPUT http://localhost:9200/%s/ -d'%s' """ % (idx_name, json.dumps(pinyin_analyzer, indent=2)))
def gen_alias(idx_name, raw_name, idx_name_pre):
    alias_update = {
        "actions": [
            { "remove": { "index": idx_name_pre, "alias": raw_name}},
            { "add": { "index":idx_name, "alias": raw_name}}
        ]
    }
    print("""
# create a alias
curl -XPUT http://localhost:9200/%s/_alias/%s
# update alias
curl -XPOST http://localhost:9200/_aliases -d'%s'
curl "http://localhost:9200/*/_alias/%s"
""" % (idx_name, raw_name, json.dumps(alias_update, indent=2), raw_name))
def gen_delete(idx_name):
    print("""
# Delete a index
curl -XDELETE http://localhost:9200/%s""" % (idx_name))
def gen_mapping(idx_name, type_name, properties):
    body = {type_name : {
            "_all": mapping_all,
            "properties": properties,
    	}
    }
    print("""
# Create mapping
curl -XPOST http://localhost:9200/%s/%s/_mapping -d'%s'""" %(idx_name, type_name, json.dumps(body, indent=2)))
iver = 5
pre_ver = "v" + str(iver-1)
ver = "v" + str(iver)
def gen_nfsearch():
    raw_name = "nfsearch"
    idx_name = raw_name + "_" + ver
    idx_name_pre = raw_name + "_" + pre_ver
    print("################# nfsearch #################")
    gen_delete(idx_name)
    gen_create(idx_name)
    props = {
        "title": field_mapping
    }
    gen_mapping(idx_name, "mediainfo", props)
    props = {
        "username": field_mapping
    }
    gen_mapping(idx_name, "userinfo", props)
    gen_alias(idx_name, raw_name, idx_name_pre)
    print("\n")
def gen_filminfo():
    raw_name = "filminfo"
    idx_name = raw_name + "_" + ver
    idx_name_pre = raw_name + "_" + pre_ver
    print("################# filminfo #################")
    gen_delete(idx_name)
    gen_create(idx_name)
    props = {
        "name": field_mapping,
        "alias": field_mapping,
        "director": field_mapping,
        "actor": field_mapping,
    }
    gen_mapping(idx_name, "filminfo", props)
    gen_alias(idx_name, raw_name, idx_name_pre)
    print("\n")
def gen_all():
    gen_nfsearch()
    gen_filminfo()
    print("""
# cat all indexes
curl 'http://localhost:9200/_cat/indices?v'
""")
gen_all()


# -*- coding: utf-8 -*-

import json

from pyload.core.utils import parse

from ..base.multi_downloader import MultiDownloader

class LeechThreeHundreedSixtyCom(MultiDownloader):
    __name__ = "LeechThreeHundreedSixtyCom"
    __type__ = "downloader"
    __version__ = "0.01"
    __status__ = "testing"

    __pattern__ = r"^unmatchable$"
    __config__ = [
        ("enabled", "bool", "Activated", True),
        ("use_premium", "bool", "Use premium account if available", True),
        ("fallback", "bool", "Fallback to free download if premium fails", False),
        ("chk_filesize", "bool", "Check file size", True),
        ("max_wait", "int", "Reconnect if waiting time is greater than minutes", 10),
        ("revert_failed", "bool", "Revert to standard download if fails", True),
    ]

    __description__ = """Leech360.com multi-downloader plugin"""
    __license__ = "GPLv3"
    __authors__ = [("GammaC0de", "nitzo2001[AT]yahoo[DOT]com")]

    def handle_premium(self, pyfile):
        json_data = self.load(
            "https://leech360.com/generate",
            get={"token": self.account.info["data"]["token"], "link": pyfile.url},
        )
        api_data = json.loads(json_data)

        if api_data["error"]:
            self.fail(api_data["error_message"])

        pyfile.name = api_data.get("filename", "") or pyfile.name
        pyfile.size = parse.bytesize(api_data.get("message", "0"))
        self.link = api_data["download_url"]


# -*- coding: utf-8 -*-

###############################################################################
###############################################################################
##                                                                           ##
##      ___  _   _   __   ____  ____   __   _   _ _____                      ##
##     / _ \| | | | /  \ |  _ \| __ \ /  \ | \ | |_   _|                     ##
##    ( |_| ) |_| |/ /\ \| |_| | -/ // /\ \|  \| | | |                       ##
##     \_  /|_____| /--\ |____/|_|\_\ /--\ |_\___| |_|                       ##
##       \/                                               v 0.0              ##
##                                                                           ##
##    Optimal Pointing Sequences in Spacecraft Formation Flying              ##
##    using Online Planning with Resource Constraints.                       ##
##                                                                           ##
##    Plotting code for single scenario results. Demonstration scenario      ##
##    results were obtained by running policy scripts for trials = 1.        ##
##                                                                           ##
##    Written by Samuel Y. W. Low.                                           ##
##    Advised by Professor Mykel J. Kochenderfer                             ##
##    First created 23-Nov-2021. Last modified 04-Apr-2022.                  ##
##                                                                           ##
###############################################################################
###############################################################################

# Import global libraries.
import numpy as np
import matplotlib.pyplot as plt
plt.close('all')

# Recorded results for random search in deterministic scenario.
p1_nodes = ['9', '3', 'S', '6', '8', '7', '5', '2',  'S',  '4'  ]
p1_times = [1938,4844,6154,7522,7818,8090,8384,10388,11565,11841]

# Recorded results for greedy (angle) search in deterministic scenario.
p2_nodes = ['5', 'S', '3', '9', 'S', '6', 'S', '7', '2',  'S',  '4',  '8'  ]
p2_times = [2214,3010,3279,5144,5894,7522,8178,8458,10388,11161,11438,11560]

# Recorded results for greedy (angle rate) search in deterministic scenario.
p3_nodes = ['7','9','8','6', '5', '3', 'S', '4','2' ]
p3_times = [294,591,882,1187,1580,2219,2986,4554,4848]

# Recorded results for one step lookahead in deterministic scenario.
p4_nodes = ['7','6','4','2', '8', '9', 'S', 'S', '3', '5' ]
p4_times = [294,585,840,1142,1624,2038,2786,2788,3060,4847]

# Recorded results for forward search (depth 2) in deterministic scenario.
p5_nodes = ['7','6','4','2', '8', 'S', '3', '5', 'S', '9' ]
p5_times = [294,585,840,1142,1624,2289,2565,4847,5689,5961]

# Recorded results for forward search (depth 3) in deterministic scenario.
p6_nodes = ['7','6','4','2', '8', 'S', '3', '5', 'S', '9' ]
p6_times = [294,585,840,1142,1624,2289,2565,4847,5689,5961]

# Put them all in a nested list.
all_nodes = [p1_nodes, p2_nodes, p3_nodes, p4_nodes, p5_nodes, p6_nodes]
all_times = [p1_times, p2_times, p3_times, p4_times, p5_times, p6_times]

# Relative run times for each script for 1 trial.
runtimes = np.array([8.4, 8.4, 8.4, 24, 97, 870])
print('Relative run-times were:', runtimes/min(runtimes))

# Create the figure subplot object.
plt.rcdefaults()
fig, ax = plt.subplots()
ax.xaxis.grid(True,linestyle='dashed')
ax.set_axisbelow(True)
cmap = plt.cm.viridis
policies_names = ('Random', 'Greedy (Angle)', 'Greedy (Rates)',
                  'Lookahead', 'Fwd Search (d=2)', 'Fwd Search (d=3)')
policies_number = np.arange( len(policies_names) )
for i in range(len(all_nodes)):
    nodes, times = all_nodes[i], all_times[i]
    plt.gca().set_prop_cycle( None )
    for j in range(len(nodes)+1):
        c = cmap( j / np.max(len(nodes)+1))
        ax.barh( i, width=times[-1*j], height=0.4, color=c )
        ax.scatter( times[-1*j], i, c='white' )
        if nodes[-1*j] != 'S':
            ax.annotate( nodes[-1*j], (times[-1*j], i+0.45))
        else:
            ax.annotate( nodes[-1*j], (times[-1*j], i+0.45), color='red')

ax.set_yticks( policies_number )
ax.set_yticklabels( policies_names )
ax.invert_yaxis()  # labels read top-to-bottom
ax.set_xlabel('Pointing mission execution time (seconds)')
ax.set_title('Pointing execution time (s) for each policy in demo scenario.')
plt.show()

# # Example data
# people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim')
# y_pos = np.arange(len(people))
# performance = 3 + 10 * np.random.rand(len(people))
# error = np.random.rand(len(people))

# ax.barh(y_pos, performance, xerr=error, align='center')
# ax.set_yticks(y_pos)
# ax.set_yticklabels(people)


import numpy as np
from scipy import signal

def filterButterBP(y, fs, division, rangef):
    """
    Pass-band Butterworth filter, 8th order, central frequencies given by "division"
    y = array to be filtered
    fs = sample rate
    division = 1 for 1/3 octave filtering and = 0 for 1/1 octave filtering
    """
    G = 2.0
    n = 3.0
    nyquistRate = fs / 2

    if division == 1:
        centerFrequency_Hz = np.array([25, 31.5, 40, 50, 63, 80, 100, 125, 160, 200, 250, 315,
                                       400, 500, 630, 800, 1000, 1250, 1600, 2000, 2500, 3150,
                                       4000, 5000, 6300, 8000, 10000, 12500, 16000, 20000])
        if rangef:
            centerFrequency_Hz = centerFrequency_Hz[rangef[0]:rangef[1] + 1]

        # Superior and inferior limits of each central band
        factor = np.power(G, 1 / (2 * n))
        lowerCutoffFrequency_Hz = centerFrequency_Hz / factor
        upperCutoffFrequency_Hz = centerFrequency_Hz * factor
        if np.where(centerFrequency_Hz == 20000)[0]:
            upperCutoffFrequency_Hz[np.where(centerFrequency_Hz == 20000)[0]] = 22049
        filteredSignal = np.zeros((len(centerFrequency_Hz), len(y)))
    else:
        centerFrequency_Hz = np.array([31.5, 63, 125, 250, 500, 1000, 2000, 4000, 8000, 16000])
        if rangef:
            centerFrequency_Hz = centerFrequency_Hz[rangef[0]:rangef[1] + 1]
        # Superior and inferior limits of each central band
        factor = np.sqrt(2)
        lowerCutoffFrequency_Hz = centerFrequency_Hz / factor
        upperCutoffFrequency_Hz = centerFrequency_Hz * factor
        if np.where(centerFrequency_Hz == 16000)[0]:
            upperCutoffFrequency_Hz[np.where(centerFrequency_Hz == 16000)[0]] = 22049
        filteredSignal = np.zeros((len(centerFrequency_Hz), len(y)))

    # Creation and application of the filter
    for lower, upper in zip(lowerCutoffFrequency_Hz, upperCutoffFrequency_Hz):  # IIR Butterworth Filter N-th order
        sos = signal.butter(N=8, Wn=np.array([lower,
                                              upper]) / nyquistRate, btype='bandpass', analog=False,
                            output='sos')

        # w, h = signal.sosfreqz(sos)  # Frequency response of filter

        #        plt.plot(w*(fs/(2*np.pi)), 20 * np.log10(abs(h)), 'b')          # Plot freq response

        index = np.where(lowerCutoffFrequency_Hz == lower)  # Finds index of iteration to store signal
        index = index[0]

        filteredSignal[index, :] = signal.sosfiltfilt(sos, y)  # Filters the input "y" signal

    return filteredSignal, centerFrequency_Hz