"""
define classes to describe observed data
"""

__author__ = 'ikibalin'
__version__ = "2019_04_16"
import os
import numpy

class ObservedDataSingle(dict):
    """
    Containt the experimental data
    """
    def __init__(self, h=None, k=None, l=None, flip_ratio=None,
                 sflip_ratio=None, wave_length=None, field=None,
                 orientation=None, file_dir=None, file_name=None):
        super(ObservedDataSingle, self).__init__()
        self._p_h = None
        self._p_k = None
        self._p_l = None
        self._p_flip_ratio = None
        self._p_sflip_ratio = None

        self._p_file_dir = None
        self._p_file_name = None

        self._p_wave_length = None
        self._p_field = None
        self._p_orientation = None

        self._refresh(h, k, l, flip_ratio, sflip_ratio, wave_length, field,
                 orientation, file_dir, file_name)

    def __repr__(self):
        ls_out = """ObservedDataSingle:\n file_dir: {:}
 file_name: {:}""".format(self._p_file_dir, self._p_file_name)
        if self._p_h is not None:
            ls_out += "\n h range: {:} --- {:}".format(
                    self._p_h.min(), self._p_h.max())
        if self._p_k is not None:
            ls_out += "\n k range: {:} --- {:}".format(
                    self._p_k.min(), self._p_k.max())
        if self._p_l is not None:
            ls_out += "\n l range: {:} --- {:}".format(
                    self._p_l.min(), self._p_l.max())
            ls_out += "\n number of reflections: {:}".format(
                    self._p_l.size)
        ls_out += "\n field: {:}".format(self._p_field)
        ls_out += "\n wave_length: {:}".format(self._p_wave_length)
        ls_out += "\n orientation: {:}".format(self._p_orientation)
        return ls_out

    def _refresh(self, h, k, l, flip_ratio, sflip_ratio, wave_length, field,
                 orientation, file_dir, file_name):
        if h is not None:
            self._p_h = h
        if k is not None:
            self._p_k = k
        if l is not None:
            self._p_l = l
        if flip_ratio is not None:
            self._p_flip_ratio = flip_ratio
        if sflip_ratio is not None:
            self._p_sflip_ratio = sflip_ratio
        if wave_length is not None:
            self._p_wave_length = wave_length
        if field is not None:
            self._p_field = field
        if orientation is not None:
            self._p_orientation = orientation
        if file_dir is not None:
            self._p_file_dir = file_dir
        if file_name is not None:
            self._p_file_name = file_name

    def set_val(self,  h=None, k=None, l=None, flip_ratio=None,
                 sflip_ratio=None, wave_length=None, field=None,
                 orientation=None, file_dir=None, file_name=None):
        self._refresh(h, k, l, flip_ratio, sflip_ratio, wave_length, field,
                 orientation, file_dir, file_name)

    def get_val(self, label):
        lab = "_p_"+label

        if lab in self.__dict__.keys():
            val = self.__dict__[lab]
            if isinstance(val, type(None)):
                self.set_val()
                val = self.__dict__[lab]
        else:
            print("The value '{:}' is not found".format(lab))
            val = None
        return val

    def list_vals(self):
        """
        give a list of parameters with small descripition
        """
        lsout = """
Parameters:
 h, k, l are the Miller indices
 flip_ratio is the flipping ratio
 sflip_ratio is the errorbar of flipping ratio
 wave_length is the neutron wave length in the Angtrems
 field is the magnetic field along z axis
 orientation is the tranbsformation matrix from local coordinate system to global (matrix U)
 file_dir is directory of file with measured data
 file_name is basename of file with measured data

        """
        print(lsout)

    def read_data(self):
        """
        read file from file
        """
        finp = os.path.join(self._p_file_dir, self._p_file_name)
        ddata = {}
        if not(os.path.isfile(finp)):
            return
        fid = open(finp,'r')
        lcontentH = fid.readlines()
        fid.close()
        lparam = [line[1:].strip() for line in lcontentH if line.startswith('#')]
        if (len(lparam) > 1):
            for line in lparam[:-1]:
                lhelp = line.strip().split()
                if (len(lhelp) > 2):
                    ddata[lhelp[0]] = [float(hh) for hh in lhelp[1:]]
                elif (len(lhelp) == 2):
                    ddata[lhelp[0]] = float(lhelp[1])
                else:
                    print("Mistake in experimental file '{:}' in line:\n {:}".format(finp, line))
                    print("The program is stopped.")
                    quit()
        lnames = lparam[-1].split()
        for name in lnames:
            ddata[name] = []
        lcontent = [line for line in lcontentH if line[0]!='#']
        for line in lcontent:
            for name, val in zip(lnames, line.strip().split()):
                ddata[name].append(val)

        field = ddata["field"]
        wave_length = ddata["wave_length"]
        l_o = ddata["orientation"]
        orientation = numpy.array([[l_o[0], l_o[1], l_o[2]],
                                   [l_o[3], l_o[4], l_o[5]],
                                   [l_o[6], l_o[7], l_o[8]]], dtype=float)

        h = numpy.array(ddata["h"], dtype=int)
        k = numpy.array(ddata["k"], dtype=int)
        l = numpy.array(ddata["l"], dtype=int)
        flip_ratio = numpy.array(ddata["FR"], dtype=float)
        sflip_ratio = numpy.array(ddata["sFR"], dtype=float)
        self.set_val(h=h, k=k, l=l, flip_ratio=flip_ratio,
                 sflip_ratio=sflip_ratio, wave_length=wave_length, field=field,
                 orientation=orientation)

    def create_input_file(self, f_inp=None):
        if f_inp is not None:
            f_dir= os.path.dirname(f_inp)
            f_name= os.path.bathename(f_inp)
            self.set_val(file_dir=f_dir, file_name=f_name)
        f_dir = self._p_file_dir
        f_name = self._p_file_name
        f_full = os.path.join(f_dir, f_name)

        s_out = """#wave_length 1.40
#field  1.000
#orientation 0.6468462   -0.6860854   0.3300297 0.2141139   -0.2557555   -0.9343334 0.7319312   0.6810804    -0.0183183
#   h    k    l        FR       sFR
    0    0    8   0.64545   0.01329 """

        fid = open(f_full, "w")
        fid.write(s_out)
        fid.close()

if (__name__ == "__main__"):
  pass


import serial
import socket
import sys
import os
from _thread import *
import serial.serialutil

last_dev = os.popen('ls /dev/ttyUSB* | tail -n 1').read()
last_dev=last_dev.replace("\n", "")
print(last_dev);
ser = serial.Serial(last_dev, 9600) # here you have to write your port. If you dont know how to find it just write ls -l /dev/tty.* in your terminal (i'm using mac)
#sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#server_address = ('192.168.137.105', 10005)

HOST = ''   # Symbolic name, meaning all available interfaces
PORT = 10101 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ('Socket created');
conn1=None

#print ('connecting')
#sock.connect(server_address)
#print ('connected')
#Bind socket to local host and port
try:

    s.bind((HOST, PORT))
except socket.error as msg:
    if msg.errno == 98:
        print ('Port is already in use');
    else:
        print ('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]);
        sys.exit()

print ('Socket bind complete');

#Start listening on socket
s.listen(10)
print ('Socket now listening');

#Function for handling connections. This will be used to create threads
def clientthread():
    #Sending message to connected client
    #message='Welcome to the server. Type something and hit enter\n'
    #conn.send(bytes(message.encode('utf-8'))) #send only takes string

    #infinite loop so that function do not terminate and thread do not end.
    while True:
        try:
            print("loop starts");
            global ser
            response = ser.readline()
            response=response.replace(b'\x02', b'')
            print("reading data")
            if(conn1):
                conn1.send(response)

            print (response);
        except serial.SerialException:
            #x=os.system("ls /dev/ttyUSB*")
            #print(x);
            #if(x==0):
            try:
                print("ok");
                last_dev = os.popen('ls /dev/ttyUSB* | tail -n 1').read()
                last_dev=last_dev.replace("\n", "")
                print(last_dev);
                ser = serial.Serial(last_dev, 9600)
            except:
                print("");
        except FileNotFoundError:
            print("");
        except KeyboardInterrupt:
            print("");
        except:
            print("");

    #ser.close()
    print("out of loop1");
    #came out of loop
    #conn1.close()

#now keep talking with the client
clientNo=0
start_new_thread(clientthread ,())
while 1:
   #wait to accept a connection - blocking call
    conn, addr = s.accept()
    if(conn1):
        conn1.close()

    conn1=conn
    clientNo=clientNo+1
    print ('Connected with ' + addr[0] + ':' + str(addr[1]));
    #start new thread takes 1st argument as a function name to be run, second is the tuple of arguments to the function.

s.close()


import os, sys
import tensorflow as tf
from utility import Utils
from dataset import Dataset
from collections import OrderedDict
import matplotlib.pyplot as plt
import numpy as np
from hook import SavedModelBuilderHook, MyLoggerHook

class NCP_Trainer():
    def __init__(self,
                 FLAGS,
                 message,
                 data,
                 model,
                 name):
        self.checkpoints_to_keep = FLAGS.checkpoints_to_keep
        self.keep_checkpoint_every_n_hours = FLAGS.keep_checkpoint_every_n_hours
        self.max_steps = FLAGS.n_epoch
        self.save_checkpoint_steps = self.max_steps / 10 if FLAGS.save_checkpoint_steps is None else FLAGS.save_checkpoint_steps
        self.aug = FLAGS.aug if hasattr(FLAGS, 'aug') and FLAGS.aug != "None" else None
        self.batch_size = FLAGS.batch_size
        self.name = name
        self.message = message
        self.data = data
        self.global_step = tf.train.get_or_create_global_step()
        self.model = model
        self.restore_dir = FLAGS.init_model
        self.util = Utils(prefix=self.name)

    def load(self):
        # Load Dataset
        self.train_x = tf.placeholder(tf.float32, [None, self.data.x_train.shape[1]], name='train_x')
        self.train_y = tf.placeholder(tf.float32, [None, 1], name='train_y')

        self.valid_x = tf.placeholder(tf.float32, [None, self.data.x_valid.shape[1]], name='valid_x')
        self.valid_y = tf.placeholder(tf.float32, [None, 1], name='valid_y')

        self.num_data = self.data.x_train.shape[0]
        self.num_valid_data = self.data.x_valid.shape[0]

        return self.train_x, self.train_y, self.valid_x, self.valid_y

    def build_logits(self, train_data, train_ans, valid_data, valid_ans):
        # train
        self.train_logits = self.model.inference(train_data)
        self.train_loss = self.model.loss(self.train_logits, train_ans, self.data.regression)
        opt_op = self.model.optimize(self.train_loss, self.global_step)
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        self.train_op = tf.group([opt_op] + update_ops)
        self.train_accuracy = self.model.evaluate(self.train_logits, train_ans)

        # test
        self.test_logits = self.model.inference(valid_data)
        self.test_loss = self.model.loss(self.test_logits, valid_ans, self.data.regression)
        #self.predict = self.model.predict(valid_data)
        self.test_accuracy = self.model.evaluate(self.test_logits, valid_ans)

        return

    def hook_append(self, metrics, signature_def_map=None):
        """
        hooksをまとめる関数
        """
        hooks = []
        hooks.append(tf.train.NanTensorHook(self.train_loss))
        hooks.append(MyLoggerHook(self.message, self.util.log_dir, metrics, every_n_iter=100))
        hooks.append(SavedModelBuilderHook(self.util.saved_model_path, signature_def_map))
        return hooks

    def summary(self):
        """
        tensorboardに表示するデータをまとめる関数
        """
        # tensorboard
        tf.summary.scalar('train/loss', self.train_loss)
        #tf.summary.scalar('train/accuracy', self.train_accuracy)
        tf.summary.scalar('train/Learning_rate', self.model.optimizer.lr)
        tf.summary.scalar('test/loss', self.test_loss)
        #tf.summary.scalar('test/accuracy', self.test_accuracy)
        return

    def before_train(self):
        # create saver
        saver = tf.train.Saver(
                max_to_keep=self.checkpoints_to_keep,
                keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours)

        scaffold = tf.train.Scaffold(
            saver=saver)

        tf.logging.set_verbosity(tf.logging.INFO)
        config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)

        # saved model
        signature_def_map = {
                        'predict': tf.saved_model.signature_def_utils.build_signature_def(
                            inputs={'inputs': tf.saved_model.utils.build_tensor_info(self.valid_x)},
                            outputs={'predict': tf.saved_model.utils.build_tensor_info(self.test_logits[0])},
                            method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME,)
                        }

        metrics = OrderedDict({
            "global_step": self.global_step,
            "train loss": self.train_loss,
            "train accuracy":self.train_accuracy,
            "test loss": self.test_loss,
            "test accuracy":self.test_accuracy})

        hooks = self.hook_append(metrics, signature_def_map)

        session = tf.train.MonitoredTrainingSession(
            config=config,
            checkpoint_dir=self.util.model_path,
            hooks=hooks,
            scaffold=scaffold,
            save_summaries_steps=1,
            save_checkpoint_steps=self.save_checkpoint_steps,
            summary_dir=self.util.tf_board)

        return session

    def feed_dict_train(self, odd=False):
        self.util.initial()
        self.util.write_configuration(self.message)

        inputs, corrects, valid_inputs, valid_corrects = self.load()

        # train
        train_log_p, _, _ = self.model.inference(inputs, corrects, valid_inputs, valid_corrects)
        self.train_loss = self.model.loss(train_log_p, None)
        opt_op = self.model.optimize(self.train_loss, self.global_step)
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        self.train_op = tf.group([opt_op] + update_ops)

        # test
        test_log_p, mu, sigma = self.model.test_inference(inputs, corrects, valid_inputs, valid_corrects)
        self.test_loss = self.model.loss(test_log_p, None)

        self.summary()

        # create saver
        saver = tf.train.Saver(
                max_to_keep=self.checkpoints_to_keep,
                keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours)

        scaffold = tf.train.Scaffold(
            saver=saver)

        tf.logging.set_verbosity(tf.logging.INFO)
        config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)

        hooks = []
        hooks.append(tf.train.NanTensorHook(self.train_loss))

        session = tf.train.MonitoredTrainingSession(
            config=config,
            checkpoint_dir=self.util.model_path,
            hooks=hooks,
            scaffold=scaffold,
            save_summaries_steps=1,
            save_checkpoint_steps=self.save_checkpoint_steps,
            summary_dir=self.util.tf_board)

        with session:
            if self.restore_dir is not None:
                ckpt = tf.train.get_checkpoint_state(self.restore_dir)
                if ckpt and ckpt.model_checkpoint_path:
                    # Restores from checkpoint
                    saver.restore(session, ckpt.model_checkpoint_path)
            for iter in range(self.max_steps):
                x_train, y_train = self.data.next_batch(self.batch_size)
                x_test, y_test = self.data.next_batch(self.batch_size * 10, test=True)
                _, train_loss, test_loss, test_input, test_answer, test_mu, test_sigma = session.run([self.train_op, self.train_loss, self.test_loss, valid_inputs, valid_corrects, mu, sigma], feed_dict={inputs:x_train, corrects:y_train, valid_inputs:x_test, valid_corrects:y_test})
                if iter % 100 == 0:
                    train_log = OrderedDict({
                                "global_step": iter,
                                "train loss": train_loss,
                                "test loss": test_loss})
                    self.util.write_log(message=train_log, cout=True)
                if iter % 500 == 0:
                    self.util.figure(np.ravel(test_input), test_answer, [np.ravel(test_mu), np.ravel(test_sigma)], iter)
        return


#!/usr/bin/env python
import tkinter as tk
from PIL import Image, ImageTk
import os
import pyocr
import pyocr.builders
import myougiden_api
import threading
tool = pyocr.get_available_tools()[0]

def best_fit(width, height, image):
	(x, y) = image.size
	scale = width / x
	if y * scale > height:
		scale = height / y
	#print(scale)
	return image.resize((int(x * scale), int(y * scale)), Image.BILINEAR)

class LookupThread(threading.Thread):
	def run(self):
		self.dirty = False
		image, app = (self._args)
		try:
			string = app.image_to_dict(image)
			if not self.dirty:
				app.draw_dict(string)

		finally:
			# Avoid a refcycle if the thread is running a function with
			# an argument that has a member that points to the thread.
			del self._target, self._args, self._kwargs

class Application(tk.Frame):
	def __init__(self, master=None):
		tk.Frame.__init__(self, master)
		if os.path.isfile("last_page"):
			last_page = open("last_page", "r")
			try:
				self.current_page = int(last_page.read())
			except:
				self.current_page = 0
			last_page.close()
		else:
			self.current_page = 0
		self.pack(fill=tk.BOTH, expand=1)
		self.createWidgets()
		self.current_page_oid = 0
		self.current_page_image = None
		self.drawing_box = False
		self.box_oid = 0
		self.box_coords = (0, 0, 0, 0)
		self.lookup = None
		self.tkimage = None
		self.rotation = 0
		self.fullscreen = False

	def createWidgets(self):
		#self.quitButton = tk.Button(self, text='Quit',
		#	command=self.quit)
		#self.quitButton.grid()
		self.update()
		(width, height) = (self.winfo_width(), self.winfo_height())
		self.frame = tk.Canvas(self, width=width, height=height, cursor="tcross")
		self.frame.pack(fill=tk.BOTH)
		self.frame.bind('<Left>', self.next_image)
		self.frame.bind('<Right>', self.prev_image)
		#self.frame.bind('<r>', self.rotate)
		self.frame.bind('<Configure>', self.resize_event)
		self.frame.focus_set()
		self.frame.bind('<Button-1>', self.start_drawing_box)
		self.frame.bind('<ButtonRelease-1>', self.stop_drawing_box)
		self.frame.bind('<Double-Button-1>', self.clear_box)
		self.frame.bind('<Button-2>', self.side_tap)
		self.frame.bind('<Motion>', self.draw_box)
		self.frame.bind('<F11>', self.toggle_fullscreen)

	def toggle_fullscreen(self, event=None):
		self.fullscreen = not self.fullscreen  # Just toggling the boolean
		self.master.attributes("-fullscreen", self.fullscreen)
		self.update_screen()

	def side_tap(self, event):
		if event.x < (self.frame.winfo_width() / 2):
			self.change_image(1)
		else:
			self.change_image(-1)

	def resize_event(self, event):
		self.frame.width = event.width   #>>>854
		self.frame.height = event.height #>>>404
		self.frame.config(width=self.frame.width, height=self.frame.height)
		self.update_screen()

	def update_screen(self):
		self.change_image(0)

	def rotate(self, event):
		self.rotation = (self.rotation + 1) % 4
		self.update_screen()

	def clear_box(self, event):
		self.drawing_box = False
		if self.box_oid != 0:
			self.frame.delete(self.box_oid)
		self.frame.delete("text")
		self.frame.delete("selection")
	def start_drawing_box(self, event):
		textbox = self.frame.bbox("text")
		selectionbox = self.frame.bbox(self.box_oid)
		for bbox in textbox, selectionbox:
			if bbox is not None:
				(x, y, x2, y2) = bbox
				mx = event.x
				my = event.y
				if mx > x and mx < x2 and my > y and my < y2:
					self.clear_box(None)
					return

		self.drawing_box = True
		x = event.x
		y = event.y
		self.box_coords = (x, y, x, y)
		if self.box_oid != 0:
			self.frame.delete(self.box_oid)
		self.box_oid = self.frame.create_rectangle(x, y, x, y, fill="black", stipple="gray50")
		self.frame.addtag_withtag("selection", self.box_oid)

	def stop_drawing_box(self, event):
		try:
			self.drawing_box = False
			(ix, iy, ix2, iy2) = self.frame.bbox(self.current_page_oid)
			(bx, by, bx2, by2) = self.frame.bbox(self.box_oid)
			px = (bx - ix) / (ix2 - ix)
			py = (by - iy) / (iy2 - iy)
			px2 = (bx2 - ix) / (ix2 - ix)
			py2 = (by2 - iy)/ (iy2 - iy)
			#print("%f, %f, %f, %f" % (px, py, px2, py2))

			(width, height) = self.current_page_image.size
			cx = int(px * width)
			cx2 = int(px2 * width)
			cy = int(py * height)
			cy2 = int(py2 * height)
			#print("%d, %d, %d, %d" % (cx, cy, cx2, cy2))
			ocr_image = self.current_page_image.crop((cx, cy, cx2, cy2))
			#draw = ImageDraw.Draw(self.current_page_image)
			#draw.rectangle([cx, cy, cx2, cy2], outline="black")
			#ocr_image = image
			if self.lookup is not None:
				self.lookup.dirty = True
			self.lookup = LookupThread(args=(ocr_image,self))
			self.lookup.start()
			#self.image_to_dict(ocr_image)
		except:
			pass

	def draw_box(self, event):
		if not self.drawing_box:
			return
		(x, y, x2, y2) = self.box_coords
		x2 = event.x
		y2 = event.y
		self.box_coords = (x, y, x2, y2)
		self.frame.delete(self.box_oid)
		self.box_oid = self.frame.create_rectangle(x, y, x2, y2, outline="#00AA00", fill="#00AA00", stipple="gray50")

	def change_image(self, amount):
		self.clear_box(None)
		new_page = self.current_page + amount
		if new_page < 0 or new_page > len(images) - 2:
			return
		self.current_page = new_page
		image = Image.open(images[self.current_page])
		if self.rotation != 0:
			image = image.rotate(-90 * self.rotation)
		(width, height) = (self.frame.winfo_width(), self.frame.winfo_height())
		image = best_fit(width, height, image)
		self.tkimage = ImageTk.PhotoImage(image)
		self.frame.delete(self.current_page_oid)
		self.current_page_oid = self.frame.create_image(int(width/2), 0, image=self.tkimage, anchor=tk.N)
		self.current_page_image = image
		last_page = open("last_page", "w")
		last_page.write(str(self.current_page))
		last_page.close()

	def prev_image(self, event):
		self.change_image(-1)

	def next_image(self, event):
		self.change_image(1)

	def image_to_dict(self, image):
		size = image.size
		image = image.resize((size[0] * 2, size[1] * 2), Image.BILINEAR)
		string = tool.image_to_string(image, lang="jpn", builder=pyocr.builders.TextBuilder(5))
		string = string.strip()
		if string != "":
			dict_entry = myougiden_api.run([string])
		else:
			dict_entry = None
			string = "No character recognized"
		#image.save("/tmp/export.png")
		if dict_entry is not None and string != "No character recognized":
			string = "\n".join(dict_entry)
		else:
			dict_entry = myougiden_api.run(string[:-1])
			dict_entry2 = myougiden_api.run(string[1:])
			dict_entry3 = myougiden_api.run(string[1:])
			result = ""
			result2 = ""
			result3 = ""
			if dict_entry is not None:
				result = "\n".join(dict_entry)
			if dict_entry2 is not None:
				result2 = "\n".join(dict_entry2)
			if dict_entry2 is not None:
				result3 = "\n".join(dict_entry3)
			string = result + result2 + result3
		print(string)
		return string

	def draw_dict(self, string):
		self.text = self.frame.create_text(5, 5, width=500, anchor=tk.NW, text=string)
		self.frame.addtag_withtag("text", self.text)
		(x, y, x2, y2) = self.frame.bbox(self.text)
		self.textbox = self.frame.create_rectangle(x, y, x2, y2, fill="white", outline="white")
		self.frame.addtag_withtag("text", self.textbox)
		self.frame.tag_lower(self.textbox, self.text)

directory = '/home/klaxa/Images/manga/'

images = sorted([os.path.join(directory, filename) for filename in os.listdir('/home/klaxa/Images/manga/')])
app = Application()
app.master.title('Yurimon reader')
app.update_screen()
app.mainloop()


import time
import numpy as np
import ctypes  # only for DLL-based instrument

import qcodes as qc
from qcodes import (Instrument, VisaInstrument,
                    ManualParameter, MultiParameter,
                    validators as vals)
from qcodes.instrument.channel import InstrumentChannel
from qcodes.instrument.parameter import ParameterWithSetpoints, Parameter

import SequenceGeneration as sqg
from qcodes.utils.delaykeyboardinterrupt import DelayedKeyboardInterrupt
import struct

class AcqChannel(InstrumentChannel):

    #initializing the array storing channel instances
    objs = []

    def __init__(self, parent: 'RFSoC', name: str, channel: int):

        """
        Args:
            parent: Instrument that this channel is bound to.
            name: Name to use for this channel.
            channel: channel on the card to use
        """
        if channel not in np.arange(1,9):
            raise ValueError('channel number must be in between 1 and 8')

        self._adc_channel = channel

        super().__init__(parent, name)

        AcqChannel.objs.append(self)

        self.add_parameter(name='status',
                           label = 'ADC{} status'.format(self._adc_channel),
                           set_cmd='ADC:ADC{} {}'.format(self._adc_channel,'{:d}'),
                           val_mapping={'on': 1, 'off': 0}
                           )

        #TODO : add allowed values of decimation and mixer frea
        self.add_parameter(name='decfact',
                           label='ADC{} decimation factor'.format(self._adc_channel),
                           #the decimation works by tiles of two adcs
                           set_cmd='ADC:TILE{}:DECFACTOR {}'.format((self._adc_channel-1)//2,'{:d}')
                           )

        self.add_parameter(name='fmixer',
                           label = 'ADC{} mixer frequency'.format(self._adc_channel),
                           set_cmd='ADC:ADC{}:MIXER {}'.format(self._adc_channel,'{:.4f}')
                           )

        self.add_parameter(name='mode',
                           label='ADC{} acquisition mode'.format(self._adc_channel),
                           vals = vals.Enum('RAW','SUM'),)

class RFSoC(VisaInstrument):

    # all instrument constructors should accept **kwargs and pass them on to
    # super().__init__
    def __init__(self, name, address, **kwargs):
        # supplying the terminator means you don't need to remove it from every
        # response
        super().__init__(name, address, terminator='\r\n', **kwargs)

        #Add the channel to the instrument
        for adc_num in np.arange(1,9):

            adc_name='ADC{}'.format(adc_num)
            adc=AcqChannel(self,adc_name,adc_num)
            self.add_submodule(adc_name, adc)

        self.add_parameter('nb_measure',
                            set_cmd='{}',
                            get_parser=int,
                            initial_value = int(1))

        self.add_parameter( name = 'output_format',
                            #Format(string) : 'BIN' or 'ASCII'
                            label='Output format',
                            vals = vals.Enum('ASCII','BIN'),
                            set_cmd='OUTPUT:FORMAT ' + '{}',
                            get_cmd='OUTPUT:FORMAT?',
                            #snapshot_get  = False,
                            get_parser=str )

    def reset_sequence(self):
    	'''
    		Delete all instances of the Pulse class from the sqg module
    	'''
    	for inst in sqg.Pulse.objs:
    		del inst

    def write_sequence_and_DAC_memory(self):

        self.log.info(__name__+ ' sending sequence'+'  \n')
        self.write(sqg.Pulse.generate_sequence_and_DAC_memory())

        for obj in sqg.PulseGeneration.objs:

            self.log.info(__name__+ ' sending DAC {} 2D memory'.format(obj.channel)+'  \n')
            self.write(obj._DAC_2D_memory)

    def reset_DAC_2D_memory(self, channel):
        """
            Reset the 2D memory of one DAC

            Input : - channel of the DAC we wantto reset
        """

        self.log.info(__name__+ ' reset the 2D memory of DAC channel'+ channel+'  \n')

        if channel in ['CH1','CH2','CH3','CH4','CH5','CH6','CH7','CH8']:

            self.write("DAC:DATA:{}:CLEAR".format(channel))

        else:
            raise ValueError('Wrong channel value')

    def reset_all_DAC_2D_memory(self):
        """
            Reset the 2D memory of all the DACs
        """
        for channel in ['CH1','CH2','CH3','CH4','CH5','CH6','CH7','CH8']:

            self.write("DAC:DATA:{}:CLEAR".format(channel))

    def ask_raw(self, cmd: str) -> str:
        """
        Overwriting the ask_raw qcodes native function to query binary

        Low-level interface to ``visa_handle.ask``.

        Args:
            cmd: The command to send to the instrument.

        Returns:
            str: The instrument's response.
        """
        with DelayedKeyboardInterrupt():
            self.visa_log.debug(f"Querying: {cmd}")
            response = self.visa_handle.query_binary_values(cmd, datatype="h", is_big_endian=True)
            self.visa_log.debug(f"Response: {response}")
        return response

    def reset_PLL(self):

    	self.write("DAC:RELAY:ALL 0")
    	self.write("PLLINIT")
    	time.sleep(5)
    	self.write("DAC:RELAY:ALL 1")

    def reset_output_data(self):

    	self.ask('OUTPUT:DATA?')

    def run_and_get_data(self):

        rep=self.ask("OUTPUT:DATA?")

        tstart = time.perf_counter()
        tick = 0.1
        duree = 2
        rep=[]

        # beginning of the sequence
        self.write("SEQ:START")
        time.sleep(2)
        while time.perf_counter()<(tstart+duree):

            time.sleep(tick)
            r = self.ask('OUTPUT:DATA?')
            if len(r)>1:
                rep = rep+r

        self.write("SEQ:STOP")

        # we ask the last packet and add it to the previous

        r = self.ask('OUTPUT:DATA?')
        if len(r)>1:
           rep = rep+r

        # data decoding
        # 8 I and Q channels
        adcdataI = [[],[],[],[],[],[],[],[]]
        adcdataQ = [[],[],[],[],[],[],[],[]]

        i=0
        TSMEM=0
        while (i + 8 )<= len(rep) : # at least one header left

            entete = np.array(rep[i:i+8])
            X =entete.astype('int16').tobytes()
            V = X[0]-1 # channel (1 to 8)
            DSPTYPE = X[1]
            #N does not have the same meaning depending on DSTYPE
            N = struct.unpack('I',X[2:6])[0]
            #number of acquisition points in continuous
            #depends on the point length
            NpCont = X[7]*256 + X[6]
            TS= struct.unpack('Q',X[8:16])[0]

            # print the header for each packet
            print("Channel={}; N={}; DSP_type={}; TimeStamp={}; Np_Cont={}; Delta_TimeStamp={}".format(V,N,DSPTYPE,TS,NpCont,TS-TSMEM))

            TSMEM=TS

            iStart=i+8
            # if not in continuous acq mode
            if ((DSPTYPE &  0x2)!=2):
                # raw adcdata for each Np points block
                if ((DSPTYPE  &  0x1)==0):
                        Np=N
                        adcdataI[V]=np.concatenate((adcdataI[V], rep[iStart:iStart+Np]))

                #in the accumulation mode, only 1 I and Q point even w mixer OFF
                #mixer ON or OFF
                if ((DSPTYPE  & 0x01)==0x1):
                    Np=8
                    D=np.array(rep[iStart:iStart+Np])
                    X = D.astype('int16').tobytes()

                    #I  dvided N and 2 bcse signed 63 bits aligned to the left
                    I=  struct.unpack('q',X[0:8])[0]/(N*2)
                    Q=  struct.unpack('q',X[8:16])[0]/(N*2)

                    #print the point
                    print("I/Q:",I,Q,"Amplitude:",np.sqrt(I*I+Q*Q),"Phase:",180*np.arctan2(I,Q)/np.pi)

                    adcdataI[V]=np.append(adcdataI[V], I)
                    adcdataQ[V]=np.append(adcdataQ[V], Q)

            # continuoous acquisition mode with accumulation (reduce the flow of data)
            elif ((DSPTYPE &  0x3)==0x3):
                # mixer OFF : onlyI @2Gs/s or 250Ms/s
                if ((DSPTYPE  & 0x20)==0x0):
                    # points are already averaged in the PS part
                    # format : 16int
                    Np = NpCont
                    adcdataI[V]=np.concatenate((adcdataI[V], rep[iStart:iStart+Np]))

                # mixer ON : I and Q present
                elif ((DSPTYPE  & 0x20)==0x20):
                    Np = NpCont
                    adcdataI[V]=np.concatenate((adcdataI[V], rep[iStart:Np:2]))
                    adcdataQ[V]=np.concatenate((adcdataQ[V], rep[iStart+1:Np:2]))

            i = iStart+Np # index of the new data block, new header

        return adcdataI,adcdataQ


# -*- coding: utf-8 -*-
import scrapy
from vulinfo.items import VulinfoItem

class VulspiderSpider(scrapy.Spider):
    name = 'vulspider'
    allowed_domains = ['www.umei.cc']
    start_urls = ['https://www.umei.cc/meinvtupian/xingganmeinv/1.htm']

    def parse(self, response):

        #本页地址列表
        localurls=response.xpath('//a[@class="TypeBigPics"]/@href').getall()
        for localurl in localurls:

            yield scrapy.Request(url=localurl, callback=self.parse)
        #下一页地址
        localnextpage=response.xpath('//div[@class="NewPages"]/ul/li/a/@href').getall()[-2]
        yield scrapy.Request(url=response.urljoin(localnextpage),callback=self.parse)

        #各个标题的item和url
        nextpage=response.xpath('//div[@class="NewPages"]/ul/li/a/@href').getall().pop()
        if nextpage =='#':
            pass
        else:

            yield scrapy.Request(url=response.urljoin(nextpage),callback=self.parse)

        item=VulinfoItem()
        item['image_urls']=response.xpath('//p[@align="center"]/a/img/@src').getall()
        item['images']=response.xpath('//p[@align="center"]/a/img/@alt').getall()
        item['image_paths']=response.xpath('//p[@align="center"]/a/img/@alt').get()

        yield item


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:

    def tree2str(self, root: Optional[TreeNode]) -> str:
        self.res = ''
        self.preorder_traversal(root)
        return self.res

    def preorder_traversal(self, node):
        if node is None: return

        self.res+=str(node.val)

        if(node.left or node.right):
            self.res+='('
            self.preorder_traversal(node.left)
            self.res+=')'

        if (node.right):
            self.res+='('
            self.preorder_traversal(node.right)
            self.res+=')'

LeetCode/606. Construct String from Binary Tree.py


"""
Run the full 13scene pipeline:  corpus -> sam topic weights -> classification results

We use the following path conventions:
Model filename:  <corpus>--<topics>T.model
Topic weights filename:  <corpus>--<topics>T.arff
Weka results:  <corpus>--<topics>T--<classifier>.results
"""

from sam.vem.cli import run_sam_batch
from sam.weka.cross_validator_cli import run_cv_batch
from sam.experiment_utils import *

# SAM
vem_config = {
    'model':[get_model_filename],
    'corpus':['13scene-gist.h5', '13scene-color-gist.h5'],
    'T':[10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
    'iterations':[100],
    'write_topic_weights':[get_topic_weight_filename],
    'condor':[''],
    }
vem_configs = enumerate_configs(vem_config)

# K-NN
knn_configs = {
    'classifier':['weka.classifiers.lazy.IBk'],
    'flags':['-K 5', '-K 10', '-K 15'],
    'data':[each['write_topic_weights'] for each in vem_configs],
    'results':[get_cv_results_filename],
    'condor':['']
}
# Logistic regression
lr_configs = {
    'classifier':['weka.classifiers.functions.SimpleLogistic'],
    'data':[each['write_topic_weights'] for each in vem_configs],
    'results':[get_cv_results_filename],
    'condor':['']
}
classifier_configs = enumerate_configs(knn_configs) + enumerate_configs(lr_configs)

if __name__ == '__main__':
    run_sam_batch(vem_configs)
    run_cv_batch(classifier_configs)


# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
# import frappe
# from _future_ import unicode_literals
import frappe
import frappe.utils
import json
import calendar
from frappe import _
from frappe.utils import getdate
from datetime import date

def get_context(context):
	EventsList=get_events('GSOD Events')
	GSSEventsList=get_events('GSS Events')
	GSYEventsList=get_events('GSY Events')
	context.GSYEventsList=GSYEventsList
	context.GSSEventsList=GSSEventsList
	context.EventsList=EventsList
	context.Advetisment=data.AdsMaxImpressionCount
	RightWidget=get_widgets('Sponsors','Right Panel','List View')
	context.RightWidget = RightWidget

def get_widgets(category,position,view):
	Widget=frappe.db.get_all('Widget Placeholder', fields=['title','category','view','name','position'],filters={'view':view,'category':category,'position':position})
	for item in Widget:
		Widgets=frappe.db.get_all('Widget Config', fields=['widget_name','widget_type','html_content','max_data','link_data','sort_order'],order_by='sort_order',filters={'parent':item.name})
		for data in Widgets:
			Content=[];
			if data.widget_type=='Dynamic':
				if data.link_data=='Sponsors':
					RelatedData=frappe.db.get_all(data.link_data,fields=['sponsor_name','company_name','logo'],order_by='name desc',limit_page_length=data.max_data)
					Content.append(RelatedData)

				else:
					RelatedData=frappe.db.get_all(data.link_data,fields=['title','route'])
					Content.append(RelatedData)
			data.Content=Content

		item.Widgets=Widgets
	return Widget

@frappe.whitelist()
def get_events(event_type):
	EventsList=frappe.db.get_all('Events'
		,fields=['name','published','route','location','start_date','end_date','banner_image','venue_type','event_venue','address_line_2','city','state','zip_code']
		, filters={'published':1,'event_type':event_type}
		, order_by='start_date desc', limit_page_length=100)
	for item in EventsList:
		item.dates=getdate(item.start_date).strftime("%d")
		if item.banner_image:
			item.image= frappe.db.get_all('File',fields=['thumb_file_1','thumb_file_2','file_url'],filters={'file_url':item.banner_image,'attached_to_doctype':'Events'})
		item.months = frappe.utils.formatdate(item.start_date, "MMM YYYY")
		item.days = calendar.day_name[item.start_date.weekday()]
		if item.venue_type=='Event Hall':
			if item.event_venue:
				venue_detail=frappe.db.get_all('Event Halls'
						,fields=['name','address_line_1','address_line_2','city','state','zip_code'],
						filters={'name':item.event_venue})[0]
				item.location=venue_detail.address_line_1
				item.address_line_2=venue_detail.address_line_2
				item.city=venue_detail.city
				item.state=venue_detail.state
				item.zip_code=venue_detail.zip_code
	return EventsList

@frappe.whitelist()
def get_ads():
	Advetisment = frappe.db.get_all('Advertisement',fields=['name'],filters={'name':'Advertisement 2'})
	return Advetisment


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import torch

from models.DSFD_vgg import build_net_vgg
from models.DSFD_resnet import build_net_resnet

def build_net(phase = 'train', num_classes = 2, model = 'vgg'):
    if phase not in ['train', 'test']:
        print("ERROR: Phase: " + phase + " not recognized")
        return

    if model not in ['vgg', 'resnet']:
        print("ERROR: model:" + model + " not recognized")
        return

    if model == 'vgg':
        return build_net_vgg(phase, num_classes)
    else:
        return build_net_resnet(phase, num_classes, model)

def basenet_factory(model='vgg'):
	if model=='vgg':
		basenet = 'vgg16_reducedfc.pth'

	elif 'resnet' in model:
		basenet = '{}.pth'.format(model)
	return basenet


from sqlalchemy import BigInteger, Column, ForeignKey
from sqlalchemy.orm import backref, relationship
from sqlalchemy_api_handler import ApiHandler

from utils.database import db

class ContentTag(ApiHandler,
                 db.Model):

    contentId = Column(BigInteger(),
                       ForeignKey('content.id'),
                       primary_key=True)

    content = relationship('Content',
                         foreign_keys=[contentId],
                         backref=backref("contentTags"))

    tagId = Column(BigInteger(),
                   ForeignKey('tag.id'),
                   primary_key=True)

    tag = relationship('Tag',
                       foreign_keys=[tagId],
                       backref=backref("contentTags"))


import boto3
import urllib3
import json
import os

urllib3.disable_warnings()

# Declaring variables
http                        =  urllib3.PoolManager()
my_headers                  = {'User-Agent': 'Baker371'}
git_token                   = os.getenv('GIT_TOKEN')
git_headers                 = {'Authorization': f'token {git_token}', 'User-Agent': 'Baker371'}
repo_url                    = 'https://api.github.com/users/k8-proxy/repos'
bucket                      = 'wmwaredata'
fileName                    = 'releases.json'
s3                          = boto3.client('s3')

def lambda_handler(event, context):
    # Listing all the repos
    repos                   = []
    resp                    = http.request('GET',repo_url,headers=my_headers)
    resm                    = resp.data
    resn                    = json.loads(resm.decode('utf8'))
    for element in resn:
      ids                   = element['id']
      repos.append(ids)

    # Getting release data from all repos
    gitdata                 =   []
    for repo in repos:
      my_url                =  f'https://api.github.com/repositories/{repo}/releases'
      res                   =  http.request('GET',my_url,headers=git_headers)
      myres                 =  json.loads(res.data)
      gitdata.append(myres)
      uploads   = bytes(json.dumps(gitdata, indent=4, sort_keys=True, default=str).encode('UTF-8'))

      # Uploading JSON file to s3 bucket
      s3.put_object(Bucket=bucket, Key=fileName, Body=uploads)

    message = {
      'message': 'JSON file succesfully created and uploaded to S3'
       }

    return {
       'statusCode': 200,
       'headers': {'Content-Type': 'application/json'},
       'body': json.dumps(message)
       }


import yaml
import os

phase = 'development'

root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), './'))

bigquery_config = yaml.load(
    open(
        os.path.join(root_path, 'config', 'bigquery.yaml'),
        'r'
    )
)

__all__ = ['phase', 'root_path', 'bigquery_config']

# CARDINAL_NUM
CARDINAL_NUM = 4

if CARDINAL_NUM == 3:
    cardinal = "3rd"
else:
    cardinal = str(CARDINAL_NUM) + "th"


# -*- coding: utf-8 -*-
import pygame
import numpy as np

import Speler # eigen class
import schotje

#fucnties
def drawall():
    gameDisplay.fill(kleur_lijnen)
    # grid
    blokgrootte = spel_config['blokgrootte']
    lijndikte = spel_config['lijndikte']
    dx = spel_config['dx']
    dy = spel_config['dy']
    for y in range(0,9):
        for x in range(0,9):
            rect = pygame.Rect(dx+x*(blokgrootte+lijndikte), dy+y*(blokgrootte+lijndikte), blokgrootte, blokgrootte)
            pygame.draw.rect(gameDisplay, kleur_achtergrond, rect)
    # pionnen
    Speler1.drawme(gameDisplay)
    select1.drawme(gameDisplay)
    Speler2.drawme(gameDisplay)
    select2.drawme(gameDisplay)
    schotje_select.drawme(gameDisplay)
    for i in range(len(schotjes)):
        (schotjes[i]).drawme(gameDisplay)

def checkmove_schotje():
    # mag niet over ander schotje
    # mag niet route afsluiten
    possiblemove = 1
    return possiblemove

def checkmove_speler(speler_id):
    zelfdeplek = 0
    if speler_id == 1:
        speler      = Speler1
        selector    = select1
        tegenspeler = Speler2
    else:
        speler      = Speler2
        selector    = select2
        tegenspeler = Speler1

    possiblemove = 0
    if not (selector.x == speler.x and selector.y == speler.y):
        if not (selector.x == tegenspeler.x and selector.y == tegenspeler.y):
            if abs(speler.x - selector.x) == 1:
                if abs(speler.y - selector.y) == 0:
                    possiblemove = 1
                elif abs(speler.y - selector.y) == 1:
                    # mag alleen als er naast de tegenspeler een verticaal schotje staat
                    possiblemove = 1 # AANPASSEN!
            elif abs(speler.x - selector.x) == 2:
                if speler.x + 0.5*(selector.x-speler.x) == tegenspeler.x and abs(selector.y-speler.y)==0:
                    possiblemove = 1
            if abs(speler.y - selector.y) == 1:
                if abs(speler.x - selector.x) == 0:
                    possiblemove = 1
                elif abs(speler.x - selector.x) == 1:
                    # mag alleen als er onder/boven de tegenspeler een horizontaal schotje staat
                    possiblemove = 1 # AANPASSEN!
            elif abs(speler.y - selector.y) == 2:
                if speler.y + 0.5*(selector.y-speler.y) == tegenspeler.y and abs(selector.x-speler.x)==0:
                    possiblemove = 1
    else:
        zelfdeplek = 1
        possiblemove = 1
    return possiblemove, zelfdeplek

# spel configuratie
kleur_achtergrond   = (200,200,200)
kleur_lijnen        = (0,0,0)
kleur_speler1       = (0,100,0)
kleur_speler2       = (100,1,100)
kleur_select        = (255,255,0)
kleur_schotje       = (255,0,0)
spel_config = {'blokgrootte': 50, 'lijndikte':10, 'dx': 130, 'dy':40}
Speler1 = Speler.Speler(5,1,spel_config,kleur_speler1,10)
select1 = Speler.Speler(Speler1.x,Speler1.y,spel_config,kleur_select,0)
Speler2 = Speler.Speler(5,9,spel_config,kleur_speler2,10)
select2 = Speler.Speler(Speler2.x,Speler2.y,spel_config,kleur_select,0)
schotje_select = schotje.schotje(1,1,'h',spel_config,kleur_lijnen)
schotjes =[]

#MAIN
# pygame settings:
pygame.init()
myfont = pygame.font.SysFont("monospace", 15)
Display_width   = 800
Display_height  = 600
gameDisplay     = pygame.display.set_mode((Display_width,Display_height))
pygame.display.set_caption('Quoridor')
clock           = pygame.time.Clock()
#$ MAIN LOOP
gamemode = 1
keydownevents = 0
keyup = 1
done = False
while not done:
    for event in pygame.event.get():
            if event.type == pygame.QUIT:
                    done = True
    if gamemode == 1:
        instructie = myfont.render("Speler 1, verzet (of niet) en sluit af met -spatie-!", False, (255,255,0))
        select1 = Speler.Speler(Speler1.x,Speler1.y,spel_config,kleur_select,5)
        gamemode = 2

    elif gamemode == 2:
        finish,keyup = select1.move(event,keyup)
        if finish == 1:
            possiblemove,zelfdeplek = checkmove_speler(1)
            if possiblemove:
                #verplaats
                Speler1 = Speler.Speler(select1.x,select1.y,spel_config,kleur_speler1,10)
                select1 = Speler.Speler(Speler1.x,Speler1.y,spel_config,kleur_select,0)
                if zelfdeplek == 1:
                    gamemode = 3
                else:
                    gamemode = 4
            else:
                finish = 0
                instructie = myfont.render("Speler 1: zet niet mogelijk", False, (255,255,0))

    elif gamemode == 3: #speler1: schotje plaatsen
        instructie = myfont.render("Speler 1: verplaats schotje. Druk x om te draaien", False, (255,255,0))
#        schotjes.append(schotje.schotje(1,1,'h',spel_config))
        #verplaats...
        schotje_select.kleur = kleur_schotje
        finish,keyup = schotje_select.move(event,keyup)
        if finish == 1:
            possiblemove = checkmove_schotje()
            if possiblemove == 1:
                schotjes.append(schotje.schotje(schotje_select.x,schotje_select.y,schotje_select.orientatie,spel_config,kleur_schotje))
                gamemode = 4
            else:
                finish = 0

    elif gamemode == 4:
        instructie = myfont.render("Speler 2, verzet (of niet) en sluit af met -spatie-!", False, (255,255,0))
        select2 = Speler.Speler(Speler2.x,Speler2.y,spel_config,kleur_select,5)
        gamemode = 5

    elif gamemode == 5:
        finish,keyup = select2.move(event,keyup)
        if finish == 1:
            possiblemove, zelfdeplek = checkmove_speler(2)
            if possiblemove:
                Speler2 = Speler.Speler(select2.x,select2.y,spel_config,kleur_speler2,10)
                select2 = Speler.Speler(Speler2.x,Speler2.y,spel_config,kleur_select,0)
                gamemode = 1
            else:
                finish = 0
                instructie = myfont.render("Speler 2: zet niet mogelijk", False, (255,255,0))

    drawall()
    gameDisplay.blit(instructie, (10, 10))
    pygame.display.flip()
    clock.tick(60)
pygame.quit()


#!/usr/bin/env python
import os
import sys
import numpy as np
from collections import OrderedDict
sys.path.append(os.pardir)
import common.layers as clay
import common.gradient as cgrad

class MultiLayerNet:
    def __init__(self, size_in, list_size_hidden, size_out,
                 lambda_l2=0., ratio_dropout=None):
        if isinstance(list_size_hidden, int):
            list_size_hidden = [list_size_hidden]
        self.size_units = [size_in] + list_size_hidden + [size_out]
        self.lambda_l2 = lambda_l2

        self.params = {}
        self.layers = OrderedDict()
        for i in range(len(self.size_units) - 1):
            affine, weight, bias, relu, dropout =\
                [k + str(i + 1) for k in ('Affine', 'W', 'b', 'Relu', 'Dropout')]
            scale = np.sqrt(2.0 / self.size_units[i])
            self.params[weight] =\
                scale * np.random.randn(self.size_units[i], self.size_units[i+1])
            self.params[bias] = np.zeros(self.size_units[i+1])

            self.layers[affine] = clay.Affine(self.params[weight], self.params[bias])
            if ratio_dropout is not None:
                self.layers[dropout] = clay.Dropout(ratio_dropout)
            if i < len(self.size_units) - 2:
                self.layers[relu] = clay.Relu()
            else:
                self.lastLayer = clay.SoftmaxWithLoss()

    def predict(self, x, flg_train=True):
        for k, layer in self.layers.items():
            if k.startswith('Dropout'):
                x = layer.forward(x, flg_train)
            else:
                x = layer.forward(x)
        return x

    def loss(self, x, t):
        y = self.predict(x)
        w = 0.
        for i in range(len(self.size_units) - 1):
            w += np.sum(self.params['W%d' % (i + 1)]**2)
        return self.lastLayer.forward(y, t) + 0.5 * self.lambda_l2 * w

    def accuracy(self, x, t):
        y = self.predict(x)
        pred = y.argmax(axis=1)
        if t.ndim != 1:
            t = t.argmax(axis=1)
        return np.sum(pred == t) / float(pred.size)

    def gradient(self, x, t):
        # forward
        self.loss(x, t)

        # backward
        dout = 1
        dout = self.lastLayer.backward(dout)
        for layer in reversed(self.layers.values()):
            dout = layer.backward(dout)

        grads = {}
        for i in range(len(self.size_units) - 1):
            affine, weight, bias = [k + str(i + 1) for k in ('Affine', 'W', 'b')]
            grads[weight] =\
                self.layers[affine].dW + self.lambda_l2 * self.params[weight]
            grads[bias] = self.layers[affine].db
        return grads


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, include, url

urlpatterns = patterns('djack',
    url(r'^create/$', 'views.create'),
    url(r'^$', 'views.list_tweets'),
    url(r'^(?P<tweet_id>\d+)/?$', 'views.detail_tweet'),
    url(r'^like/(?P<tweet_id>\d+)/?', 'views.like_process'),
    url(r'^bye/', 'views.bye_user'),
    url(r'^likers/(?P<tweet_id>\d+)/?', 'views.likers'),
    url(r'^(?P<tweet_id>\d+)/comment/?', 'views.create_comment'),
)


#!/usr/bin/python3
"""  rints all City objects from the database and his associated state  """
import sys
from model_state import Base, State
from model_city import City
from sqlalchemy import (create_engine)
from sqlalchemy.orm import sessionmaker

if __name__ == '__main__':
    engine = create_engine(
        'mysql+mysqldb://{}:{}@localhost/{}'.format(
            sys.argv[1],
            sys.argv[2],
            sys.argv[3]),
        pool_pre_ping=True
            )
    Session = sessionmaker(bind=engine)
    session = Session()
    states = session.query(
        State, City
    ).filter(
        State.id == City.state_id
    ).order_by(
        City.id.asc()
    ).all()
    for state in states:
        print("{}: ({}) {}".format(
            state[0].name,
            state[1].id,
            state[1].name
            ))


from flask import Flask, request, jsonify, send_file, safe_join
from sklearn import svm
from joblib import load, dump
from mongo import get_data2train
import os
from gevent.pywsgi import WSGIServer
import uuid
import numpy as np
import requests

app = Flask(__name__)
app.config['SECRET_KEY'] = uuid.uuid4().hex
app.config['MODEL_SVM'] = os.path.join(os.getcwd(),'model','svm_face_recognition.sav')

# send file to host [POST]
def send2host():
    with open(app.config['MODEL_SVM'],'rb') as f:
        try:
            print('send model')
            r = requests.post('http://192.168.0.252:2727/receive_model',files = {'model':f})  # ip processing machine
        except Exception as e:
            print('error post receive model:',e)
            return 'error'
        # response data
        res_data = r.json()
        if res_data['msg'] != 'success':
            print('error response data receive data')
            return 'error'

        return 'success'

# train face data
@app.route('/train_face_data',methods=['POST'])
def train_face_data():
    if request.method == 'POST':
        print('train_face_data',os.getcwd())

        # ดึงข้อมูล 2 อย่าง คือ 1. user_id 2. ข้อมูลใบหน้า
        get_data2train_var = get_data2train()

        # ประกาศตัวแปรรับข้อมูล เพื่อนำข้อมูลไปใช้ในการแยกเพื่อนำไปใช้ต่อไป
        data_train_list = list()
        data_label = list()

        for dt in get_data2train_var:
            if 'data' in dt:
                data_train_list.append(dt['data'])
                data_label.append(dt['info']['user_id'])

        # ประกาศตัวแปรเพื่อใช้ในการเตรียมข้อมูลเพื่อใช้ในการ train
        dats_train = list()
        dats_label = list()

        # ลูปเพื่อทำการตรียมข้อมูลเพื่อใช้ในการเตรียมข้อมูล
        for i, train in enumerate(data_train_list):
            # print(len(train))
            if len(train) == 5:  # จำนวนรูป
                # print(len(train))
                for sub_train in train:
                    dats_train.append(sub_train)
                    dats_label.append(data_label[i])
        #print(dats_label)

        # convert data to numpy array
        dats_train = np.array(dats_train)
        dats_label = np.array(dats_label)

        # SVM
        #check len class , have to more than 1 class
        unique_val = set(dats_label)
        if len(unique_val) > 1:
            clf = svm.SVC(probability=True)
            clf.fit(dats_train, dats_label)
            dump(clf, open('./model/svm_face_recognition.sav', 'wb'))
            # save and post to host
            res = send2host()
            if res != 'success':
                print('error send2host')
                return jsonify({"msg":"error"})
            #response status data

        else:
            return jsonify({"msg":"CLT2"}) # number of class less than 2

        # # save model
        return jsonify({"msg":"success"})

@app.route('/test_sent',methods=['POST'])
def test_sent():
    if request.method == 'POST':
        print('omg')
    return jsonify({"status":True,"msg":"success"})

if __name__ == '__main__':
    #server_train = WSGIServer(('0.0.0.0',2525),app)
    #server_train.serve_forever()

    app.run(host= '192.168.0.253',port=2525,threaded=True)


with open("./data/dictionary.txt", "r") as f:
    dictionary = set()
    for item in f:
        dictionary.add(item[:-1])

def print_table(table):
    for row in table:
        print(row)

def globaldistance(token):
    """calculate globaldistance for each word in dictionary to token using Needleman–Wunsch algorithm"""
    # [row number][col number]
    # parameters: [m, i, d, r] = [+1, -1, -1, -1]

    if token in dictionary:
        return token

    matched_word_list = []
    max_score = 0

    for word in dictionary:
        first_row = []
        dp_table = [first_row]
        for i in range(len(token)+1):
            first_row.append(-1 * i)

        for i in range(1, len(word)+1):
            dp_table.append([-1 * i])

        for row_index in range(1, len(dp_table)):
            word_index = row_index - 1
            for col_index in range(1, len(dp_table[0])):
                token_index = col_index - 1
                up, left, diag = dp_table[row_index-1][col_index], dp_table[row_index][col_index-1], dp_table[row_index-1][col_index-1]
                if word[word_index] == token[token_index]:
                    score = max(up-1, left-1, diag+1)
                else:
                    score = max(up-1, left-1, diag-1)
                dp_table[row_index].append(score)

        if dp_table[-1][-1] > max_score:
            max_score = dp_table[-1][-1]
            matched_word_list = [word]
        elif dp_table[-1][-1] == max_score:
            matched_word_list.append(word)
    return matched_word_list

with open("./data/misspell1.txt", "r") as f:
    for word in f:
        print(globaldistance(word))

# word = input('Enter misspelled word: ')
# print(globaldistance(word, {'deaden', 'blenders', 'commodity','addendum', 'end', 'leader', 'leant', 'lent', 'lemonade', 'pleading'}))


# https://www.codewars.com/kata/sort-the-odd/train/python
# Yalnızca listede bulunan tek sayıları küçükten büyüğe sıralar.
# Çift sayılar etkilenmezler ve oldukları yerde kalırlar.
# Nested loop:
# even listesindeki ilk False değeri bulunduğunda i değerindeki
# tek sayı False değerinin yerine atanır ve break ifadesiyle
# 2. döngüden çıkılarak diğer tek sayı için(i)
# 2. döngü yeniden başlar.

def sort_array(source_array):
    even = [False] * len(source_array)
    odd = []
    for i in range(len(source_array)):
    	if source_array[i] % 2 == 1:
    		odd.append(source_array[i])
    	else:
    		even[i] = source_array[i]
    odd.sort()
    for i in range(len(odd)):
    	for j in range(len(even)):
    		if even[j] is False:
    			even[j] = odd[i]
    			break
    return even

print(sort_array([5, 3, 2, 8, 1, 4])) # [1, 3, 2, 8, 5, 4]
print(sort_array([5, 3, 1, 8, 0])) # [1, 3, 5, 8, 0]
print(sort_array([])) # []

# codewars solution

def odd_sort(array):
	odds = sorted((x for x in array if x % 2 != 0), reverse=True) # sorting odd numbers big to small
	return [x if x % 2 == 0 else odds.pop() for x in array]

print(odd_sort([5, 3, 2, 8, 1, 4]))


import numpy as np
import cv2
import os
import csv
import sklearn.metrics as sm
from surf_image_processing import func,func2
from sklearn.cluster import MiniBatchKMeans
from sklearn.svm import SVC
from sklearn.grid_search import GridSearchCV
import random
import warnings
import pickle
from sklearn.naive_bayes import GaussianNB as nb
from sklearn.neighbors import KNeighborsClassifier as knn
from sklearn.linear_model import LogisticRegression as lr
from sklearn.neural_network import MLPClassifier as mlp
import numpy as np
import sklearn.metrics as sm

#initialise
path="train"
label=0
img_descs=[]
y=[]

#utility functions
def perform_data_split(X, y, training_idxs, test_idxs, val_idxs):
    """
    Split X and y into train/test/val sets
    Parameters:
    -----------
    X : eg, use img_bow_hist
    y : corresponding labels for X
    training_idxs : list/array of integers used as indicies for training rows
    test_idxs : same
    val_idxs : same
    Returns:
    --------
    X_train, X_test, X_val, y_train, y_test, y_val
    """
    X_train = X[training_idxs]
    X_test = X[test_idxs]
    X_val = X[val_idxs]

    y_train = y[training_idxs]
    y_test = y[test_idxs]
    y_val = y[val_idxs]

    return X_train, X_test, X_val, y_train, y_test, y_val

def train_test_val_split_idxs(total_rows, percent_test, percent_val):
    """
    Get indexes for training, test, and validation rows, given a total number of rows.
    Assumes indexes are sequential integers starting at 0: eg [0,1,2,3,...N]
    Returns:
    --------
    training_idxs, test_idxs, val_idxs
        Both lists of integers
    """
    if percent_test + percent_val >= 1.0:
        raise ValueError('percent_test and percent_val must sum to less than 1.0')

    row_range = range(total_rows)

    no_test_rows = int(total_rows*(percent_test))
    test_idxs = np.random.choice(row_range, size=no_test_rows, replace=False)
    # remove test indexes
    row_range = [idx for idx in row_range if idx not in test_idxs]

    no_val_rows = int(total_rows*(percent_val))
    val_idxs = np.random.choice(row_range, size=no_val_rows, replace=False)
    # remove validation indexes
    training_idxs = [idx for idx in row_range if idx not in val_idxs]

    print('Train-test-val split: %i training rows, %i test rows, %i validation rows' % (len(training_idxs), len(test_idxs), len(val_idxs)))

    return training_idxs, test_idxs, val_idxs

def cluster_features(img_descs, training_idxs, cluster_model):
    """
    Cluster the training features using the cluster_model
    and convert each set of descriptors in img_descs
    to a Visual Bag of Words histogram.
    Parameters:
    -----------
    X : list of lists of SIFT descriptors (img_descs)
    training_idxs : array/list of integers
        Indicies for the training rows in img_descs
    cluster_model : clustering model (eg KMeans from scikit-learn)
        The model used to cluster the SIFT features
    Returns:
    --------
    X, cluster_model :
        X has K feature columns, each column corresponding to a visual word
        cluster_model has been fit to the training set
    """
    n_clusters = cluster_model.n_clusters

    # # Generate the SIFT descriptor features
    # img_descs = gen_sift_features(labeled_img_paths)
    #
    # # Generate indexes of training rows
    # total_rows = len(img_descs)
    # training_idxs, test_idxs, val_idxs = train_test_val_split_idxs(total_rows, percent_test, percent_val)

    # Concatenate all descriptors in the training set together
    training_descs = [img_descs[i] for i in training_idxs]
    all_train_descriptors = [desc for desc_list in training_descs for desc in desc_list]
    all_train_descriptors = np.array(all_train_descriptors)

    print ('%i descriptors before clustering' % all_train_descriptors.shape[0])

    # Cluster descriptors to get codebook
    print ('Using clustering model %s...' % repr(cluster_model))
    print ('Clustering on training set to get codebook of %i words' % n_clusters)

    # train kmeans or other cluster model on those descriptors selected above
    cluster_model.fit(all_train_descriptors)
    print ('done clustering. Using clustering model to generate BoW histograms for each image.')

    # compute set of cluster-reduced words for each image
    img_clustered_words = [cluster_model.predict(raw_words) for raw_words in img_descs]

    # finally make a histogram of clustered word counts for each image. These are the final features.
    img_bow_hist = np.array(
        [np.bincount(clustered_words, minlength=n_clusters) for clustered_words in img_clustered_words])

    X = img_bow_hist
    print ('done generating BoW histograms.')

    return X, cluster_model

def calc_accuracy(method,label_test,pred):
    print("accuracy score for ",method,sm.accuracy_score(label_test,pred))
    print("precision_score for ",method,sm.precision_score(label_test,pred,average='micro'))
    print("f1 score for ",method,sm.f1_score(label_test,pred,average='micro'))
    print("recall score for ",method,sm.recall_score(label_test,pred,average='micro'))

def predict_svm(X_train, X_test, y_train, y_test):
    svc=SVC(kernel='linear')
    print("svm started")
    svc.fit(X_train,y_train)
    y_pred=svc.predict(X_test)
    calc_accuracy("SVM",y_test,y_pred)

def predict_lr(X_train, X_test, y_train, y_test):
    clf = lr()
    print("lr started")
    clf.fit(X_train,y_train)
    y_pred=clf.predict(X_test)
    calc_accuracy("Logistic regression",y_test,y_pred)

def predict_nb(X_train, X_test, y_train, y_test):
    clf = nb()
    print("nb started")
    clf.fit(X_train,y_train)
    y_pred=clf.predict(X_test)
    calc_accuracy("Naive Bayes",y_test,y_pred)

def predict_knn(X_train, X_test, y_train, y_test):
    clf=knn(n_neighbors=8)
    print("knn started")
    clf.fit(X_train,y_train)
    y_pred=clf.predict(X_test)
    calc_accuracy("K nearest neighbours",y_test,y_pred)

def predict_mlp(X_train, X_test, y_train, y_test):
    clf=mlp()
    print("mlp started")
    clf.fit(X_train,y_train)
    y_pred=clf.predict(X_test)
    calc_accuracy("MLP classifier",y_test,y_pred)

#creating desc for each file with label
for (dirpath,dirnames,filenames) in os.walk(path):
    for dirname in dirnames:
        print(dirname)
        for(direcpath,direcnames,files) in os.walk(path+"\\"+dirname):
            for file in files:
                actual_path=path+"\\\\"+dirname+"\\\\"+file
                print(actual_path)
                des=func2(actual_path)
                img_descs.append(des)
                y.append(label)
        label=label+1

#finding indexes of test train and validate
y=np.array(y)
training_idxs, test_idxs, val_idxs = train_test_val_split_idxs(len(img_descs), 0.4, 0.0)

#creating histogram using kmeans minibatch cluster model
X, cluster_model = cluster_features(img_descs, training_idxs, MiniBatchKMeans(n_clusters=150))

#splitting data into test, train, validate using the indexes
X_train, X_test, X_val, y_train, y_test, y_val = perform_data_split(X, y, training_idxs, test_idxs, val_idxs)

#using classification methods
predict_svm(X_train, X_test,y_train, y_test)
predict_knn(X_train, X_test,y_train, y_test)
predict_lr(X_train, X_test,y_train, y_test)
predict_nb(X_train, X_test,y_train, y_test)
predict_mlp(X_train, X_test,y_train, y_test)


from django.urls import (path,
                         include)
from .views import (loginto,
                    create_firm,
                    signup,
                    logout_view,
                    employee_login,
                    get_logged_in_user)

urlpatterns = [
    path('', loginto, name='login_default'),
    path('login', loginto, name="login"),
    path('login/<str:invalid>', loginto, name="login"),
    path('signup', signup, name="signup"),
    path('create_firm', create_firm, name="create_firm"),
    path('logout', logout_view, name="logout"),
    path('employee_login', employee_login, name="employee_login"),
    path('logged_in_user', get_logged_in_user, name="logged_in_user"),

]


from django.shortcuts import render,redirect
from django.contrib.auth.forms import UserCreationForm
from .forms import CreateUserForm,UserUpdateForm,ProfileUpdateForm
from django.contrib import messages
# Create your views here.

def register(request):
    if request.method == 'POST':
        form = CreateUserForm(request.POST)
        if form.is_valid():
            form.save()
            username = form.cleaned_data.get('username')
            messages.success(request,f'{username} ACCOUNT HAS BEEN ADDED - ({username})  Hesap Oluşturuldu')
            return redirect('user-login')

    else:
        form = CreateUserForm()

    context = {'form':form}

    return render(request,'user/register.html',context)

def profile(request):
    context = {}
    return render(request, 'user/profile.html',context)

def profile_update(request):
    if request.method == 'POST':
        user_form = UserUpdateForm(request.POST, instance = request.user)
        profile_form = ProfileUpdateForm(request.POST, request.FILES, instance = request.user.profile)
        if user_form.is_valid() and profile_form.is_valid():
            user_form.save()
            profile_form.save()
            return redirect('user-profile')
    else:
        user_form = UserUpdateForm(instance = request.user)
        profile_form = ProfileUpdateForm(instance = request.user.profile)

    context ={'user_form':user_form,'profile_form':profile_form,}

    return render(request,'user/profile_update.html',context)


#!/usr/bin/env python

"""
CABLE site run with full biogeochemistry (CNP) and POP
=======================================================

- Spin-up: using pre-industrial CO2, NDEP, PDEP. Currently this is using
           values from AmazonFACE experiment, i.e. CO2=284.7;
           NDEP-0.79 kg N ha-1 yr-1; PDEP=0.144 kg P ha-1 yr-1
- Transient: 1851-1998, varying CO2, NDEP and PDEP with recycled met data
- Simulation: actual experiment CO2, NDEP/PDEP & met

Options to turn use C/CN/CNP with POP on/off

That's all folks.

"""

__author__ = "Martin De Kauwe, Vanessa Haverd"
__version__ = "1.0 (21.07.2018)"
__email__ = "mdekauwe@gmail.com, Vanessa.Haverd@csiro.au"

import os
import sys
import glob
import shutil
import tempfile
import pandas as pd
import xarray as xr
import numpy as np
import subprocess

class RunCable(object):

    def __init__(self, experiment_id, startyear, endyear, lai_feedback,
                 site_dir, obs_dir, plot_dir, namelist_dir, param_dir, output_dir, restart_dir,
                 dump_dir, met_fname, co2_ndep_fname, nml_fn,
                 site_nml_fn,veg_param_fn,log_dir, exe, aux_dir,
                 biogeochem, call_pop, verbose):

        self.experiment_id = experiment_id
        self.startyear = startyear
        self.endyear = endyear
        self.lai_feedback = lai_feedback
        self.site_dir = site_dir
        self.obs_dir = obs_dir
        self.plot_dir = plot_dir
        self.namelist_dir = namelist_dir
        self.param_dir = param_dir
        self.output_dir = output_dir
        self.restart_dir = restart_dir
        self.dump_dir = dump_dir
        self.met_fname = met_fname
        self.co2_ndep_fname = co2_ndep_fname
        self.nml_fn = nml_fn
        self.site_nml_fn = site_nml_fn
        self.veg_param_fn = veg_param_fn
        self.cable_restart_fname = "%s_cable_rst.nc" % (self.experiment_id)
        self.casa_restart_fname = "%s_casa_rst.nc" % (self.experiment_id)
        self.pop_restart_fname = "%s_pop_rst.nc" % (self.experiment_id)
        self.climate_restart_fname = "%s_climate_rst.nc" % (self.experiment_id)
        self.log_dir = log_dir
        self.cable_exe = exe
        self.aux_dir = aux_dir
        self.verbose = verbose
        self.nyear_spinup = 30
        self.limit_labile =".FALSE."
        if biogeochem == "C":
            self.biogeochem = 1
            #self.vcmax = "standard"
            self.vcmax = "Walker2014"
            self.vcmax_feedback = ".TRUE."
        elif biogeochem == "CN":
            self.biogeochem = 2
            self.vcmax = "Walker2014"
            self.vcmax_feedback = ".TRUE."
        elif biogeochem == "CNP":
            self.biogeochem = 3
            self.vcmax = "Walker2014"
            self.vcmax_feedback = ".TRUE."
        else:
            raise ValueError("Unknown biogeochemistry option: C, CN, CNP")
        self.call_pop = call_pop
        if self.call_pop:
            self.pop_flag = ".TRUE."
        else:
            self.pop_flag = ".FALSE."
        self.debug = True

    def main(self, SPIN_UP=False, TRANSIENT=False, SIMULATION=False):

        os.chdir(site_dir + '/' + site)
        out_file=site_dir + '/' + site + "/" + output_dir + '/' + experiment_id + '_out_cable.nc'

        if not os.path.exists(restart_dir):
            os.makedirs(restart_dir)

        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        if not os.path.exists(log_dir):
            os.makedirs(log_dir)

        if not os.path.exists(dump_dir):
            os.makedirs(dump_dir)

        num = 1
        not_in_equilibrium = True

        (st_yr, en_yr,
         st_yr_trans, en_yr_trans,
         st_yr_spin, en_yr_spin) = self.get_years()

        self.initial_setup(st_yr_spin, en_yr_spin, st_yr, en_yr)

        if SPIN_UP == True:

            # initial spin
            print("Spinup")
            self.limit_labile =".TRUE."
            self.run_me()
            self.clean_up(end=False, tag="zero")
            #sys.exit()

            while num < 4:
                self.logfile="log_ccp%d" % (num)
                self.setup_re_spin(number=num)
                self.run_me()
                self.clean_up(end=False, tag="ccp%d" % (num))
                self.logfile="log_sa%d" % (num)
                self.setup_analytical_spin(number=num, st_yr_spin=st_yr_spin,
                                           en_yr_spin=en_yr_spin )
                self.run_me()
                self.clean_up(end=False, tag="saa%d" % (num))
                not_in_equilibrium = self.check_steady_state(num)
                num += 1

            not_in_equilibrium=True
            self.limit_labile =".FALSE."
            while not_in_equilibrium:

                self.logfile="log_ccp%d" % (num)
                self.setup_re_spin(number=num)
                self.run_me()
                self.clean_up(end=False, tag="ccp%d" % (num))
                #sys.exit()
                self.logfile="log_sa%d" % (num)
                self.setup_analytical_spin(number=num, st_yr_spin=st_yr_spin,
                                           en_yr_spin=en_yr_spin )
                self.run_me()
                self.clean_up(end=False, tag="saa%d" % (num))
                #sys.exit()
                not_in_equilibrium = self.check_steady_state(num)

                num += 1

            # one final spin
            self.logfile="log_ccp%d" % (num)
            self.setup_re_spin(number=num)
            self.run_me()
            self.clean_up(end=False, tag="ccp%d" % (num))

        if TRANSIENT == True:
            print("Transient")

            self.setup_transient(st_yr_trans, en_yr_trans, st_yr, en_yr)
            self.run_me()
            self.clean_up(end=False, tag="transient")

        if SIMULATION == True:
            print("Simulation")

            self.setup_simulation(st_yr, en_yr)
            self.run_me()

            print("Plotting")
            #subprocess.call(["./CABLE_plots.R", site, str(st_yr), str(en_yr), out_file, self.obs_dir])
            subprocess.call([os.path.join(self.plot_dir,"CABLE_plots.R"), self.experiment_id, str(st_yr), str(en_yr), out_file, self.obs_dir, self.plot_dir])

        self.clean_up(end=True)

    def get_years(self):
        """
        Figure out the start and end of the met file, the number of times we
        need to recycle the met data to cover the transient period and the
        start and end of the transient period.
        """
        pre_indust = 1850

        ds = xr.open_dataset(self.met_fname)

        # st_yr = pd.to_datetime(ds.time[0].values).year
        ## Note: start- and endyear now determined in the wrapper script
        st_yr = self.startyear
        en_yr = self.endyear

        # PALS met files final year tag only has a single 30 min, so need to
        # end at the previous year, which is the real file end
        # en_yr = pd.to_datetime(ds.time[-1].values).year

        # length of met record
        nrec = en_yr - st_yr + 1

        # number of times met data is recycled during transient simulation
        nloop_transient = np.ceil((st_yr - 1 - pre_indust) / nrec) - 1

        # number of times met data is recycled with a spinup run of nyear_spinup
        nloop_spin = np.ceil( self.nyear_spinup / nrec)

        st_yr_transient = st_yr - 1 - nloop_transient * nrec + 1
        en_yr_transient = st_yr_transient + nloop_transient * nrec - 1

        en_yr_spin = st_yr_transient - 1
        st_yr_spin = en_yr_spin - nloop_spin * nrec + 1

        return (st_yr, en_yr, st_yr_transient, en_yr_transient,
                st_yr_spin, en_yr_spin)

    def adjust_nml_file(self, fname, replacements):
        """
        Adjust the params/flags in the CABLE namelise file. Note this writes
        over whatever file it is given!

        Parameters:
        ----------
        fname : string
            parameter filename to be changed.
        replacements : dictionary
            dictionary of replacement values.

        """
        f = open(fname, 'r')
        param_str = f.read()
        f.close()
        new_str = self.replace_keys(param_str, replacements)
        fd, path = tempfile.mkstemp()
        os.write(fd, str.encode(new_str))
        os.close(fd)
        shutil.copy(path, fname)
        os.remove(path)

    def replace_keys(self, text, replacements_dict):
        """ Function expects to find CABLE namelist file formatted key = value.

        Parameters:
        ----------
        text : string
            input file data.
        replacements_dict : dictionary
            dictionary of replacement values.

        Returns:
        --------
        new_text : string
            input file with replacement values

        """
        lines = text.splitlines()
        for i, row in enumerate(lines):
            # skip blank lines
            if not row.strip():
                continue
            if "=" not in row:
                lines[i] = row
                continue
            elif not row.startswith("&"):
                key = row.split("=")[0]
                val = row.split("=")[1]
                lines[i] = " ".join((key.rstrip(), "=",
                                     replacements_dict.get(key.strip(),
                                     val.lstrip())))

        return '\n'.join(lines) + '\n'

    def initial_setup(self, st_yr_spin, en_yr_spin, st_yr, en_yr):
        """
        Setup CABLE namelist file for spinup from zero
        """
        shutil.copyfile(os.path.join(self.namelist_dir, "site.nml"),
                        self.site_nml_fn)
        shutil.copyfile(os.path.join(self.namelist_dir, "cable.nml"),
                        self.nml_fn)
        #self.add_missing_options_to_nml_file(self.nml_fn)

        out_fname = os.path.join(self.output_dir,
                                 "%s_out_cable_zero.nc" % (self.experiment_id))
        if os.path.isfile(out_fname):
            os.remove(out_fname)

        out_fname_CASA = os.path.join(self.output_dir,
                                 "%s_out_CASA_zero.nc" % (self.experiment_id))
        if os.path.isfile(out_fname_CASA):
            os.remove(out_fname_CASA)

        out_fname_POP = os.path.join(self.output_dir,
                                 "%s_out_POP_zero.nc" % (self.experiment_id))
        if os.path.isfile(out_fname_POP):
            os.remove(out_fname_POP)

        out_log_fname = os.path.join(self.log_dir,
                                     "%s_log_zero.txt" % (self.experiment_id))
        if os.path.isfile(out_log_fname):
            os.remove(out_log_fname)

        replace_dict = {
                        "RunType": '"spinup"',
                        "CO2NDepFile": "'%s'" % (self.co2_ndep_fname),
                        "spinstartyear": "%d" % (st_yr),
                        "spinendyear": "%d" % (en_yr),
                        "spinCO2": "284.7",
                        "spinNdep": "0.79",
                        "spinPdep": "0.144",
        }
        self.adjust_nml_file(self.site_nml_fn, replace_dict)

        replace_dict = {
                        "filename%met": "'%s'" % (self.met_fname),
                        "filename%out": "'%s'" % (out_fname),
                        "casafile%out": "'%s'" % (out_fname_CASA),
                        "cable_user%POP_outfile" : "'%s'" % (out_fname_POP),
                        "filename%log": "'%s'" % (out_log_fname),
                        "filename%restart_out": "'%s'" % os.path.join(self.restart_dir, self.cable_restart_fname),
                        "cable_user%climate_restart_out": "'%s'" % os.path.join(self.restart_dir, self.climate_restart_fname),
                        "cable_user%POP_restart_out": "'%s'" % os.path.join(self.restart_dir, self.pop_restart_fname),
                        "casafile%cnpepool": "'%s'" % os.path.join(self.restart_dir, self.casa_restart_fname),
                        "filename%restart_in": "''" ,
                        "cable_user%climate_restart_in": "''" ,
                        "cable_user%POP_restart_in": "''",
                        "filename%type": "'%s'" % (os.path.join(self.aux_dir, "offline/gridinfo_CSIRO_1x1.nc")),
                        "filename%veg": "'%s'" % os.path.join(self.param_dir, veg_param_fn),
                        "filename%soil": "'%s'" % os.path.join(self.param_dir, soil_param_fn),
                        "output%restart": ".TRUE.",
                        "casafile%phen": "'%s'" % (os.path.join(self.aux_dir, "core/biogeochem/modis_phenology_csiro.txt")),
                        "casafile%cnpbiome": "'%s'" % (os.path.join(self.param_dir, bgc_param_fn)),
                        "cable_user%RunIden": "'%s'" % (self.experiment_id),
                        "cable_user%POP_out": "'rst'",
                        "cable_user%POP_rst": "'./'",
                        "cable_user%POP_fromZero": ".T.",
                        "cable_user%CASA_fromZero": ".T.",
                        "cable_user%CLIMATE_fromZero": ".T.",
                        "cable_user%vcmax": "'%s'" % (self.vcmax),
                        "cable_user%YearStart": "%d" % (st_yr_spin),
                        "cable_user%YearEnd": "%d" % (en_yr_spin),
                        "cable_user%CASA_SPIN_STARTYEAR": "%d" % (st_yr_spin),
                        "cable_user%CASA_SPIN_ENDYEAR": "%d" % (en_yr_spin),
                        "cable_user%CALL_POP": "%s" % (self.pop_flag),
                        "output%averaging": "'monthly'",
                        "output%patch": ".TRUE.",
                        "icycle": "%d" % (self.biogeochem),
                        "leaps": ".TRUE.",
                        "l_vcmaxFeedbk": "%s" % (self.vcmax_feedback),
                        "l_laiFeedbk": ".%s." % (self.lai_feedback),
                        "cable_user%CASA_OUT_FREQ":  "'annually'" ,
                        "cable_user%limit_labile": "%s" % (self.limit_labile) ,
                        "cable_user%SRF":  ".T." ,
                        "cable_user%STRF": "'DAMM'" ,
                        "cable_user%SMRF": "'DAMM'"
          }
        self.adjust_nml_file(self.nml_fn, replace_dict)

    def setup_re_spin(self, number=None):
        """
        Adjust the CABLE namelist file with the various flags for another spin
        """
        out_log_fname = "%s_log_ccp%d.txt" % (self.experiment_id, number)
        out_log_fname = os.path.join(self.log_dir, out_log_fname)
        if os.path.isfile(out_log_fname):
            os.remove(out_log_fname)

        out_fname = "%s_out_cable_ccp%d.nc" % (self.experiment_id, number)
        out_fname = os.path.join(self.output_dir, out_fname)
        if os.path.isfile(out_fname):
            os.remove(out_fname)

        out_fname_CASA = "%s_out_CASA_ccp%d.nc" % (self.experiment_id, number)
        out_fname_CASA = os.path.join(self.output_dir, out_fname_CASA)
        if os.path.isfile(out_fname_CASA):
            os.remove(out_fname_CASA)

        out_fname_POP = "%s_out_POP_ccp%d.nc" % (self.experiment_id, number)
        out_fname_POP = os.path.join(self.output_dir, out_fname_POP)
        if os.path.isfile(out_fname_POP):
            os.remove(out_fname_POP)

        replace_dict = {
                        "filename%log": "'%s'" % (out_log_fname),
                        "filename%restart_in": "'%s'" % os.path.join(self.restart_dir, self.cable_restart_fname),
                        "cable_user%climate_restart_in": "'%s'" % os.path.join(self.restart_dir, self.climate_restart_fname),
                        "cable_user%POP_restart_in": "'%s'" % os.path.join(self.restart_dir, self.pop_restart_fname),
                        "casafile%cnpipool": "'%s'" % os.path.join(self.restart_dir,self.casa_restart_fname),
                        "cable_user%POP_fromZero": ".F.",
                        "cable_user%CASA_fromZero": ".F.",
                        "cable_user%POP_rst": "'./'",
                        "cable_user%CLIMATE_fromZero": ".F.",
                        "cable_user%CASA_DUMP_READ": ".FALSE.",
                        "cable_user%CASA_DUMP_WRITE": ".TRUE.",
                        "cable_user%CASA_NREP": "0",
                        "cable_user%SOIL_STRUC": "'sli'",
                        "icycle": "%d" % (self.biogeochem),
                        "leaps": ".TRUE.",
                        "spincasa": ".FALSE.",
                        "casafile%c2cdumppath": "' '",
                        "output%restart": ".TRUE.",
                        "filename%out": "'%s'" % (out_fname),
                        "casafile%out": "'%s'" % (out_fname_CASA),
                        "cable_user%POP_outfile" : "'%s'" % (out_fname_POP),
                        "cable_user%limit_labile": "%s" % (self.limit_labile) ,
        }
        self.adjust_nml_file(self.nml_fn, replace_dict)

    def setup_analytical_spin(self, number, st_yr_spin, en_yr_spin):
        """
        Adjust the CABLE namelist file with the various flags for the
        analytical spin step
        """
        out_log_fname = "%s_log_analytic_%d.txt" % (self.experiment_id, number)
        out_log_fname = os.path.join(self.log_dir, out_log_fname)
        if os.path.isfile(out_log_fname):
            os.remove(out_log_fname)

        out_fname_CASA = "%s_out_CASA_analytic_%d.nc" % \
                            (self.experiment_id, number)
        out_fname_CASA = os.path.join(self.output_dir, out_fname_CASA)
        if os.path.isfile(out_fname_CASA):
            os.remove(out_fname_CASA)

        out_fname_POP = "%s_out_POP_analytic_%d.nc" % (self.experiment_id, number)
        out_fname_POP = os.path.join(self.output_dir, out_fname_POP)
        if os.path.isfile(out_fname_POP):
            os.remove(out_fname_POP)

        replace_dict = {
                        "filename%log": "'%s'" % (out_log_fname),
                        "icycle": "%d" % (self.biogeochem + 10), # Need to add 10 for spinup
                        "cable_user%CASA_DUMP_READ": ".TRUE.",
                        "cable_user%CASA_DUMP_WRITE": ".FALSE.",
                        "cable_user%CASA_NREP": "1",
                        "cable_user%SOIL_STRUC": "'default'",
                        "leaps": ".FALSE.",
                        "spincasa": ".TRUE.",
                        "casafile%c2cdumppath": "'./'",
                        "cable_user%CASA_SPIN_STARTYEAR": "%d" % (st_yr_spin),
                        "cable_user%CASA_SPIN_ENDYEAR": "%d" % (en_yr_spin),
                        "casafile%out": "'%s'" % (out_fname_CASA),
                        "cable_user%POP_outfile" : "'%s'" % (out_fname_POP),
                        "cable_user%limit_labile": "%s" % (self.limit_labile) ,
        }
        self.adjust_nml_file(self.nml_fn, replace_dict)

    def setup_transient(self, st_yr_trans, en_yr_trans, st_yr, en_yr):
        """
        Adjust the CABLE namelist file for the transient run, i.e. 1850 to XXXX
        """
        replace_dict = {
                        "RunType": '"transient"',
                        "CO2NDepFile": "'%s'" % (self.co2_ndep_fname),
                        "spinstartyear": "%d" % (st_yr),
                        "spinendyear": "%d" % (en_yr),
           }
        self.adjust_nml_file(self.site_nml_fn, replace_dict)

        out_log_fname = "%s_log_transient.txt" % (self.experiment_id)
        out_log_fname = os.path.join(self.log_dir, out_log_fname)
        if os.path.isfile(out_log_fname):
            os.remove(out_log_fname)

        out_fname = "%s_out_cable_transient.nc" % (self.experiment_id)
        out_fname = os.path.join(self.output_dir, out_fname)
        if os.path.isfile(out_fname):
            os.remove(out_fname)

        out_fname_CASA = "%s_out_casa_transient.nc" % (self.experiment_id)
        out_fname_CASA = os.path.join(self.output_dir, out_fname_CASA)
        if os.path.isfile(out_fname_CASA):
            os.remove(out_fname_CASA)

        replace_dict = {
                        "filename%out": "'%s'" % (out_fname),
                        "filename%log": "'%s'" % (out_log_fname),
                        "cable_user%CASA_DUMP_READ": ".FALSE.",
                        "cable_user%CASA_DUMP_WRITE": ".FALSE.",
                        "cable_user%CASA_NREP": "0",
                        "cable_user%SOIL_STRUC": "'sli'",
                        "output%restart": ".TRUE.",
                        "output%averaging": "'monthly'",
                        "spinup": ".FALSE.",
                        "icycle": "%d" % (self.biogeochem),
                        "POPLUC": ".T.",
                        "filename%out": "'%s'" % (out_fname),
                        "casafile%out": "'%s'" % (out_fname_CASA),
                        "cable_user%YearStart": "%d" % (st_yr_trans),
                        "cable_user%YearEnd": "%d" % (en_yr_trans),
                        "filename%restart_in": "'%s'" % os.path.join(self.restart_dir, self.cable_restart_fname),
                        "cable_user%climate_restart_in": "'%s'" % os.path.join(self.restart_dir, self.climate_restart_fname),
                        "cable_user%POP_restart_in": "'%s'" % os.path.join(self.restart_dir, self.pop_restart_fname),
                        "casafile%cnpipool": "'%s'" % os.path.join(self.restart_dir,self.casa_restart_fname),
                        "cable_user%POP_fromZero": ".F.",
                        "cable_user%CASA_fromZero": ".F.",
                        "cable_user%CLIMATE_fromZero": ".F.",
        }
        self.adjust_nml_file(self.nml_fn, replace_dict)

    def setup_simulation(self, st_yr, en_yr):
        """
        Adjust the CABLE namelist file for the experiment years
        """
        replace_dict = {
                        "RunType": '"historical"',
                        "CO2NDepFile": "'%s'" % (self.co2_ndep_fname),
                        "spinstartyear": "%d" % (st_yr),
                        "spinendyear": "%d" % (en_yr)
        }
        self.adjust_nml_file(self.site_nml_fn, replace_dict)

        out_log_fname = "%s_log_simulation.txt" % (self.experiment_id)
        out_log_fname = os.path.join(self.log_dir, out_log_fname)
        if os.path.isfile(out_log_fname):
            os.remove(out_log_fname)

        out_fname = "%s_out_cable.nc" % (self.experiment_id)
        out_fname = os.path.join(self.output_dir, out_fname)

        out_fname_CASA = "%s_out_casa.nc" % (self.experiment_id)
        out_fname_CASA = os.path.join(self.output_dir, out_fname_CASA)

        if os.path.isfile(out_fname):
            os.remove(out_fname)

        replace_dict = {
                        "filename%log": "'%s'" % (out_log_fname),
                        "output%averaging": "'daily'",
                        "icycle": "%d" % (self.biogeochem),
                        "cable_user%YearStart": "%d" % (st_yr),
                        "cable_user%YearEnd": "%d" % (en_yr),
                        "filename%out": "'%s'" % (out_fname),
                        "POPLUC": ".F.",
                        "cable_user%CASA_DUMP_READ": ".FALSE.",
                        "cable_user%CASA_DUMP_WRITE": ".FALSE.",
                        "cable_user%SOIL_STRUC": "'sli'",
                        "spincasa": ".FALSE.",
                        "spinup": ".FALSE.",
                        "output%averaging": "'all'",
                        "casafile%out": "'%s'" % (out_fname_CASA),
                        "filename%restart_in": "'%s'" % os.path.join(self.restart_dir, self.cable_restart_fname),
                        "cable_user%climate_restart_in": "'%s'" % os.path.join(self.restart_dir, self.climate_restart_fname),
                        "cable_user%POP_restart_in": "'%s'" % os.path.join(self.restart_dir, self.pop_restart_fname),
                        "casafile%cnpipool": "'%s'" % os.path.join(self.restart_dir,self.casa_restart_fname),
                        "cable_user%POP_fromZero": ".F.",
                        "cable_user%CASA_fromZero": ".F.",
                        "cable_user%CLIMATE_fromZero": ".F.",
        }
        self.adjust_nml_file(self.nml_fn, replace_dict)

    def run_me(self):
        if self.verbose: # output is printed to console
            os.system("%s" % (self.cable_exe))
        else:
            # No outputs to the screen, stout and stderr to dev/null
            #os.system("%s > /dev/null 2>&1" % (self.cable_exe))
            # JK: stdout and stderr are printed to files (but not to console)
            os.system("%s > %s.out 2>%s.err" % (self.cable_exe,self.experiment_id,self.experiment_id))

    def check_steady_state(self, num):
        """
        Check whether the plant (leaves, wood and roots) and soil
        (fast, slow and active) carbon pools have reached equilibrium. To do
        this we are checking the state of the last year in the previous spin
        cycle to the state in the final year of the current spin cycle.
        """
        tol = 0.2 # This is quite high, I use 0.005 in GDAY
        g_2_kg = 0.001

        if num == 1:
            prev_cplant = 99999.9
            prev_csoil = 99999.9
        else:
            fname = "%s_out_CASA_ccp%d.nc" % (self.experiment_id, num-1)
            fname = os.path.join(self.output_dir, fname)
            ds = xr.open_dataset(fname)
            prev_cplant = ds.cplant[:,:,0].values[-1].sum() * g_2_kg
            prev_csoil = ds.csoil[:,:,0].values[-1].sum() * g_2_kg

        fname = "%s_out_CASA_ccp%d.nc" % (self.experiment_id, num)
        fname = os.path.join(self.output_dir, fname)
        ds = xr.open_dataset(fname)
        new_cplant = ds.cplant[:,:,0].values[-1].sum() * g_2_kg
        new_csoil = ds.csoil[:,:,0].values[-1].sum() * g_2_kg

        if ( np.fabs(prev_cplant - new_cplant) < tol and
             np.fabs(prev_csoil - new_csoil) < tol ):
            not_in_equilibrium = False
        else:
            not_in_equilibrium = True

        if self.debug:
            print("*", num, not_in_equilibrium,
                  "*cplant", np.fabs(prev_cplant - new_cplant),
                  "*csoil", np.fabs(prev_csoil - new_csoil))

        return not_in_equilibrium

    def clean_up(self, end=True, tag=None):
        """
        Move restart files to a directory and delete various files we no longer
        need that CABLE spits out as it spins up.
        """
        if end:
            for f in glob.glob("c2c_*_dump.nc"):
                shutil.move(f, os.path.join(self.dump_dir, f))
            f = "cnpfluxOut.csv"
            if os.path.isfile(f):
                os.remove(f)
            f = "new_sumbal"
            if os.path.isfile(f):
                os.remove(f)
            #JK: no longer remove .out files
            #for f in glob.glob("*.out"):
            #    os.remove(f)
            for f in glob.glob("restart_*.nc"):
                os.remove(f)
        else:
            old = os.path.join(self.restart_dir, self.cable_restart_fname)
            new = "%s_%s.nc" % (old[:-3], tag)
            shutil.copyfile(old, new)

            old = os.path.join(self.restart_dir, self.casa_restart_fname)
            new = "%s_%s.nc" % (old[:-3], tag)
            shutil.copyfile(old, new)

            old = os.path.join(self.restart_dir, self.climate_restart_fname)
            new = "%s_%s.nc" % (old[:-3], tag)
            shutil.copyfile(old, new)

            if self.call_pop:
                old = os.path.join(self.restart_dir, self.pop_restart_fname)
                new = "%s_%s.nc" % (old[:-3], tag)
                shutil.copyfile(old, new)

    def add_missing_options_to_nml_file(self, fname, line_start=None):
        """
        Some of the flags we may wish to change are missing from the default
        file so we can't adjust them via this script...add them
        """
        if line_start is None:
            line_start = sum(1 for line in open(fname)) - 1

        f = open(fname, "r")
        contents = f.readlines()
        f.close()

        arg = "   cable_user%GW_MODEL = .FALSE.\n"
        contents.insert(line_start, arg)
        line_start += 1

        arg = "   cable_user%or_evap = .FALSE.\n"
        contents.insert(line_start, arg)
        line_start += 1

        arg = "   filename%gw_elev = 'GSWP3_elevation_slope_stddev.nc'\n"
        contents.insert(line_start, arg)
        line_start += 1

        tmp_fname = "tmp.nml"
        f = open(tmp_fname, "w")
        contents = "".join(contents)
        f.write(contents)
        f.close()

        shutil.move(tmp_fname, fname)

if __name__ == "__main__":

    # command line parameters
    site=sys.argv[1]
    startyear=int(sys.argv[2])
    endyear=int(sys.argv[3])
    lai_feedback=sys.argv[4]
    site_dir=sys.argv[5]
    obs_dir=sys.argv[6]
    plot_dir=sys.argv[7]
    exp_name=sys.argv[8]

    cwd = os.getcwd()
    namelist_dir = "namelists"
    param_dir = "params"
    dump_dir = "dump"
    met_dir = "met"
    co2_ndep_dir = "met"
    aux_dir = "./CABLE-AUX/"
    log_dir = "logs"
    output_dir = "outputs"
    restart_dir = "restart_files"
    nml_fn = "cable.nml"
    site_nml_fn = "site.nml"
    #met_fname = os.path.join(met_dir, '%s.1.4_met.nc' % (experiment_id))
    met_fname = os.path.join(met_dir, site + '_' + str(startyear) + '_' + str(endyear) + '_2tiles.nc')
    co2_ndep_fname = os.path.join(co2_ndep_dir,
                                  "AmaFACE_co2npdepforcing_1850_2100_AMB.csv")
    veg_param_fn = "Cumberland_veg_params.txt"
    bgc_param_fn = "Cumberland_pftlookup.csv"
    soil_param_fn = "def_soil_params.txt"   # only used when soilparmnew = .FALSE. in cable.nml
    exe = "./cable"

    # special for PEST
    #veg_param_fn = "veg_params_pest.txt"
    #bgc_param_fn = "pftlookup_pest.csv"
    #param_dir = "./"

    call_pop = True
    verbose = False

    #for biogeochem in ["C", "CN", "CNP"]:
    for biogeochem in ["CNP"]:

        # experiment_id = "Cumberland_POP_%s" % (biogeochem)
        experiment_id = site + "_%s_2tiles_%s" % (biogeochem,exp_name)
        C = RunCable(experiment_id, startyear, endyear, lai_feedback,
                     site_dir, obs_dir, plot_dir,  namelist_dir,
                     param_dir,output_dir, restart_dir,dump_dir, met_fname, co2_ndep_fname,
                     nml_fn, site_nml_fn,veg_param_fn, log_dir, exe, aux_dir,
                     biogeochem, call_pop,verbose)
        C.main(SPIN_UP=True, TRANSIENT=True, SIMULATION=True)