import os
import glob
import re
import tqdm
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid

# need subprocess to run commands
import subprocess
import hashlib
import tempfile
import itertools as itt
import dask
from dask.diagnostics import ProgressBar

# import our ability to read and write MRC files
import Mrc

from collections import OrderedDict, Sequence

# import skimage components
from peaks.peakfinder import PeakFinder

from dphutils import slice_maker, scale_uint16, fft_pad, radial_profile
from dphplotting import display_grid, mip
from pyOTF.phaseretrieval import *
from pyOTF.utils import *

try:
    from pyfftw.interfaces.numpy_fft import (
        fftshift,
        ifftshift,
        fftn,
        ifftn,
        rfftn,
        fftfreq,
        rfftfreq,
    )
    import pyfftw

    # Turn on the cache for optimum performance
    pyfftw.interfaces.cache.enable()
except ImportError:
    from numpy.fft import fftshift, ifftshift, fftn, ifftn, rfftn, fftfreq, rfftfreq
from skimage.external import tifffile as tif

import logging

logger = logging.getLogger(__name__)

class PSFFinder(object):
    """Object to find and analyze subdiffractive emmitters"""

    def __init__(self, stack, psfwidth=1.3, window_width=20, **kwargs):
        """Analyze a z-stack of subdiffractive emmitters

        Parameters
        ----------
        stack : ndarray

        Kwargs
        ------
        psfwidth : float
        window_width : int"""
        self.stack = stack
        self.peakfinder = PeakFinder(stack.max(0), psfwidth, **kwargs)
        self.peakfinder.find_blobs()
        self.all_blobs = self.peakfinder.blobs
        self.window_width = window_width
        self.find_psfs(2 * psfwidth)

    def find_psfs(self, max_s=2.1, num_peaks=20):
        """Function to find and fit blobs in the max intensity image

        Blobs with the appropriate parameters are saved for further fitting.

        Parameters
        ----------
        max_s: float
            Reject all peaks with a fit width greater than this
        num_peaks: int
            The number of peaks to analyze further"""
        window_width = self.window_width
        # pull the PeakFinder object
        my_PF = self.peakfinder
        # find blobs
        my_PF.find_blobs()
        # prune blobs
        my_PF.remove_edge_blobs(window_width)
        my_PF.prune_blobs(window_width)
        # fit blobs in max intensity
        blobs_df = my_PF.fit_blobs(window_width)
        # round to make sorting a little more meaningfull
        blobs_df.SNR = blobs_df.dropna().SNR.round().astype(int)
        # sort by SNR then sigma_x after filtering for unreasonably
        # large blobs and reindex data frame here
        new_blobs_df = (
            blobs_df[blobs_df.sigma_x < max_s]
            .sort_values(["amp", "SNR", "sigma_x"], ascending=[False, False, True])
            .reset_index(drop=True)
        )
        # set the internal state to the selected blobs
        my_PF.blobs = new_blobs_df[["y0", "x0", "sigma_x", "amp"]].values.astype(int)
        self.fits = new_blobs_df

    def find_window(self, blob_num=0):
        """Finds the biggest window distance."""
        # pull all blobs
        blobs = self.all_blobs
        # three different cases
        if not len(blobs):
            # no blobs in window, raise hell
            raise RuntimeError("No blobs found, can't find window")
        else:
            # TODO: this should be refactored to use KDTrees
            # more than one blob find
            best = np.round(
                self.fits.iloc[blob_num][["y0", "x0", "sigma_x", "amp"]].values
            ).astype(int)

            def calc_r(blob1, blob2):
                """Calc euclidean distance between blob1 and blob2"""
                y1, x1, s1, a1 = blob1
                y2, x2, s2, a2 = blob2
                return np.sqrt((y1 - y2) ** 2 + (x1 - x2) ** 2)

            # calc distances
            r = np.array([calc_r(best, blob) for blob in blobs])
            # find min distances
            # remember that best is in blobs so 0 will be in the list
            # find the next value
            r.sort()
            try:
                r_min = r[1]
            except IndexError:
                # make r_min the size of the image
                r_min = min(np.concatenate((np.array(self.stack.shape[1:3]) - best[:2], best[:2])))
            # now window size equals sqrt or this
            win_size = int(round(2 * (r_min / np.sqrt(2) - best[2] * 3)))

        window = slice_maker(best[0], best[1], win_size)
        self.window = window

        return window

    def plot_all_windows(self):
        """Plot all the windows so that user can choose favorite"""
        windows = [self.find_window(i) for i in range(len(self.fits))]
        fig, axs = display_grid({i: self.peakfinder.data[win] for i, win in enumerate(windows)})
        return fig, axs

class PhaseRetriever(PSFFinder):
    """Utility to phase retrieve mutltiple emitters in a single dataset"""

    def __init__(self, stack, wl, na, ni, res, zres, **kwargs):
        """"""
        psfwidth = wl / 4 / na / res
        super().__init__(stack, psfwidth, **kwargs)
        # initialize model params
        self.params = dict(na=na, ni=ni, wl=wl, res=res, zres=zres, zsize=stack.shape[0])

    def retrieve_phase(self, blob_num=0, xysize=None, **kwargs):
        """"""
        window = self.find_window(blob_num)
        data = self.stack[[Ellipsis] + window]
        data_prepped = prep_data_for_PR(data, xysize)
        self.params.update(dict(size=data_prepped.shape[-1]))
        self.pr_result = retrieve_phase(data_prepped, self.params, **kwargs)
        return self.pr_result

class PSF2DProcessor(object):
    """An object for processing 2D PSFs and OTFs from 3D stacks"""

    def __init__(self, stack, na=0.85, pixsize=0.13, det_wl=0.585, **kwargs):
        """Find PSFs and turn them into OTFs

        Parameters
        ----------
        stack : ndarray
        na : float
        pixsize : float
        det_wl : float
        """
        # psfwidth = det_wl / 4 / na / pixsize
        self.stack = stack
        self.na = na
        self.pixsize = pixsize
        self.det_wl = det_wl

    def plot(self, **kwargs):
        """Plot all the things for this window"""
        fig, ax1 = plt.subplots(1, 1, figsize=(6, 6))
        # show mip
        mip(self.stack)
        self.gen_radialOTF(show_OTF=True, **kwargs)
        ax1.semilogy(abs(self.radprof))

    def _preprocess_stack(self):
        """Remove background and fft pad data"""
        img_raw = self.stack
        img_raw = remove_bg(center_data(img_raw), 1.0)
        nz, ny, nx = img_raw.shape
        nr = max(ny, nx)
        img = fft_pad(img_raw, (None, nr, nr), mode="constant")
        return img

    def calc_infocus_psf(self, filter_kspace=True, filter_xspace=True):
        """Calculate the infocus psf"""
        psf = self._preprocess_stack()
        # recenter
        # TODO: add this part
        # fft
        otf = fftshift(fftn(ifftshift(psf))).mean(0)
        # filter in k-space, if requested
        if filter_kspace or filter_xspace:
            yy, xx = (
                np.indices(otf.shape[-2:])
                - np.array(otf.shape[-2:])[:, np.newaxis, np.newaxis] / 2
            )
            r = np.hypot(yy, xx)
        if filter_kspace:
            # need to multiply be 1000 for the wavelength conversion (nm -> um)
            # and need to convert pixel size to k-space, equivalent
            mask = r > (2 * self.na / self.det_wl * self.pixsize) * otf.shape[-1]
            otf[mask] = 0
        # ifft
        infocus_psf = abs(fftshift(ifftn(ifftshift(otf))))
        # filter in x-space if requested
        if filter_xspace:
            mask = r > 4 * (self.det_wl / 2 * self.na / self.pixsize)
            infocus_psf[mask] = 0
        self.psf_2d = infocus_psf

    def gen_radialOTF(self, lf_cutoff=None, **kwargs):
        """Generate the Radially averaged OTF from the sample data."""
        img = self._preprocess_stack()
        # pull the max x, y size
        nx = max(img.shape[-2:])
        # calculate the um^-1/pix
        dkr = 1 / (nx * self.pixsize)
        # save dkr for later
        self.dkr = dkr
        # calculate the kspace cutoff, round up (that's what the 0.5 is for)
        diff_limit = 2 * self.na / (self.det_wl)
        krcutoff = int(diff_limit / dkr + 0.5)
        if lf_cutoff is None:
            lf_cutoff = diff_limit * 0.025
        lf_cutoff = int(lf_cutoff / dkr + 0.5)
        radprof = calc_radial_OTF(img, krcutoff, lf_cutoff, **kwargs)

        self.radprof = radprof

        print(
            "Better cutoff is {:.3f}".format(
                (abs(self.radprof[:krcutoff]).argmin() + 1) / (2 / (self.det_wl) / self.dkr)
            )
        )

    def save_radOTF_mrc(self, output_filename, **kwargs):
        # make empty header
        header = Mrc.makeHdrArray()
        # initialize it
        # set type and shape
        Mrc.init_simple(header, 4, self.radprof.shape)
        # set wavelength
        header.wave = self.det_wl * 1000
        # set number of wavelengths
        header.NumWaves = 1
        # set dimensions
        header.d = (self.dkr,) * 3
        tosave = self.radprof.astype(np.complex64)
        # save it
        tosave = tosave.reshape(1, 1, (len(tosave)))

        Mrc.save(tosave, output_filename, hdr=header, **kwargs)

class SIMOTF2D(PSFFinder):
    def __init__(self, stack, na=0.85, pixsize=0.13, det_wl=0.585, **kwargs):
        """Find PSFs and turn them into OTFs

        Parameters
        ----------
        stack : ndarray
        na : float
        pixsize : float
        det_wl : float
        """
        psfwidth = det_wl / 4 / na / pixsize
        super().__init__(stack, psfwidth, **kwargs)
        self.psfproc = PSF2DProcessor(None, na, pixsize, det_wl)

    def plot_window(self, blob_num, **kwargs):
        """Plot all the things for this window"""
        win = self.find_window(blob_num)
        self.psfproc.stack = self.stack[[Ellipsis] + win]
        return self.psfproc.plot(**kwargs)

    def calc_infocus_psf(self, **kwargs):
        """Calculate the infocus psf"""
        self.psfproc.calc_infocus_psf(**kwargs)
        self.psf = self.psfproc.psf

    def gen_radialOTF(self, **kwargs):
        """Generate the Radially averaged OTF from the sample data."""
        self.psfproc.gen_radialOTF(**kwargs)

    def save_radOTF_mrc(self, **kwargs):
        self.psfproc.save_radOTF_mrc(**kwargs)

def save_PSF_mrc(img, output_filename, pixsize=0.0975, det_wl=520):
    """
    A small utility function to save an image of a bead as an MRC

    Parameters
    ----------
    img: ndarray, rank 2
        The image to save
    output_filename: path
        The filename to output to
    pixsize: float
        the the pixel size in microns (size of the sensor pixel at the sample)
    det_wl: float
        the detection wavelength
    """

    # TODO: make sure '.mrc' is appended to files that don't have it.
    from pysegtools.mrc import MRC

    ny, nx = img.shape
    PSFmrc = MRC(output_filename, nx=nx, ny=ny, dtype=img.dtype)
    PSFmrc.header["nz"] = 1
    PSFmrc[0] = img
    PSFmrc.header["nwave"] = 1  # detection wavelength
    PSFmrc.header["wave1"] = det_wl  # detection wavelength
    # need the rest of these fields filled out otherwise header won't write.
    PSFmrc.header["wave2"] = 0
    PSFmrc.header["wave3"] = 0
    PSFmrc.header["wave4"] = 0
    PSFmrc.header["wave5"] = 0
    # fill in the pixel size
    PSFmrc.header["xlen"] = pixsize
    PSFmrc.header["ylen"] = pixsize

    # need to delete this field to let MRC know that this is an oldstyle
    # header to write
    del PSFmrc.header["cmap"]

    # write the header and close the file.
    PSFmrc.write_header()
    PSFmrc.close()

    return output_filename

def calc_radial_mrc(infile, outfile=None, na=0.85, L=8, H=22):
    """
    A simple wrapper around the radial OTF calc
    """

    # TODO: Error checking
    # make sure we have the absolute path
    infile = os.path.abspath(infile)
    if outfile is None:
        outfile = infile.replace(".mrc", "_otf2d.mrc")
    else:
        outfile = os.path.abspath(outfile)

    # write our string to send to the shell
    # 8 is the lower pixel and 22 is the higher pixel
    # 0.8 is the detection na
    # otfcalc = r'C:\newradialft\otf2d -N {na} -L {L} -H {H} {infile} {outfile}'

    # format the string
    # excstr = otfcalc.format(infile=infile, outfile=outfile, na=na, L=L, H=H)
    # send to shell
    # os.system(excstr)

    return_code = subprocess.call(
        [r"C:\newradialft\otf2d", "-N", str(na), "-L", str(L), "-H", str(H), infile, outfile]
    )

    return return_code

# Here lie all the bits for processing a 3D otf, this code I'm not proud of at
# all but it seems to work

def center_data(data, max_loc=None):
    """Utility to center the data

    Parameters
    ----------
    data : ndarray
        Array of data points

    Returns
    -------
    centered_data : ndarray same shape as data
        data with max value at the central location of the array
    """
    if max_loc is None:
        max_loc = np.unravel_index(data.argmax(), data.shape)
    # copy data
    centered_data = data.copy()
    # extract shape and max location
    data_shape = data.shape
    # iterate through dimensions and roll data to the right place
    for i, (x0, nx) in enumerate(zip(max_loc, data_shape)):
        if x0 is not None:
            centered_data = np.roll(centered_data, nx // 2 - x0, i)
    return centered_data

def makematrix(nphases, norders):
    """Make a separation matrix

    See Gustafsson 2008 Bio Phys J"""
    # phi is the phase step
    phi = 2 * np.pi / nphases
    # number of columns is determined by the number of orders
    cols = norders * 2 + 1
    # prepare an empty array to fill
    # this is very poor numpy coding but this will never be the bottleneck
    sep_mat = np.zeros(nphases * cols)
    # fill the matrix
    for j in range(nphases):
        sep_mat[0 * nphases + j] = 1.0
        for order in range(1, norders + 1):
            sep_mat[(2 * order - 1) * nphases + j] = np.cos(j * order * phi)
            sep_mat[2 * order * nphases + j] = np.sin(j * order * phi)
    return sep_mat.reshape(nphases, cols)

def rescale(otf):
    """Takes a radially averaged otf and scales by max of 0th order"""
    # assume OTF has arrangement: order, z, r
    scale_factor = abs(otf[0]).max()
    return otf / scale_factor

def rfft_trunk(nx):
    """just to test function below"""
    return nx // 2 + 1

def irfft_trunk(nx):
    """undo the trunkation along the real axis when performing an rfft"""
    return (nx - nx % 2) * 2

def test_rfft_trunc():
    """Test forward and backward truncs"""
    assert 0 == irfft_trunk(0)
    kx = 512
    assert kx == irfft_trunk(rfft_trunk(kx))

def _kspace_coords(dz, dr, shape):
    """Make the kspace coordinates for a radially averaged OTF

    Right now the function assumes that the extent of dr is the extent of dx"""
    # split open shape
    nz, nr = shape
    # calculate kz, kr freqs, assume that all data has been shifted back
    kz = fftshift(fftfreq(nz, dz))
    kr = rfftfreq(irfft_trunk(nr), dr)
    # determine delta kz
    dkz = kz[1] - kz[0]
    dkr = kr[1] - kr[0]
    # make the grids
    krr, kzz = np.meshgrid(kr, kz)
    return kzz, krr, dkz, dkr

def makemask(wl, na, ni, dz, dr, shape, offset=(0, 0), expansion=0):
    """Make a mask for the 3D OTF radially averaged

    Parameters
    ----------
    wl : float
        Wavelength of emission of data in nm
    na : float
        Numerical aperature of objective
    ni : float
        Index of refraction of media
    dz : float
        z resolution in nm
    dr : float
        x/y resolution, in nm
    shape : tuple
        shape of data
    offset : tuple
        The offset of the mask, in units of pixels
    expansion : float
        The expansion (or contraction) of the mask, in pixels

    Returns
    -------
    mask : ndarray
        A boolean array of shape `shape` that masks the otf outside its
        theoretical support
    """
    # unpack offsets
    z_offset, r_offset = offset
    # make k-space coordinates
    kz, kr, dkz, dkr = _kspace_coords(dz, dr, shape)
    kr_max = ni / wl  # the radius of the spherical shell
    kr_0 = na / wl  # the offset of the circle from the origin
    # z displacement of circle's center
    z0 = np.sqrt(kr_max ** 2 - kr_0 ** 2)
    # expand mask if requested
    kr_max += dkr * expansion
    # calculate centered kr
    cent_kr = kr - kr_0 - r_offset * dkr
    # calculate top half
    onehalf = np.hypot(cent_kr, kz - z0 - z_offset * dkz) <= kr_max
    # calculate bottom half
    otherhalf = np.hypot(cent_kr, kz + z0 - z_offset * dkz) <= kr_max
    mask = np.logical_and(otherhalf, onehalf)
    return mask

def find_origin(rad_prof):
    """Find the origin in a radially averaged profile"""
    # copy data, otherwise it will be changed
    data = rad_prof.copy()
    # grab shape
    nz, nr = data.shape
    # suppress pure DC
    data[(nz + 1) // 2, 0] = 0
    # find new max
    max_loc = np.unravel_index(abs(data).argmax(), data.shape)
    # only return kz component
    return max_loc[0]

def find_offsets(rad_prof):
    """find the offsets of the OTF bands"""
    # find potential origin
    dz = find_origin(rad_prof)
    nz, nr = rad_prof.shape
    center = nz // 2
    if dz == center:
        # if potential origin is same as center return zero
        return (0,)
    else:
        # else assume we have a band with two fold symmetry and we
        # should return the above and below parts
        return dz - center, center - dz

def mask_rad_prof(rad_prof, exp_args):
    """Mask off the radial profiles according to the experimental arguments

    Returns both the mask and the masked radial profile"""
    offsets = find_offsets(rad_prof)
    masks = np.array([makemask(*exp_args, rad_prof.shape, offset=(o, 0)) for o in offsets])
    mask = np.logical_or.reduce(masks, 0)
    return rad_prof * mask, mask

def correct_phase_angle(band, mask):
    """Remove the average phase angle"""
    # norm all values so they're on the unit circle
    # only use valid values within otf theoretical support
    valid_values = band[mask]
    assert valid_values.size, "There are not valid values!"
    normed = valid_values / abs(valid_values)
    # calculate angle of average
    phi = np.angle(normed.mean())
    # shift band to remove average phase angle
    a = band * np.exp(-1j * phi)
    # set everything outside suppor to zero
    a[np.logical_not(mask)] = 0
    return a

def average_pm_kz(data):
    """Average the top and bottom of the otf which should be conjugate
    symmetric"""
    # assume data has kz axis first
    nz = data.shape[0]
    # TODO: this part needs work!
    if nz % 2:
        # nz is odd
        data_top = data[: nz // 2]
        # + 1 here in case nz odd
        data_bottom = data[nz // 2 + 1 :]
        data_avg = (data_top + np.conj(data_bottom[::-1])) / 2
        return np.concatenate((data_avg, data[np.newaxis, nz // 2], np.conj(data_avg[::-1])))
    else:
        # nz is even
        data_top = data[1 : nz // 2]
        # + 1 here in case nz odd
        data_bottom = data[nz // 2 + 1 :]
        data_avg = (data_top + np.conj(data_bottom[::-1])) / 2
        return np.concatenate(
            (data[np.newaxis, 0], data_avg, data[np.newaxis, nz // 2], np.conj(data_avg[::-1]))
        )

class PSF3DProcessor(object):
    """An object designed to turn a 3D SIM PSF into a 3D SIM radially averaged
    OTF"""

    def __init__(self, data, exp_args):
        """Initialize the object, assumes data is already organized as:
        directions, phases, z, y, x

        exp_args holds all the experimental parameters (should be dict):
        wl, na, ni, zres, rres"""
        # set up internal data
        self.data = data
        # extract experimental args
        self.exp_args = self.wl, na, ni, dz, dr = exp_args
        # get ndirs etc
        self.ndirs, self.nphases, self.nz, self.ny, self.nx = data.shape
        # remove background
        self.data_nobg = data_nobg = remove_bg(self.data, 1.0)
        # average along directions and phases to make widefield psf
        self.conv_psf = conv_psf = data_nobg.mean((0, 1))
        # separate data
        sep_data = self.separate_data()
        # center the data using the conventional psf center
        psf_max_loc = np.unravel_index(conv_psf.argmax(), conv_psf.shape)
        cent_data = center_data(sep_data, (None,) + psf_max_loc)
        # take rfft along spatial dimensions (get seperated OTFs)
        # last fftshift isn't performed along las axis, because it's the real
        # axis
        self.cent_data_fft_sep = fftshift(
            rfftn(ifftshift(cent_data, axes=(1, 2, 3)), axes=(1, 2, 3)), axes=(1, 2)
        )
        self.avg_and_mask()
        # get spacings and save for later
        kzz, krr, self.dkz, self.dkr = _kspace_coords(dz, dr, self.masks[0].shape)
        # average bands (hard coded for convenience)
        corrected_profs = np.array(
            [correct_phase_angle(b, m) for b, m in zip(self.masked_rad_profs, self.masks)]
        )
        band0 = corrected_profs[0]
        band1 = (corrected_profs[1] + corrected_profs[2]) / 2
        band2 = (corrected_profs[3] + corrected_profs[4]) / 2
        self.bands = np.array((band0, band1, band2))
        self.bands = np.array([average_pm_kz(band) for band in self.bands])

    def separate_data(self):
        """Separate the different bands"""
        # make the separation matrix and apply it
        sep_mat = makematrix(self.nphases, self.nphases // 2)
        # add extra axis to "store" the linear combinations of the "vectors"
        # (we really need to sum along this axis to get the linear combos)
        # sum the linear combinations and take mean along directions
        # to make the broadcasting work we need to expand out the separation
        # matrix now too. The data ordering is now:
        # dirs, combos, phases, z, y, x
        sep_mat = sep_mat[np.newaxis, :, :, np.newaxis, np.newaxis, np.newaxis]
        # this multiplication recombines each phase image
        temp_data = self.data_nobg[:, np.newaxis] * sep_mat
        # sum the weighted images (linear combination) and take mean along
        # directions
        self.sep_data = temp_data.sum(2).mean(0)
        return self.sep_data

    def avg_and_mask(self):
        # radially average the OTFs
        # for each otf in the seperated data and for each kz plane calculate
        # the radial average center the radial average at 0 for last axis
        # because of real fft
        center = ((self.ny + 1) // 2, 0)
        extent = self.nx // 2 + 1
        self.r_3D = r_3D = np.array(
            [[radial_profile(o, center)[0][:extent] for o in z] for z in self.cent_data_fft_sep]
        )
        # mask OTFs and retrieve masks
        self.masked_rad_profs, masks = np.swapaxes(
            np.array([mask_rad_prof(r, self.exp_args) for r in r_3D]), 0, 1
        )
        # convert masks to bool (they've been cast to complex in the above)
        self.masks = masks.astype(bool)

    def save_radOTF_mrc(self, output_filename, **kwargs):
        # make empty header
        header = Mrc.makeHdrArray()
        # initialize it
        # set type and shape
        Mrc.init_simple(header, 4, self.bands.shape)
        # set wavelength
        header.wave = self.wl
        # set number of wavelengths
        header.NumWaves = 1
        # set dimensions
        header.d = (self.dkz * 1000, self.dkr * 1000, 0.0)
        bands = np.swapaxes(self.bands, 1, 2)
        bands = rescale(ifftshift(bands, axes=2))
        tosave = bands.astype(np.complex64)

        Mrc.save(tosave, output_filename, hdr=header, **kwargs)

def simrecon(*, input_file, output_file, otf_file, **kwargs):
    """
    A simple wrapper to Lin's sirecon.exe

    Parameters
    ----------
    input_file: path
        Path to file holding raw SIM data
    output_file: path
        Path to location to write reconstruction
    OTF_file: path
        Path to OTF file to use in reconstruction

    Options
    -------
    ndirs: int (default is 3)
        number of directions in data
    nphases: int (default is 3)
        number of phases in data
    2lenses: bool
        data acquired with 2 opposing objectives
    bessel: bool
        data acquired with Bessel beam SIM
    fastSIM: bool
        data acquired with fast live SIM, i.e. data organized into (nz, ndirs, nphases)
    recalcarray: int (default is 1)
        how many times do you want to re-calculuate overlapping arrays
    inputapo: int
        number of pixels to apodize the input data (-1 to cosine apodize)
    forcemodamp: sequence of floats (f1 f2... f_norders)
        force the modulation amplitude to be f1 and f2
                If other than 3 phases are used, the -nphases flag must be used
                 BEFORE the -forcemodamp flag
    nok0search: bool
        do not want to search for the best k0
    nokz0: bool
        do not use kz0 plane in makeoverlaps() or assembly (for bad
    k0searchAll: bool
        search for k0 for every time point in a time series
    k0angles: sequence of floats (f0 f1... f_(ndirs-1))
        user supplied pattern angle list, the -ndirs flag must be used BEFORE
        the -k0angles flag
    fitonephase: bool
        in 2D NLSIM for orders > 1, modamp's phase will be order 1 phase
        multiplied by order; default is using fitted phases for all orders
    noapodizeout: bool (False)
        do not want to apodize the output data, the default is to use a cosine
        apodization function.
    gammaApo: float (None)
        apodize the output data with a power function
    preciseapo: bool (False)
        Apply precise apo or not
    zoomfact: float (4)
        factor by which to subdivide pixels laterally
    zzoom: float (1)
        factor by which to subdivide pixels axially
    zpadto: int (0)
        how many total sections after zero padding (this hasn't been implemented properly)
    explodefact: float
        factor by which to exaggerate the order shifts for display
    nofilteroverlaps: bool (default True)
        (Used with explodefact) leave orders round
        (no filtering the overlapped regions)
    nosuppress: bool
        do not want to suppress singularity at OTF origins
    suppressR: float (default is 10)
        the radius of suppression
    dampenOrder0: bool
        dampen order 0 in filterbands
    noOrder0: bool
        do not use order 0 in assembly
    noequalize: bool
        no equalization of input data
    equalizez: bool
        to equalize images of all z sections and directions
    equalizet: bool
        to equalize all time points based on the first one
    wiener: float (default 0.01)
        set wiener constant
    wienerInr: float (default is 0.00)
        wiener constant of the final time point will be wiener + this number
    background: float (default is 515)
        set the constant background intensity
    bgInExtHdr: bool
        the background of each section is recorded in the extended header's 3rd
        float number (in Lin's scope)
    otfRA: bool
        to use radially averaged OTFs
    driftfix: bool
        to estimate and then fix drift in 3D
    driftHPcutoff: float
        the cutoff frequency (fraction of lateral resolution limit) used in
        high-pass Gaussian filter in determining drift (default 0.0)
    fix2Ddrift: bool
        to correct 2D drifts for each exposure within a pattern direction
    fixphasestep: bool
        to correct phase used in separation matrix based on within-direction
        drift correction
    noff: int
        number of switch-off images in nonlinear SIM
    usecorr: path
        use correction file to do flatfielding
    nordersout: int
        the number of orders to use in reconstruction. Use if different from
        (n_phases+1)/2
    angle0: float (1.57193)
        the starting angle (in radians) of the patterns
    negDangle: bool (False)
        use negative angle step
    ls: float (0.177)
        the illumination pattern's line spacing (in microns)
    na: float (1.36)
        the (effective) na of the objective
    nimm: float (1.515)
        the index of refraction of the immersion liquid
    saveprefiltered: path
        save separated bands into file
    savealignedraw: path
        save drift-corrected raw images into file
    saveoverlaps: path
        save overlaps by makeoverlaps() into file
    """

    # make sure the paths are absolute paths
    input_file = os.path.abspath(input_file)
    output_file = os.path.abspath(output_file)
    otf_file = os.path.abspath(otf_file)
    for file in (input_file, otf_file):
        assert os.path.exists(file), "{} doesn't exist!".format(file)

    # wrap spaces so that paths can be found
    # input_file = '"{}"'.format(input_file)
    # output_file = '"{}"'.format(output_file)
    # otf_file = '"{}"'.format(otf_file)

    # the list to pass to subprocess.call, this is just the beginning
    # execute in powershell
    exc_list = [r"C:\SIMrecon_svn\sirecon", input_file, output_file, otf_file]

    # insert default values into **kwargs here
    valid_kwargs = OrderedDict.fromkeys(
        (
            "ndirs",
            "nphases",
            "2lenses",
            "bessel",
            "fastSIM",
            "recalcarray",
            "inputapo",
            "nordersout",
            "forcemodamp",
            "nok0search",
            "nokz0",
            "k0searchAll",
            "k0angles",
            "fitonephase",
            "noapodizeout",
            "gammaApo",
            "preciseapo",
            "zoomfact",
            "zzoom",
            "zpadto",
            "explodefact",
            "nofilteroverlaps",
            "nosuppress",
            "suppressR",
            "dampenOrder0",
            "noOrder0",
            "noequalize",
            "equalizez",
            "equalizet",
            "wiener",
            "wienerInr",
            "background",
            "bgInExtHdr",
            "otfRA",
            "driftfix",
            "driftHPcutoff",
            "fix2Ddrift",
            "fixphasestep",
            "noff",
            "usecorr",
            "angle0",
            "negDangle",
            "ls",
            "na",
            "nimm",
            "saveprefiltered",
            "savealignedraw",
            "saveoverlaps",
        )
    )
    numeric = (int, float)
    valid_kwargs.update(
        {
            "ndirs": int,
            "nphases": int,
            "phaselist": Sequence,
            "2lenses": bool,
            "bessel": bool,
            "fastSIM": bool,
            "recalcarray": int,
            "inputapo": int,
            "forcemodamp": Sequence,
            "nok0search": bool,
            "nokz0": bool,
            "k0searchAll": bool,
            "k0angles": Sequence,
            "fitonephase": bool,
            "noapodizeout": bool,
            "gammaApo": float,
            "preciseapo": bool,
            "zoomfact": numeric,
            "zzoom": float,
            "zpadto": int,
            "explodefact": float,
            "nofilteroverlaps": bool,
            "nosuppress": bool,
            "suppressR": numeric,
            "dampenOrder0": bool,
            "noOrder0": bool,
            "noequalize": bool,
            "equalizez": bool,
            "equalizet": bool,
            "wiener": float,
            "wienerInr": float,
            "background": numeric,
            "bgInExtHdr": bool,
            "otfRA": bool,
            "driftfix": bool,
            "driftHPcutoff": float,
            "fix2Ddrift": bool,
            "fixphasestep": bool,
            "noff": int,
            "usecorr": "path",
            "nordersout": int,
            "angle0": float,
            "negDangle": bool,
            "ls": float,
            "na": float,
            "nimm": float,
            "saveprefiltered": "path",
            "savealignedraw": "path",
            "saveoverlaps": "path",
        }
    )

    # update kwargs with those passed by user and generate the list.
    for k, kw_type in valid_kwargs.items():
        try:
            kw_value = kwargs[k]
        except KeyError:
            # user didn't pass this one, so skip
            pass
        else:
            # test validity
            if kw_type == "path":
                pass
            else:
                assert isinstance(
                    kw_value, kw_type
                ), "{} is type {} and should have been type {}".format(
                    k, type(kw_value), repr(kw_type)
                )
            if kw_type is not bool:
                exc_list.append("-" + k)
                # exc_list.append('-' + k)
                if isinstance(kw_value, Sequence) and not isinstance(kw_value, str):
                    for v in kw_value:
                        exc_list.append(formatter(v))
                else:
                    exc_list.append(formatter(kw_value))
            else:
                # in this case the key word is a bool
                # test if bool is true
                if kw_value:
                    exc_list.append("-" + k)

    # save the output
    return_code = subprocess.run(
        exc_list, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    if return_code.stderr:
        logger.error(return_code.stderr.decode())
        logger.error(" ".join(exc_list))
    else:
        logger.debug(" ".join(exc_list))
    return return_code.stdout.decode("utf-8").split("\n")

def formatter(value):
    if isinstance(value, float):
        return "{:.16f}".format(value)
    else:
        return str(value)

def write_mrc(input_file):
    raise NotImplementedError

def calc_radial_OTF(psf, krcutoff=None, lf_cutoff=None, width=3, show_OTF=False):
    """
    Calculate radially averaged OTF given a PSF and a cutoff value.

    This is designed to work well with Lin's SIMRecon software

    Parameters
    ----------
    psf: ndarray, 2-dim, real
        The psf from which to calculate the OTF
    krcutoff: int
        The diffraction limit in pixels.
    lf_cutoff: int
        The low frequency cutoff in pixels.
    width : int
        the half width of the linear fit window to minimize the DC
        component
    show_OTF : bool
        plot the OTF for inspection

    Returns
    -------
    radprof: ndarray, 1-dim, complex
        Radially averaged OTF
    """
    # assumes background has already been removed from PSF
    # recenter
    # TODO: add this part
    # fft, switch to rfft
    otf = fftshift(fftn(ifftshift(psf)))

    if show_OTF:
        from dphplotting import slice_plot
        from matplotlib.colors import LogNorm

        # this is still wrong, need to do the mean before summing
        # really we need a slice function.
        slice_plot(abs(otf), norm=LogNorm())

    if otf.ndim > 2:
        # if we have a 3D OTF collapse it by summation along kz into a 2D OTF.
        otf = otf.mean(0)

    center = np.array(otf.shape) / 2

    radprof, _ = radial_profile(otf)[: int(center[0] + 1)]
    if krcutoff is not None:
        # calculate mean phase angle of data within diffraction limit
        temp = radprof[:krcutoff]
        temp = temp / abs(temp)
        phi = np.angle(temp.mean())
        # remove mean phase angle
        radprof *= np.exp(-1j * phi)
        # set everything beyond the diffraction limit to 0
        radprof[krcutoff:] = 0
    if lf_cutoff:
        # if given a cutoff linearly fit the points around it to a line
        # then interpolate the line back to the origin starting at the low
        # frequency
        # choose the low frequency
        lf = lf_cutoff - width
        if lf > 0:
            # if the low frequency is higher than the DC component then
            # proceed with the fit, we definitely don't want to include
            # the DC
            hf = lf_cutoff + width
            m, b = np.polyfit(np.arange(lf, hf), radprof[lf:hf], 1)
            radprof[:lf_cutoff] = np.arange(0, lf_cutoff) * m + b
        else:
            # set DC to mid_num to mid_num
            radprof[:lf_cutoff] = radprof[lf_cutoff]
    radprof /= abs(radprof).max()
    # return
    return radprof

def crop_mrc(fullpath, window=None, extension="_cropped"):
    """
    Small utility to crop MRC files

    Parameters
    ----------
    fullpath : path
        path to file
    window : slice (optional)
        crop window

    Returns
    -------
    croppath : path
        path to cropped file
    """
    # open normal MRC file
    oldmrc = Mrc.Mrc(fullpath)
    old_data = oldmrc.data
    # make the crop path
    croppath = fullpath.replace(".mrc", extension + ".mrc")
    # crop window
    if window is None:
        nz, ny, nx = old_data.shape
        window = [slice(None, None, None)] + slice_maker(ny // 2, nx // 2, max(ny, nx) // 2)
    # prepare a new file to write to
    Mrc.save(old_data[window], croppath, ifExists="overwrite", hdr=oldmrc.hdr)
    # close the old MRC file.
    oldmrc.close()
    del oldmrc
    return croppath

# split, process, recombine functions

def split_img(img, side):
    """
    A utility to split a SIM stack into substacks
    """

    # Testing input
    divisor = img.shape[-1] // side
    # Error checking
    assert side == img.shape[-1] / divisor, "Side {}, not equal to {}/{}".format(
        side, img.shape[-1], divisor
    )
    assert img.shape[-2] == img.shape[-1]
    assert img.shape[-1] % divisor == 0

    # reshape array so that it's a tiled image
    img_s0 = img.reshape(-1, divisor, side, divisor, side)
    # roll one axis so that the tile's y, x coordinates are next to each other
    img_s1 = np.rollaxis(img_s0, -3, -1)
    # combine the tile's y, x coordinates into one axis.
    img_s2 = img_s1.reshape(-1, divisor ** 2, side, side)
    # roll axis so that we can easily iterate through tiles
    return np.rollaxis(img_s2, 1, 0)

def combine_img(img_stack):
    """
    A utility to combine a processed stack.
    """
    if img_stack.ndim < 4:
        # extend array
        img_stack = img_stack[:, None]
    num_sub_imgs, zlen, ylen, xlen = img_stack.shape
    assert xlen == ylen, "{} != {}".format(xlen, ylen)
    sqrt_num_sub_imgs = np.sqrt(num_sub_imgs)
    divisor = int(sqrt_num_sub_imgs)
    assert sqrt_num_sub_imgs == divisor, "{} != {}".format(sqrt_num_sub_imgs, divisor)

    # move z to front
    img_stack = np.rollaxis(img_stack, 1)
    return np.squeeze(
        np.rollaxis(img_stack.reshape(zlen, divisor, divisor, ylen, xlen), 2, 4).reshape(
            zlen, ylen * divisor, xlen * divisor
        )
    )

def split_img_with_padding(img, side, pad_width, mode="reflect"):
    """Split SIM stack into sub-stacks with padding of pad_width"""
    # if no padding revert to simpler function.
    if pad_width == 0:
        return split_img(img, side)
    # pull the shape of the image
    # NOTE: need to refactor this so that it works well with
    # nt, np, ny, nx waves
    # nall = img.shape[:-2]
    # ny, nx = img.shape[-2], img.shape[-1]
    ny, nx = img.shape[-2:]
    # make sure the sides are equal
    assert nx == ny, "Sides are not equal, {} != {}".format(nx, ny)
    # make sure that side cleanly divides img dimensions
    assert nx % side == 0, "Sides are not mutltiples of tile size, {} % {} != 0".format(nx, side)
    # pad the whole image
    # pad_img = fft_pad(img, nall + (pad_width + ny, pad_width + nx), mode)
    assert pad_width % 2 == 0, "Pad width needs to be even"
    half_pad = pad_width // 2
    padding = ((0, 0),) * (img.ndim - 2) + ((half_pad, half_pad),) * 2
    pad_img = np.pad(img, padding, mode)
    # split the image into padded sub-images
    # don't use an array, so that we can be memory efficient
    split_pad_img = [
        pad_img[..., j * side : pad_width + (j + 1) * side, i * side : pad_width + (i + 1) * side]
        for j in tqdm.tnrange(ny // side)
        for i in range(nx // side)
    ]
    # return this
    return split_pad_img

def cosine_edge(pad_size):
    """
    Generates a cosine squared edge

    When added to its reverse it equals 1

    Parameters
    ----------
    pad_size : int
        The size of the edge (i.e. the amount of image padding)

    Returns
    -------
    edge : ndarray (1D)
        The array representing the edge

    Example
    -------
    >>> edge = cosine_edge(10)
    >>> rev_edge = edge[::-1]
    >>> np.allclose(edge + rev_edge, np.ones_like(edge))
    True
    """
    x = np.arange(pad_size)
    return np.sin(np.pi * x / (pad_size - 1) / 2) ** 2

def linear_edge(pad):
    """
    Generates a linear edge

    When added to its reverse it equals 1

    Parameters
    ----------
    pad_size : int
        The size of the edge (i.e. the amount of image padding)

    Returns
    -------
    edge : ndarray (1D)
        The array representing the edge

    Example
    -------
    >>> edge = linear_edge(10)
    >>> rev_edge = edge[::-1]
    >>> np.allclose(edge + rev_edge, np.ones_like(edge))
    True
    """
    return np.arange(pad) / (pad - 1)

def edge_window(center_size, edge_size, window_func=cosine_edge):
    """
    Generate a 1D window that ramps up through the padded region and is flat in
    the middle

    Parameters
    ----------
    center_size : int
        The size of the center part
    edge_size : int
        The size of the edge parts

    Returns
    -------
    edge_window : ndarray (1D)
        a window with a rising and falling edge
    """
    center_part = np.ones(center_size)
    left_part = window_func(edge_size)
    right_part = left_part[::-1]
    return np.concatenate((left_part, center_part, right_part))

def extend_and_window_tile(tile, pad_size, tile_num, num_tiles, window_func=cosine_edge):
    """
    Function that takes a tile that has been padded and its tile number and
    places it in the correct space of the overall image and windows the
    function before padding with zeros

    Parameters
    ----------
    tile : ndarray (img)
        the tile to pad and window
    pad_size : int
        the amount of padding in the tile
    tile_num : int
        Assumes data is from a `split_img` operation, this is the index
    num_tiles : int
        total number of tiles
    window_func : cosine_edge (default, callable)
        the window function to use

    Returns
    -------
    tile : ndarray
        A tile that has been windowed and then extended to the appropriate size
    """
    # calculate the total number of tiles in the x and y directions
    # (assumes square image)
    ytot = xtot = int(np.sqrt(num_tiles))
    assert ytot * xtot == num_tiles, "Image is not square!"
    # calculate the position of this tile
    yn, xn = tile_num // ytot, tile_num % ytot
    # calculate the unpadded size of the tile (original tile size)
    tile_size = tile.shape[-1]
    to_pad = tile_size - pad_size
    # make y window and x window
    ywin = edge_window(to_pad - pad_size, pad_size, window_func=window_func)
    xwin = ywin.copy()
    # if the tile is on an edge, don't window the edge side(s)
    if yn == 0:
        ywin[:pad_size] = 1
    elif yn == ytot - 1:
        ywin[-pad_size:] = 1
    if xn == 0:
        xwin[:pad_size] = 1
    elif xn == xtot - 1:
        xwin[-pad_size:] = 1
    # generate the 2D window
    win_2d = ywin.reshape(-1, 1).dot(xwin.reshape(1, -1))
    # reshape window so that it works for 3D and for time
    # note that this still only tiles in 2D and not a full
    # 3D tiling.
    win_2d.shape = (1,) * (tile.ndim - 2) + win_2d.shape

    # calculate the before and after padding for each direction
    ybefore = yn * to_pad
    yafter = ybefore + tile_size
    xbefore = xn * to_pad
    xafter = xbefore + tile_size
    xslice = slice(xbefore, xafter)

    yslice = slice(ybefore, yafter)
    slices = (Ellipsis, yslice, xslice)
    return tile * win_2d, slices

def combine_img_with_padding(img_stack, pad_width):
    """Reverse of split_img_with_padding"""
    if pad_width == 0:
        return combine_img(img_stack)
    assert pad_width % 2 == 0, "Padding must be even"
    half_pad = pad_width // 2
    return combine_img(img_stack[..., half_pad:-half_pad, half_pad:-half_pad])

def combine_img_with_padding_window(recon_split_data, padding, window_func=cosine_edge, zoom=1):
    """Combine a tile stack when there's padding and a window involved"""
    # get number of tiles
    num_tiles = recon_split_data.shape[0]
    # figure out the new data size from inputs
    # get unpadded yx size of data
    newdata_shape = np.array(recon_split_data.shape[-2:]) - padding * zoom
    # find number of tiles per side
    newdata_shape *= int(np.sqrt(num_tiles))
    # then increasee by padding and potential zoom factor
    newdata_shape += padding * zoom

    if recon_split_data.ndim == 4:
        # 3D data
        newdata_shape = recon_split_data.shape[1:2] + tuple(newdata_shape)
    elif recon_split_data.ndim == 3:
        # 2D data, don't do anything.
        pass
    else:
        raise RuntimeError("Unexpected data shape = {}".format(recon_split_data.shape))

    to_combine_data = np.zeros(newdata_shape, dtype=float)

    for i, d in tqdm.tqdm_notebook(enumerate(recon_split_data), "Recombining", num_tiles, False):
        current_tile, slices = extend_and_window_tile(
            d, padding * zoom, i, num_tiles, window_func=window_func
        )
        to_combine_data[slices] += current_tile

    # we need to get rid of the reflected bits
    edge_pix = (padding // 2) * zoom
    slc = slice(edge_pix, -edge_pix, None)

    # cut them here.
    return to_combine_data[..., slc, slc]

def split_process_recombine(
    fullpath, tile_size, padding, sim_kwargs, bg_estimate=None, window_func=cosine_edge
):
    """
    Method that splits then processes and then recombines images

    Returns
    -------
    total_save_path, sirecon_ouput
    """
    assert tile_size >= padding, "Tile size must be smaller than padding"
    # make copy so internals don't change.
    sim_kwargs = sim_kwargs.copy()
    # save output name for later
    outname = sim_kwargs["output_file"]
    # make sure zoomfact is integer
    zoom = sim_kwargs["zoomfact"]
    assert zoom.is_integer(), "Zoomfact is not an interger, Abort!"
    zoom = int(zoom)
    # open old Mrc
    oldmrc = Mrc.Mrc(fullpath)
    # pull data
    old_data = oldmrc.data
    # generate hex hash, will use as ID
    sha = hashlib.md5(old_data).hexdigest()
    sim_kwargs["sha"] = sha
    # split the data
    split_data = split_img_with_padding(old_data, tile_size, padding)
    num_tiles = len(split_data)
    # estimate background
    if bg_estimate:
        bgs = {}
    # make temp directory to work in
    with tempfile.TemporaryDirectory(dir="F:/") as dir_name:
        # save split data
        sirecon_ouput = []

        @dask.delayed
        def save_process(data, savepath, sim_kwargs):
            Mrc.save(data, savepath, hdr=oldmrc.hdr, ifExists="overwrite")
            return simrecon(**sim_kwargs)

        for i, data in enumerate(tqdm.tqdm_notebook(split_data, "Splitting and saving data")):
            # save subimages in sub folder, use sha as ID
            savepath = os.path.join(dir_name, "sub_image{:06d}_{}.mrc".format(i, sha))

            if bg_estimate == "min":
                bgs[i] = data.min()
            elif bg_estimate == "median":
                bgs[i] = np.median(data)
            elif bg_estimate == "mode":
                bgs[i] = np.argmax(np.bincount(data.ravel()))

            # update the kwargs to have the input file.
            sim_kwargs.update(
                {"input_file": savepath, "output_file": savepath.replace(".mrc", "_proc.mrc")}
            )
            if bg_estimate:
                sim_kwargs["background"] = float(bgs[i])

            sirecon_ouput.append(save_process(data, savepath, sim_kwargs))

        with ProgressBar():
            # process files
            sirecon_ouput = list(itt.chain.from_iterable(dask.delayed(sirecon_ouput).compute()))
        # read in processed data
        processed_paths = sorted(glob.glob(dir_name + "/sub_image*_{}_proc.mrc".format(sha)))

        @dask.delayed
        def safe_read(path):
            """Catch errors for threaded read"""
            try:
                return Mrc.Mrc(path).data
            except ValueError:
                return None

        # read in data
        recon_split_data = dask.compute(*[safe_read(path) for path in processed_paths])
        valid_paths = [p for d, p in zip(recon_split_data, processed_paths) if d is not None]

        # Check all shapes
        all_shapes = set(map(np.shape, filter(lambda x: x is not None, recon_split_data)))
        if len(all_shapes) != 1:
            raise RuntimeError("Processing failed, not all shapes the same")

        shape = all_shapes.pop()

        # recombine data
        recon_split_data = np.asarray(
            [d if d is not None else np.zeros(shape, dtype="float32") for d in recon_split_data]
        ).squeeze()
        if window_func is None or padding == 0:
            # remember the data density is doubled so padding is too
            recon_split_data_combine = combine_img_with_padding(recon_split_data, padding * zoom)
        else:
            to_combine_data = combine_img_with_padding_window(
                recon_split_data, padding, window_func, zoom
            )
            recon_split_data_combine = to_combine_data.astype(np.float32)
            # make sure the new data is the right shape
            oldy, oldx = old_data.shape[-2:]
            newy, newx = recon_split_data_combine.shape[-2:]
            assert newx == oldx * zoom, "X-dim: {} != {}".format(newx, oldx * zoom)
            assert newy == oldy * zoom, "Y-dim: {} != {}".format(newy, oldy * zoom)

        # save data
        # path is the last tile we read in, it has all the relevant metadata
        temp_mrc = Mrc.Mrc(valid_paths[-1])
        extension = "_tile{}_pad{}.mrc".format(tile_size, padding)
        total_save_path = outname.replace(".mrc", extension)
        Mrc.save(
            recon_split_data_combine,
            os.path.abspath(total_save_path),
            hdr=temp_mrc.hdr,
            ifExists="overwrite",
        )
        # clean up
        oldmrc.close()
        del oldmrc
        temp_mrc.close()
        del temp_mrc

    # directory is killed automatically.

    return total_save_path, sirecon_ouput

def process_txt_output(txt_buffer):
    """Take output from above and parse into angles"""
    # ndirs is special
    ndir_re = re.compile(r"(?<=ndirs=)\d+", flags=re.M)
    my_dirs = set(re.findall(ndir_re, txt_buffer))
    assert len(my_dirs) == 1, "my_dirs = {}".format(my_dirs)
    ndirs = int(my_dirs.pop())
    # compile the regexes
    re_dict = dict(
        angle=r"(?:amplitude:\n In.*)(?<=angle=)(-?\d+\.\d+)",
        mag=r"(?:amplitude:\n In.*)(?<=mag=)(-?\d+\.\d+)",
        amp=r"(?:amplitude:\n In.*)(?<=amp=)(-?\d+\.\d+)",
        phase=r"(?:amplitude:\n In.*)(?<=phase=)(-?\d+\.\d+)",
        ramp=r"(?:amplitude:\n In.*\n Reverse.*)(?<=amp=)(-?\d+\.\d+)",
        camp=r"(?:amplitude:\n In.*\n.*\n Combined.*)(?<=amp=)(-?\d+\.\d+)",
        ccoef=r"(?:amplitude:\n In.*\n.*\n.*\n Correlation.*)(-?\d+\.\d+)",
    )
    re_dict = {k: re.compile(v, flags=re.M) for k, v in re_dict.items()}
    # parse output
    parse_dict = {k: np.array(re.findall(v, txt_buffer)).astype(float) for k, v in re_dict.items()}
    shapes = set(map(np.shape, parse_dict.values()))
    # find sizes
    assert len(shapes) == 1, "Shapes = {}".format(shapes)
    length = shapes.pop()[0]
    assert length != 0
    nx = ny = int(np.sqrt(length // ndirs))
    assert length == nx * ny * ndirs, "{} == {} * {} * {}".format(length, ndirs, nx, ny)
    # reshape all
    parse_dict = {k: v.reshape(nx, ny, ndirs) for k, v in parse_dict.items()}
    parse_dict = {k: np.swapaxes(v, 0, -1) for k, v in parse_dict.items()}
    return parse_dict

def plot_params(data_dict):
    """Plot tiled SIM parameters
    Angle, magnitude (spatial frequency in inverse microns), phase, modulation amplitude
    First three define a wavevector
    """
    mapping = {
        "angle": "Pattern Angle",
        "phase": "Pattern Phase",
        "mag": "Pattern Spacing (µm)",
        "amp": "Modulation Amplitude",
        "ccoef": "Correlation Coefficient",
    }

    # set up figure
    norients = len(data_dict["angle"])
    nparams = len(mapping)
    fig = plt.figure(dpi=150, figsize=(norients * 3, nparams * 3))
    grid = ImageGrid(
        fig,
        111,
        nrows_ncols=(nparams, norients),
        axes_pad=0.1,
        cbar_mode="edge",
        cbar_location="left",
    )

    keys = ("angle", "phase", "mag", "amp", "ccoef")

    # plotting
    for row, key, cbar in zip(grid.axes_row, keys, grid.cbar_axes):
        # pull data
        data = data_dict[key]

        if key == "mag":
            # flip to line spacing
            data = 1 / data

        # set up keywords
        kwds = dict(vmin=data.min(), vmax=data.max(), cmap="inferno")

        # specialize
        if "angle" == key:
            kwds = dict()
        elif "phase" == key:
            kwds = dict(vmin=data.min(), vmax=data.max(), cmap="coolwarm")

        for ax, d in zip(row, data):
            im = ax.matshow(d, **kwds)
            ax.xaxis.set_major_locator(plt.NullLocator())
            ax.yaxis.set_major_locator(plt.NullLocator())
            if key == "angle":
                ax.set_title("{:.1f} Degrees".format(np.rad2deg(np.median(d))))

        cbar.colorbar(im)
        cbar.set_ylabel(mapping[key])

        if "angle" == key:
            cbar.remove()
            grid[0].set_ylabel(mapping[key])

    return fig, grid

def stitch_img(infile, labelfile, outfile, num_threads=1):
    """
    Run the stitching algorithm

    More info can be found [here](https://github.com/mkazhdan/DMG)

    Parameters
    ----------
    infile : path
        Absolute path to the input file
    labelfile : path
        Absolute path to the label image
    outfile : path
        Absolute path to the output file
    num_threads : int (default, 1)
        The number of threads to run the problem across

    Returns
    -------
    client_return_code : string
        Output from client
    server_return_code : string
        Output from server
    """
    # choose a communication port not in use
    COM_PORT = "12345"
    # set up client execution
    client_exc_str = [
        r"C:\DMG\ClientSocket.exe",  # binary location
        "--address",
        "127.0.0.1",  # address for client, points HOME
        "--port",
        COM_PORT,  # port to communicate over
        "--labels",
        labelfile,  # where's the label file, path
        "--pixels",
        infile,  # where's the input file, path
        "--threads",
        str(num_threads),  # How many threads to use for computation
        "--inCore",  # tell algo to perform all computations in memory, __DO NOT__ stream to disk
        "--out",
        outfile,  # where do you want output
        "--hdr",  # tell algorithm to use full 16 bit depth
    ]
    # set up server execution
    server_exc_str = [
        r"C:\DMG\ServerSocket.exe",  # binary location
        "--count",
        "1",  # how many connections to expect
        "--port",
        COM_PORT,  # port to communicate over
        "--verbose",  # tell me what I'm doing
        "--tileExt",
        "tif",  # what file am I working with
        "--gray",  # gray scale images, not color
    ]
    # both programs need to run concurrently so
    with subprocess.Popen(server_exc_str, stdout=subprocess.PIPE) as server:
        return_code = subprocess.check_output(client_exc_str)
        client_return_code = return_code.decode()
        server_return_code = server.stdout.read().decode()

    return client_return_code, server_return_code

def stitch_tiled_sim_img(sim_img_path, tile_size=None):
    """
    A utility function to run the DMG stitching program on SIM data.
    """
    # determin tile_size from file name if not given
    if tile_size is None:
        # make a regex
        tile_re = re.compile(r"(?<=proc)\d+")
        # apply it, there should only be one occurance
        re_result = re.findall(tile_re, sim_img_path)
        assert len(re_result) == 1, "More than one Regex found."
        tile_size = int(re_result[0])
    # prep the image to stitch
    to_stitch_path = sim_img_path.replace(".mrc", ".tif")
    # open the Mrc
    junk_mrc = Mrc.Mrc(sim_img_path)
    # pull data
    data = junk_mrc.data[0]
    # save tif version while filling up the bit depth
    tif.imsave(to_stitch_path, scale_uint16(data))
    labels = make_label_img(data.shape[-1] // 2, tile_size)
    # kill Mrc
    del junk_mrc
    head, tail = os.path.split(sim_img_path)
    label_file = head + os.path.sep + "labels.tif"
    tif.imsave(label_file, labels)
    assert os.path.exists(label_file), label_file + " doesn't exist!"
    # prep outfile
    outfile = to_stitch_path.replace(".tif", "_stitch.tif")
    # stitch
    return_codes = stitch_img(to_stitch_path, label_file, outfile)
    return return_codes

def make_label_img(img_size, tile_size):
    # double tile size because SIM
    labels = np.array(
        [
            np.ones((tile_size * 2, tile_size * 2), np.uint16) * i
            for i in range((img_size // tile_size) ** 2)
        ]
    )
    return combine_img(labels)


from typing import List

def group_anagram(elements: List[str]) -> List[List[str]]:
    groups = {}
    for element in elements:
        sorted_order = ''.join(sorted(element))
        if sorted_order not in groups:
            groups[sorted_order] = [element]
        else:
            groups[sorted_order].append(element)
    return list(groups.values())

def test():
    print(group_anagram(['yo', 'oy', 'cat', 'tac', 'atc', 'flop', 'plof']))

if __name__ == '__main__':
    test()


import requests

response = requests.get('https://comic.naver.com/webtoon/weekday.nhn')
print(response.status_code)
# HTTP GET요청으로 받아온 Content를 text데이터로 리턴
print(response.text)
# response.text에 해당하는 데이터를
# weekday.html 이라는 파일에 기록
# 다 기록 했으면 close()호출

f = open('weekday.html', 'wt')
f.write(response.text)
f.close()


import os
import glob
from PIL import Image
from tqdm import tqdm

# voc_annotations = '/media/vs/qi/data/VOC2021/Annotations/'
# yolo_txt = '/media/data/darknet/labels/'    #darknet txt文件地址
# img_path = 'ssd.pytorch/data/VOCdevkit/VOC2021/JPEGImages/'   #图片地址

voc_annotations = '/media/vs/qi/data/ssd.pytorch/data/VOCdevkit/VOC2021/Annotations/'
yolo_txt = '/media/vs/qi/data/ssd.pytorch/data/VOCdevkit/VOC2021/labels/'
img_path = '/media/vs/qi/data/ssd.pytorch/data/VOCdevkit/VOC2021/JPEGImages/'

labels = ['no helmet', 'wear helmet']  # label for datasets

# 图像存储位置
src_img_dir = img_path  # 添加你的路径
# 图像的txt文件存放位置
src_txt_dir = yolo_txt
src_xml_dir = voc_annotations
img_Lists = glob.glob(src_img_dir + '/*.jpg')
img_basenames = []

for item in img_Lists:
    img_basenames.append(os.path.basename(item))

img_names = []
for item in img_basenames:
    temp1, temp2 = os.path.splitext(item)
    img_names.append(temp1)

for img in tqdm(img_names):
    im = Image.open((src_img_dir + '/' + img + '.jpg'))
    width, height = im.size

    # 打开txt文件
    gt = open(src_txt_dir + '/' + img + '.txt').read().splitlines()
    # print(gt)
    if gt:
        # 将主干部分写入xml文件中
        xml_file = open((src_xml_dir + '/' + img + '.xml'), 'w')
        xml_file.write('<annotation>\n')
        xml_file.write('    <folder>VOC2007</folder>\n')
        xml_file.write('    <filename>' + str(img) + '.jpg' + '</filename>\n')
        xml_file.write('    <size>\n')
        xml_file.write('        <width>' + str(width) + '</width>\n')
        xml_file.write('        <height>' + str(height) + '</height>\n')
        xml_file.write('        <depth>3</depth>\n')
        xml_file.write('    </size>\n')

        # write the region of image on xml file
        for img_each_label in gt:  # txt 文件中的每一行
            spt = img_each_label.split(' ')  # 这里如果txt里面是以逗号‘，’隔开的，那么就改为spt = img_each_label.split(',')。
            # print(f'spt:{spt}')
            xml_file.write('    <object>\n')
            xml_file.write('        <name>' + str(labels[int(spt[0])]) + '</name>\n')
            xml_file.write('        <pose>Unspecified</pose>\n')
            xml_file.write('        <truncated>0</truncated>\n')
            xml_file.write('        <difficult>0</difficult>\n')
            xml_file.write('        <bndbox>\n')

            center_x = round(float(spt[1].strip()) * width)
            center_y = round(float(spt[2].strip()) * height)
            bbox_width = round(float(spt[3].strip()) * width)
            bbox_height = round(float(spt[4].strip()) * height)
            xmin = str(int(center_x - bbox_width / 2))
            ymin = str(int(center_y - bbox_height / 2))
            xmax = str(int(center_x + bbox_width / 2))
            ymax = str(int(center_y + bbox_height / 2))

            xml_file.write('            <xmin>' + xmin + '</xmin>\n')
            xml_file.write('            <ymin>' + ymin + '</ymin>\n')
            xml_file.write('            <xmax>' + xmax + '</xmax>\n')
            xml_file.write('            <ymax>' + ymax + '</ymax>\n')
            xml_file.write('        </bndbox>\n')
            xml_file.write('    </object>\n')

        xml_file.write('</annotation>')


#importing libraries
import numpy as np
import pandas as pd

# Import Libraries for visualization
import matplotlib.pyplot as plt
import seaborn as sns

#importing data set using pandas
from google.colab import files
uploaded = files.upload()
import io
agri_data = pd.read_csv('C:/Users/Kummera Sneha/OneDrive/Desktop/csv.csv')

print(agri_data.shape)
agri_data.head()
agri_data.info()

c_data = agri_data.copy()
c_data.columns

#checking null value
c_data.isnull().sum()
price_data = c_data[['Price Date','Min Price (Rs./Quintal)','Max Price (Rs./Quintal)','Modal Price (Rs./Quintal)']]
#printing price data    TASK (A)
price_data
c_data
plt.figure(figsize=(10,5))
c_data.groupby(['Market Name'])['Modal Price (Rs./Quintal)'].count().sort_values(ascending=False).head(10).plot.bar()
plt.ylabel('Price of potatoes')
plt.title("Market comparision",fontsize=20)
plt.show()
#analyzing minimuim and maximuim value of potatoes in year
df1 = agri_data.copy()
df1.sort_values('Min Price (Rs./Quintal)',inplace=True)
df1.head(20)
df1.tail()
# Import libraries for train test split
from sklearn.model_selection import train_test_split

# import Ilbrary for Scaling
from sklearn.preprocessing import StandardScaler

# import Ilbrary for Model Building
from sklearn.linear_model import LinearRegression

#creating copy of data
ml_data = agri_data.copy()

#Applying mean encoding for Market name
name = ml_data.groupby('Market Name')['Modal Price (Rs./Quintal)'].mean()
ml_data['Market Name'] = ml_data['Market Name'].map(name)

print(ml_data['Market Name'])
#dropping unwanted columns
ml_data = ml_data.drop(['Sl no.','District Name','Commodity','Variety','Grade','Price Date'],axis = 1)
ml_data.info()
agri_train,agri_test = train_test_split(ml_data,train_size = 0.7,random_state = 42)

print(agri_train.shape)
print(agri_test.shape)
# Divide tarin set into Dependent and independent variables
y_train = agri_train.pop('Modal Price (Rs./Quintal)')

X_train = agri_train

# Divide test set into Dependent and independent variables
y_test = agri_test.pop('Modal Price (Rs./Quintal)')

X_test = agri_test

# Scale the train
scaler = StandardScaler()

X_train[X_train.columns] = scaler.fit_transform(X_train[X_train.columns])

X_train.describe()
# Scale the test
X_test[X_test.columns] = scaler.transform(X_test[X_test.columns])

X_test.describe()
# Build the model
lr = LinearRegression()

agri_reg = lr.fit(X_train,y_train)

# the r2 score
agri_reg.score(X_train,y_train)
# r2 for test data
agri_reg.score(X_test,y_test)
# Plot Distribution plot of Residuals
plt.figure(figsize=(10,5))
y_train_pred = agri_reg.predict(X_train)
res = y_train - y_train_pred
sns.distplot(res)
plt.xlabel('Residuals')
plt.title("Residual Analysis",fontsize=20)
plt.show()
sns.scatterplot(x=res,y=y_train_pred)
plt.xlabel('Residuals')
plt.title("Residual Analysis",fontsize=20)
plt.show()
# Print coef
print("Coef are:",agri_reg.coef_)

#print intercept
print("Intercept is",agri_reg.intercept_)
model = str(agri_reg.intercept_)

for i in range(len(agri_reg.coef_)):
    model = model +' + '  +(str(agri_reg.coef_[i])) + ' * ' +(str(X_train.columns[i]))
print(model)


from tkinter import *
# import the necessary packages
from imutils.video import VideoStream
from source.shapedetector import ShapeDetector
import numpy as np
import argparse
import cv2
import imutils
from PIL import Image
from PIL import ImageTk
import threading
import sqlite3 as sq

class camera:
    def __init__(self, window, vs, pts, layar):

        self.panel = None
        self.foto = layar
        self.frame2 = window
        self.vs = vs
        self.pts = pts

        self.stopEvent = threading.Event()
        self.thread = threading.Thread(target=self.videoLoop, args=())
        self.thread.start()

        current_row = 0
        #create a label for databases entry
        self.judul_label = Label(self.frame2, text="Judul: ", font=18)
        self.judul_label.grid(row=current_row, column=0)
        self.judul_text = StringVar()
        self.judul_entry = Entry(self.frame2, textvariable=self.judul_text, font=18)
        self.judul_entry.grid(row=current_row, column=1)
        current_row += 1

        self.pengarang_label = Label(self.frame2, text="Pengarang: ", font=18)
        self.pengarang_label.grid(row=current_row, column=0)
        self.pengarang_text = StringVar()
        self.pengarang_entry = Entry(self.frame2, textvariable=self.pengarang_text, font=18)
        self.pengarang_entry.grid(row=current_row, column=1)
        current_row += 1

        self.genre_label = Label(self.frame2, text="Genre: ", font=18)
        self.genre_label.grid(row=current_row, column=0)
        self.genre_text = StringVar()
        self.genre_entry = Entry(self.frame2, textvariable=self.genre_text, font=18)
        self.genre_entry.grid(row=current_row, column=1)
        current_row += 1

        self.tahun_label = Label(self.frame2, text="Tahun: ", font=18)
        self.tahun_label.grid(row=current_row, column=0)
        self.tahun_text = StringVar()
        self.tahun_entry = Entry(self.frame2, textvariable=self.tahun_text, font=18)
        self.tahun_entry.grid(row=current_row, column=1)
        current_row += 1

        self.penerbit_label = Label(self.frame2, text="Penerbit: ", font=18)
        self.penerbit_label.grid(row=current_row, column=0)
        self.penerbit_text = StringVar()
        self.penerbit_entry = Entry(self.frame2, textvariable=self.penerbit_text, font=18)
        self.penerbit_entry.grid(row=current_row, column=1)
        current_row += 1

        self.start = Button(self.frame2, text="Run Again", command=self.resume)
        self.start.grid(row=current_row, column=0)
        self.frame2.pack(expand="yes", side="left", fill='y')

    def videoLoop(self):
        # construct the argument parse and parse the arguments
        ap = argparse.ArgumentParser()
        ap.add_argument("-v", "--video",
                        help="path to the (optional) video file")
        ap.add_argument("-b", "--buffer", type=int, default=64,
                        help="max buffer size")
        args = vars(ap.parse_args())

        # define the lower and upper boundaries of the "green"
        # ball in the HSV color space, then initialize the
        # list of tracked points
        lower = {'blue': (110, 100, 100), 'red': (-10, 100, 100), 'green': (40, 70, 70),
                 'yellow': (20, 100, 117)}  # assign new item lower['blue'] = (93, 10, 0)
        upper = {'blue': (130, 255, 255), 'red': (10, 255, 255), 'green': (70, 255, 255), 'yellow': (40, 255, 255)}

        # database session
        con = sq.connect('data_comvis.db')  # dB browser for sqlite needed
        con.text_factory = str
        cmd_db = con.cursor()  # SQLite command, to connect to db so 'execute' method can be called

        # make a call for ShapeDetector class
        sd = ShapeDetector()

        stopEvent = threading.Event()

        # keep looping
        while not self.stopEvent.is_set():
            # grab the current frame
            self.foto = self.vs.read()
            # handle the frame from VideoCapture or VideoStream
            # resize the frame, blur it, and convert it to the HSV
            # color space
            self.frame = imutils.resize(self.foto, width=300)
            blurred = cv2.GaussianBlur(self.frame, (11, 11), 0)
            hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV)
            # construct a mask for the color "green", then perform
            # a series of dilations and erosions to remove any small
            # blobs left in the mask

            for key, value in upper.items():
                mask = cv2.inRange(hsv, lower[key], upper[key])
                mask = cv2.erode(mask, None, iterations=2)
                mask = cv2.dilate(mask, None, iterations=2)

                # find contours in the mask and initialize the current
                # (x, y) center of the ball
                cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
                                        cv2.CHAIN_APPROX_SIMPLE)
                cnts = imutils.grab_contours(cnts)
                center = None
                # only proceed if at least one contour was found
                if len(cnts) > 0:
                    # find the largest contour in the mask, then use
                    # it to compute the minimum enclosing circle and
                    # centroid
                    c = max(cnts, key=cv2.contourArea)
                    (x, y, w, h) = cv2.boundingRect(c)
                    M = cv2.moments(c)
                    center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
                    # only proceed if the radius meets a minimum size
                    if (x + y + w + h) > 400:
                        # Do when reach the minimum contours length
                        shape = sd.detect(c)
                        # draw the circle and centroid on the frame,
                        # then update the list of tracked points
                        cv2.putText(self.frame, key, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 2)
                        cv2.drawContours(self.frame, [c], -1, (0, 0, 0), 2)

                        cv2.putText(self.frame, shape, (x + 15, y + 15), cv2.FONT_HERSHEY_SIMPLEX,
                                    0.5, (0, 0, 0), 2)
                        # cv2.circle(frame, (int(x), int(y)), int(radius),
                        #           (0, 255, 255), 2)
                        # cv2.circle(frame, center, 5, (0, 0, 255), -1)
                        # Gets data from database
                        text = key + " " + shape
                        plus = 'FROM buku INNER JOIN kode_buku ON buku.id = kode_buku.id WHERE kode_buku.kode = "' + text + '"'
                        cmd_db.execute('SELECT * ' + plus)

                        data = cmd_db.fetchall()  # Gets the data from the table

                        for row in data:
                            self.judul_text.set(row[1])
                            self.pengarang_text.set(row[2])
                            self.genre_text.set(row[3])
                            self.tahun_text.set(row[4])
                            self.penerbit_text.set(row[5])
                            self.penerbit_text.set(row[6])

                        con.commit()
                        self.stopEvent.set()
                # update the points queue
                # pts.appendleft(center)

            # loop over the set of tracked points
            for i in range(1, len(self.pts)):
                # if either of the tracked points are None, ignore
                # them
                if self.pts[i - 1] is None or self.pts[i] is None:
                    continue
                # otherwise, compute the thickness of the line and
                # draw the connecting lines
                thickness = int(np.sqrt(args["buffer"] / float(i + 1)) * 2.5)
                cv2.line(self.frame, self.pts[i - 1], self.pts[i], (0, 0, 255), thickness)
            # show the frame to our screen

            # Show image in GUI
            #self.frame = cv2.cvtColor(self.frame, cv2.COLOR_HSV2BGR)
            #self.frame = cv2.cvtColor(self.frame, cv2.COLOR_BGR2RGB)
            self.frame = Image.fromarray(self.frame)
            self.frame = ImageTk.PhotoImage(self.frame)
            #cv2.imshow("Frame", self.frame)
            #key = cv2.waitKey(1) & 0xFF

            # if the panel is not None, we need to initialize it
            if self.panel is None:
                self.panel = Label(image=self.frame)
                self.panel.image = self.frame
                self.panel.pack(side="left", padx=10, pady=10)

            # otherwise, simply update the panel
            else:
                self.panel.configure(image=self.frame)
                self.panel.image = self.frame

    def resume(self):
        # Resume the Thread
        kosong = ""
        self.judul_text.set(kosong)
        self.pengarang_text.set(kosong)
        self.genre_text.set(kosong)
        self.tahun_text.set(kosong)
        self.penerbit_text.set(kosong)
        self.penerbit_text.set(kosong)

        self.stopEvent = threading.Event()
        self.thread = threading.Thread(target=self.videoLoop, args=())
        self.thread.start()


import os
from celery import Celery

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cycle.settings')

app = Celery('cycle')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
app.conf.update(result_expires=3600, enable_utc=True, timezone='UTC')
app.conf.beat_schedule = {}
app.conf.timezone = 'UTC'


"""
Utility funcions for plotting response analysis

Authors: Dylan Paiton, Santiago Cadena
"""

import os
import sys

import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import matplotlib.ticker as plticker
from matplotlib.offsetbox import (OffsetImage, AnnotationBbox)
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
import numpy as np
import proplot as pro

ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),'..','..'))
if ROOT_DIR not in sys.path: sys.path.append(ROOT_DIR)

import response_contour_analysis.utils.dataset_generation as data_utils

def clear_axis(ax, spines='none'):
    for ax_loc in ['top', 'bottom', 'left', 'right']:
        ax.spines[ax_loc].set_color(spines)
    ax.set_yticklabels([])
    ax.set_xticklabels([])
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    ax.tick_params(axis='both', bottom=False, top=False, left=False, right=False)
    return ax

def set_size(width, fraction=1, subplot=[1, 1]):
    """
    Set aesthetic figure dimensions to avoid scaling in latex.
    Parameters:
        width: float
                Width in pts
        fraction: float
                Fraction of the width which you wish the figure to occupy
    Returns:
      fig_dim: tuple
              Dimensions of figure in inches
    Usage:
        figsize = set_size(text_width, fraction=1, subplot=[1, 1])
    Code obtained from: https://jwalton.info/Embed-Publication-Matplotlib-Latex/
    """
    fig_width_pt = width * fraction # Width of figure
    inches_per_pt = 1 / 72.27 # Convert from pt to inches
    golden_ratio = (5**.5 - 1) / 2 # Golden ratio to set aesthetic figure height
    fig_width_in = fig_width_pt * inches_per_pt # Figure width in inches
    fig_height_in = fig_width_in * golden_ratio * (subplot[0] / subplot[1]) # Figure height in inches
    fig_dim = (fig_width_in, fig_height_in) # Final figure dimensions
    return fig_dim

def plot_group_iso_contours(analysis_dict, num_levels, show_contours=True, targets_comparisons=[2,3], text_width=200, width_fraction=1.0, dpi=100):
    arrow_width = 0.0
    arrow_linewidth = 1
    arrow_headsize = 0.15
    arrow_head_length = 0.15
    arrow_head_width = 0.15
    gs0_hspace = 0.5
    gs0_wspace = -0.6
    phi_k_text_x_offset = 0.6 / width_fraction
    phi_k_text_y_offset = -1.2 / width_fraction
    phi_j_text_x_offset = 0.9 / width_fraction
    phi_j_text_y_offset = 0.3 / width_fraction
    nu_text_x_offset = -0.56 / width_fraction
    nu_text_y_offset = 0.3 / width_fraction
    num_plots_y = targets_comparisons[0] # num target neurons
    num_plots_x = targets_comparisons[1] # num comparison planes
    gs0 = gridspec.GridSpec(num_plots_y, num_plots_x, wspace=gs0_wspace, hspace=gs0_hspace)
    vmin = np.min(analysis_dict['activations'])
    vmax = np.max(analysis_dict['activations'])
    levels = np.linspace(vmin, vmax, num_levels)
    cmap = plt.get_cmap('cividis')
    cNorm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
    scalarMap = mpl.cm.ScalarMappable(norm=cNorm, cmap=cmap)
    fig = plt.figure(figsize=set_size(text_width, width_fraction, [num_plots_y, num_plots_x]), dpi=dpi)
    contour_handles = []
    curve_axes = []
    for plot_id in np.ndindex((num_plots_y, num_plots_x)):
        (y_id, x_id) = plot_id
        neuron_index = y_id
        orth_index = x_id
        inner_gs = gridspec.GridSpecFromSubplotSpec(1, 1, gs0[plot_id])
        curve_axes.append(clear_axis(fig.add_subplot(inner_gs[0])))
        #curve_axes[-1].set_title(str(plot_id)) # TODO: change title
        # plot colored mesh points
        norm_activity = analysis_dict['activations'][neuron_index, orth_index, ...]
        x_mesh, y_mesh = np.meshgrid(
            analysis_dict['contour_dataset']['x_pts'],
            analysis_dict['contour_dataset']['y_pts'])
        if show_contours:
            contsf = curve_axes[-1].contourf(x_mesh, y_mesh, norm_activity,
                levels=levels, vmin=vmin, vmax=vmax, alpha=1.0, antialiased=True, cmap=cmap)
        else:
            contsf = curve_axes[-1].scatter(x_mesh, y_mesh,
                vmin=vmin, vmax=vmax, cmap=cmap, marker='s', alpha=1.0, c=norm_activity, s=30.0)
        contour_handles.append(contsf)
        # plot target neuron arrow & label
        proj_target = analysis_dict['contour_dataset']['proj_target_vect'][neuron_index][orth_index]
        target_vector_x = proj_target[0].item()
        target_vector_y = proj_target[1].item()
        curve_axes[-1].arrow(0, 0, target_vector_x, target_vector_y,
            width=arrow_width, head_width=arrow_head_width, head_length=arrow_head_length,
            fc='k', ec='k', linestyle='-', linewidth=arrow_linewidth)
        tenth_range_shift = ((max(analysis_dict['x_range']) - min(analysis_dict['x_range']))/10) # For shifting labels
        text_handle = curve_axes[-1].text(
            target_vector_x+(tenth_range_shift*phi_k_text_x_offset),
            target_vector_y+(tenth_range_shift*phi_k_text_y_offset),
            r'$\Phi_{k}$', horizontalalignment='center', verticalalignment='center')
        # plot comparison neuron arrow & label
        proj_comparison = analysis_dict['contour_dataset']['proj_comparison_vect'][neuron_index][orth_index]
        comparison_vector_x = proj_comparison[0].item()
        comparison_vector_y = proj_comparison[1].item()
        curve_axes[-1].arrow(0, 0, comparison_vector_x, comparison_vector_y,
            width=arrow_width, head_width=arrow_head_width, head_length=arrow_head_length,
            fc='k', ec='k', linestyle='-', linewidth=arrow_linewidth)
        text_handle = curve_axes[-1].text(
            comparison_vector_x+(tenth_range_shift*phi_j_text_x_offset),
            comparison_vector_y+(tenth_range_shift*phi_j_text_y_offset),
            r'$\Phi_{j}$', horizontalalignment='center', verticalalignment='center')
        # Plot orthogonal vector Nu
        proj_orth = analysis_dict['contour_dataset']['proj_orth_vect'][neuron_index][orth_index]
        orth_vector_x = proj_orth[0].item()
        orth_vector_y = proj_orth[1].item()
        curve_axes[-1].arrow(0, 0, orth_vector_x, orth_vector_y,
            width=arrow_width, head_width=arrow_head_width, head_length=arrow_head_length,
            fc='k', ec='k', linestyle='-', linewidth=arrow_linewidth)
        text_handle = curve_axes[-1].text(
            orth_vector_x+(tenth_range_shift*nu_text_x_offset),
            orth_vector_y+(tenth_range_shift*nu_text_y_offset),
            r'$\nu$', horizontalalignment='center', verticalalignment='center')
        # Plot axes
        curve_axes[-1].set_aspect('equal')
        curve_axes[-1].plot(analysis_dict['x_range'], [0,0], color='k', linewidth=arrow_linewidth/2)
        curve_axes[-1].plot([0,0], analysis_dict['y_range'], color='k', linewidth=arrow_linewidth/2)
    # Add colorbar
    scalarMap._A = []
    cbar_ax = inset_axes(curve_axes[-1],
        width='5%',
        height='100%',
        loc='lower left',
        bbox_to_anchor=(1.05, 0., 1, 1),
        bbox_transform=curve_axes[-1].transAxes,
        borderpad=0,
        )
    cbar = fig.colorbar(scalarMap, cax=cbar_ax, ticks=[vmin, vmax])
    cbar.ax.tick_params(labelleft=False, labelright=True, left=False, right=True)
    cbar.ax.set_yticklabels(['{:.0f}'.format(vmin), '{:.0f}'.format(vmax)])
    plt.show()
    return fig, contour_handles

def plot_curvature_histograms(hist_list, label_list, color_list, bin_centers, title, xlabel,
                              text_width=200, width_ratio=1.0, dpi=100):
    """
    hist_list [nested list of floats] containing histogram values that is indexed by
        [curvature type]
        [dataset type]
        [target neuron id]
        [num_bins - 1]
    label_list [nested list of strings] containing labels that shares its structure with hist_list
    color_list [nested list of strings] containing color hash values that shares its structure with hist_list
    bin_centers [nested list of floats] containing bin center values that is indexed by
        [curvature type]
        [num_bins]
    title [string]
    xlabel [string]
    """
    gs0_wspace = 0.5
    hspace_hist = 0.7
    wspace_hist = 0.10
    num_y_plots = 1
    num_x_plots = 1
    fig = plt.figure(figsize=set_size(text_width, width_ratio, [num_y_plots, num_x_plots]), dpi=dpi)
    gs_base = gridspec.GridSpec(num_y_plots, num_x_plots, wspace=gs0_wspace)
    num_hist_y_plots = 2
    num_hist_x_plots = 2
    gs_hist = gridspec.GridSpecFromSubplotSpec(num_hist_y_plots, num_hist_x_plots, gs_base[0],
        hspace=hspace_hist, wspace=wspace_hist)
    orig_ax = fig.add_subplot(gs_hist[0,0])
    axes = []
    for sub_plt_y in range(0, num_hist_y_plots):
        axes.append([])
        for sub_plt_x in range(0, num_hist_x_plots):
            if (sub_plt_x, sub_plt_y) == (0,0):
                axes[sub_plt_y].append(orig_ax)
            else:
                axes[sub_plt_y].append(fig.add_subplot(gs_hist[sub_plt_y, sub_plt_x], sharey=orig_ax))
    all_x_lists = zip(hist_list, label_list, color_list, bin_centers, title)
    for axis_x, (type_hist, sub_label, sub_color, sub_bins, sub_title) in enumerate(all_x_lists):
        max_hist_val = 0.001
        min_hist_val = 100
        all_y_lists = zip(type_hist, sub_label, sub_color, xlabel)
        for axis_y, (dataset_hist, axis_labels, axis_colors, sub_xlabel) in enumerate(all_y_lists):
            axes[axis_y][axis_x].spines['top'].set_visible(False)
            axes[axis_y][axis_x].spines['right'].set_visible(False)
            axes[axis_y][axis_x].set_xticks(sub_bins, minor=True)
            axes[axis_y][axis_x].set_xticks(sub_bins[::int(len(sub_bins)/4)], minor=False)
            axes[axis_y][axis_x].xaxis.set_major_formatter(plticker.FormatStrFormatter('%0.3f'))
            for hist, label, color in zip(dataset_hist, axis_labels, axis_colors):
                axes[axis_y][axis_x].plot(sub_bins, hist, color=color, linestyle='-',
                    drawstyle='steps-mid', label=label)
                axes[axis_y][axis_x].set_yscale('log')
                if np.max(hist) > max_hist_val:
                    max_hist_val = np.max(hist)
                if np.min(hist) < min_hist_val:
                    min_hist_val = np.min(hist)
            axes[axis_y][axis_x].axvline(0.0, color='black', linestyle='dashed', linewidth=1)
            if axis_y == 0:
                axes[axis_y][axis_x].set_title(sub_title)
            axes[axis_y][axis_x].set_xlabel(sub_xlabel)
            if axis_x == 0:
                axes[axis_y][axis_x].set_ylabel('Relative\nFrequency')
                ax_handles, ax_labels = axes[axis_y][axis_x].get_legend_handles_labels()
                legend = axes[axis_y][axis_x].legend(handles=ax_handles, labels=ax_labels,
                    loc='upper right', ncol=1, borderaxespad=0., borderpad=0.,
                    handlelength=0., columnspacing=-0.5, labelspacing=0., bbox_to_anchor=(0.95, 0.95))
                legend.get_frame().set_linewidth(0.0)
                for text, color in zip(legend.get_texts(), axis_colors):
                    text.set_color(color)
                for item in legend.legendHandles:
                    item.set_visible(False)
            if axis_x == 1:
                axes[axis_y][axis_x].tick_params(axis='y', labelleft=False)
        #axes[axis_y][axis_x].set_ylim([0, 1.0])
    plt.show()
    return fig

def add_arrow(ax, vect, xrange, yx_offset=[1,1], linestyle='-', label='', text_color='k'):
    """
    adds an arrow to a given plot, defined by the vect input
    ax [matplotlib axis]
    vect [list] with entries [vect_x, vect_y]
    xrange [float] range of x values, to be used to rescale
    yx_offset [list of a pair of ints] offset label from arrow
    linestyle [str] same as matplotlib kwarg
    label [str] label to be given to the arrow
    text_color [str] same as color matplotlib kwarg
    """
    arrow_width = 0.0
    arrow_linewidth = 1
    arrow_headsize = 0.15
    arrow_head_length = 0.15
    arrow_head_width = 0.15
    vect_x = vect[0].item()
    vect_y = vect[1].item()
    ax.arrow(0, 0, vect_x, vect_y,
        width=arrow_width, head_width=arrow_head_width, head_length=arrow_head_length,
        fc='k', ec='k', linestyle=linestyle, linewidth=arrow_linewidth, length_includes_head=True)
    tenth_range_shift = xrange / 10 # For shifting labels
    text_handle = ax.text(
        vect_x + (tenth_range_shift * yx_offset[1]),
        vect_y + (tenth_range_shift * yx_offset[0]),
        label,
        weight='bold',
        color=text_color,
        horizontalalignment='center',
        verticalalignment='center')

def plot_contours(ax, activity, yx_pts, yx_range, proj_vects=None, num_levels=10, contours=None, fits=None, cmap='cividis', vlim=None, title=''):
    """
    ax [matplotlib axis]
    activity [numpy ndarray] neuron activation values
    yx_pts [list] returned from dataset_genration utilities
    yx_range [list] containing [y_range, x_range], returned from dataset_generation utilities
    proj_vects [list or None] if not None then plot target, comparison, and orthogonal vectors
    num_levels [int] number of contour levels
    contours [list of datapoints or None] if it is not None then the exctracted target contour points will be included
    fits [list of datapoints or None] if it is not None then the polynomial fit points will be included
    cmap [str] matplotlib kwarg
    vlim [tuple] containing (vmin, vmax)
    title [str] plot title
    """
    if vlim is None:
        vmin = np.min(activity)
        vmax = np.max(activity)
    else:
        vmin, vmax = vlim
    cmap = plt.get_cmap(cmap)
    cNorm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
    scalarMap = mpl.cm.ScalarMappable(norm=cNorm, cmap=cmap)
    # Plot contours
    x_mesh, y_mesh = np.meshgrid(*yx_pts[::-1])
    levels = np.linspace(vmin, vmax, num_levels)
    contsf = ax.contourf(x_mesh, y_mesh, activity,
        levels=levels, vmin=vmin, vmax=vmax, antialiased=True, cmap=cmap)#, alpha=1.0)
    if proj_vects is not None:
        # Add arrows
        proj_target = proj_vects[0]
        xrange = max(yx_range[1]) - min(yx_range[1])
        add_arrow(ax, proj_target, xrange, linestyle='-')
        proj_comparison = proj_vects[1]
        add_arrow(ax, proj_comparison, xrange, linestyle='--')
        proj_orth = proj_vects[2]
        add_arrow(ax, proj_orth, xrange, linestyle='-')
    # Add axis grid
    ax.set_aspect('equal')
    ax.plot(yx_range[1], [0,0], color='k', linewidth=0.5) # vertical
    ax.plot([0,0], yx_range[0], color='k', linewidth=0.5) # horizontal
    if contours is not None:
        ax.scatter(contours[0], contours[1], s=4, color='r')
    if fits is not None:
        ax.scatter(fits[0], fits[1], s=3, marker='*', color='k')
    ax.format(
        ylim=[np.min(yx_pts[0]), np.max(yx_pts[0])],
        xlim=[np.min(yx_pts[1]), np.max(yx_pts[1])],
        title=title
    )
    return contsf

def overlay_image(ax, images, y_pos, x_pos, yx_range, offset, arrowprops=None, cmap=None, vmin=None, vmax=None):
    """
    ax [matplotlib axis]
    images [np.ndarray] of shape [num_images_per_edge, num_images_per_edge, channels, height, weidth] images for each mesh point
    """
    num_images_per_edge = images.shape[0]
    image_y_pos = data_utils.remap_axis_index_to_dataset_index(
        axis_index=y_pos,
        axis_min=yx_range[0][0],
        axis_max=yx_range[0][1],
        num_images=num_images_per_edge)
    image_x_pos = data_utils.remap_axis_index_to_dataset_index(
        axis_index=x_pos,
        axis_min=yx_range[1][0],
        axis_max=yx_range[1][1],
        num_images=num_images_per_edge)
    height_ratio = images.shape[3] / np.maximum(*images.shape[3:])
    width_ratio = images.shape[4] / np.maximum(*images.shape[3:])
    inset_height = 0.35 * height_ratio
    inset_width = 0.35 * width_ratio
    arr_img = np.squeeze(images[image_y_pos, image_x_pos, ...].transpose((1,2,0)))
    imagebox = OffsetImage(arr_img, zoom=0.75, cmap=cmap)
    img = imagebox.get_children()[0]
    img.set_clim(vmin=vmin, vmax=vmax)
    imagebox.image.axes = ax
    if arrowprops is None:
        arrowprops = dict(
            linestyle='--',
            lw=1,
            arrowstyle='->',
            color='r'
        )
    ab = AnnotationBbox(imagebox,
        xy=[x_pos, y_pos],
        xybox=offset,
        xycoords=ax.transData,
        boxcoords='offset points',
        pad=0.0,
        arrowprops=arrowprops
    )
    ax.add_artist(ab)
    return arr_img


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import itertools

import numpy as np
import tensorflow as tf
from system import ODESystem
from spaces import GRF

import deepxde as dde
from utils import merge_values, trim_to_65535, mean_squared_error_outlier, safe_test

lr = 0.001
epochs = 50000
problem = "ode"
nn = "opnn"
activation = "relu"
initializer = "Glorot normal"  # "He normal" or "Glorot normal"
dim_x = 1 if problem in ["ode", "lt"] else 2

net = dde.maps.OpNN(
    [100, 40, 40],
    [1, 40, 40],
    activation,
    initializer,
    use_bias=True,
    stacked=False,
    )
dat = np.loadtxt('data_12.dat')
dat = dat[1:100001,:]
y = np.array([ele[-1] for ele in dat])
u = np.array([ele[0:-2] for ele in dat])
x = np.array([ele[-2] for ele in dat])
index = np.random.choice(100000,100000)
train_index = index[0:10000]
test_index = index[10000:100000]
x_train = x[train_index][:,None]
x_test = x[test_index][:,None]
y_train = y[train_index][:,None]
y_test = y[test_index][:,None]
u_train = u[train_index,:]
u_test = u[test_index,:]
x_train = [u_train,x_train]
x_test = [u_test,x_test]
#print(x_train.shape)
#np.savetxt("x_train.dat",x_train)
data = dde.data.OpDataSet(
            X_train=x_train, y_train=y_train, X_test=x_test, y_test=y_test
        )
model = dde.Model(data, net)
model.compile("adam", lr=lr, metrics=[mean_squared_error_outlier])

checker = dde.callbacks.ModelCheckpoint(
        "model/model.ckpt", save_better_only=True, period=1000
    )
losshistory, train_state = model.train(epochs=epochs, callbacks=[checker])
print("# Parameters:", np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]))
dde.saveplot(losshistory, train_state, issave=True, isplot=False)
model.restore("model/model.ckpt-" + str(train_state.best_step), verbose=1)
# model.restore(f"model/model-{train_state.best_step}.ckpt", verbose=1)
safe_test(model, data, x_test, y_test)

### TEST
def ode_system(T,ic):
    """ODE"""

    def g(s,u,x):
        # Antiderivative
        # return u
        # Nonlinear ODE
        # return -s**2 + u
        # Gravity pendulum
        # k = 1
        # return [s[1], - k * np.sin(s[0]) + u]
        # Rober Problem
        k1,k2,k3 = 4, 3, 1
        return [(- k1 * s[0] + k3 * s[1] * s[2]) ,
                (k1 * s[0] - k2 * s[1] * s[1] - k3 * s[1] * s[2]) ,
                (k2 * s[1] * s[1]),
                u]

    # s0 = [0]
    s0 = ic  # Gravity pendulum
    return ODESystem(g, s0, T)
T = 1
m = 100
sensors = np.linspace(0, T, num=m)[:, None]
space = GRF(T, length_scale=0.2, N=1000 * T, interp="cubic")
ics = np.column_stack((np.array([1,0.5,0.7,0.9]),
                        np.array([0,0.5,0.2,0.2]),
                        np.array([0,0.5,0.1,0.2]),
                        np.array([0,0,0,0])))
system = ode_system(T,ics[0])
a, y1_test = system.gen_sensors_data(space,1,100,0)
a, y2_test = system.gen_sensors_data(space,1,100,1)
X_test = [np.tile(y1_test.T,(100,1)),sensors]
y2_pred = model.predict(data.transform_inputs(X_test))
np.savetxt("testPred.dat", np.hstack((sensors, y2_test, y2_pred)))
print("L2relative error:", dde.metrics.l2_relative_error(y2_test, y2_pred))


from flask import Flask, request, make_response
from urllib import parse
import requests
import re

# definitions
server_addr = "127.0.0.1"
server_port = "5000"

app = Flask(__name__)

# fix links
def fix_html(html_str):
    new_str = re.sub("https:\/\/", "/https://", html_str)
    new_str = re.sub("http:\/\/", "/http://", new_str)
    return new_str

# get request data
def fetch_url(r):
    if r.encoding and r.encoding.lower() == "utf-8":
        return fix_html(r.text).encode("utf-8")
    else:
        return r.content

# main
@app.route("/<path:url>", methods=["GET", "POST"])
def proxy(url):
    # handle the proxy url cookie
    if request.cookies.get("py_rproxy_url"):
        rproxy_urls = request.cookies.get("py_rproxy_url").split()
    else:
        rproxy_urls = []

    if url.startswith("http://") or url.startswith("https://"):
        new_url = url
        if request.method == "GET":
            req = requests.get(new_url, cookies=request.cookies)
        elif request.method == "POST":
            req = requests.post(new_url, request.form, cookies=request.cookies)
        parsed_url = parse.urlparse(req.url).scheme + "://" + parse.urlparse(req.url).netloc
        if parsed_url in rproxy_urls:
            del rproxy_urls[rproxy_urls.index(parsed_url)]
        rproxy_urls.append(parsed_url)
    else:
        # handle local urls passed to the server
        if request.cookies.get("py_rproxy_url"):
            # indexer
            i = -1
            # check if we can use url per each subdomain
            while True:
                new_url = rproxy_urls[i] + "/" + url
                if request.method == "GET":
                    req = requests.get(new_url, cookies=request.cookies)
                elif request.method == "POST":
                    req = requests.post(new_url, request.form, cookies=request.cookies)
                i = i - 1
                if ( req.status_code>=200 and req.status_code<300 ) or abs(i) >= len(rproxy_urls):
                    break

    resp = make_response(fetch_url(req), req.status_code)

    # auto set cookies
    for cookie in req.cookies.keys():
        resp.set_cookie(cookie, req.cookies[cookie])

    # handle normal urls passed to the server
    resp.set_cookie("py_rproxy_url", " ".join(rproxy_urls))

    # auto set header
    #for header in req.headers.keys():
    #    resp.headers[header] = req.headers[header]

    # fix up some headers I don't want
    #del resp.headers["X-Frame-Options"]
    #resp.headers["Host"] = server_addr + ":" + server_port

    resp.headers["Content-Type"] = req.headers["content-type"]

    return resp

# handle root
@app.route("/", methods=["GET", "POST"])
def proxy_root():
    return proxy("/")

if __name__ == "__main__":
    app.run(host=server_addr, port=server_port, threaded=True)