import os, sys
import glob
import shutil

from mic import msger
from mic.utils import fs_related, rpmmisc, runner, misc
from mic.utils.errors import CreatorError
from mauimic import kickstart
from mauimic.fs import *
from mauimic.imager.loop import LoopImageCreator

class LiveImageCreatorBase(LoopImageCreator):
    """A base class for LiveCD image creators.

        This class serves as a base class for the architecture-specific LiveCD
        image creator subclass, LiveImageCreator.

        LiveImageCreator creates a bootable ISO containing the system image,
        bootloader, bootloader configuration, kernel and initramfs.
    """

    def __init__(self, creatoropts=None, pkgmgr=None,
                 title="Linux", product="Linux"):
        """Initialise a LiveImageCreator instance.

           This method takes the same arguments as ImageCreator.__init__().
        """
        LoopImageCreator.__init__(self, creatoropts, pkgmgr)

        # Controls whether to use squashfs to compress the image.
        self.skip_compression = False

        # Controls whether an image minimizing snapshot should be created.
        #
        # This snapshot can be used when copying the system image from the ISO in
        # order to minimize the amount of data that needs to be copied; simply,
        # it makes it possible to create a version of the image's filesystem with
        # no spare space.
        self.skip_minimize = False

        # A flag which indicates i act as a convertor default false
        self.actasconvertor = False

        # The bootloader timeout from kickstart
        if self.ks:
            self._timeout = kickstart.get_timeout(self.ks, 10)
        else:
            self._timeout = 10

        # The default kernel type from kickstart
        if self.ks:
            self._default_kernel = kickstart.get_default_kernel(self.ks,
                                                                "kernel")
        else:
            self._default_kernel = None

        if self.ks:
            parts = kickstart.get_partitions(self.ks)
            if len(parts) > 1:
                raise CreatorError("Can't support multi partitions in ks file "
                                   "for this image type")
            # FIXME: rename rootfs img to self.name,
            # else can't find files when create iso
            self._instloops[0]['name'] = self.name + ".img"

        self.__isodir = None

        self.__modules = ["=ata", "sym53c8xx", "aic7xxx", "=usb", "=firewire",
                          "=mmc", "=pcmcia", "mptsas", "udf", "virtio_blk",
                          "virtio_pci", "virtio_scsi", "virtio_net", "virtio_mmio",
                          "virtio_balloon", "virtio-rng"]
        if self.ks:
            self.__modules.extend(kickstart.get_modules(self.ks))

        self._dep_checks.extend(["isohybrid",
                                 "unsquashfs",
                                 "mksquashfs",
                                 "dd",
                                 "genisoimage"])

        self._isohybrid = True

        self._isofstype = "iso9660"
        self.base_on = False

        self.title = title
        self.product = product

    #
    # Hooks for subclasses
    #
    def _configure_bootloader(self, isodir):
        """Create the architecture specific booloader configuration.

            This is the hook where subclasses must create the booloader
            configuration in order to allow a bootable ISO to be built.

            isodir -- the directory where the contents of the ISO are to
                      be staged
        """
        raise CreatorError("Bootloader configuration is arch-specific, "
                           "but not implemented for this arch!")

    def _get_menu_options(self):
        """Return a menu options string for syslinux configuration.
        """
        if self.ks is None:
            return "liveinst autoinst"
        r = kickstart.get_menu_args(self.ks)
        return r

    def _get_kernel_options(self):
        """Return a kernel options string for bootloader configuration.

            This is the hook where subclasses may specify a set of kernel
            options which should be included in the images bootloader
            configuration.

            A sensible default implementation is provided.
        """

        r = kickstart.get_kernel_args(self.ks)

        if "splash" not in r:
            for path in ("/usr/bin/plymouth", "/usr/bin/ply-image"):
                if os.path.exists(self._instroot + path):
                    r += " splash"
                    break

        return r

    def _get_mkisofs_options(self, isodir):
        """Return the architecture specific mkisosfs options.

            This is the hook where subclasses may specify additional arguments
            to mkisofs, e.g. to enable a bootable ISO to be built.

            By default, an empty list is returned.
        """
        return []

    #
    # Helpers for subclasses
    #
    def _has_checkisomd5(self):
        """Check whether checkisomd5 is available in the install root."""
        def exists(instroot, path):
            return os.path.exists(instroot + path)

        if (exists(self._instroot, "/usr/lib/anaconda-runtime/checkisomd5") or
            exists(self._instroot, "/usr/bin/checkisomd5")):
            return True

        return False

    def _mount_instroot(self, base_on = None):
        LoopImageCreator._mount_instroot(self, base_on)
        self.__write_initrd_conf(self._instroot + "/etc/sysconfig/mkinitrd")
        self.__write_dracut_conf(self._instroot + "/etc/dracut.conf.d/02livecd.conf")

    def _unmount_instroot(self):
        self.__restore_file(self._instroot + "/etc/sysconfig/mkinitrd")
        self.__restore_file(self._instroot + "/etc/dracut.conf.d/02livecd.conf")
        LoopImageCreator._unmount_instroot(self)

    def __ensure_isodir(self):
        if self.__isodir is None:
            self.__isodir = self._mkdtemp("iso-")
        return self.__isodir

    def _get_isodir(self):
        return self.__ensure_isodir()

    def _set_isodir(self, isodir = None):
        self.__isodir = isodir

    def _create_bootconfig(self):
        """Configure the image so that it's bootable."""
        self._configure_bootloader(self.__ensure_isodir())

    def _get_post_scripts_env(self, in_chroot):
        env = LoopImageCreator._get_post_scripts_env(self, in_chroot)

        if not in_chroot:
            env["LIVE_ROOT"] = self.__ensure_isodir()

        return env

    def __extra_filesystems(self):
        return "vfat msdos isofs ext4 xfs btrfs"

    def __extra_drivers(self):
        retval = "sr_mod sd_mod ide-cd cdrom "
        for module in self.__modules:
            if module == "=usb":
                retval = retval + "ehci_hcd uhci_hcd ohci_hcd "
                retval = retval + "usb_storage usbhid "
            elif module == "=firewire":
                retval = retval + "firewire-sbp2 firewire-ohci "
                retval = retval + "sbp2 ohci1394 ieee1394 "
            elif module == "=mmc":
                retval = retval + "mmc_block sdhci sdhci-pci "
            elif module == "=pcmcia":
                retval = retval + "pata_pcmcia "
            else:
                retval = retval + module + " "
        return retval

    def __restore_file(self,path):
        try:
            os.unlink(path)
        except:
            pass
        if os.path.exists(path + '.rpmnew'):
            os.rename(path + '.rpmnew', path)

    def __write_initrd_conf(self, path):
        if not os.path.exists(os.path.dirname(path)):
            makedirs(os.path.dirname(path))
        f = open(path, "a")
        f.write('LIVEOS="yes"\n')
        f.write('PROBE="no"\n')
        f.write('MODULES+="' + self.__extra_filesystems() + '"\n')
        f.write('MODULES+="' + self.__extra_drivers() + '"\n')
        f.close()

    def __write_dracut_conf(self, path):
        # XXX: Hack! Mer doesn't have /lib64 because it doesn't support
        # mixing 32-bit and 64-bit binaries but the lack of /lib64 is a
        # problem for us because dracut won't find ld-linux-x86_64.so
        # without it. With this hack we link /lib64 to /lib until
        # we updated the filesystem package.
        if self.target_arch == "x86_64":
            os.symlink("lib", self._instroot + "/lib64")

        if not os.path.exists(os.path.dirname(path)):
            makedirs(os.path.dirname(path))
        f = open(path, "a")
        f.write('filesystems+="' + self.__extra_filesystems() + ' "\n')
        f.write('drivers+="' + self.__extra_drivers() + ' "\n')
        f.write('add_dracutmodules+=" dmsquash-live pollcdrom "\n')
        f.write('hostonly="no"\n')
        f.write('dracut_rescue_image="no"\n')
        f.close()

    def __create_iso(self, isodir):
        iso = self._outdir + "/" + self.name + ".iso"
        genisoimage = fs_related.find_binary_path("genisoimage")
        args = [genisoimage,
                "-J", "-r",
                "-hide-rr-moved", "-hide-joliet-trans-tbl",
                "-V", self.fslabel,
                "-o", iso]

        args.extend(self._get_mkisofs_options(isodir))

        args.append(isodir)

        if runner.show(args) != 0:
            raise CreatorError("ISO creation failed!")

        """ It should be ok still even if you haven't isohybrid """
        isohybrid = None
        if self._isohybrid:
            try:
                isohybrid = fs_related.find_binary_path("isohybrid")
                msger.info("isohybrid found")
            except:
                msger.warning("isohybrid NOT found")

        if isohybrid:
            args = [isohybrid, "-partok", iso ]
            if runner.show(args) != 0:
             	raise CreatorError("Hybrid ISO creation failed!")
            else:
                msger.info("Hybrid ISO created successfully")

        self.__implant_md5sum(iso)

    def __implant_md5sum(self, iso):
        """Implant an isomd5sum."""
        if os.path.exists("/usr/bin/implantisomd5"):
            implantisomd5 = "/usr/bin/implantisomd5"
        elif os.path.exists("/usr/lib/anaconda-runtime/implantisomd5"):
            implantisomd5 = "/usr/lib/anaconda-runtime/implantisomd5"
        else:
            msger.warning("isomd5sum not installed; not setting up mediacheck")
            implantisomd5 = ""
            return

        runner.show([implantisomd5, iso])

    def _stage_final_image(self):
        try:
            fs_related.makedirs(self.__ensure_isodir() + "/LiveOS")

            minimal_size = self._resparse()

            if not self.skip_minimize:
                fs_related.create_image_minimizer(self.__isodir + \
                                                      "/LiveOS/osmin.img",
                                                  self._image,
                                                  minimal_size)

            if self.skip_compression:
                shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img")
            else:
                fs_related.makedirs(os.path.join(
                                        os.path.dirname(self._image),
                                        "LiveOS"))
                shutil.move(self._image,
                            os.path.join(os.path.dirname(self._image),
                                         "LiveOS", "ext3fs.img"))
                fs_related.mksquashfs(os.path.dirname(self._image),
                           self.__isodir + "/LiveOS/squashfs.img")

            self.__create_iso(self.__isodir)

            if self.pack_to:
                isoimg = os.path.join(self._outdir, self.name + ".iso")
                packimg = os.path.join(self._outdir, self.pack_to)
                misc.packing(packimg, isoimg)
                os.unlink(isoimg)

        finally:
            shutil.rmtree(self.__isodir, ignore_errors = True)
            self.__isodir = None

class x86LiveImageCreator(LiveImageCreatorBase):
    """ImageCreator for x86 machines"""
    def __init__(self, *args, **kwargs):
        LiveImageCreatorBase.__init__(self, *args, **kwargs)
        self._efiarch = None
        self._isolinux_debug = False
        self._isohybrid = not self._isolinux_debug

    def _get_mkisofs_options(self, isodir):
        isolinuxbin = "isolinux/isolinux.bin"
        if self._isolinux_debug:
            isolinuxbin = "isolinux/isolinux-debug.bin"
        return [ "-b", isolinuxbin,
                 "-c", "isolinux/boot.cat",
                 "-no-emul-boot", "-boot-info-table",
                 "-boot-load-size", "4" ]

    def _get_required_packages(self):
        return ["syslinux", "syslinux-extlinux"] + \
               LiveImageCreatorBase._get_required_packages(self)

    def _get_isolinux_stanzas(self, isodir):
        return ""

    def __find_syslinux_menu(self):
        for menu in ["vesamenu.c32", "menu.c32"]:
            for dir in ("/usr/lib/syslinux/", "/usr/share/syslinux/"):
                if os.path.isfile(self._instroot + dir + menu):
                    return menu

        raise CreatorError("syslinux not installed : "
                           "no suitable *menu.c32 found")

    def __find_syslinux_mboot(self):
        #
        # We only need the mboot module if we have any xen hypervisors
        #
        if not glob.glob(self._instroot + "/boot/xen.gz*"):
            return None

        return "mboot.c32"

    def __copy_syslinux_files(self, isodir, menu, mboot = None):
        files = ["isolinux.bin", "ldlinux.c32", "libcom32.c32", "libutil.c32", menu]
        if self._isolinux_debug:
            files += ["isolinux-debug.bin"]
        if mboot:
            files += [mboot]

        for f in files:
            if os.path.exists(self._instroot + "/usr/lib/syslinux/" + f):
                path = self._instroot + "/usr/lib/syslinux/" + f
            elif os.path.exists(self._instroot + "/usr/share/syslinux/" + f):
                path = self._instroot + "/usr/share/syslinux/" + f
            if not os.path.isfile(path):
                raise CreatorError("syslinux not installed : "
                                   "%s not found" % path)

            shutil.copy(path, isodir + "/isolinux/")

    def __copy_syslinux_background(self, isodest):
        files = ["/usr/share/pixmaps/syslinux-splash.png",
                 "/usr/share/anaconda/boot/syslinux-vesa-splash.png",
                 "/usr/lib/anaconda-runtime/syslinux-vesa-splash.png"]

        for f in files:
            if os.path.exists(self._instroot + f):
                shutil.copyfile(self._instroot + f, isodest)
                return True

        return False

    def __copy_kernel_and_initramfs(self, isodir, version, index):
        bootdir = self._instroot + "/boot"

        shutil.copyfile(bootdir + "/vmlinuz-" + version,
                        isodir + "/isolinux/vmlinuz" + index)

        isDracut = False
        if os.path.exists(bootdir + "/initramfs-" + version + ".img"):
            shutil.copyfile(bootdir + "/initramfs-" + version + ".img",
                            isodir + "/isolinux/initrd" + index + ".img")
            isDracut = True
        elif os.path.exists(bootdir + "/initrd-" + version + ".img"):
            shutil.copyfile(bootdir + "/initrd-" + version + ".img",
                            isodir + "/isolinux/initrd" + index + ".img")
        else:
            msger.error("No initrd or initramfs found for %s" % (version,))

        is_xen = False
        if os.path.exists(bootdir + "/xen.gz-" + version[:-3]):
            shutil.copyfile(bootdir + "/xen.gz-" + version[:-3],
                            isodir + "/isolinux/xen" + index + ".gz")
            is_xen = True

        return (is_xen, isDracut)

    def __is_default_kernel(self, kernel, kernels):
        if len(kernels) == 1:
            return True

        if kernel == self._default_kernel:
            return True

        if kernel.startswith("kernel-") and kernel[7:] == self._default_kernel:
            return True

        return False

    def __get_basic_syslinux_config(self, **args):
        return """
default %(menu)s
timeout %(timeout)d

menu background %(background)s
menu autoboot Starting %(title)s in # second{,s}. Press any key to interrupt.

menu clear
menu title %(title)s
menu width 78
menu margin 4
menu rows 7
menu vshift 10
menu tabmsgrow 14
menu cmdlinerow 14
menu helpmsgrow 16
menu helpmsgendrow 29

# Refer to http://syslinux.zytor.com/wiki/index.php/Doc/menu

menu color border * #00000000 #00000000 none
menu color sel 0 #ff3a6496 #00000000 none
menu color title 0 #ff7ba3d0 #00000000 none
menu color tabmsg 0 #ff3a6496 #00000000 none
menu color unsel 0 #ff347ead #00000000 none
menu color hotsel 0 #ff64b0ea #00000000 none
menu color hotkey 0 #ffffffff #00000000 none
menu color help 0 #993677bc #00000000 none
menu color scrollbar 0 #ffffffff #ff355594 none
menu color timeout 0 #ff999999 #00000000 none
menu color timeout_msg 0 #ff444b54 #00000000 none
menu color cmdmark 0 #844bb2e5 #00000000 none
menu color cmdline 0 #ffffffff #00000000 none

menu tabmsg Press Tab for full configuration options on menu items.
""" % args

    def __get_image_stanza(self, is_xen, isDracut, **args):
        if isDracut:
            args["rootlabel"] = "live:CDLABEL=%(fslabel)s" % args
        else:
            args["rootlabel"] = "CDLABEL=%(fslabel)s" % args

        if not is_xen:
            template = """label %(short)s
  menu label %(long)s
  kernel vmlinuz%(index)s
  append initrd=initrd%(index)s.img root=%(rootlabel)s rootfstype=%(isofstype)s %(liveargs)s %(extra)s
"""
        else:
            template = """label %(short)s
  menu label %(long)s
  kernel mboot.c32
  append xen%(index)s.gz --- vmlinuz%(index)s root=%(rootlabel)s rootfstype=%(isofstype)s %(liveargs)s %(extra)s --- initrd%(index)s.img
"""

        if args.get("help"):
            template += """  text help
      %(help)s
  endtext
"""

        return template % args

    def __get_image_stanzas(self, isodir):
        kernels = self._get_kernel_versions()
        kernel_options = self._get_kernel_options()
        checkisomd5 = self._has_checkisomd5()

        # Stanzas for insertion into the config template
        linux = []
        basic = []
        check = []

        index = "0"
        for kernel, version in ((k,v) for k in kernels for v in kernels[k]):
            (is_xen, isDracut) = self.__copy_kernel_and_initramfs(isodir, version, index)
            if index == "0":
                self._isDracut = isDracut

            default = self.__is_default_kernel(kernel, kernels)

            if default:
                long = self.product
            elif kernel.startswith("kernel-"):
                long = "%s (%s)" % (self.product, kernel[7:])
            else:
                long = "%s (%s)" % (self.product, kernel)

            # tell dracut not to ask for LUKS passwords or activate mdraid sets
            if isDracut:
                kern_opts = kernel_options + " rd.luks=0 rd.md=0 rd.dm=0"
            else:
                kern_opts = kernel_options

            linux.append(self.__get_image_stanza(is_xen, isDracut,
                                           fslabel = self.fslabel,
                                           isofstype = "auto",
                                           liveargs = kern_opts,
                                           long = "^Start " + long,
                                           short = "linux" + index,
                                           extra = "",
                                           help = "",
                                           index = index))

            if default:
                linux[-1] += "  menu default\n"

            basic.append(self.__get_image_stanza(is_xen, isDracut,
                                           fslabel = self.fslabel,
                                           isofstype = "auto",
                                           liveargs = kern_opts,
                                           long = "Start " + long + " in ^basic graphics mode.",
                                           short = "basic" + index,
                                           extra = "nomodeset",
                                           help = "Try this option out if you're having trouble starting.",
                                           index = index))

            if checkisomd5:
                check.append(self.__get_image_stanza(is_xen, isDracut,
                                               fslabel = self.fslabel,
                                               isofstype = "auto",
                                               liveargs = kern_opts,
                                               long = "^Test this media & start " + long,
                                               short = "check" + index,
                                               extra = "rd.live.check",
                                               help = "",
                                               index = index))
            else:
                check.append(None)

            index = str(int(index) + 1)

        return (linux, basic, check)

    def __get_memtest_stanza(self, isodir):
        memtest = glob.glob(self._instroot + "/boot/memtest86*")
        if not memtest:
            return ""

        shutil.copyfile(memtest[0], isodir + "/isolinux/memtest")

        return """label memtest
  menu label Run a ^memory test
  text help
    If your system is having issues, an problem with your
    system's memory may be the cause. Use this utility to
    see if the memory is working correctly.
  endtext
  kernel memtest
"""

    def __get_local_stanza(self, isodir):
        return """label local
  menu label Boot from ^local drive
  localboot 0xffff
"""

    def _configure_syslinux_bootloader(self, isodir):
        """configure the boot loader"""
        makedirs(isodir + "/isolinux")

        menu = self.__find_syslinux_menu()

        self.__copy_syslinux_files(isodir, menu,
                                   self.__find_syslinux_mboot())

        background = ""
        if self.__copy_syslinux_background(isodir + "/isolinux/splash.png"):
            background = "splash.png"

        cfg = self.__get_basic_syslinux_config(menu = menu,
                                               background = background,
                                               title = self.title,
                                               timeout = self._timeout * 10)
        cfg += "menu separator\n"

        linux, basic, check = self.__get_image_stanzas(isodir)
        # Add linux stanzas to main menu
        for s in linux:
            cfg += s
        cfg += "menu separator\n"

        cfg += """menu begin ^Troubleshooting
  menu title Troubleshooting
"""
        # Add basic video and check to submenu
        for b, c in zip(basic, check):
            cfg += b
            if c:
                cfg += c

        cfg += self.__get_memtest_stanza(isodir)
        cfg += "menu separator\n"

        cfg += self.__get_local_stanza(isodir)
        cfg += self._get_isolinux_stanzas(isodir)

        cfg += """menu separator
label returntomain
  menu label Return to ^main menu.
  menu exit
menu end
"""
        cfgf = open(isodir + "/isolinux/isolinux.cfg", "w")
        cfgf.write(cfg)
        cfgf.close()

    @property
    def efiarch(self):
        if not self._efiarch:
            # for most things, we want them named boot$efiarch
            efiarch = {"i386": "IA32", "x86_64": "X64"}
            self._efiarch = efiarch[rpmmisc.getBaseArch()]
        return self._efiarch

    def __copy_efi_files(self, isodir):
        """ Copy the efi files into /EFI/BOOT/
            If any of them are missing, return False.
            requires:
              shim.efi
              gcdx64.efi
              fonts/unicode.pf2
        """
        fail = False
        missing = []
        files = [("/boot/efi/EFI/*/shim.efi", "/EFI/BOOT/BOOT%s.efi" % (self.efiarch,)),
                 ("/boot/efi/EFI/*/gcdx64.efi", "/EFI/BOOT/grubx64.efi"),
                 ("/boot/efi/EFI/*/fonts/unicode.pf2", "/EFI/BOOT/fonts/"),
                ]
        makedirs(isodir+"/EFI/BOOT/fonts/")
        for src, dest in files:
            src_glob = glob.glob(self._instroot + src)
            if not src_glob:
                missing.append("Missing EFI file (%s)" % (src,))
                fail = True
            else:
                shutil.copy(src_glob[0], isodir+dest)
        map(msger.error, missing)
        return fail

    def __get_basic_efi_config(self, **args):
        return """
set default="1"

function load_video {
  insmod efi_gop
  insmod efi_uga
  insmod video_bochs
  insmod video_cirrus
  insmod all_video
}

load_video
set gfxpayload=keep
insmod gzio
insmod part_gpt
insmod ext2

set timeout=%(timeout)d
### END /etc/grub.d/00_header ###

search --no-floppy --set=root -l '%(isolabel)s'

### BEGIN /etc/grub.d/10_linux ###
""" %args

    def __get_efi_image_stanza(self, **args):
        if self._isDracut:
            args["rootlabel"] = "live:LABEL=%(fslabel)s" % args
        else:
            args["rootlabel"] = "CDLABEL=%(fslabel)s" % args
        return """menuentry '%(long)s' --class fedora --class gnu-linux --class gnu --class os {
	linuxefi /isolinux/vmlinuz%(index)s root=%(rootlabel)s %(liveargs)s %(extra)s
	initrdefi /isolinux/initrd%(index)s.img
}
""" %args

    def __get_efi_image_stanzas(self, isodir, name):
        # FIXME: this only supports one kernel right now...

        kernel_options = self._get_kernel_options()
        checkisomd5 = self._has_checkisomd5()

        cfg = ""

        for index in range(0, 9):
            # we don't support xen kernels
            if os.path.exists("%s/EFI/BOOT/xen%d.gz" %(isodir, index)):
                continue
            cfg += self.__get_efi_image_stanza(fslabel = self.fslabel,
                                               liveargs = kernel_options,
                                               long = "Start " + self.product,
                                               extra = "", index = index)
            if checkisomd5:
                cfg += self.__get_efi_image_stanza(fslabel = self.fslabel,
                                                   liveargs = kernel_options,
                                                   long = "Test this media & start " + self.product,
                                                   extra = "rd.live.check",
                                                   index = index)
            cfg += """
submenu 'Troubleshooting -->' {
"""
            cfg += self.__get_efi_image_stanza(fslabel = self.fslabel,
                                               liveargs = kernel_options,
                                               long = "Start " + self.product + " in basic graphics mode",
                                               extra = "nomodeset", index = index)

            cfg+= """}
"""
            break

        return cfg

    def _configure_efi_bootloader(self, isodir):
        """Set up the configuration for an EFI bootloader"""
        if self.__copy_efi_files(isodir):
            shutil.rmtree(isodir + "/EFI")
            msger.warning("Failed to copy EFI files, no EFI Support will be included.")
            return

        cfg = self.__get_basic_efi_config(isolabel = self.fslabel,
                                          timeout = self._timeout)
        cfg += self.__get_efi_image_stanzas(isodir, self.name)

        cfgf = open(isodir + "/EFI/BOOT/grub.cfg", "w")
        cfgf.write(cfg)
        cfgf.close()

        # first gen mactel machines get the bootloader name wrong apparently
        if rpmUtils.arch.getBaseArch() == "i386":
            os.link(isodir + "/EFI/BOOT/BOOT%s.efi" % (self.efiarch),
                    isodir + "/EFI/BOOT/BOOT.efi")

    def _configure_bootloader(self, isodir):
        self._configure_syslinux_bootloader(isodir)
        # TODO: Enable EFI configuration when we actually have grub2
        #self._configure_efi_bootloader(isodir)

arch = rpmmisc.getBaseArch()
if arch in ("i386", "x86_64"):
    LiveCDImageCreator = x86LiveImageCreator
elif arch.startswith("arm"):
    LiveCDImageCreator = LiveImageCreatorBase

else:
    raise CreatorError("Architecture not supported!")


import logging
import os
import numpy as np
import torch
from tqdm import tqdm, trange
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import AdamW, get_linear_schedule_with_warmup
from sklearn.metrics import classification_report
from task_ner.processor import real_labels
from utils.util import set_seed, flatten
from models.model import save_model
import warnings

warnings.filterwarnings("ignore")
logger = logging.getLogger(__name__)

def train(args, train_dataset, eval_dataset, model):
    train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
    t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs

    no_decay = ['bias', 'LayerNorm.weight', 'transitions']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
         'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

    logger.info("***** 训练中 *****")
    logger.info("  Num examples = %d", len(train_dataset))
    logger.info("  Num Epochs = %d", args.num_train_epochs)
    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    logger.info("  Total optimization steps = %d", t_total)

    global_step = 0
    tr_loss, logging_loss = 0.0, 0.0
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc="Epoch")
    set_seed(args)
    best_f1 = 0.
    for _ in train_iterator:
        model.train()
        epoch_iterator = tqdm(train_dataloader, desc="Iteration")
        for step, batch in enumerate(epoch_iterator):
            batch = tuple(t.to(args.device) for t in batch)
            inputs = {'input_ids': batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2],
                      'labels': batch[3]}
            outputs = model(**inputs)
            loss, pre_tag = outputs[0], outputs[1]

            if args.gradient_accumulation_steps > 1:
                loss = loss / args.gradient_accumulation_steps
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
            logging_loss += loss.item()
            tr_loss += loss.item()
            if 0 == (step + 1) % args.gradient_accumulation_steps:
                optimizer.step()
                scheduler.step()
                model.zero_grad()
                global_step += 1
                logger.info("EPOCH [%d/%d] global_step=%d   loss=%f", _+1, args.num_train_epochs, global_step, logging_loss)
                logging_loss = 0.0

                # 每相隔 args.save_steps 步，评估一次
                if global_step % args.save_steps == 0:
                    # best_f1 = evaluate_and_save_model(args, model, eval_dataset, _, global_step, best_f1)
                    save_model(args, model, global_step)

    # 最后循环结束, 评估一次
    best_f1 = evaluate_and_save_model(args, model, eval_dataset, _, global_step, best_f1)
    return best_f1

def evaluate_and_save_model(args, model, eval_dataset, epoch, global_step, best_f1):
    ret = evaluate(args, model, eval_dataset)

    precision_b = ret['1']['precision']
    recall_b = ret['1']['recall']
    f1_b = ret['1']['f1-score']
    support_b = ret['1']['support']

    precision_i = ret['2']['precision']
    recall_i = ret['2']['recall']
    f1_i = ret['2']['f1-score']
    support_i = ret['2']['support']

    weight_b = support_b / (support_b + support_i)
    weight_i = 1 - weight_b

    avg_precision = precision_b * weight_b + precision_i * weight_i
    avg_recall = recall_b * weight_b + recall_i * weight_i
    avg_f1 = f1_b * weight_b + f1_i * weight_i
    all_avg_precision = ret['macro avg']['precision']
    all_avg_recall = ret['macro avg']['recall']
    all_avg_f1 = ret['macro avg']['f1-score']

    logger.info("Evaluating EPOCH = [%d/%d] global_step = %d", epoch+1,args.num_train_epochs, global_step)
    logger.info("B-LOC precision = %f recall = %f  f1 = %f support = %d", precision_b, recall_b, f1_b, support_b)
    logger.info("I-LOC precision = %f recall = %f  f1 = %f support = %d", precision_i, recall_i, f1_i, support_i)
    logger.info("attention AVG:precision = %f recall = %f  f1 = %f ", avg_precision, avg_recall, avg_f1)
    logger.info("all AVG:precision = %f recall = %f  f1 = %f ", all_avg_precision, all_avg_recall, all_avg_f1)

    if avg_f1 > best_f1:
        best_f1 = avg_f1
        torch.save(model.state_dict(), os.path.join(args.output_dir, "pytorch_model.bin"))
        logging.info("save the best model %s,avg_f1= %f", os.path.join(args.output_dir, "pytorch_model.bin"), best_f1)

    return best_f1

def evaluate(args, model, eval_dataset):
    eval_output_dirs = args.output_dir
    if not os.path.exists(eval_output_dirs):
        os.makedirs(eval_output_dirs)
    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
    model.eval()

    logger.info("***** 验证中 *****")
    loss = []
    y_true_list = []
    y_pred_list = []
    for batch in tqdm(eval_dataloader, desc="Evaluating"):
        model.eval()
        batch = tuple(t.to(args.device) for t in batch)
        with torch.no_grad():
            inputs = {'input_ids': batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2],
                      'labels': batch[3]
                      }
            outputs = model(**inputs)
        eval_loss, logits = outputs[:2]
        list_tmp = []
        list_tmp.append(eval_loss.tolist())
        loss.extend(list_tmp)

        logits = model.crf.decode(logits, inputs['attention_mask'])
        labels_pred = logits.squeeze(0).cpu().tolist()
        labels_true = batch[3].tolist()

        labels_pred_real = list(real_labels(inputs['attention_mask'], labels_pred))
        labels_true_real = list(real_labels(inputs['attention_mask'], labels_true))
        y_pred_list.extend(labels_pred_real)
        y_true_list.extend(labels_true_real)

    y_true = np.array(flatten(y_true_list))
    y_pred = np.array(flatten(y_pred_list))
    assert y_true.shape == y_pred.shape, "y_true vs y_pred {} vs {}".format(len(y_true), len(y_pred))
    ret = classification_report(y_true=y_true, y_pred=y_pred, output_dict=True)
    return ret

'''
#               precision    recall  f1-score   support
#
#            0   0.998345  0.996229  0.997286     89638
#            1   0.993100  0.989685  0.991389      9016
#            2   0.992506  0.997225  0.994860     46483
#
#    micro avg   0.996142  0.996142  0.996142    145137
#    macro avg   0.994650  0.994380  0.994512    145137
# weighted avg   0.996149  0.996142  0.996143    145137
'''
def test(args, test_dataset, model):
    sampler = RandomSampler(test_dataset)
    data_loader = DataLoader(test_dataset, sampler=sampler, batch_size=2)
    y_true_list = []
    y_pred_list = []

    for batch in tqdm(data_loader, desc="test"):
        model.eval()
        batch = tuple(t.to(args.device) for t in batch)
        with torch.no_grad():
            inputs = {'input_ids': batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2],
                      'labels': None
                      }
            outputs = model(**inputs)
        logits = outputs[0]
        logits = model.crf.decode(logits, inputs['attention_mask'])
        labels_pred = logits.squeeze(0).cpu().tolist()
        labels_true = batch[3].tolist()

        labels_pred_real = list(real_labels(inputs['attention_mask'], labels_pred))
        labels_true_real = list(real_labels(inputs['attention_mask'], labels_true))
        y_pred_list.extend(labels_pred_real)
        y_true_list.extend(labels_true_real)

    y_pred = np.array(flatten(y_pred_list))
    y_true = np.array(flatten(y_true_list))
    assert y_true.shape == y_pred.shape
    ret = classification_report(y_true=y_true, y_pred=y_pred, digits=6)
    return ret


import torch.nn as nn
import torch.nn.functional as F
from torchvision import models

class Transfer(nn.Module):

    def __init__(self, n_class):
        super().__init__()
        self.n_class = n_class
        mdeep = models.segmentation.deeplabv3_resnet101(pretrained=True)
        backbone = mdeep.backbone
        l4 = backbone.pop('layer4')
        l3 = backbone.pop('layer3')
        # Layer two ouputs appropriate size

        for param in backbone.parameters():
            # False implies no retraining
            param.requires_grad=False

        self.encoder = backbone

        self.decoder = nn.ModuleList([

            nn.Sequential(
                nn.ConvTranspose2d(512, 512, kernel_size=3, stride=2, padding=1, dilation=1, output_padding=1),
                nn.BatchNorm2d(512),
                nn.ReLU(inplace=True),),

            nn.Sequential(
                nn.ConvTranspose2d(512, 256, kernel_size=3, stride=2, padding=1, dilation=1, output_padding=1),
                nn.BatchNorm2d(256),
                nn.ReLU(inplace=True),
                ),

            nn.Sequential(
                nn.ConvTranspose2d(256, 128, kernel_size=3, stride=2, padding=1, dilation=1, output_padding=1),
                nn.BatchNorm2d(128),
                nn.ReLU(inplace=True),
                ),

            nn.Sequential(
                nn.ConvTranspose2d(128, 64, kernel_size=3, stride=2, padding=1, dilation=1, output_padding=1),
                nn.BatchNorm2d(64),
                nn.ReLU(inplace=True),
                ),

            nn.Sequential(
                nn.ConvTranspose2d(64, 32, kernel_size=3, stride=2, padding=1, dilation=1, output_padding=1),
                nn.BatchNorm2d(32),
                nn.ReLU(inplace=True),
                )
        ])

        # Final output layer to have a 30 output classes
        self.classifier = nn.Conv2d(32, self.n_class, kernel_size=1)

    def forward(self, x):

        out_encoder = self.encoder(x)

        out_decoder = out_encoder

        for layer in self.decoder:
            out_decoder = layer(out_decoder)

        score = self.classifier(out_decoder)

        return score


import json
from lib import base64
from lib import authz
from lib import ncr_util
from lib.logger import logger
from lib.dict_merge import dict_merge
from lib.lambda_decorator.email_decorator import email_decorator
from lib.lambda_decorator.decorator import api_decorator
from lib.lambda_decorator import exceptions
from lib.lambda_decorator.scan_id_decorator import get_scan_id_decorator
from lib.dynamodb import exclusions_table, audit_table, ncr_table, requirements_table, config_table
from lib.exclusions import exclusions
from states.exclude import update_ncr_exclusion

@api_decorator
@email_decorator
def get_exclusions_handler(event, context):
    authz.require_is_admin(event.get('userRecord'))
    scan_params = {}
    querystring_parameters = event.get('queryStringParameters') or {}
    try:
        next_token = querystring_parameters.get('nextToken')
        if next_token:
            scan_params['ExclusiveStartKey'] = json.loads(base64.base64_to_string(next_token))
    except Exception: # pylint: disable=broad-except
        pass
    try:
        limit = querystring_parameters.get('limit')
        if limit:
            scan_params['Limit'] = int(limit)
    except Exception: # pylint: disable=broad-except
        pass

    result = exclusions_table.scan(**scan_params)
    exclusions = result['Items']
    for exclusion in exclusions:
        exclusion['exclusionId'] = exclusions_table.get_exclusion_id(exclusion)
    last_evaluated_key = None
    if 'LastEvaluatedKey' in result:
        last_evaluated_key = base64.string_to_base64(json.dumps(result['LastEvaluatedKey']))

    return {
        'exclusions': exclusions,
        'nextToken': last_evaluated_key,
    }

@api_decorator
@email_decorator
@get_scan_id_decorator
def put_exclusions_handler(event, context):
    user_record = event.get('userRecord', {})
    authz.require_is_admin(user_record)
    body = event.get('body', {})
    current_exclusion_id = body.get('exclusionId', '')
    update_request = body.get('exclusion', {})
    if current_exclusion_id:
        _, requirement_id, _ = split_exclusion_id(current_exclusion_id)
    else:
        requirement_id = update_request.get('requirementId', '')

    # input validation
    if not update_request:
        raise exceptions.HttpInvalidException('Must supply exclusion to put')

    # data validation
    current_exclusion = get_current_exclusion(current_exclusion_id)
    prospective_exclusion = dict_merge(current_exclusion, update_request)
    target_account_id = prospective_exclusion.get('accountId')
    delete_exclusion = current_exclusion if update_requires_replacement(current_exclusion, update_request) else {}
    requirement = requirements_table.get(requirement_id)
    exclusion_type = requirement.get('exclusionType')
    exclusion_config = config_table.get_config(config_table.EXCLUSIONS).get(exclusion_type)
    logger.debug('%s', json.dumps({
        'current_exclusion': current_exclusion,
        'prospective_exclusion': prospective_exclusion,
        'target_account_id': target_account_id,
        'delete_exclusion': delete_exclusion,
        'requirement': requirement,
        'exclusion_type': exclusion_type,
        'exclusion_config': exclusion_config,
    }, default=str))
    if not requirement:
        raise exceptions.HttpNotFoundException(f'Requirement not found: {requirement_id}')
    if not exclusion_config:
        raise exceptions.HttpInvalidException(f'Cannot find exclusion type: {exclusion_type}')

    # authorization
    authz.require_can_request_exclusion(user_record, target_account_id)

    new_exclusion = exclusions.update_exclusion(current_exclusion, update_request, exclusion_config, True)
    if new_exclusion:
        new_exclusion['exclusionId'] = exclusions_table.get_exclusion_id(new_exclusion)
        new_exclusion['lastModifiedByAdmin'] = user_record.get('email')
        new_exclusion['rqrmntId_rsrceRegex'] = '#'.join([requirement_id, new_exclusion['resourceId']])
        new_exclusion['type'] = exclusion_type
    if delete_exclusion:
        delete_exclusion['exclusionId'] = exclusions_table.get_exclusion_id(delete_exclusion)
    exclusions_table.update_exclusion(new_exclusion, delete_exclusion)

    if user_record.get('email'):
        audit_table.put_audit_trail(user_record['email'], audit_table.PUT_EXCLUSION_ADMIN, {
            'updateRequest': update_request,
            'newExclusion': new_exclusion,
            'deleteExclusion': delete_exclusion,
        })

    return {
        'newExclusion': new_exclusion,
        'deleteExclusion': delete_exclusion,
    }

@api_decorator
@email_decorator
@get_scan_id_decorator
def put_exclusions_for_user_handler(event, context):
    latest_scan_id = event.get('scanId', '')
    user_record = event.get('userRecord', {})
    body = event.get('body', {})
    ncr_id = body.get('ncrId', '')
    update_request = body.get('exclusion', {})
    scan_id, account_id, resource_id, requirement_id = split_ncr_id(ncr_id)

    # input validation
    if scan_id != latest_scan_id:
        raise exceptions.HttpInvalidException('Can only exclude ncrs from latest scans')
    if not update_request:
        raise exceptions.HttpInvalidException('Must supply exclusion to put')

    # data validation
    requirement = requirements_table.get(requirement_id)
    ncr = ncr_table.get_ncr(scan_id, account_id, resource_id, requirement_id)
    current_exclusion = exclusions_table.get_exclusion(account_id=account_id, requirement_id=requirement_id, resource_id=resource_id)
    exclusion_type = requirement.get('exclusionType')
    exclusion_types = config_table.get_config(config_table.EXCLUSIONS)
    exclusion_config = exclusion_types.get(exclusion_type, {})
    if not requirement:
        raise exceptions.HttpNotFoundException(f'Requirement not found: {requirement_id}')
    if not ncr:
        raise exceptions.HttpNotFoundException(f'NCR does not exist: {ncr_id}')
    if not exclusion_config:
        raise exceptions.HttpInvalidException(f'Cannot find exclusion type: {exclusion_type}')

    # authorization
    if exclusions.is_wildcard_exclusion(current_exclusion):
        raise exceptions.HttpForbiddenException('Wildcard exclusion applied to ncr')
    allowed_actions = ncr_util.get_allowed_actions(user_record, account_id, requirement, current_exclusion)
    prospective_exclusion = dict_merge(current_exclusion, update_request)
    prospective_state = exclusions.get_state(prospective_exclusion)
    if prospective_state in exclusions.REQUEST_EXCLUSION_STATES:
        if not allowed_actions['requestExclusion']:
            raise exceptions.HttpForbiddenException('Cannot requestExclusion')
    if prospective_state in exclusions.REQUEST_EXCLUSION_CHANGE_STATES:
        if not allowed_actions['requestExclusionChange']:
            raise exceptions.HttpForbiddenException('Cannot requestExclusionChange')

    # update
    new_exclusion = exclusions.update_exclusion(current_exclusion, update_request, exclusion_config, False)
    new_exclusion['accountId'] = account_id
    new_exclusion['resourceId'] = resource_id
    new_exclusion['requirementId'] = requirement_id
    new_exclusion['type'] = exclusion_type
    new_exclusion['exclusionId'] = exclusions_table.get_exclusion_id(new_exclusion)
    new_exclusion['lastModifiedByUser'] = user_record.get('email')
    new_exclusion['rqrmntId_rsrceRegex'] = f'{new_exclusion["requirementId"]}#{new_exclusion["resourceId"]}'
    exclusions_table.update_exclusion(new_exclusion, {})

    if user_record.get('email'):
        audit_table.put_audit_trail(user_record['email'], audit_table.PUT_EXCLUSION_USER, {
            'updateRequest': update_request,
            'newExclusion': new_exclusion,
            'deleteExclusion': {},
        })

    new_allowed_actions = ncr_util.get_allowed_actions(user_record, account_id, requirement, new_exclusion)
    updated_ncr = update_ncr_exclusion(ncr, new_exclusion, exclusion_types)
    logger.debug('Updated ncr: %s', json.dumps(updated_ncr, default=str))
    ncr_table.put_item(Item=updated_ncr)

    return {
        'newExclusion': new_exclusion,
        'newNcr': {
            'ncrId': ncr_id,
            'resource': updated_ncr,
            'allowedActions': new_allowed_actions,
        }
    }

def split_exclusion_id(exclusion_id: str) -> tuple:
    try:
        parts = exclusions_table.parse_exclusion_id(exclusion_id)
    except TypeError:
        raise exceptions.HttpInvalidException('Invalid exclusionId')
    account_id, requirement_id, resource_id = parts
    return account_id, requirement_id, resource_id

def split_ncr_id(ncr_id: str) -> tuple:
    try:
        parts = ncr_table.parse_ncr_id(ncr_id)
    except TypeError:
        raise exceptions.HttpInvalidException('Invalid ncrId')
    scan_id_date, scan_id_randomness, account_id, resource_id, requirement_id = parts
    scan_id = '#'.join([scan_id_date, scan_id_randomness])
    return scan_id, account_id, resource_id, requirement_id

def get_current_exclusion(exclusion_id: str) -> dict:
    if exclusion_id:
        current_exclusion = exclusions_table.get_exclusion(exclusion_id=exclusion_id)
        if not current_exclusion:
            raise exceptions.HttpNotFoundException(f'Exclusion {exclusion_id} not found')
        return current_exclusion
    else:
        return {}

def update_requires_replacement(current_exclusion, update_request) -> bool:
    if not current_exclusion:
        return False
    key_updates = {'accountId', 'requirementId', 'resourceId'} & set(update_request.keys())
    if key_updates:
        logger.debug('Requested update of a key part/value')
        prospective_merge = dict_merge(current_exclusion, update_request)
        if exclusions_table.get_exclusion_id(prospective_merge) != exclusions_table.get_exclusion_id(current_exclusion):
            logger.debug('Provided an exclusion update that would update the exclusion ID, adding delete_exclusion to transaction items')
            return True
    return False


import torch
import numpy as np
from args import get_parser
import pickle
import os
from torchvision import transforms
from model import get_model, label2onehot
import json
import sys
import random
from modules.utils import LogResults, Accuracy, store_features, maintain_cnn_features, print_arg_info, unit_test_portions_units
from data_modules.recipe_loader import get_recipe_loader
from data_modules.ingr_loader import get_ingr_loader
from data_modules.data_aux import transformation_functions

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
map_loc = None if torch.cuda.is_available() else 'cpu'

# -------------------------------------------

def main(args):

    checkpoints_dir = args.save_dir
    args.use_predicted = True

    vars_to_replace = ['batch_size', 'eval_split', 'save_dir', 'aux_data_dir',
                       'recipe1m_dir', 'use_lmdb', 'ckpt_epoch', 'subsequent_calorie',
                       'use_predicted', 'evaluate_given_images', 'images_folder']

    store_dict = {}
    for var in vars_to_replace:
        store_dict[var] = getattr(args, var)

    if not args.store_cnn_feat:
        do_calorie_scale = args.do_calorie_scale
        splits = args.splits
        model_output = args.model_output
        subsequent_calorie = args.subsequent_calorie
        args = pickle.load(open(os.path.join(checkpoints_dir, 'args.pkl'), 'rb'))
        args.splits = splits
        args.do_calorie_scale = do_calorie_scale
        args.model_output = model_output
        args.subsequent_calorie = subsequent_calorie

    for var in vars_to_replace:
        setattr(args, var, store_dict[var])

    model_path = os.path.join(checkpoints_dir, 'model{}.ckpt'.format(args.ckpt_epoch))

    torch.set_printoptions(profile="full")

    data_dir = args.recipe1m_dir

    train_transform = transformation_functions(args, run_mode="train")
    eval_transform = transformation_functions(args, run_mode="eval")
    transforms = {"train": train_transform, "eval": eval_transform}

    if "simple" in args.model_type:
        is_simple = True
        data_loader, dataset = get_ingr_loader(data_dir, args.aux_data_dir, args.splits,
                                      run_mode='eval', transforms=transforms,
                                      batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
                                      drop_last=False, suff=args.suff, args=args)
    else:
        is_simple = False
        data_loader, dataset = get_recipe_loader(data_dir, args.aux_data_dir, args.splits,
                                      run_mode='eval', transforms=transforms,
                                      batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
                                      drop_last=False, suff=args.suff, args=args)

    ingr_vocab_size = dataset.get_ingrs_vocab_size()
    if args.dataset=="recipe1m":
        unit_vocab_size = dataset.get_unit_vocab_size()
    else:
        unit_vocab_size = -1
    dish_vocab_size = dataset.get_dish_vocab_size()
    states_vocab_size = dataset.get_state_vocab_size()
    calorie_classes_num = dataset.get_calorie_classes_num()

    # Build the model
    model = get_model(args, ingr_vocab_size, unit_vocab_size, states_vocab_size, calorie_classes_num, dish_vocab_size, dataset.bottom_up_feat_size, dataset.pretrained_emb, loss_weights_matrix=dataset.weights_tensor, flattened_weights=dataset.flattened_weights)

    if not args.store_cnn_feat:
        # Load the trained model parameters
        model.load_state_dict(torch.load(model_path, map_location=map_loc))
        print ("loaded epoch {} checkpoint of the model.".format(args.ckpt_epoch))

    log_results = LogResults(dataset, is_simple, dataset.run_mode, args.evaluate_given_images)

    # prints settings of the current run
    print_arg_info(args)

    if args.just_plot_error:
        log_results.plot_errors()
        exit()

    model = model.to(device)
    model.eval()

    # dictionary for storing cnn features for each instance in the dataset
    if args.store_cnn_feat:
        imgid_to_features = {}
        stored_imgids = set({})
        store_batch_num = 1

    accuracy_log = Accuracy(args, dataset.data_size, is_simple, dataset.run_mode, dataset.unscale_batch)
    for step, (ques_idx, tar_idx, tar_values, tar_calories, tar_states, tar_calorie, mean_prior_calories_gt, portion_distrib, calorie_class_idx, dish_idxs, image_input, img_id, qids, path) in enumerate(data_loader):

        if args.store_cnn_feat:
            update_batch_num = store_features(args.features_dir, dataset.run_mode, imgid_to_features, store_batch_num)
            if update_batch_num!=store_batch_num:
                store_batch_num = update_batch_num
                imgid_to_features = {}

        # ---------- move all data loaded from dataloader to gpu
        # input indice referring to ingredients
        if not ques_idx is None:
            ques_idx = ques_idx.to(device)
        # input indice referring to a dish name
        if not dish_idxs is None:
            dish_idxs = dish_idxs.to(device)
        # target state idx
        if not tar_states is None:
            tar_states = tar_states.to(device)
        # input image or image features
        if not image_input is None:
            image_input = image_input.to(device)
        # target indice referring to unit classes (N=6)
        if not tar_idx is None:
            tar_idx = tar_idx.to(device)
        # target portion values per input (ingredient)
        if not tar_values is None:
            tar_values = tar_values.to(device)
        # target portion outputs referring to each unit type (N=6)
        if not portion_distrib is None:
            portion_distrib = portion_distrib.to(device)
        # target calorie class associated with the calorie value.
        if not calorie_class_idx is None:
            calorie_class_idx = calorie_class_idx.to(device)
        # target per recipe calorie
        if not tar_calorie is None:
            tar_calorie = tar_calorie.to(device)
        # individual calories referring to ingredients
        if not tar_calories is None:
            tar_calories = tar_calories.to(device)
        # ----------
        if not mean_prior_calories_gt is None:
            mean_prior_calories_gt = mean_prior_calories_gt.to(device)

        if "simple" in args.model_type:
            ingr_calorie_vec_mask = label2onehot(ques_idx, dataset.pad_value)
            ingr_calorie_vec = label2onehot(ques_idx, dataset.pad_value, tar_calorie.squeeze())
        else:
            ingr_calorie_vec, ingr_calorie_vec_mask = None, None

        # create a mask for image featuers if the number of image feature vectors used is variable
        if args.image_encoder_type=="bottom-up":
            image_mask = (torch.sum(torch.abs(image_input), dim=-1) == 0)
        else:
            image_mask = None

        # create mask for ingredients
        ques_mask = (ques_idx == dataset.pad_value)

        # feed-forward data in the model
        losses, outputs = model(ques_idx, ques_mask, None, tar_idx, tar_values, tar_calorie, tar_calories, tar_states,
                                mean_prior_calories_gt, portion_distrib, dish_idxs, image_input, image_mask,
                                ingr_calorie_vec_mask, ingr_calorie_vec, calorie_class_idx=calorie_class_idx)

        if args.store_cnn_feat:
            maintain_cnn_features(img_id, outputs, imgid_to_features, stored_imgids)
            continue

        # logging acuracy
        pred_idx = accuracy_log.update(outputs, tar_idx, tar_values, tar_calorie, tar_calories, tar_states, ques_mask)

        log_results.update(outputs, pred_idx, tar_idx, tar_values, tar_calorie, tar_calories, tar_states, qids, img_id, ques_idx, is_simple)

    if args.store_cnn_feat:
        store_features(args.features_dir, dataset.run_mode, imgid_to_features, store_batch_num, last_batch=True)

    log_results.finalize(args, is_simple)

    # mean accuracy over all testing data
    accuracy_log.epoch_finalize()

    print ('******************* Testing Ended *******************')

if __name__ == '__main__':
    args = get_parser()
    torch.manual_seed(1234)
    torch.cuda.manual_seed(1234)
    random.seed(1234)
    np.random.seed(1234)
    main(args)


# -*- coding: utf-8 -*-

##################
#   LIBRAIRIES   #
##################

import numpy as np
import math
from random import randint
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

##################
#   CONSTANTES   #
##################

#Nombre de classes
NB_CLASS = 10
#Dimension choisie pour la projection
DIMENSION = 9

################
#   FICHIERS   #
################

#Ouvertures des différents fichiers
#Et chargement des données provenant des fichiers dans des tableaux
#Corpus d'apprentissage
f = open('appr.ascii', 'r')
array_appr = np.loadtxt(f)
#Liste des classes du corpus d'apprentissage
f = open('appr_cl.ascii', 'r')
arraycl = np.loadtxt(f)
#Corpus de développement
f = open('dev.ascii', 'r')
array_dev = np.loadtxt(f)
#Liste des classes du corpus de développement
f = open('dev_cl.ascii', 'r')
array_dev_cl = np.loadtxt(f)
#COrpus d'évaluation
f = open('eval.ascii', 'r')
array_eval = np.loadtxt(f)

#Fermeture du fichier
f.close()

########################################
#   Analyse en Composante Principale   #
########################################

array_appr = StandardScaler().fit_transform(array_appr)
pca = PCA(n_components=DIMENSION)
array_appr = pca.fit_transform(array_appr)
array_dev = StandardScaler().fit_transform(array_dev)
pca = PCA(n_components=DIMENSION)
array_dev = pca.fit_transform(array_dev)
array_eval = StandardScaler().fit_transform(array_eval)
pca = PCA(n_components=DIMENSION)
array_eval = pca.fit_transform(array_eval)

################################
#   Declaration de tableaux    #
################################

#array_meanByClass contient la moyenne des valeurs pour les différentes classes
array_meanByClass = [[0 for x in range(0, DIMENSION)] for y in range(0, NB_CLASS)]
#array_nbImageByClass contient les images triées par classes
#list_dev[0] contient les images de la classe 0
array_imageByClass = [[] for x in range(0, NB_CLASS)]

array_matricesByClass = [[0 for y in range(0, DIMENSION)] for z in range(0, NB_CLASS)]

##############################
#   Phase d'apprentissage    #
##############################

for x in range(0, len(array_appr)):
    #On traite une nouvelle ligne
    tmp_class = int(arraycl[x])
    #On range les images par classes
    array_imageByClass[tmp_class].append(array_appr[x])
    #On ajoute la valeur de chaque image à la somme pour calculer la moeynne
    for y in range(0, DIMENSION):
        array_meanByClass[tmp_class][y] += array_appr[x][y]

#Calcul des moyennes par classes
for x in range(0, NB_CLASS):
    #Division de la somme des valeurs par le nombre d'images
    for y in range(0, DIMENSION):
        array_meanByClass[x][y] /= len(array_imageByClass[int(arraycl[x])])

#On calcule chaque matrice de covariance par classe
for x in range(0, NB_CLASS):
    array_matricesByClass[x] = np.cov(np.transpose(array_imageByClass[x]))

externecycle = 1
internecycle = len(array_eval)
#f = open('eval_cl.ascii', 'w')
v = [0] * externecycle
for ttt in range(0, externecycle):
    count = 0
    matricedeconfusion = [[] for ii in range(0, NB_CLASS)]
    for tt in range(0, internecycle):
        index = randint(0, len(array_dev) - 1)
        test = array_eval[tt]
        answer = array_dev_cl[index]
        elems = [0] * NB_CLASS
        for x in range(0, NB_CLASS):
            X = test
            Y = array_meanByClass[x]
            Z = X - Y  # vector hor
            Z1 = np.transpose(Z)  # vector vert
            Z2 = np.linalg.inv(array_matricesByClass[x])  # matrice
            Z4 = np.dot(Z1, Z2)  # vector
            Z5 = np.dot(Z4, Z)  # valeur
            e = math.pow(math.e, -Z5 / 2)
            m1 = math.pow(2 * math.pi, float(DIMENSION) / 2)
            m2 = math.pow(np.linalg.det(array_matricesByClass[x]), (1.0 / 2))
            p = e / (m1 * m2)
            elems[x] = p
        maxim = elems[0]
        i = 0
        for x in range(0, NB_CLASS):
            if elems[x] > maxim:
                maxim = elems[x]
                i = x
        # print(str(answer)+'est probablement '+str(i))
        #f.write(str(i)+'\n')
        matricedeconfusion[int(answer)].append(i)
        if answer == i:
            count += 1
    # print(count)
    v[ttt] = count
    print('pourcentage de vraisemblance:' + str(float(count) / internecycle))
    #for ii in range(0, 10):
    #    print(matricedeconfusion[ii])
m = v[0]
mi = v[0]
for x in range(0, externecycle):
    if v[x] < mi:
        mi = v[x]
    if v[x] > m:
        m = v[x]
print('max:' + str(m))
print('min:' + str(mi))


#/usr/bin/env python3
#Author: Zhu Yong-kai (yongkai_zhu@hotmail.com)
import argparse
import os

import numpy as np
import pandas as pd
from PIL import Image
from astropy.io import fits
from matplotlib import pyplot as plt

class PSD:
    """
    Caculate the 2-dimension power spectral density.
    """
    def __init__(self, image, pixel=1.0, step=1):
        self.image = np.array(image, dtype = float)
        self.shape = self.image.shape
        self.pixel = pixel
        if self.shape[0] % 2 ==0:
            self.center = [self.shape[0] / 2, self.shape[1] / 2]
        else:
            self.center = [(self.shape[0] - 1) / 2, (self.shape[1] - 1) / 2]
        self.step = step

    def radii(self):
        shape = self.shape
        points1 = np.arange(0, shape[0])
        points2 = np.arange(0, shape[1])
        x, y = np.meshgrid(points1, points2)
        x_center, y_center = self.center
        distances = np.sqrt((x-x_center)**2+(y-y_center)**2)
        return distances

    def freq(self, r, d=1.0):
        n = self.shape[0]
        f = r / n / d
        return f

    def cal_psd2d(self):
        I = self.image
        Ifft2 = np.fft.fft2(I)
        Pxx_shift = np.fft.fftshift(Ifft2)
        Pxx = (abs(Pxx_shift))**2 / Pxx_shift.shape[0] / Pxx_shift.shape[1]\
            / self.pixel / self.pixel
        self.psd2d = Pxx
        return self.psd2d

    def cal_psd1d(self):
        OD = []
        nu = []
        rho = self.radii()
        Pxx = self.cal_psd2d()
        for r in np.arange(0, self.shape[0] / 2, self.step):
            r1 = r
            r2 = r + 1
            if r2 <= self.shape[0]:
                R = (rho >= r1).astype(int) * (rho < r2).astype(int)
                Pxx_R = R * Pxx
                nu.append(r)
                OD.append(np.median(Pxx_R[R.nonzero()[0],R.nonzero()[1]]))
        nu = np.array(nu)
        OD = np.array(OD)
        k = self.freq(nu)
        self.k = k
        self.psd1d = OD
        return self.k, self.psd1d

    def plot(self,outfile):
        """
        plot the 1-D radial power spectrum.
        """
        k = self.k
        OD = self.psd1d
        fig = plt.figure()
        plt.loglog(k, OD, '.-')
        plt.savefig(outfile)
        plt.show()

    def save(self, outfile):
        dataframe = pd.DataFrame({'k':self.k, 'K*K':self.psd1d})
        dataframe.to_csv(outfile, index=False, sep='\t')

def main():
    parser = argparse.ArgumentParser(
       description="Caculate the power spectrum distribution of\
       the input image")
    parser.add_argument("-i", "--inputfile", dest="inputfile",
                        required=True,
                        help="inputfile image file")
    parser.add_argument("-oc", "--outfile1", dest="outfile1",
                        required=True,
                        help="output CSV file")
    parser.add_argument("-oj", "--outfile2", dest="outfile2",
                        required=True,
                        help="output JPG file")
    parser.add_argument("-s", "--step", dest="step",
                        type=float,
                        help="sampling interval")
    parser.add_argument("-p", "--pixel", dest="pixel",
                        type=float,
                        help="pixel of the image")
    parser.add_argument("-c", "--clobber",
                        action='store_true',
                        default=False,
                        help="Overwrite original file.( default:False)")
    args = parser.parse_args()
    for i in [args.outfile1, args.outfile2]:
        if os.path.exists(i) and not args.clobber:
            raise FileExistsError(f'file already exists:{i}')

    img = fits.open(args.inputfile)
    img = fits.open(inputfile)
    img = img[0].data
    psd = PSD(image=img, pixel=args.pixel, step=args.step)
    psd.cal_psd1d()
    psd.plot(args.outfile2)
    psd.save(args.outfile1)

if __name__=="__main__":
    main()


from django.contrib import admin
from django.urls import path,include
from dot_learn_app.views import *
urlpatterns = [
    path('',display_data,name='home'),
    path('branch/<str:branch_slug>/',display_data,name="branch"),
    path('branch/<str:branch_slug>/subject/<str:subject_slug>/',display_data,name="branch+sub"),
    path('branch/<str:branch_slug>/subject/<str:subject_slug>/chapter/<str:chapter_slug>',display_data,name="branch+sub+chap"),
    path("topic/<str:topic_slug>/",display_data,name="branch+sub+chap+topic"),
    path('practice',temp,name='practice'),
    path('material',temp,name='material'),
    path('contact',temp,name='contact'),
    path('topic/',question_list,name='topic'),
]


from airflow.hooks.postgres_hook import PostgresHook
from airflow.contrib.hooks.aws_hook import AwsHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from helpers import SqlQueries

import logging
log = logging.getLogger(__name__)

class StageToRedshiftOperator(BaseOperator):
    '''
    Copy files from s3 to redshift.
    '''
    ui_color = '#358140'
    @apply_defaults

    def __init__(self,
                 redshift_conn_id="redshift",
                 aws_conn_id="aws_credentials",
                 source_location="",
                 targ_table="",
                 file_type="json",
                 json_path="",
                 *args, **kwargs):

        super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
        self.redshift_conn_id = redshift_conn_id
        self.aws_conn_id = aws_conn_id
        self.source_location = source_location
        self.targ_table = targ_table
        self.file_type = file_type
        self.json_path = json_path

    def execute(self, context):
        if self.file_type in ["json", "csv"]:
            redshift_hook = PostgresHook(self.redshift_conn_id)
            aws_hook = AwsHook(self.aws_conn_id)
            credentials = aws_hook.get_credentials()

            self.log.info(f'StageToRedshiftOperator loading data to staging table.')
            redshift_hook.run(SqlQueries.truncate_table.format(self.targ_table))
            if self.file_type == "json":
                if self.json_path != "":
                    redshift_hook.run (
                        SqlQueries.copy_json_with_json_path_to_redshift.format (
                            self.targ_table,
                            self.source_location,
                            credentials.access_key,
                            credentials.secret_key,
                            self.json_path
                        )
                    )
                else:
                    redshift_hook.run (
                        SqlQueries.copy_json_to_redshift.format (
                            self.targ_table,
                            self.source_location,
                            credentials.access_key,
                            credentials.secret_key
                        )
                    )
            elif self.file_type == "csv":
                redshift_hook.run (
                    SqlQueries.copy_csv_to_redshift.format (
                    self.targ_table,
                    self.source_location,
                    credentials.access_key,
                    credentials.secret_key)
                )

            self.log.info(f'StageToRedshiftOperator completed data load to staging table.')
        else:
            raise ValueError("file_type must be json or csv")


## Process FLORET Matlab csv Files to have them in my FISTA format

import os,sys
import numpy

print("-------------------------------------------------------------")
print("Processing Files")
# Kpos=open("C:\\Users\\AC243636\\Documents\\FLORET\\csv_files\\NewSandroFev3\\ktraj.csv",'r')
# Kpos = open("E:\\Experiments_DATA\\Sodium\\Sodium_Aachen_1_03_16\\Data_FISTA\\Acq1\\Ktraj.csv",'r')
Kpos = open("Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\ktraj.csv",'r')

ind=0
# 300 spokes 1373 points
# Sandro's july tests 279 spokes of 1676 points

# SaltIT pipeline has 300 spokes of 1561 points
for line in Kpos:
        # print ind
        if (ind==0):
            x=line.split(',')
        if (ind==1):
            y=line.split(',')
        if (ind==2):
            z=line.split(',')
        ind=ind+1
print("number of points in trajectory = ", int(len(x)))

Kpos.close()
kdatan=[]
# Kdata=open("C:\\Users\\AC243636\\Documents\\FLORET\\csv_files\\NewSandroFev3\\rawdata.csv",'r')
# Kdata=open("E:\\Experiments_DATA\\Sodium\\Sodium_Aachen_1_03_16\\Data_FISTA\\Acq1\\rawdata.csv",'r')
Kdata=open("Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\rawdata.csv",'r')
# kdata=[]
for line in Kdata:
            kdata=line.split(',')
            nb_spokes = int(len(kdata))
            for pts in kdata:
                kdatan.append(pts.split(','))
kdatac=[]
for i in range(300):
    a= kdatan[i::300]
    for j in range(len(a)):
        kdatac.append(a[j])

print("number of values in trajectory = ", int(len(kdatan)))
Kdata.close()
print("Writting Fista readable File")

coil=1
# exponent=0
# for i in range(len(kdatan)):
    # a=str(kdatan[i])
    # print a
    # print (int(len(a)))
    # for i in range(int(len(a))):
        # if str(a[i]) == '-':
            # signepos=i
        # if str(a[i]) == '+':
            # signepos=i
        # if str(a[i]) == 'e':
            # exponent=i
    # print signepos
    # print exponent
    # if exponent==0 :
        # print a[2:signepos]
        # print a[signepos+1:len(a)-3]
    # if exponent!=0 :
        # print a[2:signepos]
        # print a[signepos+1:exponent+2]
# if os.path.isfile("E:\\Experiments_DATA\\Sodium\\Sodium_Aachen_1_03_16\\Data_FISTA\\Acq1\\FLORET_posval_Sodium_invivo.csv") :
if os.path.isfile("Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\FLORET_posval_Sodium_invivo.csv") :
    print("File already exists !")
# if not os.path.isfile("E:\\Experiments_DATA\\Sodium\\Sodium_Aachen_1_03_16\\Data_FISTA\\Acq1\\FLORET_posval_Sodium_invivo.csv") :
if not os.path.isfile("Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\FLORET_posval_Sodium_invivo.csv") :
    f=open("Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\FLORET_posval_Sodium_invivo.csv","w")
    for point in range(int(len(kdatac))):
    # print "spoke ", int(round(point/nb_spokes)), "point " , int(point), x[point], y[point] , z[point]
        f.write(str(coil))
        f.write(',')
        f.write(str(int(round(point/1561))))
        f.write(',')
        f.write(str(point))
        f.write(',')
        f.write(str(float(x[point])))
        f.write(',')
        f.write(str(float(y[point])))
        f.write(',')
        f.write(str(float(z[point])))
        f.write(',')
        a = str(kdatac[point])
        f.write(str(a[2:len(kdatac[point])-3]))
        f.write("\n")
    f.close()
# print "[DONE] --> E:\\Experiments_DATA\\Sodium\\Sodium_Aachen_1_03_16\\Data_FISTA\\Acq1\\FLORET_posval_Sodium_invivo.csv"
print("[DONE] --> Z:\\arthur_coste\\ToBeProcessedMatlabLaptop\\FloretKspace\\FLORET_posval_Sodium_invivo.csv")
print("-------------------------------------------------------------")


from aiogram import types, Dispatcher
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton

from config import bot

async def quiz_2(call: types.CallbackQuery):
    markup = InlineKeyboardMarkup()
    button_2 = InlineKeyboardButton("NEXT", callback_data="button_2")
    question = "Сколько областей в Кыргызстане?"
    markup.add(button_2)
    answer = [
        "12",
        "3",
        "7",
        "0",
        "-10",
        "999",
    ]
    await bot.send_poll(
        chat_id=call.from_user.id,
        question=question,
        options=answer,
        is_anonymous=False,
        type='quiz',
        correct_option_id=2,
        explanation="Стыдно не знать",
        open_period=5,
        reply_markup=markup
    )

async def quiz_3(call: types.CallbackQuery):
    question = "сколько вам будет лет если вы родились бы 9 лет назад?"
    answer = [
        '10',
        '9',
        '5',
        '2',
        '5',
    ]

    await bot.send_poll(
        chat_id=call.from_user.id,
        question=question,
        options=answer,
        is_anonymous=False,
        type='quiz',
        correct_option_id=1,
        explanation="Стыдно не знать",
        # open_period=5,
    )

def register_handlers_callback(dp: Dispatcher):
    dp.register_callback_query_handler(quiz_2, text="button_1")
    dp.register_callback_query_handler(quiz_3, text="button_2")


# -*- coding: utf-8 -*-

import xml.dom.minidom as xml
import Config
import os

class XMLWriter():

    def __init__(self,isAppend=False,udid=None,mappingudid=None,recordFilePath=None):
        if not isAppend:
            self.implementation = xml.getDOMImplementation()
            self.document = self.implementation.createDocument(None,'Record',None)
            self.rootElement = self.document.documentElement
            self.deviceElement = self.document.createElement(Config.RECORD_XML_TAG_DEVICE)
            self.deviceElement.setAttribute(Config.RECORD_XML_DEVICE_ATTRIBUTE_ID,udid)
            self.deviceElement.setAttribute(Config.RECORD_XML_DEVICE_ATTRUBUTE_MAPPINGID,mappingudid)
            self.setps = [];
        else:
            self.recordFilePath = recordFilePath
            self.document = xml.parse(recordFilePath)
            self.rootElement = self.document.documentElement

    def handleScriptStart(self,scriptName=None,scriptStartTime=None,scriptCurrentLoop=None,scriptRoundFolder=None):
        self.scriptElement = self.document.createElement(Config.RECORD_XML_TAG_SCRIPT)
        self.scriptElement.setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_NAME,scriptName)
        self.scriptElement.setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_STARTTIME,scriptStartTime)
        self.scriptElement.setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_LOOP,scriptCurrentLoop)
        self.scriptElement.setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_ROUND,scriptRoundFolder)
        self.scriptResultElement = self.document.createElement(Config.RECORD_XML_TAG_SCRIPTRESULT)
        self.scriptLogElement = self.document.createElement(Config.RECORD_XML_TAG_LOG)
        self.scriptLogElement.setAttribute(Config.RECORD_XML_LOG_ATTRIBUTE_FOLDER,'Time[' + scriptStartTime + ']_Loop[' + scriptCurrentLoop + ']')
        self.scriptOKInfoElement = self.document.createElement(Config.RECORD_XML_TAG_OKINFO)
        self.scriptErrorInfoElement = self.document.createElement(Config.RECORD_XML_TAG_ERROR_INFO)

    def handleScriptEnd(self,scriptResult=None,scriptEndTime=None):
        self.scriptElement.setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_ENDTIME,scriptEndTime)
        self.scriptResultElement.setAttribute(Config.RECORD_XML_SCRIPTRESULT_ATTRIBUTE_VALUE,scriptResult)

    def handleAppendScriptEnd(self,scriptResult=None,scriptEndTime=None):
        scriptElements = self.document.getElementsByTagName(Config.RECORD_XML_TAG_SCRIPT)
        scriptElements[0].setAttribute(Config.RECORD_XML_SCRIPT_ATTRIBUTE_ENDTIME,scriptEndTime)
        resultElemens = self.document.getElementsByTagName(Config.RECORD_XML_TAG_SCRIPTRESULT)
        resultElemens[0].setAttribute(Config.RECORD_XML_SCRIPTRESULT_ATTRIBUTE_VALUE,scriptResult)

    def handleScriptExecute(self,stepContent=None,stepResult=None,relatedDevice=None):
        stepElement = self.document.createElement(Config.RECORD_XML_TAG_SCRIPTSTEP)
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_CONTENT,stepContent)
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_RESULT,str(stepResult))
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_DEVICE,str(relatedDevice))
        self.setps.append(stepElement)

    def handleAppendScriptExecute(self,stepContent=None,stepResult=None,relatedDevice=None):
        scriptElements = self.rootElement.getElementsByTagName(Config.RECORD_XML_TAG_SCRIPT)
        stepElement = self.document.createElement(Config.RECORD_XML_TAG_SCRIPTSTEP)
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_CONTENT,stepContent)
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_RESULT,str(stepResult))
        stepElement.setAttribute(Config.RECORD_XML_SCRIPTSTEP_ATTRIBUTE_DEVICE,str(relatedDevice))
        scriptElements[0].appendChild(stepElement)

    def handleErrorInfo(self,error=None):
        errorInfoNode = self.document.createTextNode(error)
        self.scriptErrorInfoElement.appendChild(errorInfoNode)

    def handleAppendedErrorInfo(self,error=None):
        errorInfoElements = self.rootElement.getElementsByTagName(Config.RECORD_XML_TAG_ERROR_INFO)
        errorInfoNode = self.document.createTextNode(error)
        errorInfoElements[0].appendChild(errorInfoNode)

    def handleOKInfo(self,ok=None):
        okInfoNode = self.document.createTextNode(ok)
        self.scriptOKInfoElement.appendChild(okInfoNode)

    def generateRecord(self,recordPath=None):
        self.scriptElement.appendChild(self.scriptResultElement)
        self.scriptElement.appendChild(self.scriptLogElement)
        self.scriptElement.appendChild(self.scriptOKInfoElement)
        self.scriptElement.appendChild(self.scriptErrorInfoElement)
        for verStep in self.setps:
            self.scriptElement.appendChild(verStep)
        self.deviceElement.appendChild(self.scriptElement)
        self.rootElement.appendChild(self.deviceElement)
        file = open(os.path.join(recordPath,Config.RECORD_XML_NAME),'a')
        self.document.writexml(file)
        file.close()

    def saveRecord(self):
        file = open(self.recordFilePath,'w')
        self.document.writexml(file)
        file.close()


#!user/bin/python
#-*- coding: utf-8 -*-
__author__: 'huangsonglin@dcpai.cn'
__Time__: '2020/4/20 11:12'

import os
import sys

curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import socket

class GetIp(object):

	def __init__(self):
		self._ip = socket.gethostname()

	@property
	def get_ip(self):
		return self._ip


"""
    Problem 15
    Lattice paths
"""
def p015():
    print("Lattice paths")
    grid_length = 20 # Given Grid Length
    path = 1  # Path holder

    # Loop possible routes
    # wiki: http://en.wikipedia.org/wiki/Binomial_coefficient
    # using binomial coefficient
    for idx in range(0, grid_length):
        path = (path * (grid_length * 2 - idx)/(idx+1))

    # Output
    return(f'p015 Ans: {path}')


from db import db_config
from typing import List
from logger.logging import get_logger
logger = get_logger(__name__)

async def process_log_message(log_message):
    logger.info(f"Processing log message: {log_message}")
    try:
        # Store in MongoDB
        insert_result = await db_config.logs_collection.insert_one(log_message)
        logger.info(f"Inserted log message into MongoDB with ID: {insert_result.inserted_id}")

        # Prepare document for ElasticSearch
        es_document = log_message.copy()
        es_document_id = str(insert_result.inserted_id)
        del es_document['_id']  # Remove _id from the document body

        # Also index in ElasticSearch
        await db_config.es.index(index="log-index", id=es_document_id, document=es_document)
        logger.info(f"Indexed log message in Elasticsearch with ID: {es_document_id}")
    except Exception as e:
        logger.error(f"Error processing log message: {e}")

async def delete_index(index):
    res = await db_config.es.indices.delete(index=index.index, ignore=[400, 404])
    return res

async def elastc_search(query):
    return await db_config.es.search(index="log-index", body=query)


import unittest
import json
import os
from memcnn.experiment.factory import load_experiment_config, experiment_config_parser
from memcnn.config import Config
import memcnn.config

class ConfigTestCase(unittest.TestCase):

    class ConfigTest(Config):
        @staticmethod
        def get_filename():
            return os.path.join(Config.get_dir(), "config.json.example")

    def setUp(self):
        self.config = ConfigTestCase.ConfigTest()

        self.config_fname = os.path.join(os.path.dirname(__file__), "..", "config.json.example")
        self.experiments_fname = os.path.join(os.path.dirname(__file__), "..", "experiments.json")

        def load_json_file(fname):
            with open(fname, 'r') as f:
                data = json.load(f)
            return data

        self.load_json_file = load_json_file

    def test_loading_main_config(self):
        self.assertTrue(os.path.exists(self.config.get_filename()))
        data = self.config
        self.assertTrue(isinstance(data, dict))
        self.assertTrue("data_dir" in data)
        self.assertTrue("results_dir" in data)

    def test_loading_experiments_config(self):
        self.assertTrue(os.path.exists(self.experiments_fname))
        data = self.load_json_file(self.experiments_fname)
        self.assertTrue(isinstance(data, dict))

    def test_experiment_configs(self):
        data = self.load_json_file(self.experiments_fname)
        config = self.config
        keys = data.keys()
        for key in keys:
            result = load_experiment_config(self.experiments_fname, [key])
            self.assertTrue(isinstance(result, dict))
            if "dataset" in result:
                experiment_config_parser(result, config['data_dir'])

    def test_config_get_filename(self):
        self.assertEqual(Config.get_filename(), os.path.join(os.path.dirname(memcnn.config.__file__), "config.json"))

    def test_config_get_dir(self):
        self.assertEqual(Config.get_dir(), os.path.dirname(memcnn.config.__file__))

    def test_verbose(self):
        ConfigTestCase.ConfigTest(verbose=True)

if __name__ == '__main__':
    unittest.main()


import enum
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
import open3d as o3d
import numpy as np
import trimesh

# Custom dataset class
class ColoredMeshesDataset(Dataset):
    def __init__(self, split, dataset_path):
        assert split in ['train', 'val', 'overfit']

        # Read the lines from the split or overfit file and separate view, sample, and class elements
        with open(f"./utils/{split}.txt", "r") as file:
            lines = [line.strip().split() for line in file]

        # Unpack the elements into separate variables using list comprehension
        self.items = [(view, sample, class_name) for view, sample, class_name in lines]
        self.dataset_path = dataset_path

    def __len__(self):
        return len(self.items)

    def __getitem__(self, idx):
        view_id, shape_id, class_name = self.items[idx]

        complete_mesh = ColoredMeshesDataset.get_target_mesh(dataset_path=self.dataset_path,
                                                        class_name=class_name,
                                                        shape_id=shape_id)

        incomplete_mesh = ColoredMeshesDataset.get_incomplete_mesh(dataset_path=self.dataset_path,
                                                                  class_name=class_name,
                                                                  shape_id=shape_id,
                                                                  view_id=view_id)
        return {
            "target_mesh": complete_mesh,
            "incomplete_mesh": incomplete_mesh
        }
    @staticmethod
    def send_data_to_device(batch_val, device):
        batch_val["target_mesh"] = batch_val["target_mesh"].to(device)
        batch_val["incomplete_mesh"] = batch_val["incomplete_mesh"].to(device)
    @staticmethod
    def get_target_mesh(dataset_path, class_name, shape_id):
        target_path = dataset_path+ '/GT/colorized_meshes/' + class_name + f"/{shape_id}.obj"
        mesh = trimesh.load(target_path)
        vertices = np.asarray(mesh.vertices).T
        colors = np.asarray(mesh.visual.vertex_colors).T[0:3]
        full_mesh = np.concatenate((vertices,colors[0:3]),axis=0)
        num_points = colors.shape[1]
        num_points = num_points - num_points % 16
        # Ensure that the number of points in the point clouds is a multiple of 16, and ensure that the shapes remain
        # consistent after passing through four layers of convolution and convolution transpose
        return colors[:, :num_points]

    @staticmethod
    def get_incomplete_mesh(dataset_path, class_name, shape_id, view_id):
        target_path = dataset_path+ '/GT/colorized_meshes/' + class_name + f"/{shape_id}.obj"
        incomplete_path = (dataset_path + "/InputSamples/colorized_meshes/" + class_name + f"/{shape_id}/" + f"{view_id}.obj")
        mesh_complete = trimesh.load(target_path)
        mesh_incomplete = trimesh.load(incomplete_path)
        mesh_incomplete.visual = mesh_incomplete.visual.to_texture()
        mesh_complete.visual = mesh_incomplete.visual.copy()
        v_idx = mesh_incomplete.kdtree.query(mesh_complete.vertices.copy())[1]
        mesh_complete.visual.uv = mesh_incomplete.visual.uv[v_idx]
        mesh_complete.visual = mesh_complete.visual.to_color()
        vertices = np.asarray(mesh_complete.vertices).T
        colors =np.asarray(mesh_complete.visual.vertex_colors).T[0:3]
        full_mesh = np.concatenate((vertices,colors[0:3]),axis=0)
        num_points = colors.shape[1]
        num_points = num_points - num_points % 16
        # Ensure that the number of points in the point clouds is a multiple of 16, and ensure that the shapes remain
        # consistent after passing through four layers of convolution and convolution transpose
        return colors[:, :num_points]

def collate_fn(batch):
    batch = sorted(batch, key=lambda x: len(x['target_mesh']), reverse=True)
    targets = [torch.from_numpy(item['target_mesh']) for item in batch]
    inputs = [torch.from_numpy(item['incomplete_mesh']) for item in batch]
    max_length = max(seq.shape[1] for seq in targets)
    target_padded_batch = []
    input_padded_batch = []
    for i, target in enumerate(targets):
        padding_length = max_length - target.shape[1]
        if padding_length>0:
            padded_target = torch.cat((target, torch.zeros([3,padding_length])),dim=1)
            padded_input = torch.cat((inputs[i], torch.zeros([3,padding_length])),dim=1)
            target_padded_batch.append(padded_target)
            input_padded_batch.append(padded_input)
        else:
            target_padded_batch.append(target)
            input_padded_batch.append(inputs[i])

    return {
            "target_mesh": torch.stack(target_padded_batch),
            "incomplete_mesh": torch.stack(input_padded_batch)
        }

if __name__ == "__main__":
    from torch.utils.data import DataLoader
    dataset = ColoredMeshesDataset("overfit","/media/rauldds/TOSHIBA EXT/ML3G/Full_Project_Dataset")
    train_dataloader = DataLoader(dataset, batch_size=1, shuffle=True,collate_fn=collate_fn)
    for batch in train_dataloader:
        print(len(batch['target_mesh']))
        for i, target in enumerate(batch["target_mesh"]):
            print(target.shape)
            print(batch['incomplete_mesh'][i].shape)


from menu_item import MenuItem

def show_user_menu():
    choose = input("A)Add an Item\n D) Delete an Item \n V) Delete an Item \nX)Exit \n").upper()
    return choose

def add_an_item():
    new_item = input ("Add an item")
    new_price= int(input("Add price"))
    new_one= MenuItem(new_item, new_price)
    return new_one
    print("Item was added successfully.")

def remove_item_from_menu():
    new_item = input ("What item would you like to delete")
    new_delete= MenuItem.delete(new_item)
    print("Item was deleted successfully.")

def main():
    choice = show_user_menu()
    while choice != "X":
        if choice == "A":
             add_an_item()
        #    return add
        elif choice == "D":
            remove = remove_item_from_menu()
            return remove
        elif choice == "V":
            print(MenuItem.all())
    print(MenuItem.all())
    return

def load_manager():
    menu=MenuItem("Fries", 5)

main()
load_manager()


from flask import Flask, jsonify, request
import pickle
import pandas as pd
from sklearn.neighbors import NearestNeighbors

app = Flask("__name__")

# Load model, user data, and restaurant data
with open("model.pkl", "rb") as f:
    model = pickle.load(f)

user_df = pd.read_parquet("../dataset/user.parquet")
restaurant_df = pd.read_parquet("../dataset/restaurant.mapping.parquet").set_index("index")

def recommend_restaurants(user_id):
    # Find 20 nearest neighbors to recommend restaurants
    user_data = user_df[user_df["user_id"] == user_id].drop(columns="user_id")

    if user_data.empty:
        return {"error": "User not found"}, 404

    dist, ind = model.kneighbors(user_data, n_neighbors=20)
    recommended_restaurant_ids = restaurant_df.iloc[ind[0]]["restaurant_id"].tolist()
    return {"restaurant": recommended_restaurant_ids}, 200

@app.route('/recommend/<user_id>', methods=['GET'])
def recommend(user_id):
    if request.method == 'GET':
        result, status_code = recommend_restaurants(user_id)
        return jsonify(result), status_code

@app.route('/ready', methods=['GET'])
def ready():
    return 'OK'

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=5005)


from espywrapper import EsPyWrapper

esx = EsPyWrapper()

SQL_QUERY = '''select
    *
    from "test-*"
'''

results = esx.query_sql(SQL_QUERY)

# print python dict
print(results.dict())

# print json
print(results.json())

# print pandas dataframe
print(results.df())


def Search2DMatrix(mtr, target):
    if mtr[0][0] > target:
        return False
    def hello(target, start=0, end=len(mtr) -1):
        mid = round((start + end) / 2)
        if (mtr[mid][-1]) >= target and mtr[mid][0] <= target:
                return mtr[mid]
        if start > end or start == end:
            return False
        if target >= (mtr[mid][-1]):
            return hello(target, start=mid + 1, end=end)
        elif target <= (mtr[mid][-1]):
            return hello(target, start=start, end=mid - 1)

    row = hello(target)
    if row != False:
        def checkTarget(target, start=0, end=len(row) - 1):
            middle = round((start + end) / 2)
            if row[middle] == target:
                return True
            if start == end:
                return False
            if row[middle] <= target:
                return checkTarget(target, start=middle + 1, end=end)
            if row[middle] >= target:
                return checkTarget(target, start, end=middle - 1)
        return checkTarget(target)

    return False
print(
Search2DMatrix([[1,3,5,7,8],[9,12,15,19,21],[22,24,27,29,30],[32,34,36,37,38],[40,42,46,47,48]],38)
)