import copy
import os
import re
import traceback
import warnings
import weakref

from astro_metadata_translator import fix_header
from lsst.utils import doImport
import lsst.daf.persistence as dafPersist
from . import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
import lsst.daf.base as dafBase
import lsst.afw.geom as afwGeom
import lsst.afw.image as afwImage
import lsst.afw.table as afwTable
from lsst.afw.fits import readMetadata
import lsst.afw.cameraGeom as afwCameraGeom
import lsst.log as lsstLog
import lsst.pex.exceptions as pexExcept
from .exposureIdInfo import ExposureIdInfo
from .makeRawVisitInfo import MakeRawVisitInfo
from .utils import createInitialSkyWcs, InitialSkyWcsError
from lsst.utils import getPackageDir
from ._instrument import Instrument

__all__ = ["CameraMapper", "exposureFromImage"]

class CameraMapper(dafPersist.Mapper):

    """CameraMapper is a base class for mappers that handle images from a
    camera and products derived from them.  This provides an abstraction layer
    between the data on disk and the code.

    Public methods: keys, queryMetadata, getDatasetTypes, map,
    canStandardize, standardize

    Mappers for specific data sources (e.g., CFHT Megacam, LSST
    simulations, etc.) should inherit this class.

    The CameraMapper manages datasets within a "root" directory. Note that
    writing to a dataset present in the input root will hide the existing
    dataset but not overwrite it.  See #2160 for design discussion.

    A camera is assumed to consist of one or more rafts, each composed of
    multiple CCDs.  Each CCD is in turn composed of one or more amplifiers
    (amps).  A camera is also assumed to have a camera geometry description
    (CameraGeom object) as a policy file, a filter description (Filter class
    static configuration) as another policy file.

    Information from the camera geometry and defects are inserted into all
    Exposure objects returned.

    The mapper uses one or two registries to retrieve metadata about the
    images.  The first is a registry of all raw exposures.  This must contain
    the time of the observation.  One or more tables (or the equivalent)
    within the registry are used to look up data identifier components that
    are not specified by the user (e.g. filter) and to return results for
    metadata queries.  The second is an optional registry of all calibration
    data.  This should contain validity start and end entries for each
    calibration dataset in the same timescale as the observation time.

    Subclasses will typically set MakeRawVisitInfoClass and optionally the
    metadata translator class:

    MakeRawVisitInfoClass: a class variable that points to a subclass of
    MakeRawVisitInfo, a functor that creates an
    lsst.afw.image.VisitInfo from the FITS metadata of a raw image.

    translatorClass: The `~astro_metadata_translator.MetadataTranslator`
    class to use for fixing metadata values.  If it is not set an attempt
    will be made to infer the class from ``MakeRawVisitInfoClass``, failing
    that the metadata fixup will try to infer the translator class from the
    header itself.

    Subclasses must provide the following methods:

    _extractDetectorName(self, dataId): returns the detector name for a CCD
    (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given
    a dataset identifier referring to that CCD or a subcomponent of it.

    _computeCcdExposureId(self, dataId): see below

    _computeCoaddExposureId(self, dataId, singleFilter): see below

    Subclasses may also need to override the following methods:

    _transformId(self, dataId): transformation of a data identifier
    from colloquial usage (e.g., "ccdname") to proper/actual usage
    (e.g., "ccd"), including making suitable for path expansion (e.g. removing
    commas). The default implementation does nothing.  Note that this
    method should not modify its input parameter.

    getShortCcdName(self, ccdName): a static method that returns a shortened
    name suitable for use as a filename. The default version converts spaces
    to underscores.

    _mapActualToPath(self, template, actualId): convert a template path to an
    actual path, using the actual dataset identifier.

    The mapper's behaviors are largely specified by the policy file.
    See the MapperDictionary.paf for descriptions of the available items.

    The 'exposures', 'calibrations', and 'datasets' subpolicies configure
    mappings (see Mappings class).

    Common default mappings for all subclasses can be specified in the
    "policy/{images,exposures,calibrations,datasets}.yaml" files. This
    provides a simple way to add a product to all camera mappers.

    Functions to map (provide a path to the data given a dataset
    identifier dictionary) and standardize (convert data into some standard
    format or type) may be provided in the subclass as "map_{dataset type}"
    and "std_{dataset type}", respectively.

    If non-Exposure datasets cannot be retrieved using standard
    daf_persistence methods alone, a "bypass_{dataset type}" function may be
    provided in the subclass to return the dataset instead of using the
    "datasets" subpolicy.

    Implementations of map_camera and bypass_camera that should typically be
    sufficient are provided in this base class.

    Notes
    -----
    .. todo::

       Instead of auto-loading the camera at construction time, load it from
       the calibration registry

    Parameters
    ----------
    policy : daf_persistence.Policy,
        Policy with per-camera defaults already merged.
    repositoryDir : string
        Policy repository for the subclassing module (obtained with
        getRepositoryPath() on the per-camera default dictionary).
    root : string, optional
        Path to the root directory for data.
    registry : string, optional
        Path to registry with data's metadata.
    calibRoot : string, optional
        Root directory for calibrations.
    calibRegistry : string, optional
        Path to registry with calibrations' metadata.
    provided : list of string, optional
        Keys provided by the mapper.
    parentRegistry : Registry subclass, optional
        Registry from a parent repository that may be used to look up
        data's metadata.
    repositoryCfg : daf_persistence.RepositoryCfg or None, optional
        The configuration information for the repository this mapper is
        being used with.
    """
    packageName = None

    # a class or subclass of MakeRawVisitInfo, a functor that makes an
    # lsst.afw.image.VisitInfo from the FITS metadata of a raw image
    MakeRawVisitInfoClass = MakeRawVisitInfo

    # a class or subclass of PupilFactory
    PupilFactoryClass = afwCameraGeom.PupilFactory

    # Class to use for metadata translations
    translatorClass = None

    # Gen3 instrument corresponding to this mapper
    # Can be a class or a string with the full name of the class
    _gen3instrument = None

    def __init__(self, policy, repositoryDir,
                 root=None, registry=None, calibRoot=None, calibRegistry=None,
                 provided=None, parentRegistry=None, repositoryCfg=None):

        dafPersist.Mapper.__init__(self)

        self.log = lsstLog.Log.getLogger("CameraMapper")

        if root:
            self.root = root
        elif repositoryCfg:
            self.root = repositoryCfg.root
        else:
            self.root = None

        repoPolicy = repositoryCfg.policy if repositoryCfg else None
        if repoPolicy is not None:
            policy.update(repoPolicy)

        # Levels
        self.levels = dict()
        if 'levels' in policy:
            levelsPolicy = policy['levels']
            for key in levelsPolicy.names(True):
                self.levels[key] = set(levelsPolicy.asArray(key))
        self.defaultLevel = policy['defaultLevel']
        self.defaultSubLevels = dict()
        if 'defaultSubLevels' in policy:
            self.defaultSubLevels = policy['defaultSubLevels']

        # Root directories
        if root is None:
            root = "."
        root = dafPersist.LogicalLocation(root).locString()

        self.rootStorage = dafPersist.Storage.makeFromURI(uri=root)

        # If the calibRoot is passed in, use that. If not and it's indicated in
        # the policy, use that. And otherwise, the calibs are in the regular
        # root.
        # If the location indicated by the calib root does not exist, do not
        # create it.
        calibStorage = None
        if calibRoot is not None:
            calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
            calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
                                                          create=False)
        else:
            calibRoot = policy.get('calibRoot', None)
            if calibRoot:
                calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
                                                              create=False)
        if calibStorage is None:
            calibStorage = self.rootStorage

        self.root = root

        # Registries
        self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath",
                                            self.rootStorage, searchParents=False,
                                            posixIfNoSql=(not parentRegistry))
        if not self.registry:
            self.registry = parentRegistry
        needCalibRegistry = policy.get('needCalibRegistry', None)
        if needCalibRegistry:
            if calibStorage:
                self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy,
                                                         "calibRegistryPath", calibStorage,
                                                         posixIfNoSql=False)  # NB never use posix for calibs
            else:
                raise RuntimeError(
                    "'needCalibRegistry' is true in Policy, but was unable to locate a repo at "
                    f"calibRoot ivar:{calibRoot} or policy['calibRoot']:{policy.get('calibRoot', None)}")
        else:
            self.calibRegistry = None

        # Dict of valid keys and their value types
        self.keyDict = dict()

        self._initMappings(policy, self.rootStorage, calibStorage, provided=None)
        self._initWriteRecipes()

        # Camera geometry
        self.cameraDataLocation = None  # path to camera geometry config file
        self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)

        # Filter translation table
        self.filters = None

        # verify that the class variable packageName is set before attempting
        # to instantiate an instance
        if self.packageName is None:
            raise ValueError('class variable packageName must not be None')

        self.makeRawVisitInfo = self.MakeRawVisitInfoClass(log=self.log)

        # Assign a metadata translator if one has not been defined by
        # subclass. We can sometimes infer one from the RawVisitInfo
        # class.
        if self.translatorClass is None and hasattr(self.makeRawVisitInfo, "metadataTranslator"):
            self.translatorClass = self.makeRawVisitInfo.metadataTranslator

    def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
        """Initialize mappings

        For each of the dataset types that we want to be able to read, there
        are methods that can be created to support them:
        * map_<dataset> : determine the path for dataset
        * std_<dataset> : standardize the retrieved dataset
        * bypass_<dataset> : retrieve the dataset (bypassing the usual
          retrieval machinery)
        * query_<dataset> : query the registry

        Besides the dataset types explicitly listed in the policy, we create
        additional, derived datasets for additional conveniences,
        e.g., reading the header of an image, retrieving only the size of a
        catalog.

        Parameters
        ----------
        policy : `lsst.daf.persistence.Policy`
            Policy with per-camera defaults already merged
        rootStorage : `Storage subclass instance`
            Interface to persisted repository data.
        calibRoot : `Storage subclass instance`
            Interface to persisted calib repository data
        provided : `list` of `str`
            Keys provided by the mapper
        """
        # Sub-dictionaries (for exposure/calibration/dataset types)
        imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
            "obs_base", "ImageMappingDefaults.yaml", "policy"))
        expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
            "obs_base", "ExposureMappingDefaults.yaml", "policy"))
        calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
            "obs_base", "CalibrationMappingDefaults.yaml", "policy"))
        dsMappingPolicy = dafPersist.Policy()

        # Mappings
        mappingList = (
            ("images", imgMappingPolicy, ImageMapping),
            ("exposures", expMappingPolicy, ExposureMapping),
            ("calibrations", calMappingPolicy, CalibrationMapping),
            ("datasets", dsMappingPolicy, DatasetMapping)
        )
        self.mappings = dict()
        for name, defPolicy, cls in mappingList:
            if name in policy:
                datasets = policy[name]

                # Centrally-defined datasets
                defaultsPath = os.path.join(getPackageDir("obs_base"), "policy", name + ".yaml")
                if os.path.exists(defaultsPath):
                    datasets.merge(dafPersist.Policy(defaultsPath))

                mappings = dict()
                setattr(self, name, mappings)
                for datasetType in datasets.names(True):
                    subPolicy = datasets[datasetType]
                    subPolicy.merge(defPolicy)

                    if not hasattr(self, "map_" + datasetType) and 'composite' in subPolicy:
                        def compositeClosure(dataId, write=False, mapper=None, mapping=None,
                                             subPolicy=subPolicy):
                            components = subPolicy.get('composite')
                            assembler = subPolicy['assembler'] if 'assembler' in subPolicy else None
                            disassembler = subPolicy['disassembler'] if 'disassembler' in subPolicy else None
                            python = subPolicy['python']
                            butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
                                                                         disassembler=disassembler,
                                                                         python=python,
                                                                         dataId=dataId,
                                                                         mapper=self)
                            for name, component in components.items():
                                butlerComposite.add(id=name,
                                                    datasetType=component.get('datasetType'),
                                                    setter=component.get('setter', None),
                                                    getter=component.get('getter', None),
                                                    subset=component.get('subset', False),
                                                    inputOnly=component.get('inputOnly', False))
                            return butlerComposite
                        setattr(self, "map_" + datasetType, compositeClosure)
                        # for now at least, don't set up any other handling for
                        # this dataset type.
                        continue

                    if name == "calibrations":
                        mapping = cls(datasetType, subPolicy, self.registry, self.calibRegistry, calibStorage,
                                      provided=provided, dataRoot=rootStorage)
                    else:
                        mapping = cls(datasetType, subPolicy, self.registry, rootStorage, provided=provided)

                    if datasetType in self.mappings:
                        raise ValueError(f"Duplicate mapping policy for dataset type {datasetType}")
                    self.keyDict.update(mapping.keys())
                    mappings[datasetType] = mapping
                    self.mappings[datasetType] = mapping
                    if not hasattr(self, "map_" + datasetType):
                        def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
                            return mapping.map(mapper, dataId, write)
                        setattr(self, "map_" + datasetType, mapClosure)
                    if not hasattr(self, "query_" + datasetType):
                        def queryClosure(format, dataId, mapping=mapping):
                            return mapping.lookup(format, dataId)
                        setattr(self, "query_" + datasetType, queryClosure)
                    if hasattr(mapping, "standardize") and not hasattr(self, "std_" + datasetType):
                        def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
                            return mapping.standardize(mapper, item, dataId)
                        setattr(self, "std_" + datasetType, stdClosure)

                    def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
                        """Set convenience methods on CameraMapper"""
                        mapName = "map_" + datasetType + "_" + suffix
                        bypassName = "bypass_" + datasetType + "_" + suffix
                        queryName = "query_" + datasetType + "_" + suffix
                        if not hasattr(self, mapName):
                            setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
                        if not hasattr(self, bypassName):
                            if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
                                bypassImpl = getattr(self, "bypass_" + datasetType)
                            if bypassImpl is not None:
                                setattr(self, bypassName, bypassImpl)
                        if not hasattr(self, queryName):
                            setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))

                    # Filename of dataset
                    setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
                               [os.path.join(location.getStorage().root, p) for p in location.getLocations()])
                    # Metadata from FITS file
                    if subPolicy["storage"] == "FitsStorage":  # a FITS image
                        def getMetadata(datasetType, pythonType, location, dataId):
                            md = readMetadata(location.getLocationsWithRoot()[0])
                            fix_header(md, translator_class=self.translatorClass)
                            return md

                        setMethods("md", bypassImpl=getMetadata)

                        # Add support for configuring FITS compression
                        addName = "add_" + datasetType
                        if not hasattr(self, addName):
                            setattr(self, addName, self.getImageCompressionSettings)

                        if name == "exposures":
                            def getSkyWcs(datasetType, pythonType, location, dataId):
                                fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
                                return fitsReader.readWcs()

                            setMethods("wcs", bypassImpl=getSkyWcs)

                            def getRawHeaderWcs(datasetType, pythonType, location, dataId):
                                """Create a SkyWcs from the un-modified raw
                                FITS WCS header keys."""
                                if datasetType[:3] != "raw":
                                    raise dafPersist.NoResults("Can only get header WCS for raw exposures.",
                                                               datasetType, dataId)
                                return afwGeom.makeSkyWcs(readMetadata(location.getLocationsWithRoot()[0]))

                            setMethods("header_wcs", bypassImpl=getRawHeaderWcs)

                            def getPhotoCalib(datasetType, pythonType, location, dataId):
                                fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
                                return fitsReader.readPhotoCalib()

                            setMethods("photoCalib", bypassImpl=getPhotoCalib)

                            def getVisitInfo(datasetType, pythonType, location, dataId):
                                fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
                                return fitsReader.readVisitInfo()

                            setMethods("visitInfo", bypassImpl=getVisitInfo)

                            # TODO: deprecate in DM-27170, remove in DM-27177
                            def getFilter(datasetType, pythonType, location, dataId):
                                fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
                                return fitsReader.readFilter()

                            setMethods("filter", bypassImpl=getFilter)

                            # TODO: deprecate in DM-27177, remove in DM-27811
                            def getFilterLabel(datasetType, pythonType, location, dataId):
                                fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
                                storedFilter = fitsReader.readFilterLabel()

                                # Apply standardization used by full Exposure
                                try:
                                    # mapping is local to enclosing scope
                                    idFilter = mapping.need(['filter'], dataId)['filter']
                                except dafPersist.NoResults:
                                    idFilter = None
                                bestFilter = self._getBestFilter(storedFilter, idFilter)
                                if bestFilter is not None:
                                    return bestFilter
                                else:
                                    return storedFilter

                            setMethods("filterLabel", bypassImpl=getFilterLabel)

                            setMethods("detector",
                                       mapImpl=lambda dataId, write=False:
                                           dafPersist.ButlerLocation(
                                               pythonType="lsst.afw.cameraGeom.CameraConfig",
                                               cppType="Config",
                                               storageName="Internal",
                                               locationList="ignored",
                                               dataId=dataId,
                                               mapper=self,
                                               storage=None,
                                           ),
                                       bypassImpl=lambda datasetType, pythonType, location, dataId:
                                           self.camera[self._extractDetectorName(dataId)]
                                       )

                            def getBBox(datasetType, pythonType, location, dataId):
                                md = readMetadata(location.getLocationsWithRoot()[0], hdu=1)
                                fix_header(md, translator_class=self.translatorClass)
                                return afwImage.bboxFromMetadata(md)

                            setMethods("bbox", bypassImpl=getBBox)

                        elif name == "images":
                            def getBBox(datasetType, pythonType, location, dataId):
                                md = readMetadata(location.getLocationsWithRoot()[0])
                                fix_header(md, translator_class=self.translatorClass)
                                return afwImage.bboxFromMetadata(md)
                            setMethods("bbox", bypassImpl=getBBox)

                    if subPolicy["storage"] == "FitsCatalogStorage":  # a FITS catalog

                        def getMetadata(datasetType, pythonType, location, dataId):
                            md = readMetadata(os.path.join(location.getStorage().root,
                                              location.getLocations()[0]), hdu=1)
                            fix_header(md, translator_class=self.translatorClass)
                            return md

                        setMethods("md", bypassImpl=getMetadata)

                    # Sub-images
                    if subPolicy["storage"] == "FitsStorage":
                        def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
                            subId = dataId.copy()
                            del subId['bbox']
                            loc = mapping.map(mapper, subId, write)
                            bbox = dataId['bbox']
                            llcX = bbox.getMinX()
                            llcY = bbox.getMinY()
                            width = bbox.getWidth()
                            height = bbox.getHeight()
                            loc.additionalData.set('llcX', llcX)
                            loc.additionalData.set('llcY', llcY)
                            loc.additionalData.set('width', width)
                            loc.additionalData.set('height', height)
                            if 'imageOrigin' in dataId:
                                loc.additionalData.set('imageOrigin',
                                                       dataId['imageOrigin'])
                            return loc

                        def querySubClosure(key, format, dataId, mapping=mapping):
                            subId = dataId.copy()
                            del subId['bbox']
                            return mapping.lookup(format, subId)
                        setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)

                    if subPolicy["storage"] == "FitsCatalogStorage":
                        # Length of catalog

                        def getLen(datasetType, pythonType, location, dataId):
                            md = readMetadata(os.path.join(location.getStorage().root,
                                              location.getLocations()[0]), hdu=1)
                            fix_header(md, translator_class=self.translatorClass)
                            return md["NAXIS2"]

                        setMethods("len", bypassImpl=getLen)

                        # Schema of catalog
                        if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
                            setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
                                       afwTable.Schema.readFits(os.path.join(location.getStorage().root,
                                                                             location.getLocations()[0])))

    def _computeCcdExposureId(self, dataId):
        """Compute the 64-bit (long) identifier for a CCD exposure.

        Subclasses must override

        Parameters
        ----------
        dataId : `dict`
            Data identifier with visit, ccd.
        """
        raise NotImplementedError()

    def _computeCoaddExposureId(self, dataId, singleFilter):
        """Compute the 64-bit (long) identifier for a coadd.

        Subclasses must override

        Parameters
        ----------
        dataId  : `dict`
            Data identifier with tract and patch.
        singleFilter  : `bool`
            True means the desired ID is for a single-filter coadd, in which
            case dataIdmust contain filter.
        """
        raise NotImplementedError()

    def _search(self, path):
        """Search for path in the associated repository's storage.

        Parameters
        ----------
        path : string
            Path that describes an object in the repository associated with
            this mapper.
            Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The
            indicator will be stripped when searching and so will match
            filenames without the HDU indicator, e.g. 'foo.fits'. The path
            returned WILL contain the indicator though, e.g. ['foo.fits[1]'].

        Returns
        -------
        string
            The path for this object in the repository. Will return None if the
            object can't be found. If the input argument path contained an HDU
            indicator, the returned path will also contain the HDU indicator.
        """
        return self.rootStorage.search(path)

    def backup(self, datasetType, dataId):
        """Rename any existing object with the given type and dataId.

        The CameraMapper implementation saves objects in a sequence of e.g.:

        - foo.fits
        - foo.fits~1
        - foo.fits~2

        All of the backups will be placed in the output repo, however, and will
        not be removed if they are found elsewhere in the _parent chain.  This
        means that the same file will be stored twice if the previous version
        was found in an input repo.
        """

        # Calling PosixStorage directly is not the long term solution in this
        # function, this is work-in-progress on epic DM-6225. The plan is for
        # parentSearch to be changed to 'search', and search only the storage
        # associated with this mapper. All searching of parents will be handled
        # by traversing the container of repositories in Butler.

        def firstElement(list):
            """Get the first element in the list, or None if that can't be
            done.
            """
            return list[0] if list is not None and len(list) else None

        n = 0
        newLocation = self.map(datasetType, dataId, write=True)
        newPath = newLocation.getLocations()[0]
        path = dafPersist.PosixStorage.search(self.root, newPath, searchParents=True)
        path = firstElement(path)
        oldPaths = []
        while path is not None:
            n += 1
            oldPaths.append((n, path))
            path = dafPersist.PosixStorage.search(self.root, "%s~%d" % (newPath, n), searchParents=True)
            path = firstElement(path)
        for n, oldPath in reversed(oldPaths):
            self.rootStorage.copyFile(oldPath, "%s~%d" % (newPath, n))

    def keys(self):
        """Return supported keys.

        Returns
        -------
        iterable
            List of keys usable in a dataset identifier
        """
        return iter(self.keyDict.keys())

    def getKeys(self, datasetType, level):
        """Return a dict of supported keys and their value types for a given
        dataset type at a given level of the key hierarchy.

        Parameters
        ----------
        datasetType :  `str`
            Dataset type or None for all dataset types.
        level :  `str` or None
            Level or None for all levels or '' for the default level for the
            camera.

        Returns
        -------
        `dict`
            Keys are strings usable in a dataset identifier, values are their
            value types.
        """

        # not sure if this is how we want to do this. what if None was
        # intended?
        if level == '':
            level = self.getDefaultLevel()

        if datasetType is None:
            keyDict = copy.copy(self.keyDict)
        else:
            keyDict = self.mappings[datasetType].keys()
        if level is not None and level in self.levels:
            keyDict = copy.copy(keyDict)
            for lev in self.levels[level]:
                if lev in keyDict:
                    del keyDict[lev]
        return keyDict

    def getDefaultLevel(self):
        return self.defaultLevel

    def getDefaultSubLevel(self, level):
        if level in self.defaultSubLevels:
            return self.defaultSubLevels[level]
        return None

    @classmethod
    def getCameraName(cls):
        """Return the name of the camera that this CameraMapper is for."""
        className = str(cls)
        className = className[className.find('.'):-1]
        m = re.search(r'(\w+)Mapper', className)
        if m is None:
            m = re.search(r"class '[\w.]*?(\w+)'", className)
        name = m.group(1)
        return name[:1].lower() + name[1:] if name else ''

    @classmethod
    def getPackageName(cls):
        """Return the name of the package containing this CameraMapper."""
        if cls.packageName is None:
            raise ValueError('class variable packageName must not be None')
        return cls.packageName

    @classmethod
    def getGen3Instrument(cls):
        """Return the gen3 Instrument class equivalent for this gen2 Mapper.

        Returns
        -------
        instr : `type`
            A `~lsst.obs.base.Instrument` class.
        """
        if cls._gen3instrument is None:
            raise NotImplementedError("Please provide a specific implementation for your instrument"
                                      " to enable conversion of this gen2 repository to gen3")
        if isinstance(cls._gen3instrument, str):
            # Given a string to convert to an instrument class
            cls._gen3instrument = doImport(cls._gen3instrument)
        if not issubclass(cls._gen3instrument, Instrument):
            raise ValueError(f"Mapper {cls} has declared a gen3 instrument class of {cls._gen3instrument}"
                             " but that is not an lsst.obs.base.Instrument")
        return cls._gen3instrument

    @classmethod
    def getPackageDir(cls):
        """Return the base directory of this package"""
        return getPackageDir(cls.getPackageName())

    def map_camera(self, dataId, write=False):
        """Map a camera dataset."""
        if self.camera is None:
            raise RuntimeError("No camera dataset available.")
        actualId = self._transformId(dataId)
        return dafPersist.ButlerLocation(
            pythonType="lsst.afw.cameraGeom.CameraConfig",
            cppType="Config",
            storageName="ConfigStorage",
            locationList=self.cameraDataLocation or "ignored",
            dataId=actualId,
            mapper=self,
            storage=self.rootStorage
        )

    def bypass_camera(self, datasetType, pythonType, butlerLocation, dataId):
        """Return the (preloaded) camera object.
        """
        if self.camera is None:
            raise RuntimeError("No camera dataset available.")
        return self.camera

    def map_expIdInfo(self, dataId, write=False):
        return dafPersist.ButlerLocation(
            pythonType="lsst.obs.base.ExposureIdInfo",
            cppType=None,
            storageName="Internal",
            locationList="ignored",
            dataId=dataId,
            mapper=self,
            storage=self.rootStorage
        )

    def bypass_expIdInfo(self, datasetType, pythonType, location, dataId):
        """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure"""
        expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
        expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
        return ExposureIdInfo(expId=expId, expBits=expBits)

    def std_bfKernel(self, item, dataId):
        """Disable standardization for bfKernel

        bfKernel is a calibration product that is numpy array,
        unlike other calibration products that are all images;
        all calibration images are sent through _standardizeExposure
        due to CalibrationMapping, but we don't want that to happen to bfKernel
        """
        return item

    def std_raw(self, item, dataId):
        """Standardize a raw dataset by converting it to an Exposure instead
        of an Image"""
        return self._standardizeExposure(self.exposures['raw'], item, dataId,
                                         trimmed=False, setVisitInfo=True)

    def map_skypolicy(self, dataId):
        """Map a sky policy."""
        return dafPersist.ButlerLocation("lsst.pex.policy.Policy", "Policy",
                                         "Internal", None, None, self,
                                         storage=self.rootStorage)

    def std_skypolicy(self, item, dataId):
        """Standardize a sky policy by returning the one we use."""
        return self.skypolicy

###############################################################################
#
# Utility functions
#
###############################################################################

    def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
                       posixIfNoSql=True):
        """Set up a registry (usually SQLite3), trying a number of possible
        paths.

        Parameters
        ----------
        name : string
            Name of registry.
        description: `str`
            Description of registry (for log messages)
        path : string
            Path for registry.
        policy : string
            Policy that contains the registry name, used if path is None.
        policyKey : string
            Key in policy for registry path.
        storage : Storage subclass
            Repository Storage to look in.
        searchParents : bool, optional
            True if the search for a registry should follow any Butler v1
            _parent symlinks.
        posixIfNoSql : bool, optional
            If an sqlite registry is not found, will create a posix registry if
            this is True.

        Returns
        -------
        lsst.daf.persistence.Registry
            Registry object
        """
        if path is None and policyKey in policy:
            path = dafPersist.LogicalLocation(policy[policyKey]).locString()
            if os.path.isabs(path):
                raise RuntimeError("Policy should not indicate an absolute path for registry.")
            if not storage.exists(path):
                newPath = storage.instanceSearch(path)

                newPath = newPath[0] if newPath is not None and len(newPath) else None
                if newPath is None:
                    self.log.warn("Unable to locate registry at policy path (also looked in root): %s",
                                  path)
                path = newPath
            else:
                self.log.warn("Unable to locate registry at policy path: %s", path)
                path = None

        # Old Butler API was to indicate the registry WITH the repo folder,
        # New Butler expects the registry to be in the repo folder. To support
        # Old API, check to see if path starts with root, and if so, strip
        # root from path. Currently only works with PosixStorage
        try:
            root = storage.root
            if path and (path.startswith(root)):
                path = path[len(root + '/'):]
        except AttributeError:
            pass

        # determine if there is an sqlite registry and if not, try the posix
        # registry.
        registry = None

        def search(filename, description):
            """Search for file in storage

            Parameters
            ----------
            filename : `str`
                Filename to search for
            description : `str`
                Description of file, for error message.

            Returns
            -------
            path : `str` or `None`
                Path to file, or None
            """
            result = storage.instanceSearch(filename)
            if result:
                return result[0]
            self.log.debug("Unable to locate %s: %s", description, filename)
            return None

        # Search for a suitable registry database
        if path is None:
            path = search("%s.pgsql" % name, "%s in root" % description)
        if path is None:
            path = search("%s.sqlite3" % name, "%s in root" % description)
        if path is None:
            path = search(os.path.join(".", "%s.sqlite3" % name), "%s in current dir" % description)

        if path is not None:
            if not storage.exists(path):
                newPath = storage.instanceSearch(path)
                newPath = newPath[0] if newPath is not None and len(newPath) else None
                if newPath is not None:
                    path = newPath
            localFileObj = storage.getLocalFile(path)
            self.log.info("Loading %s registry from %s", description, localFileObj.name)
            registry = dafPersist.Registry.create(localFileObj.name)
            localFileObj.close()
        elif not registry and posixIfNoSql:
            try:
                self.log.info("Loading Posix %s registry from %s", description, storage.root)
                registry = dafPersist.PosixRegistry(storage.root)
            except Exception:
                registry = None

        return registry

    def _transformId(self, dataId):
        """Generate a standard ID dict from a camera-specific ID dict.

        Canonical keys include:
        - amp: amplifier name
        - ccd: CCD name (in LSST this is a combination of raft and sensor)
        The default implementation returns a copy of its input.

        Parameters
        ----------
        dataId : `dict`
            Dataset identifier; this must not be modified

        Returns
        -------
        `dict`
            Transformed dataset identifier.
        """

        return dataId.copy()

    def _mapActualToPath(self, template, actualId):
        """Convert a template path to an actual path, using the actual data
        identifier.  This implementation is usually sufficient but can be
        overridden by the subclass.

        Parameters
        ----------
        template : `str`
            Template path
        actualId : `dict`
            Dataset identifier

        Returns
        -------
        `str`
            Pathname
        """

        try:
            transformedId = self._transformId(actualId)
            return template % transformedId
        except Exception as e:
            raise RuntimeError("Failed to format %r with data %r: %s" % (template, transformedId, e))

    @staticmethod
    def getShortCcdName(ccdName):
        """Convert a CCD name to a form useful as a filename

        The default implementation converts spaces to underscores.
        """
        return ccdName.replace(" ", "_")

    def _extractDetectorName(self, dataId):
        """Extract the detector (CCD) name from the dataset identifier.

        The name in question is the detector name used by lsst.afw.cameraGeom.

        Parameters
        ----------
        dataId : `dict`
            Dataset identifier.

        Returns
        -------
        `str`
            Detector name
        """
        raise NotImplementedError("No _extractDetectorName() function specified")

    def _setAmpDetector(self, item, dataId, trimmed=True):
        """Set the detector object in an Exposure for an amplifier.

        Defects are also added to the Exposure based on the detector object.

        Parameters
        ----------
        item : `lsst.afw.image.Exposure`
            Exposure to set the detector in.
        dataId : `dict`
            Dataset identifier
        trimmed : `bool`
            Should detector be marked as trimmed? (ignored)
        """

        return self._setCcdDetector(item=item, dataId=dataId, trimmed=trimmed)

    def _setCcdDetector(self, item, dataId, trimmed=True):
        """Set the detector object in an Exposure for a CCD.

        Parameters
        ----------
        item : `lsst.afw.image.Exposure`
            Exposure to set the detector in.
        dataId : `dict`
            Dataset identifier
        trimmed : `bool`
            Should detector be marked as trimmed? (ignored)
        """
        if item.getDetector() is not None:
            return

        detectorName = self._extractDetectorName(dataId)
        detector = self.camera[detectorName]
        item.setDetector(detector)

    @staticmethod
    def _resolveFilters(definitions, idFilter, filterLabel):
        """Identify the filter(s) consistent with partial filter information.

        Parameters
        ----------
        definitions : `lsst.obs.base.FilterDefinitionCollection`
            The filter definitions in which to search for filters.
        idFilter : `str` or `None`
            The filter information provided in a data ID.
        filterLabel : `lsst.afw.image.FilterLabel` or `None`
            The filter information provided by an exposure; may be incomplete.

        Returns
        -------
        filters : `set` [`lsst.obs.base.FilterDefinition`]
            The set of filters consistent with ``idFilter``
            and ``filterLabel``.
        """
        # Assume none of the filter constraints actually wrong/contradictory.
        # Then taking the intersection of all constraints will give a unique
        # result if one exists.
        matches = set(definitions)
        if idFilter is not None:
            matches.intersection_update(definitions.findAll(idFilter))
        if filterLabel is not None and filterLabel.hasPhysicalLabel():
            matches.intersection_update(definitions.findAll(filterLabel.physicalLabel))
        if filterLabel is not None and filterLabel.hasBandLabel():
            matches.intersection_update(definitions.findAll(filterLabel.bandLabel))
        return matches

    def _getBestFilter(self, storedLabel, idFilter):
        """Estimate the most complete filter information consistent with the
        file or registry.

        Parameters
        ----------
        storedLabel : `lsst.afw.image.FilterLabel` or `None`
            The filter previously stored in the file.
        idFilter : `str` or `None`
            The filter implied by the data ID, if any.

        Returns
        -------
        bestFitler : `lsst.afw.image.FilterLabel` or `None`
            The complete filter to describe the dataset. May be equal to
            ``storedLabel``. `None` if no recommendation can be generated.
        """
        try:
            # getGen3Instrument returns class; need to construct it.
            filterDefinitions = self.getGen3Instrument()().filterDefinitions
        except NotImplementedError:
            filterDefinitions = None

        if filterDefinitions is not None:
            definitions = self._resolveFilters(filterDefinitions, idFilter, storedLabel)
            self.log.debug("Matching filters for id=%r and label=%r are %s.",
                           idFilter, storedLabel, definitions)
            if len(definitions) == 1:
                newLabel = list(definitions)[0].makeFilterLabel()
                return newLabel
            elif definitions:
                self.log.warn("Multiple matches for filter %r with data ID %r.", storedLabel, idFilter)
                # Can we at least add a band?
                # Never expect multiple definitions with same physical filter.
                bands = {d.band for d in definitions}  # None counts as separate result!
                if len(bands) == 1 and storedLabel is None:
                    band = list(bands)[0]
                    return afwImage.FilterLabel(band=band)
                else:
                    return None
            else:
                # Unknown filter, nothing to be done.
                self.log.warn("Cannot reconcile filter %r with data ID %r.", storedLabel, idFilter)
                return None

        # Not practical to recommend a FilterLabel without filterDefinitions

        return None

    def _setFilter(self, mapping, item, dataId):
        """Set the filter information in an Exposure.

        The Exposure should already have had a filter loaded, but the reader
        (in ``afw``) had to act on incomplete information. This method
        cross-checks the filter against the data ID and the standard list
        of filters.

        Parameters
        ----------
        mapping : `lsst.obs.base.Mapping`
            Where to get the data ID filter from.
        item : `lsst.afw.image.Exposure`
            Exposure to set the filter in.
        dataId : `dict`
            Dataset identifier.
        """
        if not (isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureI)
                or isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureD)):
            return

        itemFilter = item.getFilterLabel()  # may be None
        try:
            idFilter = mapping.need(['filter'], dataId)['filter']
        except dafPersist.NoResults:
            idFilter = None

        bestFilter = self._getBestFilter(itemFilter, idFilter)
        if bestFilter is not None:
            if bestFilter != itemFilter:
                item.setFilterLabel(bestFilter)
            # Already using bestFilter, avoid unnecessary edits
        elif itemFilter is None:
            # Old Filter cleanup, without the benefit of FilterDefinition
            if self.filters is not None and idFilter in self.filters:
                idFilter = self.filters[idFilter]
            try:
                # TODO: remove in DM-27177; at that point may not be able
                # to process IDs without FilterDefinition.
                with warnings.catch_warnings():
                    warnings.filterwarnings("ignore", category=FutureWarning)
                    item.setFilter(afwImage.Filter(idFilter))
            except pexExcept.NotFoundError:
                self.log.warn("Filter %s not defined.  Set to UNKNOWN.", idFilter)

    def _standardizeExposure(self, mapping, item, dataId, filter=True,
                             trimmed=True, setVisitInfo=True):
        """Default standardization function for images.

        This sets the Detector from the camera geometry
        and optionally set the Filter. In both cases this saves
        having to persist some data in each exposure (or image).

        Parameters
        ----------
        mapping : `lsst.obs.base.Mapping`
            Where to get the values from.
        item : image-like object
            Can be any of lsst.afw.image.Exposure,
            lsst.afw.image.DecoratedImage, lsst.afw.image.Image
            or lsst.afw.image.MaskedImage

        dataId : `dict`
            Dataset identifier
        filter : `bool`
            Set filter? Ignored if item is already an exposure
        trimmed : `bool`
            Should detector be marked as trimmed?
        setVisitInfo : `bool`
            Should Exposure have its VisitInfo filled out from the metadata?

        Returns
        -------
        `lsst.afw.image.Exposure`
            The standardized Exposure.
        """
        try:
            exposure = exposureFromImage(item, dataId, mapper=self, logger=self.log,
                                         setVisitInfo=setVisitInfo)
        except Exception as e:
            self.log.error("Could not turn item=%r into an exposure: %s" % (repr(item), e))
            raise

        if mapping.level.lower() == "amp":
            self._setAmpDetector(exposure, dataId, trimmed)
        elif mapping.level.lower() == "ccd":
            self._setCcdDetector(exposure, dataId, trimmed)

        # We can only create a WCS if it doesn't already have one and
        # we have either a VisitInfo or exposure metadata.
        # Do not calculate a WCS if this is an amplifier exposure
        if mapping.level.lower() != "amp" and exposure.getWcs() is None and \
           (exposure.getInfo().getVisitInfo() is not None or exposure.getMetadata().toDict()):
            self._createInitialSkyWcs(exposure)

        if filter:
            self._setFilter(mapping, exposure, dataId)

        return exposure

    def _createSkyWcsFromMetadata(self, exposure):
        """Create a SkyWcs from the FITS header metadata in an Exposure.

        Parameters
        ----------
        exposure : `lsst.afw.image.Exposure`
            The exposure to get metadata from, and attach the SkyWcs to.
        """
        metadata = exposure.getMetadata()
        fix_header(metadata, translator_class=self.translatorClass)
        try:
            wcs = afwGeom.makeSkyWcs(metadata, strip=True)
            exposure.setWcs(wcs)
        except pexExcept.TypeError as e:
            # See DM-14372 for why this is debug and not warn (e.g. calib
            # files without wcs metadata).
            self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
                           " %s", e.args[0])
        # ensure any WCS values stripped from the metadata are removed in the
        # exposure
        exposure.setMetadata(metadata)

    def _createInitialSkyWcs(self, exposure):
        """Create a SkyWcs from the boresight and camera geometry.

        If the boresight or camera geometry do not support this method of
        WCS creation, this falls back on the header metadata-based version
        (typically a purely linear FITS crval/crpix/cdmatrix WCS).

        Parameters
        ----------
        exposure : `lsst.afw.image.Exposure`
            The exposure to get data from, and attach the SkyWcs to.
        """
        # Always use try to use metadata first, to strip WCS keys from it.
        self._createSkyWcsFromMetadata(exposure)

        if exposure.getInfo().getVisitInfo() is None:
            msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs."
            self.log.warn(msg)
            return
        try:
            newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector())
            exposure.setWcs(newSkyWcs)
        except InitialSkyWcsError as e:
            msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s"
            self.log.warn(msg, e)
            self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e))
            if e.__context__ is not None:
                self.log.debug("Root-cause Exception was: %s",
                               traceback.TracebackException.from_exception(e.__context__))

    def _makeCamera(self, policy, repositoryDir):
        """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing
        the camera geometry

        Also set self.cameraDataLocation, if relevant (else it can be left
        None).

        This implementation assumes that policy contains an entry "camera"
        that points to the subdirectory in this package of camera data;
        specifically, that subdirectory must contain:
        - a file named `camera.py` that contains persisted camera config
        - ampInfo table FITS files, as required by
          lsst.afw.cameraGeom.makeCameraFromPath

        Parameters
        ----------
        policy : `lsst.daf.persistence.Policy`
             Policy with per-camera defaults already merged
             (PexPolicy only for backward compatibility).
        repositoryDir : `str`
            Policy repository for the subclassing module (obtained with
            getRepositoryPath() on the per-camera default dictionary).
        """
        if 'camera' not in policy:
            raise RuntimeError("Cannot find 'camera' in policy; cannot construct a camera")
        cameraDataSubdir = policy['camera']
        self.cameraDataLocation = os.path.normpath(
            os.path.join(repositoryDir, cameraDataSubdir, "camera.py"))
        cameraConfig = afwCameraGeom.CameraConfig()
        cameraConfig.load(self.cameraDataLocation)
        ampInfoPath = os.path.dirname(self.cameraDataLocation)
        return afwCameraGeom.makeCameraFromPath(
            cameraConfig=cameraConfig,
            ampInfoPath=ampInfoPath,
            shortNameFunc=self.getShortCcdName,
            pupilFactoryClass=self.PupilFactoryClass
        )

    def getRegistry(self):
        """Get the registry used by this mapper.

        Returns
        -------
        Registry or None
            The registry used by this mapper for this mapper's repository.
        """
        return self.registry

    def getImageCompressionSettings(self, datasetType, dataId):
        """Stuff image compression settings into a daf.base.PropertySet

        This goes into the ButlerLocation's "additionalData", which gets
        passed into the boost::persistence framework.

        Parameters
        ----------
        datasetType : `str`
            Type of dataset for which to get the image compression settings.
        dataId : `dict`
            Dataset identifier.

        Returns
        -------
        additionalData : `lsst.daf.base.PropertySet`
            Image compression settings.
        """
        mapping = self.mappings[datasetType]
        recipeName = mapping.recipe
        storageType = mapping.storage
        if storageType not in self._writeRecipes:
            return dafBase.PropertySet()
        if recipeName not in self._writeRecipes[storageType]:
            raise RuntimeError("Unrecognized write recipe for datasetType %s (storage type %s): %s" %
                               (datasetType, storageType, recipeName))
        recipe = self._writeRecipes[storageType][recipeName].deepCopy()
        seed = hash(tuple(dataId.items())) % 2**31
        for plane in ("image", "mask", "variance"):
            if recipe.exists(plane + ".scaling.seed") and recipe.getScalar(plane + ".scaling.seed") == 0:
                recipe.set(plane + ".scaling.seed", seed)
        return recipe

    def _initWriteRecipes(self):
        """Read the recipes for writing files

        These recipes are currently used for configuring FITS compression,
        but they could have wider uses for configuring different flavors
        of the storage types. A recipe is referred to by a symbolic name,
        which has associated settings. These settings are stored as a
        `PropertySet` so they can easily be passed down to the
        boost::persistence framework as the "additionalData" parameter.

        The list of recipes is written in YAML. A default recipe and
        some other convenient recipes are in obs_base/policy/writeRecipes.yaml
        and these may be overridden or supplemented by the individual obs_*
        packages' own policy/writeRecipes.yaml files.

        Recipes are grouped by the storage type. Currently, only the
        ``FitsStorage`` storage type uses recipes, which uses it to
        configure FITS image compression.

        Each ``FitsStorage`` recipe for FITS compression should define
        "image", "mask" and "variance" entries, each of which may contain
        "compression" and "scaling" entries. Defaults will be provided for
        any missing elements under "compression" and "scaling".

        The allowed entries under "compression" are:

        * algorithm (string): compression algorithm to use
        * rows (int): number of rows per tile (0 = entire dimension)
        * columns (int): number of columns per tile (0 = entire dimension)
        * quantizeLevel (float): cfitsio quantization level

        The allowed entries under "scaling" are:

        * algorithm (string): scaling algorithm to use
        * bitpix (int): bits per pixel (0,8,16,32,64,-32,-64)
        * fuzz (bool): fuzz the values when quantising floating-point values?
        * seed (long): seed for random number generator when fuzzing
        * maskPlanes (list of string): mask planes to ignore when doing
          statistics
        * quantizeLevel: divisor of the standard deviation for STDEV_* scaling
        * quantizePad: number of stdev to allow on the low side (for
          STDEV_POSITIVE/NEGATIVE)
        * bscale: manually specified BSCALE (for MANUAL scaling)
        * bzero: manually specified BSCALE (for MANUAL scaling)

        A very simple example YAML recipe:

            FitsStorage:
              default:
                image: &default
                  compression:
                    algorithm: GZIP_SHUFFLE
                mask: *default
                variance: *default
        """
        recipesFile = os.path.join(getPackageDir("obs_base"), "policy", "writeRecipes.yaml")
        recipes = dafPersist.Policy(recipesFile)
        supplementsFile = os.path.join(self.getPackageDir(), "policy", "writeRecipes.yaml")
        validationMenu = {'FitsStorage': validateRecipeFitsStorage, }
        if os.path.exists(supplementsFile) and supplementsFile != recipesFile:
            supplements = dafPersist.Policy(supplementsFile)
            # Don't allow overrides, only supplements
            for entry in validationMenu:
                intersection = set(recipes[entry].names()).intersection(set(supplements.names()))
                if intersection:
                    raise RuntimeError("Recipes provided in %s section %s may not override those in %s: %s" %
                                       (supplementsFile, entry, recipesFile, intersection))
            recipes.update(supplements)

        self._writeRecipes = {}
        for storageType in recipes.names(True):
            if "default" not in recipes[storageType]:
                raise RuntimeError("No 'default' recipe defined for storage type %s in %s" %
                                   (storageType, recipesFile))
            self._writeRecipes[storageType] = validationMenu[storageType](recipes[storageType])

def exposureFromImage(image, dataId=None, mapper=None, logger=None, setVisitInfo=True):
    """Generate an Exposure from an image-like object

    If the image is a DecoratedImage then also set its WCS and metadata
    (Image and MaskedImage are missing the necessary metadata
    and Exposure already has those set)

    Parameters
    ----------
    image : Image-like object
        Can be one of lsst.afw.image.DecoratedImage, Image, MaskedImage or
        Exposure.

    Returns
    -------
    `lsst.afw.image.Exposure`
        Exposure containing input image.
    """
    translatorClass = None
    if mapper is not None:
        translatorClass = mapper.translatorClass

    metadata = None
    if isinstance(image, afwImage.MaskedImage):
        exposure = afwImage.makeExposure(image)
    elif isinstance(image, afwImage.DecoratedImage):
        exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
        metadata = image.getMetadata()
        fix_header(metadata, translator_class=translatorClass)
        exposure.setMetadata(metadata)
    elif isinstance(image, afwImage.Exposure):
        exposure = image
        metadata = exposure.getMetadata()
        fix_header(metadata, translator_class=translatorClass)
    else:  # Image
        exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))

    # set VisitInfo if we can
    if setVisitInfo and exposure.getInfo().getVisitInfo() is None:
        if metadata is not None:
            if mapper is None:
                if not logger:
                    logger = lsstLog.Log.getLogger("CameraMapper")
                logger.warn("I can only set the VisitInfo if you provide a mapper")
            else:
                exposureId = mapper._computeCcdExposureId(dataId)
                visitInfo = mapper.makeRawVisitInfo(md=metadata, exposureId=exposureId)

                exposure.getInfo().setVisitInfo(visitInfo)

    return exposure

def validateRecipeFitsStorage(recipes):
    """Validate recipes for FitsStorage

    The recipes are supplemented with default values where appropriate.

    TODO: replace this custom validation code with Cerberus (DM-11846)

    Parameters
    ----------
    recipes : `lsst.daf.persistence.Policy`
        FitsStorage recipes to validate.

    Returns
    -------
    validated : `lsst.daf.base.PropertySet`
        Validated FitsStorage recipe.

    Raises
    ------
    `RuntimeError`
        If validation fails.
    """
    # Schemas define what should be there, and the default values (and by the
    # default value, the expected type).
    compressionSchema = {
        "algorithm": "NONE",
        "rows": 1,
        "columns": 0,
        "quantizeLevel": 0.0,
    }
    scalingSchema = {
        "algorithm": "NONE",
        "bitpix": 0,
        "maskPlanes": ["NO_DATA"],
        "seed": 0,
        "quantizeLevel": 4.0,
        "quantizePad": 5.0,
        "fuzz": True,
        "bscale": 1.0,
        "bzero": 0.0,
    }

    def checkUnrecognized(entry, allowed, description):
        """Check to see if the entry contains unrecognised keywords"""
        unrecognized = set(entry.keys()) - set(allowed)
        if unrecognized:
            raise RuntimeError(
                "Unrecognized entries when parsing image compression recipe %s: %s" %
                (description, unrecognized))

    validated = {}
    for name in recipes.names(True):
        checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
        rr = dafBase.PropertySet()
        validated[name] = rr
        for plane in ("image", "mask", "variance"):
            checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
                              name + "->" + plane)

            for settings, schema in (("compression", compressionSchema),
                                     ("scaling", scalingSchema)):
                prefix = plane + "." + settings
                if settings not in recipes[name][plane]:
                    for key in schema:
                        rr.set(prefix + "." + key, schema[key])
                    continue
                entry = recipes[name][plane][settings]
                checkUnrecognized(entry, schema.keys(), name + "->" + plane + "->" + settings)
                for key in schema:
                    value = type(schema[key])(entry[key]) if key in entry else schema[key]
                    rr.set(prefix + "." + key, value)
    return validated


from channels.consumer import SyncConsumer, AsyncConsumer
from time import sleep
from channels.exceptions import StopConsumer
import asyncio

class MySyncConsumer(SyncConsumer):

    def websocket_connect(self, event):
        print("sync connect method +++++++++")
        self.send({
            'type': 'websocket.accept',
        })

    def websocket_receive(self, event):
        print("sync receive method +++++++++++", event)
        print("sync message ", event['text'])
        for i in range(20):
            self.send({
                'type':"websocket.send",
                "text": f"message count{i}"
            })
            sleep(1)

    def websocket_disconnect(self, event):
        print("sync disconnect method ++++++++++++")
        raise StopConsumer()

class MyAsyncConsumer(AsyncConsumer):

    async def websocket_connect(self, event):
        print("async connect method +++++++++")
        await self.send({
            'type': 'websocket.accept',
        })

    async def websocket_receive(self, event):
        print("async receive method +++++++++++", event)
        print("async message ", event['text'])
        for i in range(20):
            await self.send({
                'type':"websocket.send",
                "text": f"message count{i}"
            })
            await asyncio.sleep(1)

    async def websocket_disconnect(self, event):
        print("async disconnect method ++++++++++++")
        raise StopConsumer()


# -*- coding: utf-8 -*-

import sys, os
import time

sys.path.append(os.getcwd().split("test")[0])

import FitFile_Handler
import DB_Handler
import Graphic_store

import unittest

import xml.etree.ElementTree as ET
import numpy as np

from Logger import Logger

class Test_process(unittest.TestCase):

    def setUp(self):

        logger = Logger('test_process')
        logger.add_StreamHandler()
        logger.add_FileHandler('Initial_test.log')
        self.logger = logger.logger

        self.parameters =  {}
        tree = ET.parse("parameters.xml")
        root = tree.getroot()
        for child in root:
            for sub_child in child:
                self.parameters[sub_child.tag] = sub_child.test.encode('utf8')

        self.REPO_DIR = os.getcwd().split("test")[0]
        self.parameters["REPO_DIR"] = self.REPO_DIR
        self.fit_files = {}
        self.graphic = Graphic_store.Graphic_store(self.parameters, self.logger)

    def test_basic_process(self):
        files = os.listdir(os.path.join(self.REPO_DIR, 'source_data'))
        fit_files = [os.path.join(self.REPO_DIR, 'source_data', file) for file in files if file[-4:].lower()=='.fit']

        for file in fit_files:
            self.fit_files[file] = FitFile_Handler.FitFile_handler(self.parameters, self.logger, file)
            self.fit_files[file].load_file()

        for file, handler in self.fit_files.items():
            handler.parse()
        # Create db
        self.db_user = DB_Handler.DB_Handler('users', self.parameters, self.logger)
        self.db_event = DB_Handler.DB_Handler("event", self.parameters, self.logger)

        # Insert data into db
        for file, handler in self.fit_files.items():
            for v in handler.data:
                v['user_id'] = 1
                self.db_event.insert_data(v)

        # Reload those data from db
        sql_query = "SELECT position_lat, position_long, power, altitude FROM event WHERE user_id=1"
        rows = self.db_event.load_data(sql_query)

        longitude, latitude, power, altitude = np.array([]), np.array([]), np.array([]), np.array([])
        for row in rows:
            longitude = np.append(longitude, row[0])
            latitude = np.append(latitude, row[1])
            power = np.append(power, row[2])
            altitude = np.append(altitude, row[3])

        self.graphic.draw_2D_enriched('test_power.jpeg', latitude, longitude, power)
        self.graphic.draw_3d('test_power.jpeg', latitude, longitude, altitude)

        self.db_user.off()
        self.db_event.off()

if __name__=="__main__":
    unittest.main()


from selenium.webdriver.common.by import By
from selenium import webdriver
import time

try:
    line = 'https://suninjuly.github.io/execute_script.html'
    browser = webdriver.Chrome()
    browser.get(line)
    button = browser.find_element_by_tag_name("button")
    button.sc
    time.sleep(1)

    browser.execute_script('return arguments[0].scrollIntoView(true);', button)
    time.sleep(1)

    assert True

finally:
    time.sleep(5)
    browser.quit()


__author__      = "Martine Lenders"
__copyright__   = "Copyright (C) 2016 Freie Universität Berlin"
__license__     = "LGPLv2.1"
__email__       = "m.lenders@fu-berlin.de"

import argparse
import json
import networkx as nx
from networkx.readwrite import json_graph
import sys

from .node import Node

def _default_json_dumper(obj):
    return getattr(obj.__class__, "to_json", json.JSONEncoder().default)(obj)

def store_mesh(filename, mesh):
    json.dump(json_graph.node_link_data(mesh), open(filename, 'w'),
              default=_default_json_dumper)

def load_mesh(filename):
    mesh_dict = json.load(open(filename))
    nodes = []
    for node in mesh_dict['nodes']:
        n_obj = {}
        n_app = None
        if ('application' in node['id']):
            n_app = eval(node['id']['application']['class'])(**(node['id']['application']))
        n_obj['id'] = Node(application=n_app, *node['id']['sock_addr'])
        nodes.append(n_obj)
    mesh_dict['nodes'] = nodes
    return json_graph.node_link_graph(mesh_dict)

def load():
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--dump-pcap", help="Dumps pcap of traffic to stdout", action="store_true")
    parser.add_argument("-i", "--input-file", help="JSON input file")
    parser.add_argument("-n", "--number",
                        help="Number of nodes to generate (if no input file is given)",
                        type=int, default=5)
    parser.add_argument("-c", "--connectedness",
                        help="Connectedness [0.2, 1.0) of the network (if now input file is given)",
                        type=float, default=0.75)
    args = parser.parse_args()
    if (args.input_file):
        mesh = load_mesh(args.input_file)
    else:
        mesh = nx.connected_watts_strogatz_graph(args.number,
                int(args.number * args.connectedness), args.connectedness)
    return mesh, args


from torchvision import transforms as T
import numpy as np
import torch
import json

def mean(arrs):
    return sum(arrs) * 1.0 / len(arrs)

def set_seed(seed):
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)

transform = T.Compose(
    [
        T.Resize([50, 50]),
        # T.Grayscale(),
        T.ToTensor(),
        T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ]
)

def dump_list(path, lst):
    with open(path, "w") as f:
        for x in lst:
            f.write(f"{x}\n")

def dump_json(data, file: str):
    json.dump(data, open(file, "w"), indent=4)

def get_param_cnt(model):
    return sum([p.numel() for p in model.parameters()])


import numpy as np
from tensorflow import keras
from tensorflow.keras.models import Sequential, Model, load_model
from PIL import Image
import sys

import tensorflow as tf
def memory_limit():
    physical_devices = tf.config.list_physical_devices('GPU')
    if len(physical_devices) > 0:
        for device in physical_devices:
            tf.config.experimental.set_memory_growth(device, True)
            print('{} memory growth: {}'.format(device, tf.config.experimental.get_memory_growth(device)))
    else:
        print("Not enough GPU hardware devices available")

memory_limit()

# パラメーターの初期化
classes = ["cow", "chicken", "pig"]
num_classes = len(classes)
image_size = 224

# 引数から画像ファイルを参照して読み込む
image = Image.open(sys.argv[1])
image = image.convert("RGB")
image = image.resize((image_size, image_size))
data = np.asarray(image) / 255.0
X = []
X.append(data)
X = np.array(X)

# モデルのロード
model = load_model('./vgg16_transfer.h5')

result = model.predict([X])[0]
predicted = result.argmax() # 最大値のインデックスの値を取る
percentage = int(result[predicted] * 100)

cow_res = str(round(result[0] * 100, 1))
chicken_res = str(round(result[1] * 100, 1))
pig_res = str(round(result[2] * 100, 1))

# print(classes[predicted], percentage, "%")
print("cow: " + cow_res + "%\n" +
    "chicken: " + chicken_res + "%\n" +
    "pig: " + pig_res + "%")


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np

class Net(nn.Module):
    def __init__(self, width=19, height=19):
        super().__init__()

        self.width = width
        self.height = height

        # common layers
        self.conv1 = nn.Conv2d(4, 32, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, padding=1)
        # action policy layers
        self.act_conv1 = nn.Conv2d(128, 4, kernel_size=1)
        self.act_fc1 = nn.Linear(4*width*height,
                                 width*height)
        # state value layers
        self.val_conv1 = nn.Conv2d(128, 2, kernel_size=1)
        self.val_fc1 = nn.Linear(2*width*height, 64)
        self.val_fc2 = nn.Linear(64, 1)

    def forward(self, state_input):

        # common layers
        x = F.relu(self.conv1(state_input))
        x = F.relu(self.conv2(x))
        x = F.relu(self.conv3(x))
        # action policy layers
        x_act = F.relu(self.act_conv1(x))
        x_act = x_act.view(-1, 4*self.width*self.height)
        x_act = F.log_softmax(self.act_fc1(x_act))
        # state value layers
        x_val = F.relu(self.val_conv1(x))
        x_val = x_val.view(-1, 2*self.width*self.height)
        x_val = F.relu(self.val_fc1(x_val))
        x_val = torch.tanh(self.val_fc2(x_val))
        return x_act, x_val

class PolicyNetwork():
    def __init__(self, width=8, height=8, model_file=None, use_gpu=False):
        self.use_gpu = use_gpu
        self.board_width = width
        self.board_height = height

        if self.use_gpu:
            self.policy_net = Net(width, height).cuda()
        else:
            self.policy_net = Net(width, height)

        self.l2_const = 1e-4

        self.optimizer = optim.Adam(self.policy_net.parameters(), weight_decay=self.l2_const)

        # Initialize using the parameters in the file,
        # or use pytorch's own method to initialize parameters.
        if model_file:
            self.load_file(model_file)

    def batch_policy_fn(self, board_state_batch):
        '''
        Input: a batch of board states
        Output: a batch of action probabilities, and state values
        '''
        if self.use_gpu:
            board_state_batch = Variable(torch.FloatTensor(board_state_batch).cuda())
            log_act_probs, value = self.policy_net(board_state_batch)
            act_probs = np.exp(log_act_probs.data.cpu().numpy())
            return act_probs, value.data.cpu().numpy()
        else:
            board_state_batch = Variable(torch.FloatTensor(board_state_batch))
            log_act_probs, value = self.policy_net(board_state_batch)
            act_probs = np.exp(log_act_probs.data.numpy())
            return act_probs, value

    def policy_fn(self, board):
        '''Gomoku board state evaluation function, which is the core improvements in the AlphaZero version.

        Input: the board state, input size: 4*width*heigth.
        Output: probability of next move and the evaluation of the state of the board.
        '''
        legal_positions = board.availables
        current_state = np.ascontiguousarray(board.current_state().reshape(
                -1, 4, self.board_width, self.board_height))
        if self.use_gpu:
            log_act_probs, value = self.policy_net(
                    Variable(torch.from_numpy(current_state)).cuda().float())
            act_probs = np.exp(log_act_probs.data.cpu().numpy().flatten())
        else:
            log_act_probs, value = self.policy_net(
                    Variable(torch.from_numpy(current_state)).float())
            act_probs = np.exp(log_act_probs.data.numpy().flatten())
        act_probs = zip(legal_positions, act_probs[legal_positions])
        value = value.data[0][0]
        return act_probs, value

    def train_step(self, board_state_batch, mcts_probs, winner_batch, lr=0.1):
        '''This function is used to train the network.

        Input: board_state_batch, mcts_probs, winner_batch
        Output: the value of loss and entropy
        '''
        if self.use_gpu:
            board_state_batch = Variable(torch.FloatTensor(board_state_batch).cuda())
            mcts_probs = Variable(torch.FloatTensor(mcts_probs).cuda())
            winner_batch = Variable(torch.FloatTensor(winner_batch).cuda())
        else:
            board_state_batch = Variable(torch.FloatTensor(board_state_batch))
            mcts_probs = Variable(torch.FloatTensor(mcts_probs))
            winner_batch = Variable(torch.FloatTensor(winner_batch))

        self.optimizer.zero_grad()

        log_act_probs, value = self.policy_net(board_state_batch)
        # loss = (z-v)^2 - pi^T * log(p) + c * theta^2
        value_loss = F.mse_loss(value.view(-1), winner_batch)
        policy_loss = -torch.mean(torch.sum(mcts_probs*log_act_probs))
        loss = value_loss + policy_loss

        loss.backward()
        for param_group in self.optimizer.param_groups:
            param_group['lr'] = lr
        self.optimizer.step()

        entropy = -torch.mean(torch.sum(torch.exp(log_act_probs) * log_act_probs, 1))

        return loss.item(), entropy.item()

    def get_policy_param(self):
        return self.policy_net.state_dict()

    def save_model(self, model_file):
        net_params = self.get_policy_param()
        torch.save(net_params, model_file)

    def load_file(self, model_file):
        '''The smallest gomoku board size is 8 by 8.

        If the board size is bigger than 8 by 8 and you want to use some policy to the bigger gomoku board,
        use the following code.
        '''
        net_params = torch.load(model_file) # 8 * 8 = 64

        cur = self.board_height * self.board_width
        if net_params['act_fc1.weight'].shape[1] != 4*cur:
            print("Try to apply the 8x8 model to a larger gomoku board!")
            a = torch.zeros(self.board_width*self.board_height, 4*self.board_width*self.board_height)
            b = torch.zeros(self.board_width*self.board_height)
            c = torch.zeros(64, 2*self.board_width*self.board_height)
            a[(cur-64)//2:(cur+64)//2, 2*(cur-64):2*(cur+64)] = net_params['act_fc1.weight']
            b[(cur-64)//2:(cur+64)//2] = net_params['act_fc1.bias']
            c[:,cur-64:cur+64] = net_params['val_fc1.weight']
            net_params['act_fc1.weight'] = a # cur, 4*cur
            net_params['act_fc1.bias'] = b # cur
            net_params['val_fc1.weight'] = c # 64, 2*cur

            net_params['conv1.weight'].required_grad = False
            net_params['conv1.bias'].required_grad = False
            net_params['conv2.weight'].required_grad = False
            net_params['conv2.bias'].required_grad = False
            net_params['conv3.weight'].required_grad = False
            net_params['conv3.bias'].required_grad = False
            net_params['act_conv1.weight'].required_grad = False
            net_params['act_conv1.bias'].required_grad = False

        self.policy_net.load_state_dict(net_params)

if __name__ == '__main__':
    # a = PolicyNetwork(10, 10,'./model/model100_10x10.model')
    a = PolicyNetwork(10, 10,'./best_model/best_model_8x8')


# Importing Libraries
import serial
import time
arduino = serial.Serial(port='/dev/ttyACM0', baudrate=9600, timeout=.1)
def write_read(x):
    arduino.write(bytes(x, 'utf-8'))
    arduino.flush()
    time.sleep(0.3)
    data = arduino.readline().decode('utf-8').rstrip()
    arduino.flush()
    return data
while True:
    num = "0.15/0.22/0.34/0.44/0.54/0.63/0.73/0.86/0.98/0.10/0.11/0.12\n"
    value = write_read(num)
    print(value)


import os
import argparse
from data_loder import data_loader_augment_color_for_transfer as data_loader
from torch.backends import cudnn
import torch
from model.model import FeatureExtractNet, SNResIdDiscriminator
from model.model import NoTrackExpressionGenerater as ExpressionGenerater
from torchvision.utils import save_image
import sys
from utils.Logger import Logger
from torch.nn.functional import softplus
from pytorch_msssim import ssim

device = None

def denorm(x):
    """Convert the range from [-1, 1] to [0, 1]."""
    out = (x + 1) / 2
    return out.clamp_(0, 1)

def to_image(x):
    """Convert the range from [-1, 1] to [0, 1]."""
    x = denorm(x.data.cpu())
    ndarr = x.mul(255).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy()
    im = ndarr
    return im

def gradient_penalty(y, x):
    """Compute gradient penalty: (L2_norm(dy/dx) - 1)**2."""
    weight = torch.ones(y.size()).to(device)
    dydx = torch.autograd.grad(outputs=y,
                               inputs=x,
                               grad_outputs=weight,
                               retain_graph=True,
                               create_graph=True,
                               only_inputs=True)[0]

    dydx = dydx.view(dydx.size(0), -1)
    dydx_l2norm = torch.sqrt(torch.sum(dydx**2, dim=1))
    return torch.mean((dydx_l2norm-1)**2)

def main(config):
    # For fast training.
    cudnn.benchmark = True
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    if len(sys.argv) > 1:
        os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu
    else:
        os.environ["CUDA_VISIBLE_DEVICES"] = "5"
    global device
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    version = config.version
    beta1= 0.5
    beta2 = 0.999

    loader = data_loader.get_loader("/media/data2/laixc/AI_DATA/expression_transfer/face12/crop_face",
                                         "/media/data2/laixc/AI_DATA/expression_transfer/face12/points_face", config)
    G = ExpressionGenerater()
    FEN = FeatureExtractNet()
    color_D = SNResIdDiscriminator()

    #######   载入预训练网络   ######
    resume_iter = config.resume_iter
    ckpt_dir = "/media/data2/laixc/Facial_Expression_GAN/ckpt-{}".format(version)
    if not os.path.isdir(ckpt_dir):
        os.mkdir(ckpt_dir)
    log = Logger(os.path.join(ckpt_dir,
                              'log.txt'))
    if os.path.exists(os.path.join(ckpt_dir,
                                  '{}-G.ckpt'.format(resume_iter))):
        G_path = os.path.join(ckpt_dir,
                              '{}-G.ckpt'.format(resume_iter))
        G.load_state_dict(torch.load(G_path, map_location=lambda storage, loc: storage))

        IdD_path = os.path.join(ckpt_dir,
                              '{}-idD.ckpt'.format(resume_iter))
        color_D.load_state_dict(torch.load(IdD_path, map_location=lambda storage, loc: storage))

    else:
        resume_iter = 0

    #####  训练face2keypoint   ####
    G_optimizer = torch.optim.Adam(G.parameters(), lr=0.0001, betas=(0.5, 0.9))
    idD_optimizer = torch.optim.Adam(color_D.parameters(), lr=0.001, betas=(0.5, 0.9))
    G.to(device)
    color_D.to(device)
    FEN.to(device)

    FEN.eval()

    log.print(config)

    # Start training from scratch or resume training.
    start_iters = resume_iter
    trigger_rec = 1
    data_iter = iter(loader)

    # Start training.
    print('Start training...')
    for i in range(start_iters, 150000):
        # =================================================================================== #
        #                             1. Preprocess input data                                #
        # =================================================================================== #

        #faces, origin_points = next(data_iter)
        #_, target_points = next(data_iter)
        try:
            color_faces, faces = next(data_iter)
        except StopIteration:
            data_iter = iter(loader)
            color_faces, faces = next(data_iter)
        rand_idx = torch.randperm(faces.size(0))
        condition_faces = faces[rand_idx]

        faces = faces.to(device)
        color_faces = color_faces.to(device)
        condition_faces = condition_faces.to(device)

        # =================================================================================== #
        #                               3. Train the discriminator                            #
        # =================================================================================== #

        # ID Dis
        id_real_loss = torch.mean(softplus(-color_D(faces, condition_faces)))  # big for real
        faces_fake = G(color_faces, condition_faces)
        id_fake_loss = torch.mean(softplus(color_D(faces_fake, condition_faces)))  # small for fake

        id_Dis_loss = id_real_loss + id_fake_loss

        idD_optimizer.zero_grad()
        id_Dis_loss.backward()
        idD_optimizer.step()

        # =================================================================================== #
        #                               3. Train the generator                                #
        # =================================================================================== #

        n_critic = 1
        if (i + 1) % n_critic == 0:
            # Original-to-target domain.
            faces_fake = G(color_faces, condition_faces)

            g_id_loss = torch.mean(softplus(-color_D(faces_fake, condition_faces)))

            l1_loss = torch.mean(torch.abs(faces_fake - faces)) + (1 - ssim(faces_fake, faces))

            feature_loss = torch.mean(torch.abs(FEN(faces_fake) - FEN(faces)))

            lambda_l1 = config.lambda_l1
            lambda_id = config.lambda_id
            lambda_feature = config.lambda_feature
            g_loss =  lambda_id * g_id_loss + lambda_l1 * l1_loss + lambda_feature*feature_loss

            G_optimizer.zero_grad()
            g_loss.backward()
            G_optimizer.step()

            # Print out training information.
            if (i + 1) % 4 == 0:
                log.print("iter {} - id_real_loss {:.2}, "
                      "id_fake_loss {:.2} ,  g_id_loss {:.2}, L1_loss {:.2}, feature_loss {:.2}".format(i, id_real_loss.item(), id_fake_loss.item()
                                                                ,lambda_id * g_id_loss.item(), lambda_l1 * l1_loss,
                                                                                lambda_feature *feature_loss.item()))

            sample_dir = "gan-sample-{}".format(version)
            if not os.path.isdir(sample_dir):
                os.mkdir(sample_dir)
            if (i + 1) % 24 == 0:
                with torch.no_grad():
                    fake_face = faces_fake[0]
                    condition_face = condition_faces[0]
                    color_face = color_faces[0]
                    #reconstruct = reconstructs[0]

                    sample_path_face = os.path.join(sample_dir, '{}-image-face.jpg'.format(i + 1))
                    save_image(denorm(condition_face.data.cpu()), sample_path_face)

                    sample_path_rec = os.path.join(sample_dir, '{}-image-color.jpg'.format(i + 1))
                    save_image(denorm(color_face.data.cpu()), sample_path_rec)

                    sample_path_fake = os.path.join(sample_dir, '{}-image-fake.jpg'.format(i + 1))
                    save_image(denorm(fake_face.data.cpu()), sample_path_fake)

                    print('Saved real and fake images into {}...'.format(sample_path_face))

        # Save model checkpoints.
        model_save_dir = "ckpt-{}".format(version)

        if (i + 1) % 1000 == 0:
            if not os.path.isdir(model_save_dir):
                os.mkdir(model_save_dir)
            G_path = os.path.join(model_save_dir, '{}-G.ckpt'.format(i + 1))
            torch.save(G.state_dict(), G_path)
            idD_path = os.path.join(model_save_dir, '{}-idD.ckpt'.format(i + 1))
            torch.save(color_D.state_dict(), idD_path)
            print('Saved model checkpoints into {}...'.format(model_save_dir))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    # Model configuration.
    parser.add_argument('--resume_iter', type=int, default=14000)
    parser.add_argument('--version', type=str, default="256-level1")
    parser.add_argument('--dataset_level', '--list', nargs='+',
                        default=['easy', 'middle', 'hard'])
    parser.add_argument('--gpu', type=str, default="2")

    parser.add_argument('--lambda_l1', type=float, default=4)
    parser.add_argument('--lambda_id', type=float, default=0.1)
    parser.add_argument('--lambda_feature', type=float, default=2)

    config = parser.parse_args()
    main(config)


from __future__ import absolute_import, division, print_function

import re

import six

from inspire_schemas.api import LiteratureBuilder

from refextract.documents.pdf import replace_undesirable_characters
from refextract.documents.text import wash_line

# Differentiate authors from affiliations:
# authors have a number after name, affiliations lines start with a number
split_authors_affs_pattern = re.compile(r'(.*?\d)\n+(\d.*)', flags=re.DOTALL)
# Differentiate affiliations from one another, line break is required
split_affs_pattern = re.compile(r'(\d+\.?[\s\t]*.*)\n*', flags=re.UNICODE)

def authorlist(text):
    """Call the relevant formatter depending on the existence of affiliations.

    Input can be in two different formats:
        F. Lastname, F.M. Otherlastname
    or
        F. Lastname1, F.M. Otherlastname1,2
        1 CERN
        2 Otheraffiliation

    There should always be only one affiliation per line and the affiliation
    ids in author names should be separated with commas.
    """
    if not text:
        return {}
    if not isinstance(text, six.text_type):
        text = text.decode('utf-8')
    # Do some pre-cleaning of the input string
    text = text.replace('\r', '')  # Input from the form contains unwanted \r's
    text = text.replace(u'†', '')
    text = text.replace(u'∗', '')
    text = re.sub(r'( *\n+ *)', r'\n', text)
    text = replace_undesirable_characters(text)
    text = wash_line(text)

    # Assume that if there are numbers in author name, they are affilation ids.
    # Also assume that if one author has an affiliation, everyone else has too.
    if re.search(r'([a-zA-Z_-]+[\n]*\d+)', text):
        return authorlist_with_affiliations(text)
    return authorlist_without_affiliations(text)

def authorlist_without_affiliations(text):
    """Return a record containing the authors."""
    builder = LiteratureBuilder()
    authors = text.replace(' and ', ', ').split(', ')
    for author in authors:
        builder.add_author(builder.make_author(author))

    return {'authors': builder.record['authors']}

def authorlist_with_affiliations(text):
    """Return a record containing the authors, including affiliations."""
    def parse_author_string(author):
        """Get fullname and affiliation ids."""
        name, affs = re.search(
            r'(.+?)(\d+[\,\d]*)', author, flags=re.UNICODE
        ).groups()
        aff_ids = [aff for aff in affs.split(',') if aff.isdigit()]

        return name, aff_ids

    def get_author_affiliations(author, affiliations):
        """Get affiliations belonging to author."""
        try:
            fullname, aff_ids = parse_author_string(author)
        except AttributeError:
            raise AttributeError(
                'Cannot split affiliation IDs from author name. This author '
                'might not have an affiliation at all', author
            )
        try:
            author_affiliations = [affiliations[aff_id] for aff_id in aff_ids]
        except KeyError:
            raise KeyError(
                'There might be multiple affiliations per line or '
                'affiliation IDs might not be separated with commas or '
                'the affiliation is missing. '
                'Problematic author and affiliations', author, aff_ids,
                affiliations
            )

        return (fullname, author_affiliations)

    # Try to work with badly formatted input
    # There should be commas between different affiliation ids in author
    # names and there should be only one affiliation name per line.
    split_auths_and_affs = split_authors_affs_pattern.search(text)
    if not split_auths_and_affs:
        raise AttributeError('Could not find affiliations')
    authors_raw = split_auths_and_affs.group(1)
    # ensure comma between affid and next author name:
    authors_raw = re.sub(r'(\d+)[\n\s](\D)', r'\1, \2', authors_raw)
    authors_raw = authors_raw.replace('\n', '')
    # ensure no comma between author name and its affids
    authors_raw = re.sub(r'(\D)\,[\n\s]*(\d)', r'\1\2', authors_raw)
    # ensure space between comma and next author name:
    authors_raw = re.sub(r'(\d+)\,(\S\D)', r'\1, \2', authors_raw)
    # ensure no spaces between affids of an author
    authors_raw = re.sub(r'(\d+)\,\s(\d+)', r'\1,\2', authors_raw)

    authors = authors_raw.replace(' and ', ', ').split(', ')

    # Extract affiliations:
    affs_raw = split_auths_and_affs.group(2)
    affs_list = split_affs_pattern.findall(affs_raw)
    affs_list = [aff.replace('\n', ' ').strip() for aff in affs_list]
    affiliations = {}
    for aff in affs_list:
        try:
            # Note that affiliation may contain numbers
            aff_id, aff_name = re.search(r'^(\d+)\.?\s?(.*)$', aff).groups()
            affiliations[aff_id] = aff_name
        except (ValueError, AttributeError):
            raise ValueError('Cannot parse affiliation', aff)

    builder = LiteratureBuilder()
    for author in authors:
        fullname, author_affs = get_author_affiliations(author, affiliations)
        builder.add_author(builder.make_author(fullname, raw_affiliations=author_affs))

    return {'authors': builder.record['authors']}


import wget
import re
import os

import pandas as pd
import wget
import re
import os

import pandas as pd

import json

data=pd.read_csv('../cim_codes.csv')
data = data

i=1
outdata={}
for d in data.to_numpy():
    try:
        os.remove("data.html")
    except FileNotFoundError:
        pass
    cim=d[0].lower()
    print("process code ("+str(i)+"): '"+cim+"'")

    url = 'https://www.aideaucodage.fr/cim-'+cim
    print(url)
    try:
        wget.download(url, out="data.html")
    except:
        continue

    f=open('data.html',"r", encoding="latin-1")
    content = f.read()
    f.close()

    content=content.replace("\n","")
    content=content.replace("\t","")
    content=content.replace("\r","")

    #print("---- ACTES CCAM Liés ----")
    m=re.search("
Actes CCAM(.*)

(.*)
(.*)
",content)
    try:
        tabledata=m.group(2)
        m=re.findall("(\w+?)	(['()\-,\w\s]+?)

",tabledata, re.UNICODE)
        ccam={e[1]:(float(e[3])-2) for e in m}
        #print(ccam)
    except AttributeError:
        ccam={}

    #print("---- Diagnostics CIM Liés ----")

    try:
        m=re.search("
Diagnostics CIM\-10 \(DAS\) associés à (\w*)

(.*)
",content, re.UNICODE)
        tabledata=m.group(2)
        m=re.findall("(\w+[.]{0,1}\w*)	(\d+)	(['()\-,\w\s]+?)

",tabledata, re.UNICODE)
        cimd={e[1]:(float(e[4])-2) for e in m}
        #print(cimd)
    except AttributeError:
        cimd={}

    #print("---- GHM associés ----")
    try:
        m=re.search("
GHM associés avec (\w*)

(.*)
",content, re.UNICODE)
        tabledata=m.group(2)
        m=re.findall("(\w+)	(['()\-,\w\s]+?)

",tabledata, re.UNICODE)
        ghm={e[1]:(float(e[3])-2) for e in m}
        #print(ghm)
    except AttributeError:
        ghm={}

    outdata[cim.upper()]={"ccam":ccam,"cim":cimd,"ghm":ghm}

    if i%100==0:
        with open('result.json', 'w') as fp:
            json.dump(outdata, fp)
    i+=1