"""
Logic for cleaning and processing zipcode data
"""
import pandas as pd
import string
import zipcodes

# Mapping of zipcode to Municipality for all zipcodes that are uniquely associated with one municipality, calculated in `zipcode_muni_mapping.ipynb`
ZIP_MUNI_MAP = {
    '01001': 'Agawam',
    '01003': 'Amherst',
    '01004': 'Amherst',
    '01005': 'Barre',
    '01007': 'Belchertown',
    '01008': 'Blandford',
    '01009': 'Palmer',
    '01010': 'Brimfield',
    '01011': 'Chester',
    '01012': 'Chesterfield',
    '01013': 'Chicopee',
    '01014': 'Chicopee',
    '01020': 'Chicopee',
    '01021': 'Chicopee',
    '01022': 'Chicopee',
    '01026': 'Cummington',
    '01028': 'East Longmeadow',
    '01029': 'Otis',
    '01030': 'Agawam',
    '01031': 'Hardwick',
    '01032': 'Goshen',
    '01033': 'Granby',
    '01034': 'Granville',
    '01035': 'Hadley',
    '01036': 'Hampden',
    '01037': 'Hardwick',
    '01038': 'Hatfield',
    '01039': 'Williamsburg',
    '01040': 'Holyoke',
    '01041': 'Holyoke',
    '01050': 'Huntington',
    '01053': 'Northampton',
    '01054': 'Leverett',
    '01056': 'Ludlow',
    '01057': 'Monson',
    '01059': 'Amherst',
    '01060': 'Northampton',
    '01061': 'Northampton',
    '01062': 'Northampton',
    '01063': 'Northampton',
    '01066': 'Hatfield',
    '01068': 'Oakham',
    '01069': 'Palmer',
    '01070': 'Plainfield',
    '01071': 'Russell',
    '01072': 'Shutesbury',
    '01073': 'Southampton',
    '01074': 'Barre',
    '01075': 'South Hadley',
    '01077': 'Southwick',
    '01079': 'Palmer',
    '01080': 'Palmer',
    '01081': 'Wales',
    '01082': 'Ware',
    '01083': 'Warren',
    '01084': 'Chesterfield',
    '01086': 'Westfield',
    '01088': 'Hatfield',
    '01089': 'West Springfield',
    '01090': 'West Springfield',
    '01092': 'Warren',
    '01093': 'Whately',
    '01094': 'Hardwick',
    '01095': 'Wilbraham',
    '01096': 'Williamsburg',
    '01097': 'Russell',
    '01098': 'Worthington',
    '01101': 'Springfield',
    '01102': 'Springfield',
    '01103': 'Springfield',
    '01104': 'Springfield',
    '01105': 'Springfield',
    '01106': 'Longmeadow',
    '01107': 'Springfield',
    '01108': 'Springfield',
    '01109': 'Springfield',
    '01111': 'Springfield',
    '01115': 'Springfield',
    '01116': 'Longmeadow',
    '01118': 'Springfield',
    '01119': 'Springfield',
    '01128': 'Springfield',
    '01129': 'Springfield',
    '01133': 'Springfield',
    '01138': 'Springfield',
    '01139': 'Springfield',
    '01144': 'Springfield',
    '01151': 'Springfield',
    '01152': 'Springfield',
    '01195': 'Springfield',
    '01199': 'Springfield',
    '01201': 'Pittsfield',
    '01202': 'Pittsfield',
    '01203': 'Pittsfield',
    '01220': 'Adams',
    '01222': 'Sheffield',
    '01224': 'Lanesborough',
    '01225': 'Cheshire',
    '01227': 'Dalton',
    '01229': 'Stockbridge',
    '01230': 'Great Barrington',
    '01235': 'Hinsdale',
    '01236': 'West Stockbridge',
    '01238': 'Lee',
    '01240': 'Lenox',
    '01242': 'Lenox',
    '01243': 'Middlefield',
    '01244': 'New Marlborough',
    '01245': 'Monterey',
    '01252': 'Egremont',
    '01253': 'Otis',
    '01254': 'Richmond',
    '01255': 'Sandisfield',
    '01256': 'Savoy',
    '01257': 'Sheffield',
    '01259': 'New Marlborough',
    '01260': 'Lee',
    '01262': 'Stockbridge',
    '01263': 'Stockbridge',
    '01264': 'Tyringham',
    '01266': 'West Stockbridge',
    '01267': 'Williamstown',
    '01270': 'Windsor',
    '01301': 'Greenfield',
    '01302': 'Greenfield',
    '01330': 'Ashfield',
    '01338': 'Buckland',
    '01340': 'Colrain',
    '01341': 'Conway',
    '01342': 'Deerfield',
    '01343': 'Florida',
    '01344': 'Erving',
    '01346': 'Heath',
    '01347': 'Montague',
    '01349': 'Montague',
    '01350': 'Monroe',
    '01351': 'Montague',
    '01354': 'Gill',
    '01355': 'New Salem',
    '01360': 'Northfield',
    '01364': 'Orange',
    '01366': 'Petersham',
    '01367': 'Rowe',
    '01368': 'Royalston',
    '01370': 'Shelburne',
    '01373': 'Deerfield',
    '01375': 'Sunderland',
    '01376': 'Montague',
    '01378': 'Warwick',
    '01379': 'Wendell',
    '01380': 'Wendell',
    '01420': 'Fitchburg',
    '01430': 'Ashburnham',
    '01431': 'Ashby',
    '01432': 'Ayer',
    '01434': 'Ayer',
    '01436': 'Templeton',
    '01438': 'Templeton',
    '01440': 'Gardner',
    '01441': 'Westminster',
    '01450': 'Groton',
    '01451': 'Harvard',
    '01452': 'Hubbardston',
    '01453': 'Leominster',
    '01460': 'Littleton',
    '01462': 'Lunenburg',
    '01463': 'Pepperell',
    '01464': 'Shirley',
    '01467': 'Harvard',
    '01468': 'Templeton',
    '01469': 'Townsend',
    '01470': 'Groton',
    '01471': 'Groton',
    '01472': 'Groton',
    '01473': 'Westminster',
    '01474': 'Townsend',
    '01475': 'Winchendon',
    '01477': 'Winchendon',
    '01501': 'Auburn',
    '01503': 'Berlin',
    '01504': 'Blackstone',
    '01505': 'Boylston',
    '01506': 'Brookfield',
    '01507': 'Charlton',
    '01508': 'Charlton',
    '01509': 'Charlton',
    '01510': 'Clinton',
    '01515': 'East Brookfield',
    '01516': 'Douglas',
    '01517': 'Princeton',
    '01518': 'Sturbridge',
    '01519': 'Grafton',
    '01520': 'Holden',
    '01521': 'Holland',
    '01522': 'Holden',
    '01523': 'Lancaster',
    '01524': 'Leicester',
    '01525': 'Northbridge',
    '01526': 'Sutton',
    '01527': 'Millbury',
    '01529': 'Millville',
    '01531': 'New Braintree',
    '01532': 'Northborough',
    '01534': 'Northbridge',
    '01535': 'North Brookfield',
    '01536': 'Grafton',
    '01537': 'Oxford',
    '01538': 'Uxbridge',
    '01540': 'Oxford',
    '01541': 'Princeton',
    '01542': 'Leicester',
    '01543': 'Rutland',
    '01545': 'Shrewsbury',
    '01546': 'Shrewsbury',
    '01550': 'Southbridge',
    '01560': 'Grafton',
    '01561': 'Lancaster',
    '01562': 'Spencer',
    '01564': 'Sterling',
    '01566': 'Sturbridge',
    '01568': 'Upton',
    '01569': 'Uxbridge',
    '01570': 'Webster',
    '01571': 'Dudley',
    '01580': 'Westborough',
    '01581': 'Westborough',
    '01582': 'Westborough',
    '01583': 'West Boylston',
    '01585': 'West Brookfield',
    '01586': 'Millbury',
    '01588': 'Mansfield',
    '01590': 'Sutton',
    '01601': 'Worcester',
    '01602': 'Worcester',
    '01603': 'Worcester',
    '01604': 'Worcester',
    '01605': 'Worcester',
    '01606': 'Worcester',
    '01607': 'Worcester',
    '01608': 'Worcester',
    '01609': 'Worcester',
    '01610': 'Worcester',
    '01611': 'Leicester',
    '01612': 'Paxton',
    '01613': 'Worcester',
    '01614': 'Worcester',
    '01615': 'Worcester',
    '01653': 'Worcester',
    '01654': 'Worcester',
    '01655': 'Worcester',
    '01701': 'Framingham',
    '01702': 'Framingham',
    '01703': 'Framingham',
    '01704': 'Framingham',
    '01705': 'Framingham',
    '01718': 'Acton',
    '01719': 'Boxborough',
    '01720': 'Acton',
    '01721': 'Ashland',
    '01730': 'Bedford',
    '01731': 'Bedford',
    '01740': 'Bolton',
    '01741': 'Carlisle',
    '01742': 'Concord',
    '01745': 'Southborough',
    '01746': 'Holliston',
    '01747': 'Hopedale',
    '01748': 'Hopkinton',
    '01749': 'Hudson',
    '01752': 'Marlborough',
    '01754': 'Maynard',
    '01756': 'Mendon',
    '01757': 'Milford',
    '01760': 'Natick',
    '01770': 'Sherborn',
    '01772': 'Southborough',
    '01773': 'Lincoln',
    '01775': 'Stow',
    '01776': 'Sudbury',
    '01778': 'Wayland',
    '01784': 'Hopkinton',
    '01801': 'Woburn',
    '01803': 'Burlington',
    '01805': 'Burlington',
    '01806': 'Woburn',
    '01807': 'Woburn',
    '01808': 'Woburn',
    '01810': 'Andover',
    '01812': 'Andover',
    '01813': 'Woburn',
    '01815': 'Woburn',
    '01821': 'Billerica',
    '01822': 'Billerica',
    '01824': 'Chelmsford',
    '01826': 'Dracut',
    '01827': 'Dunstable',
    '01830': 'Haverhill',
    '01831': 'Haverhill',
    '01832': 'Haverhill',
    '01833': 'Georgetown',
    '01834': 'Groveland',
    '01835': 'Haverhill',
    '01840': 'Lawrence',
    '01841': 'Lawrence',
    '01842': 'Lawrence',
    '01843': 'Lawrence',
    '01844': 'Methuen',
    '01845': 'North Andover',
    '01850': 'Lowell',
    '01851': 'Lowell',
    '01852': 'Lowell',
    '01853': 'Lowell',
    '01854': 'Lowell',
    '01860': 'Merrimac',
    '01862': 'Billerica',
    '01863': 'Chelmsford',
    '01864': 'North Reading',
    '01865': 'Billerica',
    '01866': 'Billerica',
    '01867': 'Reading',
    '01876': 'Tewksbury',
    '01879': 'Tyngsborough',
    '01880': 'Wakefield',
    '01885': 'Boxford',
    '01886': 'Westford',
    '01887': 'Wilmington',
    '01888': 'Woburn',
    '01889': 'North Reading',
    '01890': 'Winchester',
    '01899': 'Andover',
    '01901': 'Lynn',
    '01902': 'Lynn',
    '01903': 'Lynn',
    '01904': 'Lynn',
    '01905': 'Lynn',
    '01906': 'Saugus',
    '01907': 'Swampscott',
    '01908': 'Nahant',
    '01910': 'Lynn',
    '01913': 'Amesbury',
    '01915': 'Beverly',
    '01921': 'Boxford',
    '01922': 'Newbury',
    '01923': 'Danvers',
    '01929': 'Essex',
    '01930': 'Gloucester',
    '01931': 'Gloucester',
    '01936': 'Hamilton',
    '01937': 'Hathorne',
    '01938': 'Ipswich',
    '01940': 'Lynnfield',
    '01944': 'Manchester-by-the-Sea',
    '01945': 'Marblehead',
    '01949': 'Middleton',
    '01950': 'Newburyport',
    '01951': 'Newbury',
    '01952': 'Salisbury',
    '01960': 'Peabody',
    '01961': 'Peabody',
    '01965': 'Beverly',
    '01966': 'Rockport',
    '01969': 'Rowley',
    '01970': 'Salem',
    '01971': 'Salem',
    '01982': 'Hamilton',
    '01983': 'Topsfield',
    '01984': 'Wenham',
    '01985': 'West Newbury',
    '02018': 'Norwell',
    '02019': 'Bellingham',
    '02020': 'Nantucket',
    '02021': 'Canton',
    '02025': 'Cohasset',
    '02026': 'Dedham',
    '02027': 'Dedham',
    '02030': 'Dover',
    '02031': 'Mansfield',
    '02032': 'Walpole',
    '02035': 'Foxborough',
    '02038': 'Franklin',
    '02040': 'Scituate',
    '02041': 'Duxbury',
    '02043': 'Hingham',
    '02044': 'Hingham',
    '02045': 'Hull',
    '02047': 'Scituate',
    '02048': 'Mansfield',
    '02050': 'Marshfield',
    '02051': 'Marshfield',
    '02052': 'Medfield',
    '02053': 'Medway',
    '02054': 'Millis',
    '02055': 'Scituate',
    '02056': 'Norfolk',
    '02059': 'Marshfield',
    '02060': 'Scituate',
    '02061': 'Norwell',
    '02062': 'Norwood',
    '02065': 'Marshfield',
    '02066': 'Scituate',
    '02067': 'Sharon',
    '02070': 'Wrentham',
    '02071': 'Walpole',
    '02072': 'Stoughton',
    '02081': 'Walpole',
    '02090': 'Westwood',
    '02093': 'Wrentham',
    '02108': 'Boston',
    '02109': 'Boston',
    '02110': 'Boston',
    '02111': 'Boston',
    '02112': 'Boston',
    '02113': 'Boston',
    '02114': 'Boston',
    '02115': 'Boston',
    '02116': 'Boston',
    '02117': 'Boston',
    '02118': 'Boston',
    '02119': 'Boston',
    '02120': 'Boston',
    '02121': 'Boston',
    '02122': 'Boston',
    '02123': 'Boston',
    '02124': 'Boston',
    '02125': 'Boston',
    '02126': 'Boston',
    '02127': 'Boston',
    '02128': 'Boston',
    '02129': 'Boston',
    '02130': 'Boston',
    '02131': 'Boston',
    '02132': 'Boston',
    '02133': 'Boston',
    '02134': 'Boston',
    '02135': 'Boston',
    '02136': 'Boston',
    '02137': 'Boston',
    '02138': 'Cambridge',
    '02139': 'Cambridge',
    '02140': 'Cambridge',
    '02141': 'Cambridge',
    '02142': 'Cambridge',
    '02143': 'Somerville',
    '02144': 'Somerville',
    '02145': 'Somerville',
    '02148': 'Malden',
    '02149': 'Everett',
    '02150': 'Chelsea',
    '02151': 'Revere',
    '02152': 'Winthrop',
    '02153': 'Medford',
    '02155': 'Medford',
    '02156': 'Medford',
    '02163': 'Boston',
    '02169': 'Quincy',
    '02170': 'Quincy',
    '02171': 'Quincy',
    '02176': 'Melrose',
    '02180': 'Stoneham',
    '02184': 'Braintree',
    '02185': 'Braintree',
    '02186': 'Milton',
    '02187': 'Milton',
    '02188': 'Weymouth',
    '02189': 'Weymouth',
    '02190': 'Weymouth',
    '02191': 'Weymouth',
    '02196': 'Boston',
    '02199': 'Boston',
    '02201': 'Boston',
    '02203': 'Boston',
    '02204': 'Boston',
    '02205': 'Boston',
    '02206': 'Boston',
    '02207': 'Boston',
    '02210': 'Boston',
    '02211': 'Boston',
    '02212': 'Boston',
    '02215': 'Boston',
    '02216': 'Boston',
    '02217': 'Boston',
    '02222': 'Boston',
    '02228': 'Boston',
    '02238': 'Cambridge',
    '02239': 'Cambridge',
    '02241': 'Boston',
    '02266': 'Boston',
    '02269': 'Quincy',
    '02283': 'Boston',
    '02284': 'Boston',
    '02293': 'Boston',
    '02295': 'Boston',
    '02297': 'Boston',
    '02298': 'Boston',
    '02301': 'Brockton',
    '02302': 'Brockton',
    '02303': 'Brockton',
    '02304': 'Brockton',
    '02305': 'Brockton',
    '02322': 'Avon',
    '02324': 'Bridgewater',
    '02325': 'Bridgewater',
    '02327': 'Pembroke',
    '02330': 'Carver',
    '02331': 'Duxbury',
    '02332': 'Duxbury',
    '02333': 'East Bridgewater',
    '02334': 'Easton',
    '02337': 'East Bridgewater',
    '02338': 'Halifax',
    '02339': 'Hanover',
    '02340': 'Hanover',
    '02341': 'Hanson',
    '02343': 'Holbrook',
    '02344': 'Middleborough',
    '02345': 'Plymouth',
    '02346': 'Middleborough',
    '02347': 'Lakeville',
    '02348': 'Lakeville',
    '02349': 'Middleborough',
    '02350': 'Hanson',
    '02351': 'Abington',
    '02355': 'Carver',
    '02356': 'Easton',
    '02357': 'Easton',
    '02358': 'Pembroke',
    '02359': 'Pembroke',
    '02360': 'Plymouth',
    '02361': 'Plymouth',
    '02362': 'Plymouth',
    '02364': 'Kingston',
    '02366': 'Carver',
    '02367': 'Plympton',
    '02368': 'Randolph',
    '02370': 'Rockland',
    '02375': 'Easton',
    '02379': 'West Bridgewater',
    '02381': 'Plymouth',
    '02382': 'Whitman',
    '02420': 'Lexington',
    '02421': 'Lexington',
    '02445': 'Brookline',
    '02446': 'Brookline',
    '02447': 'Brookline',
    '02451': 'Waltham',
    '02452': 'Waltham',
    '02453': 'Waltham',
    '02454': 'Waltham',
    '02455': 'Waltham',
    '02456': 'Littleton',
    '02457': 'Wellesley',
    '02458': 'Newton',
    '02459': 'Newton',
    '02460': 'Newton',
    '02461': 'Newton',
    '02462': 'Newton',
    '02464': 'Newton',
    '02465': 'Newton',
    '02466': 'Newton',
    '02467': 'Newton',
    '02468': 'Newton',
    '02471': 'Watertown',
    '02472': 'Watertown',
    '02474': 'Arlington',
    '02475': 'Arlington',
    '02476': 'Arlington',
    '02477': 'Watertown',
    '02478': 'Belmont',
    '02479': 'Belmont',
    '02481': 'Wellesley',
    '02482': 'Wellesley',
    '02492': 'Needham',
    '02493': 'Weston',
    '02494': 'Needham',
    '02495': 'Newton',
    '02532': 'Bourne',
    '02534': 'Bourne',
    '02536': 'Falmouth',
    '02537': 'Sandwich',
    '02538': 'Wareham',
    '02539': 'Edgartown',
    '02540': 'Falmouth',
    '02541': 'Falmouth',
    '02542': 'Bourne',
    '02543': 'Falmouth',
    '02552': 'Chilmark',
    '02553': 'Bourne',
    '02554': 'Nantucket',
    '02556': 'Falmouth',
    '02557': 'Oak Bluffs',
    '02558': 'Wareham',
    '02559': 'Bourne',
    '02561': 'Bourne',
    '02562': 'Bourne',
    '02563': 'Sandwich',
    '02564': 'Nantucket',
    '02565': 'Falmouth',
    '02568': 'Tisbury',
    '02571': 'Wareham',
    '02573': 'Tisbury',
    '02574': 'Falmouth',
    '02575': 'West Tisbury',
    '02576': 'Wareham',
    '02584': 'Nantucket',
    '02601': 'Barnstable',
    '02630': 'Barnstable',
    '02631': 'Brewster',
    '02632': 'Barnstable',
    '02633': 'Chatham',
    '02634': 'Barnstable',
    '02635': 'Barnstable',
    '02636': 'Barnstable',
    '02637': 'Barnstable',
    '02638': 'Dennis',
    '02639': 'Dennis',
    '02641': 'Dennis',
    '02642': 'Eastham',
    '02643': 'Orleans',
    '02644': 'Sandwich',
    '02645': 'Harwich',
    '02646': 'Harwich',
    '02647': 'Barnstable',
    '02648': 'Barnstable',
    '02649': 'Mashpee',
    '02650': 'Chatham',
    '02651': 'Eastham',
    '02652': 'Truro',
    '02653': 'Orleans',
    '02655': 'Barnstable',
    '02657': 'Provincetown',
    '02659': 'Chatham',
    '02660': 'Dennis',
    '02661': 'Harwich',
    '02662': 'Orleans',
    '02663': 'Wellfleet',
    '02664': 'Yarmouth',
    '02666': 'Truro',
    '02667': 'Wellfleet',
    '02668': 'Barnstable',
    '02669': 'Chatham',
    '02670': 'Dennis',
    '02671': 'Harwich',
    '02672': 'Barnstable',
    '02673': 'Yarmouth',
    '02675': 'Yarmouth',
    '02702': 'Freetown',
    '02703': 'Attleboro',
    '02712': 'Norton',
    '02713': 'Gosnold',
    '02714': 'Dartmouth',
    '02715': 'Dighton',
    '02717': 'Freetown',
    '02718': 'Taunton',
    '02719': 'Fairhaven',
    '02720': 'Fall River',
    '02721': 'Fall River',
    '02722': 'Fall River',
    '02723': 'Fall River',
    '02724': 'Fall River',
    '02725': 'Somerset',
    '02726': 'Somerset',
    '02738': 'Marion',
    '02739': 'Mattapoisett',
    '02740': 'New Bedford',
    '02741': 'New Bedford',
    '02742': 'New Bedford',
    '02743': 'Acushnet',
    '02744': 'New Bedford',
    '02745': 'New Bedford',
    '02746': 'New Bedford',
    '02747': 'Dartmouth',
    '02748': 'Dartmouth',
    '02760': 'North Attleboro',
    '02761': 'North Attleboro',
    '02762': 'Plainville',
    '02763': 'Attleboro',
    '02764': 'Dighton',
    '02766': 'Norton',
    '02767': 'Raynham',
    '02768': 'Raynham',
    '02769': 'Rehoboth',
    '02770': 'Rochester',
    '02771': 'Seekonk',
    '02777': 'Swansea',
    '02779': 'Berkley',
    '02780': 'Taunton',
    '02783': 'Taunton',
    '02790': 'Westport',
    '02791': 'Westport',
    '05501': 'Andover',
    '05544': 'Andover'
 }

# Mappings of city name to Municipality name ONLY for cities that contain zipcodes that map to multiple municipalities.
# Used to disambiguate multi-muni zipcodes when City field is present.
CITY_MUNI_MAP = {
    'amherst': 'Amherst',
    'easthampton': 'Easthampton',
    'westfield': 'Westfield',
    'becket': 'Becket',
    'dalton': 'Dalton',
    'lanesboro': 'Lanesborough',
    'north adams': 'North Adams',
    'south egremont': 'Egremont',
    'athol': 'Athol',
    'bernardston': 'Bernardston',
    'charlemont': 'Charlemont',
    'chilmark': 'Chilmark',
    'alford': 'Alford',
    'aquinnah': 'Aquinnah',
    'clarksburg': 'Clarksburg',
    'hancock': 'Hancock',
    'hawley': 'Hawley',
    'leyden': 'Leyden',
    'montgomery': 'Montgomery',
    'mount washington': 'Mount Washington',
    'new ashford': 'New Ashford',
    'pelham': 'Pelham',
    'phillipston': 'Phillipston',
    'washington': 'Washington',
    'westhampton': 'Westhampton'
 }

def clean(zip_series: pd.Series) -> pd.DataFrame:
    """Standardize zipcode formatting in a pandas series

    Pandas will likely load zipcodes from an excel file as an object series
    mixing string and numeric types.  This function will cast all entries to
    string types, strip whitespace, left pad to a minimum of 5 characters with zeros, then
    validate the entry contains a valid zipcode using a regular expression.

    Return: pandas.DataFrame
    - Pandas series of cleaned zipcodes: str
    - Pandas series of cleaned zip4, or empty string if missing: str
    - Pandas series of valid zipcode indicators: boolean
    """
    # Valid zipcodes are 3-5 numeric digits, followed by an optional dash and 4 more digits, or an optional ".0" if the data has been cast as floats.
    valid_zipcode_regex = r"^([0-9]{3,5})(?:[.]0)?(?:-([0-9]{4})|-)?$"
    #The extract function will match this pattern, and extract the zip5 group into column 0 and the zip4 group into column 1.  NaN if group is not present or pattern isn't matched.
    res = zip_series.astype(str).str.strip().str.extract(valid_zipcode_regex)

    cleaned_zipcode_series = res[0].str.zfill(5).fillna('')
    cleaned_zip4_series = res[1].fillna('')

    valid_zipcode_series = cleaned_zipcode_series.str.match(valid_zipcode_regex)

    #Replace invalid value rows with original inputs
    cleaned_zipcode_series.loc[~valid_zipcode_series] = zip_series[~valid_zipcode_series]
    return pd.DataFrame(data={"zip_cleaned": cleaned_zipcode_series, "zip4_cleaned": cleaned_zip4_series, "zip_valid": valid_zipcode_series})

def validate_zip_town_row(row: dict, town_field: str, zip_field: str) -> pd.Series:
    """
    Use zipcodes library to append town data from zipcodes and validate existing zip/town inputs
    """
    INVALID_RESULT = pd.Series({
        "town": "INVALID",
        "municipality": "INVALID",
        "zip_exists": False,
        "town_valid": False,
    })

    muni = ZIP_MUNI_MAP.get(row[zip_field])
    if not muni:
        #Try looking up town_field in City/Muni map
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        muni = CITY_MUNI_MAP.get(standardized_raw_town_name)
        if not muni:
            return INVALID_RESULT

    # From here on, we were at least able to look up a muni.  But we still want to determine whether the raw town data is valid
    INVALID_RESULT = pd.Series({
            "municipality": muni,
            "town": string.capwords(str(row[town_field]).strip()),
            "zip_exists": False,
            "town_valid": False,
        })

    try:
        zip_results = zipcodes.matching(row[zip_field])
    except TypeError:
        return INVALID_RESULT
    except ValueError:
        return INVALID_RESULT

    if len(zip_results) > 1:
        raise ValueError(f"Multimatch zipcode {row[zip_field]} encountered!")
    elif len(zip_results) == 0:
        return INVALID_RESULT
    else: # len(zip_results)==1

        zip_results = zip_results[0]
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        standardized_town_name = zip_results["city"].lower().strip()
        standardized_acceptable_towns = [x.lower().strip() for x in zip_results["acceptable_cities"]]
        standardized_acceptable_towns.append(standardized_town_name)

        #Strict exact matching for now
        town_valid = standardized_raw_town_name in standardized_acceptable_towns
        output = {
            "municipality": muni,
            "town": zip_results["city"],
            "zip_exists": True,
            "town_valid": town_valid
        }
    return pd.Series(output)

def validate_zip_town(df: pd.DataFrame, town_field: str, zip_field: str) -> pd.DataFrame:
    """
    Use zipcodes library to append town data from zipcodes and validate existing zip/town inputs

    Return: pandas.DataFrame
    - Pandas series of town name from zipcodes library: str
    - Pandas series indicating zipcode exists: boolean
    - Pandas series indicating raw data town is valid given zipcode: boolean
    """
    return df.merge(df.apply(
                        lambda row: validate_zip_town_row(row, town_field, zip_field), axis=1
                        ),
                    left_index=True, right_index=True
                )

def valid_town_row(row: dict, town_field: str) -> pd.Series:
    standardized_town = string.capwords(row[town_field].strip())
    zip_list = zipcodes.filter_by(city=standardized_town, state='MA')
    town_valid = (len(zip_list)>0)

    muni_list = [ZIP_MUNI_MAP.get(z['zip_code']) for z in zip_list]
    unique_munis = list(pd.Series([m for m in muni_list if m is not None]).unique())
    if len(unique_munis) == 1:
        muni = unique_munis[0]
    else:
        muni = None

    if not muni:
        #Try looking up town_field in City/Muni map
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        muni = CITY_MUNI_MAP.get(standardized_raw_town_name)
        if not muni:
            muni = "INVALID"
            town_valid = False

    output = {
            "municipality": muni,
            "town": standardized_town,
            "zip_exists": False,
            "zip_cleaned": "",
            "zip4_cleaned": "",
            "zip_valid": False,
            "town_valid": town_valid
        }
    return pd.Series(output)

def validate_town(df: pd.DataFrame, town_field: str) -> pd.DataFrame:
    """
    Use zipcodes library to verify town name is valid and append cleaned fields

    Return: pandas.DataFrame
    - Pandas series of standardize (Capitalized) town name from zipcodes library: str
    - Blank columns for zip_cleaned, zip4_cleaned, and zip_valid=False to comply with clean data schema
    - Pandas series indicating raw data town is valid in the sense of existing in the zipcode lookup table.
    """
    return df.merge(df.apply(lambda row: valid_town_row(row, town_field), axis=1), left_index=True, right_index=True)


from pyiges.IGESCore import IGEStorage
import pyiges.IGESGeomLib as IGES

filename = "102_x.igs"

"""
1H,,1H;,5HF102X,9HF102X.IGS,13H{unspecified},13H{unspecified},16,6,15,  G      1
13,15,5HF102X,3.,1,4HINCH,8,0.016,15H20060617.212531,0.0001,15.5,       G      2
22H Dennette@WiZ-WORX.com,15H www.IGES5x.org,11,3,13H920717.080000,     G      3
22HMIL-PRF-28000B Class 2;                                              G      4
     406       1                                                00020000D      1
     406       2               1      16                                D      2
     410       2                                                00020300D      3
     410       2               1                                        D      4
     404       3                                                00000300D      5
     404       2               1                                        D      6
     124       4       0       0       0       0       0       000000001D      7
     124       2       2       1       0                                D      8
     116       5       0       0       0       0       0       000010001D      9
     116       2       3       1       0                                D     10
     110       6       0       1       0       0       0       000010001D     11
     110       2       4       1       0                                D     12
     100       7       0       1       0       0       7       000010001D     13
     100       2       2       1       0                                D     14
     112       8       0       1       0       0       0       000010001D     15
     112       2       7       5       0                                D     16
     102      13       0       0       0       0       0       000000001D     17
     102       2       6       1       0                                D     18
406,2,6.,8.;                                                           1P      1
410,1,3.,0,0,0,0,0,0;                                                  3P      2
404,1,3,0.,0.,0,0,1,1;                                                 5P      3
124,-1.,0.,0.,3.5,0.,1.,0.,15.,0.,0.,-1.,0.;                           7P      4
116,3.5,15.,0.,;                                                       9P      5
110,3.5,15.,0.,3.5,15.5,0.;                                           11P      6
100,0.,0.,0.,0.,0.5,-0.5,0.;                                          13P      7
112,3,1,3,2,0.,1.,2.,4.,0.5625,0.,-0.0625,15.,-0.312501,              15P      8
1.430510000000000E-006,0.0624995,0.,0.,0.,0.,4.5,0.375,-0.1875,       15P      9
0.0625,14.75,-0.125,0.1875,-0.0625005,0.,0.,0.,0.,4.75,0.1875,        15P     10
0.,0.375,14.75,0.062499,-2.861020000000000E-006,-0.375003,0.,0.,      15P     11
0.,0.;                                                                15P     12
102,4,9,11,13,15;                                                     17P     13
"""

system = IGEStorage()
system.GlobalSection.ParameterDelimiterCharacter = ","
system.GlobalSection.RecordDelimiter = ";"
system.GlobalSection.ProductIdentificationFromSender = filename.split(".")[0]
system.GlobalSection.FileName = filename
system.GlobalSection.NativeSystemID = filename
system.GlobalSection.PreprocessorVersion = "pyIGES"
system.GlobalSection.IntegerBits = 16
system.GlobalSection.SPMagnitude = 6
system.GlobalSection.SPSignificance = 15
system.GlobalSection.DPMagnitude = 13
system.GlobalSection.DPSignificance = 15
system.GlobalSection.ProductIdentificationForReceiver = filename.split(".")[0]
system.GlobalSection.ModelSpaceScale = 3.0
system.GlobalSection.Units.setInches()
system.GlobalSection.MaxNumberLineWeightGrads = 8
system.GlobalSection.WidthMaxLineWeightUnits = 0.016
system.GlobalSection.MaxUserResolution = 0.0001
system.GlobalSection.MaxCoordValue = 15.5
system.GlobalSection.NameOfAuthor = "github.com"
system.GlobalSection.AuthorOrg = "pyIGES"
system.GlobalSection.VersionFlag = 11
system.GlobalSection.DraftStandardFlag = 3
system.GlobalSection.AppProtocol = "pyIGES"

prop = IGES.IGESPropertyEntity([2,6.,8.], formNumber=16)
prop.StatusNumber.Subordinate.setLogicallyDependent()
system.Commit(prop)

view = IGES.IGESViewEntity([1,3.,0,0,0,0,0,0])
view.StatusNumber.EntityUseFlag.setOther()
view.StatusNumber.Subordinate.setLogicallyDependent()
system.Commit(view)

drawing = IGES.IGESDrawingEntity([1,3,0.,0.,0,0,1,1], formNumber=1)
drawing.StatusNumber.EntityUseFlag.setOther()
system.Commit(drawing)

trans = IGES.IGESGeomTransform([-1.,0.,0.,
                            3.5,0.,1.,
                            0.,15.,0.,
                            0.,-1.,0.], formNumber=1)
trans.StatusNumber.Hierachy.setGlobalDefer()
trans.LineWeightNum = 2
trans.Color.setCyan()
system.Commit(trans)

point = IGES.IGESGeomPoint(IGES.IGESPoint(3.5, 15.0, 0.0))
point.LineFontPattern.setNone()
point.LineWeightNum = 2
point.Color.setGreen()
point.StatusNumber.Hierachy.setGlobalDefer()
point.StatusNumber.EntityUseFlag.setGeometry()
point.StatusNumber.Subordinate.setPhysicallyDependent()
point.StatusNumber.Visablilty.setVisible()
system.Commit(point)

line = IGES.IGESGeomLine(IGES.IGESPoint(3.5, 15.0, 0.0), IGES.IGESPoint(3.5, 15.5, 0.0))
line.LineFontPattern.setSolid()
line.LineWeightNum = 2
line.Color.setBlue()
line.StatusNumber.Hierachy.setGlobalDefer()
line.StatusNumber.EntityUseFlag.setGeometry()
line.StatusNumber.Subordinate.setPhysicallyDependent()
line.StatusNumber.Visablilty.setVisible()
system.Commit(line)

arc = IGES.IGESGeomArc(0.0, IGES.IGESPoint(0.0, 0.0, 0.0),  IGES.IGESPoint(0.0, 0.5, 0.0), IGES.IGESPoint(-0.5, 0.0, 0.0))
arc.TransfrmMat = trans.DirectoryDataPointer.data
arc.LineFontPattern.setSolid()
arc.LineWeightNum = 2
arc.Color.setRed()
arc.StatusNumber.Hierachy.setGlobalDefer()
arc.StatusNumber.EntityUseFlag.setGeometry()
arc.StatusNumber.Subordinate.setPhysicallyDependent()
arc.StatusNumber.Visablilty.setVisible()
system.Commit(arc)

scurve = IGES.IGESSplineCurve(3, 1, 3)
scurve.addSegment(0.0, [4.,0.5625,0.,-0.0625,15.,-0.312501,1.430510000000000E-006,0.0624995,0.,0.,0.,0.])
scurve.addSegment(1.0, [4.5,0.375,-0.1875,0.0625,14.75,-0.125,0.1875,-0.0625005,0., 0., 0., 0.])
scurve.addSegment(2.0, [4.75, 0.1875, 0., 0.375, 14.75, 0.062499, -2.861020000000000E-006, -0.375003, 0., 0., 0., 0.])
scurve.LineFontPattern.setSolid()
scurve.LineWeightNum = 2
scurve.Color.setCyan()
scurve.StatusNumber.Hierachy.setGlobalDefer()
scurve.StatusNumber.EntityUseFlag.setGeometry()
scurve.StatusNumber.Subordinate.setPhysicallyDependent()
scurve.StatusNumber.Visablilty.setVisible()
system.Commit(scurve)

compCurve = IGES.IGESGeomCompositeCurve(point, line, arc, scurve)
compCurve.LineWeightNum = 2
compCurve.Color.setMagenta()
compCurve.StatusNumber.Hierachy.setGlobalDefer()
system.Commit(compCurve)

system.save(filename = filename)


import random

class Bichinho:
    def __init__(self, nome, fome, saude, idade, tedio):
        self.nome = nome
        self.fome = fome
        self.saude = saude
        self.idade = idade
        self.tedio = tedio

    def __str__(self):
        return f"""
        Nome: {self.nome}
        Fome: {self.fome}
        Saúde: {self.saude}
        Idade: {self.idade}
        Tédio: {self.tedio}
        """

    def humor(self):
        return (self.fome + self.saude) / 2

    def alimentar(self, quantidade_comida):
        self.fome -= quantidade_comida
        self.saude += quantidade_comida / 2

    def brincar(self, tempo_brincadeira):
        self.tedio -= tempo_brincadeira
        if self.tedio < 0:
            self.tedio = 0
        self.saude += tempo_brincadeira / 2

    def dormir(self):
        self.fome += 1
        self.saude += 2
        self.idade += 1

class FazendaDeBichinhos:
    def __init__(self, quantidade_bichinhos):
        self.bichinhos = []
        for i in range(quantidade_bichinhos):
            nome = f"Bichinho{i}"
            fome = random.randint(0, 10)
            saude = random.randint(0, 10)
            idade = random.randint(0, 10)
            tedio = random.randint(0, 10)
            bichinho = Bichinho(nome, fome, saude, idade, tedio)
            self.bichinhos.append(bichinho)

    def alimentar_todos(self, quantidade_comida):
        for bichinho in self.bichinhos:
            bichinho.alimentar(quantidade_comida)

    def brincar_todos(self, tempo_brincadeira):
        for bichinho in self.bichinhos:
            bichinho.brincar(tempo_brincadeira)

    def dormir_todos(self):
        for bichinho in self.bichinhos:
            bichinho.dormir()

    def mostrar_bichinhos(self):
        for bichinho in self.bichinhos:
            print(bichinho)

fazenda = FazendaDeBichinhos(3)

opcao = 0

opcao = 0

while opcao != 5:
    print("----- CUIDANDO DA FAZENDA DE BICHINHOS -----")
    print("1 - Alimentar todos os bichinhos")
    print("2 - Brincar com todos os bichinhos")
    print("3 - Fazer todos os bichinhos dormir")
    print("4 - Mostrar todos os bichinhos")
    print("5 - Sair")
    opcao = int(input("Escolha uma opção: "))
    if opcao == 1:
        quantidade_comida = int(input("Quantidade de comida a dar para cada bichinho: "))
        fazenda.alimentar_todos(quantidade_comida)
    elif opcao == 2:
        tempo_brincadeira = int(input("Tempo de brincadeira (em horas): "))
        fazenda.brincar_todos(tempo_brincadeira)
    elif opcao == 3:
        fazenda.dormir_todos()
    elif opcao == 4:
        fazenda.mostrar_bichinhos()
    elif opcao == 42: # opção secreta para mostrar os valores exatos dos atributos do objeto
        print("Valores exatos dos atributos dos bichinhos:")
        for bichinho in fazenda.bichinhos:
            print(bichinho.__dict__)
    elif opcao == 5:
        print("Até mais!")
    else:
        print("Opção inválida. Escolha novamente.")


"""
hhpy.ipython.py
~~~~~~~~~~~~~~~

Contains convenience wrappers for ipython

"""
# ---- imports
# --- standard imports
import pandas as pd
# --- third party imports
from IPython.display import display, HTML
# --- local imports
from hhpy.main import export, assert_list, list_exclude

# ---- functions
# --- export
@export
def wide_notebook(width: int = 90):
    """
    makes the jupyter notebook wider by appending html code to change the width,
     based on https://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-jupyter-
     ipython-notebook-in-my-browser

    :param: width in percent, default 90 [optional]
    :return: None
    """
    # noinspection PyTypeChecker
    display(
        HTML(f""))

@export
def hide_code():
    """
    hides the code and introduces a toggle button
     based on https://stackoverflow.com/questions/27934885/how-to-hide-code-from-cells-in-ipython-notebook-visualized
     -with-nbviewer

    :return: None
    """

    # noinspection PyTypeChecker
    display(HTML('''

        The raw code for this IPython notebook is by default hidden for easier reading.
        To toggle on/off the raw code, click here.
    '''))

@export
def display_full(*args, rows=None, cols=None, **kwargs):
    """
    wrapper to display a pandas DataFrame with all rows and columns

    :param rows: number of rows to display, defaults to all
    :param cols: number of columns to display, defaults to all
    :param args: passed to display
    :param kwargs: passed to display
    :return: None
    """
    with pd.option_context('display.max_rows', rows, 'display.max_columns', cols):
        display(*args, **kwargs)

@export
def pd_display(*args, number_format='{:,.2f}', full=True, **kwargs):
    """
    wrapper to display a pandas DataFrame with a specified number format

    :param args: passed to display
    :param number_format: the number format to apply
    :param full: whether to use :func:`~display_full` (True) or standard display (False)
    :param kwargs: passed to display
    :return: None
    """
    pd.set_option('display.float_format', number_format.format)

    if full:
        display_full(*args, **kwargs)
    else:
        display(*args, **kwargs)

    pd.reset_option('display.float_format')

@export
def display_df(df, int_format=',', float_format=',.2f', exclude=None, full=True, **kwargs):
    """
    Wrapper to display a pandas DataFrame with separate options for int / float, also adds an option to exclude columns

    :param df: pandas DataFrame to display
    :param int_format: format for integer columns
    :param float_format: format for float columns
    :param exclude: columns to exclude
    :param full: whether to show all rows and columns or keep default behaviour
    :param kwargs: passed to display
    :return: None
    """

    if exclude is None:
        exclude = []
    else:
        exclude = assert_list(exclude)
    # avoid inplace
    df = pd.DataFrame(df).copy()

    _cols_int = list_exclude(df.select_dtypes(int).columns, exclude)
    _cols_float = list_exclude(df.select_dtypes(float).columns, exclude)

    for _col in _cols_int:
        df[_col] = df[_col].apply(lambda _: format(
            _, int_format) if not pd.isna(_) else '<NaN>')
    for _col in _cols_float:
        df[_col] = df[_col].apply(lambda _: format(
            _, float_format) if not pd.isna(_) else '<NaN>')

    if full:
        display_full(df, **kwargs)
    else:
        display(df, **kwargs)

# --- pandas styles
@export
def highlight_max(df: pd.DataFrame, color: str = 'xkcd:cyan') -> pd.DataFrame:
    """
    highlights the largest value in each column of a pandas DataFrame

    :param df: pandas DataFrame
    :param color: color used for highlighting
    :return: the pandas DataFrame with the style applied to it
    """
    def cond_max(s: pd.Series):
        return ['background-color: {}'.format(color) if v else '' for v in s == s.max()]

    return df.style.apply(cond_max)

@export
def highlight_min(df: pd.DataFrame, color: str = 'xkcd:light red') -> pd.DataFrame:
    """
    highlights the smallest value in each column of a pandas DataFrame

    :param df: pandas DataFrame
    :param color: color used for highlighting
    :return: the pandas DataFrame with the style applied to it
    """
    def cond_min(s: pd.Series):
        return ['background-color: {}'.format(color) if v else '' for v in s == s.min()]

    return df.style.apply(cond_min)

@export
def highlight_max_min(df: pd.DataFrame, max_color: str = 'xkcd:cyan', min_color: str = 'xkcd:light red'):
    """
    highlights the largest and smallest value in each column of a pandas DataFrame

    :param df: pandas DataFrame
    :param max_color: color used for highlighting largest value
    :param min_color: color used for highlighting smallest value
    :return: the pandas DataFrame with the style applied to it
    """
    def cond_max_min(s):

        _out = []

        for _i in s:

            if _i == s.max():
                _out.append('background-color: {}'.format(max_color))
            elif _i == s.min():
                _out.append('background-color: {}'.format(min_color))
            else:
                _out.append('')

        return _out

    return df.style.apply(cond_max_min)


import time
import math
import collections

class Display:
    """Write on terminal statistics in a fancy way.

    Colors are used to signal variations in the data.

    Example:

    display = Display("Step {step}/{}   loss: {loss:.2f}")
    display.disp(10, 100, loss=3.14159)

    It would print the message:

    Step 10/100    loss 3.14

    with "3.14" colored according to historical variation of the loss
    value.

    Named fields (such as "loss") are tracked and displayed in color.
    Unnamed fields are not tracked.  "step" is a special untracked field,
    and "steps_s" is a tracked field that is automatically computed.

    """
    def __init__(self, format_string):
        """Create the display object.

        The format string encodes how information should be displayed.
        """
        self.fmt = format_string
        self.vars_ = collections.defaultdict(_DisplayVar)
        self.steps_s = _DisplayVar()
        self.last_step = None
        self.last_time = None

    def message(self, step, *fields, **data):
        """Compose a message with the given information."""
        self._update_steps_s(step)
        d = dict((k, self._update_var(k, v)) for (k, v) in data.items())
        return self.fmt.format(*fields, step=step, steps_s=self.steps_s, **d)

    def disp(self, step, *fields, **data):
        """Print on stdout the given information according the the format of the display."""
        print(self.message(step, *fields, **data))

    def _update_var(self, k, v):
        dv = self.vars_[k]
        dv.add(v)
        return dv

    def _update_steps_s(self, step):
        tm = time.perf_counter()
        if self.last_step is None or self.last_time >= tm:
            speed = float("nan")
        else:
            speed = (step - self.last_step) / (tm - self.last_time)
        self.last_time = tm
        self.last_step = step
        self.steps_s.add(speed)

class _DisplayVar:
    """Track the history of a value and format its last value accordingly."""

    # Ansi codes for colors and styles
    MIN = "\x1B[1;32m"    # bold green
    LOW = "\x1B[0;32m"    # green
    NORMAL = "\x1B[0;33m" # yellow
    HIGH = "\x1B[0;31m"   # red
    MAX = "\x1B[1;31m"    # bold red
    NAN = "\x1B[1;36m"    # cyan
    RESET = "\x1B[0m"     # default style

    def __init__(self, history_len=10):
        """Initialize the object.

        Remembers up to history_len values.
        """
        self.history = collections.deque(maxlen=history_len)
        self.minval = self.maxval = None
        self.lastvalue = float("nan")
        self.state = self.NAN

    def add(self, value):
        """Add a new value to the series."""
        self.lastvalue = value
        if math.isnan(value):
            self.state = self.NAN
        elif not self.history:
            self.state = self.NORMAL
            self.history.append(value)
            self.minval = self.maxval = value
        else:
            _, s = min((min(self.history), self.NORMAL), (value, self.LOW))
            _, s = max((max(self.history), s), (value, self.HIGH))
            self.maxval, _, s = max((self.maxval, 1, s), (value, 0, self.MAX))
            self.minval, _, s = min((self.minval, 0, s), (value, 1, self.MIN))
            self.state = s
            self.history.append(value)

    def __format__(self, spec):
        """Format the last added value."""
        s = self.lastvalue.__format__(spec)
        return self.state + s + self.RESET

def _demo():
    import random
    fmt = "Step: {step:3d}/{}  Loss: {loss:6.3f}  {steps_s:6.4f} steps/s"
    display = Display(fmt)
    for step in range(1, 101):
        time.sleep(1)
        display.disp(step, 100, loss=random.random() * 100)

if __name__ == "__main__":
    _demo()
    # session = Session("model_dir", save_every=100, save_count=5, state=[model, optimizer], max_epocs=3)
    # session.add_state(model)
    # session.add_state(optimizer)
    # session.restore()
    # for x ,y in session.train_loop(loader):
    #     session.step
    #     session.epoc


import os
from pathlib import Path
from typing import Tuple
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img

TRAIN_PATH = os.path.join(
    os.path.dirname(__file__),
    "D:\Computer Vision\Rock-Paper-Scissors-Classification\data\data\\test",
)

TEST_PATH = os.path.join(
    os.path.dirname(__file__),
    "D:\Computer Vision\Rock-Paper-Scissors-Classification\data\data\\train",
)

IMAGE_WIDTH = 224
IMAGE_HEIGHT = 224
BATCH_SIZE = 32

class MyImageDataGenerator(object):
    def __init__(self, batch_size=BATCH_SIZE, input_shape=(IMAGE_WIDTH, IMAGE_HEIGHT)):

        self.batch_size = batch_size
        self.input_shape = input_shape
        super(MyImageDataGenerator, self).__init__()

    def image_pipeline(self):

        data_gen = ImageDataGenerator(
            rescale=1.0 / 255.0,
            zca_epsilon=1e-3,
            rotation_range=15,
            shear_range=0.1,
            zoom_range=0.2,
            fill_mode="nearest",
            horizontal_flip=True,
            vertical_flip=False,
            width_shift_range=0.1,
            height_shift_range=0.1,
        )

        train_generator = data_gen.flow_from_directory(
            TRAIN_PATH,
            target_size=self.input_shape,
            batch_size=self.batch_size,
            class_mode="categorical",
        )

        validation_generator = data_gen.flow_from_directory(
            TEST_PATH,
            target_size=self.input_shape,
            batch_size=self.batch_size,
            class_mode="categorical",
        )
        train_samples = train_generator.samples
        validation_samples = validation_generator.samples
        batch_size = train_generator.batch_size

        return (
            train_generator,
            validation_generator,
            train_samples,
            validation_samples,
            batch_size,
        )

def main():
    images = MyImageDataGenerator()
    images.image_pipeline()

if __name__ == "__main__":
    main()


import json
import base64

from xcalar.compute.coretypes.LibApisCommon.ttypes import *
from xcalar.compute.coretypes.LibApisCommon.constants import *
from xcalar.compute.coretypes.LibApisEnums.ttypes import *
from xcalar.compute.coretypes.LibApisConstants.ttypes import *
from xcalar.compute.coretypes.DagTypes.ttypes import *
from xcalar.compute.coretypes.DagTypes.constants import *
from xcalar.compute.coretypes.XcalarApiVersionSignature.ttypes import XcalarApiVersionT
from xcalar.compute.coretypes.UdfTypes.ttypes import *
from xcalar.compute.coretypes.DataTargetTypes.ttypes import *
from xcalar.compute.coretypes.DataTargetTypes.constants import *
from xcalar.compute.coretypes.DataTargetEnums.constants import *
from xcalar.compute.coretypes.DataFormatEnums.ttypes import *
from xcalar.compute.coretypes.DataFormatEnums.constants import *
from xcalar.compute.coretypes.OrderingEnums.constants import *
from xcalar.compute.coretypes.JoinOpEnums.constants import *
from xcalar.compute.coretypes.UnionOpEnums.constants import *

#
# This file defines convenience classes for constructing Xcalar API work
# items.
#

# Base class for all work items. Defines a few basic things.
class WorkItem(object):
    def __init__(self, userName=None, userIdUnique=None, sessionName=None):
        self.workItem = XcalarApiWorkItemT()
        self.workItem.apiVersionSignature = XcalarApiVersionT.XcalarApiVersionSignature
        if sessionName:
            self.workItem.sessionName = sessionName
        if userIdUnique:
            self.workItem.userIdUnique = userIdUnique
        if userName:
            self.workItem.userId = userName

    # Gives the work item a chance to define how output should look to callers.
    def output(self, executeOutput):
        return executeOutput

class WorkItemGetNumNodes(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemGetNumNodes, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetNumNodes

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getNumNodesOutput

class WorkItemTop(WorkItem):
    def __init__(self,
                 measureIntervalInMs=None,
                 cacheValidityInMs=None,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemTop, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiTop

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.topInput = XcalarApiTopInputT()
        assert (measureIntervalInMs is not None)
        assert (cacheValidityInMs is not None)
        self.workItem.input.topInput.measureIntervalInMs = measureIntervalInMs
        self.workItem.input.topInput.cacheValidityInMs = cacheValidityInMs

    def output(self, executeOutput):
        return executeOutput.output.outputResult.topOutput

    def updateMeasureIntervalInMs(self, measureIntervalInMs=None):
        assert (measureIntervalInMs is not None)
        self.workItem.input.topInput.measureIntervalInMs = measureIntervalInMs

    def updateCacheValidity(self, cacheValidityInMs=None):
        assert (cacheValidityInMs is not None)
        self.workItem.input.topInput.cacheValidityInMs = \
            cacheValidityInMs

class WorkItemListDataset(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        # XXX super() in python 3.
        super(WorkItemListDataset, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiListDatasets

    def output(self, executeOutput):
        return executeOutput.output.outputResult.listDatasetsOutput

class WorkItemGetDatasetsInfo(WorkItem):
    def __init__(self, datasetsNamePattern, userName=None, userIdUnique=None):
        super(WorkItemGetDatasetsInfo, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetDatasetsInfo

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.getDatasetsInfoInput = XcalarApiGetDatasetsInfoInputT(
        )
        self.workItem.input.getDatasetsInfoInput.datasetsNamePattern = datasetsNamePattern

    def output(self, executeOutput):
        return executeOutput.output.outputResult

class WorkItemListDagInfo(WorkItem):
    def __init__(self, namePattern, srcType, userName=None, userIdUnique=None):
        super(WorkItemListDagInfo, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiListDagNodeInfo

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.listDagNodesInput = XcalarApiDagNodeNamePatternInputT(
        )
        self.workItem.input.listDagNodesInput.srcType = srcType
        self.workItem.input.listDagNodesInput.namePattern = namePattern

    def output(self, executeOutput):
        return executeOutput.output.outputResult.listNodesOutput

class WorkItemQueryState(WorkItem):
    def __init__(self,
                 queryName,
                 detailedStats=True,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemQueryState, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiQueryState

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.queryStateInput = XcalarApiQueryNameInputT()
        self.workItem.input.queryStateInput.queryName = queryName
        self.workItem.input.queryStateInput.detailedStats = detailedStats

    def output(self, executeOutput):
        return executeOutput.output.outputResult.queryStateOutput

class WorkItemQueryCancel(WorkItem):
    def __init__(self, queryName, userName=None, userIdUnique=None):
        super(WorkItemQueryCancel, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiQueryCancel

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.queryStateInput = XcalarApiQueryNameInputT()
        self.workItem.input.queryStateInput.queryName = queryName

class WorkItemQueryDelete(WorkItem):
    def __init__(self, queryName="", userName=None, userIdUnique=None):
        super(WorkItemQueryDelete, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiQueryDelete

        self.workItem.input = XcalarApiInputT()
        # Not a typo. Query delete uses queryStateInput for some reason...
        self.workItem.input.queryStateInput = XcalarApiQueryNameInputT()
        self.workItem.input.queryStateInput.queryName = queryName

class WorkItemCancelOp(WorkItem):
    def __init__(self, name):
        super(WorkItemCancelOp, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiCancelOp

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.dagTableNameInput = XcalarApiDagTableNameInputT()
        self.workItem.input.dagTableNameInput.tableInput = name

class WorkItemGetOpStats(WorkItem):
    def __init__(self, name, userName=None, userIdUnique=None):
        super(WorkItemGetOpStats, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetOpStats

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.dagTableNameInput = XcalarApiDagTableNameInputT()
        self.workItem.input.dagTableNameInput.tableInput = name

    def output(self, executeOutput):
        return executeOutput.output.outputResult.opStatsOutput

class WorkItemDeleteDagNode(WorkItem):
    def __init__(self,
                 namePattern,
                 srcType,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None,
                 deleteCompletely=False):
        super(WorkItemDeleteDagNode, self).__init__(userName, userIdUnique,
                                                    sessionName)
        self.workItem.api = XcalarApisT.XcalarApiDeleteObjects

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.deleteDagNodeInput = XcalarApiDagNodeNamePatternInputT(
        )
        self.workItem.input.deleteDagNodeInput.namePattern = namePattern
        self.workItem.input.deleteDagNodeInput.srcType = srcType
        self.workItem.input.deleteDagNodeInput.deleteCompletely = deleteCompletely

    def output(self, executeOutput):
        return executeOutput.output.outputResult.deleteDagNodesOutput

class WorkItemRenameNode(WorkItem):
    def __init__(self, oldName, newName, userName=None, userIdUnique=None):
        super().__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiRenameNode

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.renameNodeInput = XcalarApiRenameNodeInputT()
        self.workItem.input.renameNodeInput.oldName = oldName
        self.workItem.input.renameNodeInput.newName = newName

class WorkItemArchiveTables(WorkItem):
    def __init__(self,
                 archive,
                 tableNames,
                 allTables=False,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemArchiveTables, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiArchiveTables

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.archiveTablesInput = XcalarApiArchiveTablesInputT()
        self.workItem.input.archiveTablesInput.archive = archive
        self.workItem.input.archiveTablesInput.allTables = allTables
        self.workItem.input.archiveTablesInput.tableNames = tableNames

    def output(self, executeOutput):
        return executeOutput.output.outputResult.archiveTablesOutput

class WorkItemSessionNew(WorkItem):
    def __init__(self,
                 name,
                 fork,
                 forkedSession=None,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemSessionNew, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionNew

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionNewInput = XcalarApiSessionNewInputT()
        self.workItem.input.sessionNewInput.sessionName = name
        self.workItem.input.sessionNewInput.fork = fork
        self.workItem.input.sessionNewInput.forkedSessionName = forkedSession

class WorkItemSessionDelete(WorkItem):
    def __init__(self, namePattern, userName=None, userIdUnique=None):
        super(WorkItemSessionDelete, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionDelete

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionDeleteInput = XcalarApiSessionDeleteInputT()
        self.workItem.input.sessionDeleteInput.sessionName = namePattern
        self.workItem.input.sessionDeleteInput.noCleanup = False

class WorkItemSessionActivate(WorkItem):
    def __init__(self, name, userName=None, userIdUnique=None):
        super(WorkItemSessionActivate, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionActivate

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionActivateInput = XcalarApiSessionActivateInputT(
        )
        self.workItem.input.sessionActivateInput.sessionName = name

class WorkItemSessionInact(WorkItem):
    def __init__(self, name, userName=None, userIdUnique=None):
        super(WorkItemSessionInact, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionInact

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionDeleteInput = XcalarApiSessionDeleteInputT()
        self.workItem.input.sessionDeleteInput.sessionName = name
        self.workItem.input.sessionDeleteInput.noCleanup = False

class WorkItemSessionList(WorkItem):
    def __init__(self, pattern, userName=None, userIdUnique=None):
        super(WorkItemSessionList, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionList

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionListInput = XcalarApiSessionListArrayInputT(
        )
        self.workItem.input.sessionListInput.sesListInput = pattern

    def output(self, executeOutput):
        return executeOutput.output.outputResult.sessionListOutput

class WorkItemSessionRename(WorkItem):
    def __init__(self, newName, oldName, userName=None, userIdUnique=None):
        super().__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionRename

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionRenameInput = XcalarApiSessionRenameInputT()
        self.workItem.input.sessionRenameInput.sessionName = newName
        self.workItem.input.sessionRenameInput.origSessionName = oldName

class WorkItemSessionPersist(WorkItem):
    def __init__(self, name, userName=None, userIdUnique=None):
        super().__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionPersist

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionDeleteInput = XcalarApiSessionDeleteInputT()
        self.workItem.input.sessionDeleteInput.sessionName = name

class WorkItemSessionDownload(WorkItem):
    def __init__(self,
                 name,
                 pathToAdditionalFiles,
                 userName=None,
                 userIdUnique=None):
        super().__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSessionDownload

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionDownloadInput = XcalarApiSessionDownloadInputT(
        )
        self.workItem.input.sessionDownloadInput.sessionName = name
        self.workItem.input.sessionDownloadInput.pathToAdditionalFiles = pathToAdditionalFiles

    def output(self, executeOutput):
        downloadStr = executeOutput.output.outputResult.sessionDownloadOutput.sessionContent
        return base64.b64decode(downloadStr)

class WorkItemSessionUpload(WorkItem):
    def __init__(self,
                 name,
                 sessionContent,
                 pathToAdditionalFiles,
                 userName=None,
                 userIdUnique=None):
        super().__init__(userName, userIdUnique)

        encodedSessionContent = base64.b64encode(sessionContent).decode(
            "ascii")

        self.workItem.api = XcalarApisT.XcalarApiSessionUpload

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.sessionUploadInput = XcalarApiSessionUploadInputT()
        self.workItem.input.sessionUploadInput.sessionName = name
        self.workItem.input.sessionUploadInput.sessionContent = encodedSessionContent
        self.workItem.input.sessionUploadInput.sessionContentCount = len(
            encodedSessionContent)
        self.workItem.input.sessionUploadInput.pathToAdditionalFiles = pathToAdditionalFiles

    def output(self, executeOutput):
        return executeOutput.output.outputResult.sessionNewOutput

class WorkItemGetTableMeta(WorkItem):
    def __init__(self,
                 isTable,
                 name,
                 isPrecise,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemGetTableMeta, self).__init__(userName, userIdUnique,
                                                   sessionName)
        self.workItem.api = XcalarApisT.XcalarApiGetTableMeta

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.getTableMetaInput = XcalarApiGetTableMetaInputT()
        self.workItem.input.getTableMetaInput.tableNameInput = XcalarApiNamedInputT(
        )
        self.workItem.input.getTableMetaInput.tableNameInput.isTable = isTable
        self.workItem.input.getTableMetaInput.tableNameInput.name = name
        self.workItem.input.getTableMetaInput.tableNameInput.xid = XcalarApiXidInvalidT
        self.workItem.input.getTableMetaInput.isPrecise = isPrecise

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getTableMetaOutput

class WorkItemPreview(WorkItem):
    def __init__(self,
                 sourceArgs,
                 numBytes,
                 offset,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemPreview, self).__init__(userName, userIdUnique)
        inputObj = {
            "func": "preview",
            "sourceArgs": {
                "targetName": sourceArgs.targetName,
                "path": sourceArgs.path,
                "fileNamePattern": sourceArgs.fileNamePattern,
                "recursive": sourceArgs.recursive,
            },
            "offset": offset,
            "bytesRequested": numBytes
        }
        self.workItem.api = XcalarApisT.XcalarApiPreview
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.previewInput = XcalarApiPreviewInputT()
        self.workItem.input.previewInput.inputJson = json.dumps(inputObj)

    def output(self, executeOutput):
        return executeOutput.output.outputResult.previewOutput

class WorkItemAddTarget2(WorkItem):
    def __init__(self,
                 targetTypeId,
                 targetName,
                 targetParams,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemAddTarget2, self).__init__(userName, userIdUnique)
        inputObj = {
            "func": "addTarget",
            "targetTypeId": targetTypeId,
            "targetName": targetName,
            "targetParams": targetParams
        }
        self.workItem.api = XcalarApisT.XcalarApiTarget
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.targetInput = XcalarApiTargetInputT()
        self.workItem.input.targetInput.inputJson = json.dumps(inputObj)

    def output(self, executeOutput):
        return json.loads(
            executeOutput.output.outputResult.targetOutput.outputJson)

class WorkItemDeleteTarget2(WorkItem):
    def __init__(self, targetName, userName=None, userIdUnique=None):
        super(WorkItemDeleteTarget2, self).__init__(userName, userIdUnique)
        inputObj = {"func": "deleteTarget", "targetName": targetName}
        self.workItem.api = XcalarApisT.XcalarApiTarget
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.targetInput = XcalarApiTargetInputT()
        self.workItem.input.targetInput.inputJson = json.dumps(inputObj)

    def output(self, executeOutput):
        return json.loads(
            executeOutput.output.outputResult.targetOutput.outputJson)

class WorkItemListTargets2(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemListTargets2, self).__init__(userName, userIdUnique)
        inputObj = {"func": "listTargets"}
        self.workItem.api = XcalarApisT.XcalarApiTarget
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.targetInput = XcalarApiTargetInputT()
        self.workItem.input.targetInput.inputJson = json.dumps(inputObj)

    def output(self, executeOutput):
        return json.loads(
            executeOutput.output.outputResult.targetOutput.outputJson)

class WorkItemListTargetTypes2(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemListTargetTypes2, self).__init__(userName, userIdUnique)
        inputObj = {"func": "listTypes"}
        self.workItem.api = XcalarApisT.XcalarApiTarget
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.targetInput = XcalarApiTargetInputT()
        self.workItem.input.targetInput.inputJson = json.dumps(inputObj)

    def output(self, executeOutput):
        return json.loads(
            executeOutput.output.outputResult.targetOutput.outputJson)

class WorkItemDatasetCreate(WorkItem):
    def __init__(self,
                 name,
                 loadArgs,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemDatasetCreate, self).__init__(userName, userIdUnique,
                                                    sessionName)
        self.workItem.api = XcalarApisT.XcalarApiDatasetCreate
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.datasetCreateInput = XcalarApiDatasetCreateInputT()
        self.workItem.input.datasetCreateInput.dest = name
        self.workItem.input.datasetCreateInput.loadArgs = loadArgs

class WorkItemLoad(WorkItem):
    def __init__(self,
                 name,
                 loadArgs,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemLoad, self).__init__(userName, userIdUnique, sessionName)
        self.workItem.api = XcalarApisT.XcalarApiBulkLoad
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.loadInput = XcalarApiBulkLoadInputT()
        self.workItem.input.loadInput.dest = name
        self.workItem.input.loadInput.loadArgs = loadArgs

    def output(self, executeOutput):
        return executeOutput.output.outputResult.loadOutput

class WorkItemDatasetGetMeta(WorkItem):
    def __init__(self,
                 name,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemDatasetGetMeta, self).__init__(userName, userIdUnique,
                                                     sessionName)
        self.workItem.api = XcalarApisT.XcalarApiDatasetGetMeta
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.datasetGetMetaInput = XcalarApiDatasetGetMetaInputT(
        )
        self.workItem.input.datasetGetMetaInput.datasetName = name

    def output(self, executeOutput):
        return executeOutput.output.outputResult.datasetGetMetaOutput

class WorkItemDatasetUnload(WorkItem):
    def __init__(self,
                 namePattern,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemDatasetUnload, self).__init__(userName, userIdUnique,
                                                    sessionName)
        self.workItem.api = XcalarApisT.XcalarApiDatasetUnload
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.datasetUnloadInput = XcalarApiDatasetUnloadInputT()
        self.workItem.input.datasetUnloadInput.datasetNamePattern = namePattern

    def output(self, executeOutput):
        return executeOutput.output.outputResult.datasetUnloadOutput

class WorkItemDatasetDelete(WorkItem):
    def __init__(self,
                 name,
                 userName=None,
                 userIdUnique=None,
                 sessionName=None):
        super(WorkItemDatasetDelete, self).__init__(userName, userIdUnique,
                                                    sessionName)
        self.workItem.api = XcalarApisT.XcalarApiDatasetDelete
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.datasetDeleteInput = XcalarApiDatasetDeleteInputT()
        self.workItem.input.datasetDeleteInput.datasetName = name

class WorkItemIndex(WorkItem):
    def __init__(self,
                 datasetName,
                 dstTableName,
                 keyNames,
                 dhtName,
                 ordering,
                 isTable,
                 fatptrPrefixName="",
                 delaySort=False,
                 keyFieldNames=None):
        super(WorkItemIndex, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiIndex
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.indexInput = XcalarApiIndexInputT()

        self.workItem.input.indexInput.source = datasetName
        self.workItem.input.indexInput.dest = dstTableName
        self.workItem.input.indexInput.key = []

        if (isinstance(keyNames, list)):
            for ii in range(len(keyNames)):
                key = XcalarApiKeyT()
                key.name = keyNames[ii]
                key.type = "DfUnknown"

                if keyFieldNames:
                    key.keyFieldName = keyFieldNames[ii]

                if (isinstance(ordering, list)):
                    key.ordering = XcalarOrderingTStr[ordering[ii]]
                else:
                    key.ordering = XcalarOrderingTStr[ordering]

                self.workItem.input.indexInput.key.append(key)
        else:
            key = XcalarApiKeyT()
            key.name = keyNames
            if keyFieldNames:
                key.keyFieldName = keyFieldNames

            key.type = "DfUnknown"
            key.ordering = XcalarOrderingTStr[ordering]

            self.workItem.input.indexInput.key.append(key)

        self.workItem.input.indexInput.dhtName = dhtName
        self.workItem.input.indexInput.prefix = fatptrPrefixName
        self.workItem.input.indexInput.delaySort = delaySort

    def output(self, executeOutput):
        return executeOutput.output.outputResult.indexOutput.tableName

class WorkItemGetRowNum(WorkItem):
    def __init__(self, srcTable, dstTable, newFieldName):
        super(WorkItemGetRowNum, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiGetRowNum
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.getRowNumInput = XcalarApiGetRowNumInputT()
        self.workItem.input.getRowNumInput.source = srcTable
        self.workItem.input.getRowNumInput.dest = dstTable
        self.workItem.input.getRowNumInput.newField = newFieldName

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getRowNumOutput.tableName

class WorkItemMap(WorkItem):
    def __init__(self, srcTable, dstTable, evalStrs, newFieldNames):
        super(WorkItemMap, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiMap
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.mapInput = XcalarApiMapInputT()
        self.workItem.input.mapInput.source = srcTable
        self.workItem.input.mapInput.dest = dstTable

        self.workItem.input.mapInput.eval = []

        if (isinstance(evalStrs, list)):
            for ii in range(0, len(evalStrs)):
                eval = XcalarApiEvalT()
                eval.evalString = evalStrs[ii]
                eval.newField = newFieldNames[ii]

                self.workItem.input.mapInput.eval.append(eval)
        else:
            eval = XcalarApiEvalT()
            eval.evalString = evalStrs
            eval.newField = newFieldNames

            self.workItem.input.mapInput.eval.append(eval)

    def output(self, executeOutput):
        return executeOutput.output.outputResult.mapOutput.tableName

class WorkItemFilter(WorkItem):
    def __init__(self, srcTable, dstTable, filterStr):
        super(WorkItemFilter, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiFilter
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.filterInput = XcalarApiFilterInputT()
        self.workItem.input.filterInput.source = srcTable
        self.workItem.input.filterInput.dest = dstTable
        self.workItem.input.filterInput.eval = []
        eval = XcalarApiEvalT()
        eval.evalString = filterStr
        self.workItem.input.filterInput.eval.append(eval)

    def output(self, executeOutput):
        return executeOutput.output.outputResult.filterOutput.tableName

class WorkItemJoin(WorkItem):
    def __init__(self,
                 leftTable,
                 rightTable,
                 joinTable,
                 joinType,
                 leftColumns,
                 rightColumns,
                 filterString,
                 keepAllColumns=True):
        super(WorkItemJoin, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiJoin
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.joinInput = XcalarApiJoinInputT()

        self.workItem.input.joinInput.source = [leftTable, rightTable]
        self.workItem.input.joinInput.dest = joinTable

        self.workItem.input.joinInput.joinType = JoinOperatorTStr[joinType]
        self.workItem.input.joinInput.columns = [leftColumns, rightColumns]
        self.workItem.input.joinInput.evalString = filterString
        self.workItem.input.joinInput.keepAllColumns = keepAllColumns

    def output(self, executeOutput):
        return executeOutput.output.outputResult.joinOutput.tableName

class WorkItemUnion(WorkItem):
    def __init__(self, sources, dest, columns, dedup, unionType):
        super(WorkItemUnion, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiUnion
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.unionInput = XcalarApiUnionInputT()

        self.workItem.input.unionInput.source = sources
        self.workItem.input.unionInput.dest = dest
        self.workItem.input.unionInput.columns = columns
        self.workItem.input.unionInput.dedup = dedup
        self.workItem.input.unionInput.unionType = UnionOperatorTStr[unionType]

    def output(self, executeOutput):
        return executeOutput.output.outputResult.unionOutput.tableName

class WorkItemAggregate(WorkItem):
    def __init__(self,
                 srcTableName,
                 dstTableName,
                 evalStr,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemAggregate, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiAggregate
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.aggregateInput = XcalarApiAggregateInputT()

        self.workItem.input.aggregateInput.source = srcTableName
        self.workItem.input.aggregateInput.dest = dstTableName
        self.workItem.input.aggregateInput.eval = []
        eval = XcalarApiEvalT()
        eval.evalString = evalStr
        self.workItem.input.aggregateInput.eval.append(eval)

    def output(self, executeOutput):
        return json.loads(executeOutput.output.outputResult.aggregateOutput.
                          jsonAnswer)['Value']

class WorkItemGroupBy(WorkItem):
    def __init__(self,
                 srcTable,
                 dstTable,
                 evalStrs,
                 newFieldNames,
                 includeSrcTableSample=False,
                 groupAll=False):
        super(WorkItemGroupBy, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiGroupBy
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.groupByInput = XcalarApiGroupByInputT()
        self.workItem.input.groupByInput.source = srcTable
        self.workItem.input.groupByInput.dest = dstTable
        self.workItem.input.groupByInput.eval = []

        if (isinstance(evalStrs, list)):
            for ii in range(0, len(evalStrs)):
                eval = XcalarApiEvalT()
                eval.evalString = evalStrs[ii]
                eval.newField = newFieldNames[ii]

                self.workItem.input.groupByInput.eval.append(eval)
        else:
            eval = XcalarApiEvalT()
            eval.evalString = evalStrs
            eval.newField = newFieldNames

            self.workItem.input.groupByInput.eval.append(eval)

        self.workItem.input.groupByInput.includeSample = includeSrcTableSample
        self.workItem.input.groupByInput.icv = False
        self.workItem.input.groupByInput.newKeyField = ""
        self.workItem.input.groupByInput.groupAll = groupAll

    def output(self, executeOutput):
        return executeOutput.output.outputResult.groupByOutput.tableName

class WorkItemProject(WorkItem):
    def __init__(self, srcTable, dstTable, columns):
        super(WorkItemProject, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiProject
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.projectInput = XcalarApiProjectInputT()
        self.workItem.input.projectInput.source = srcTable
        self.workItem.input.projectInput.dest = dstTable
        self.workItem.input.projectInput.columns = columns

    def output(self, executeOutput):
        return executeOutput.output.outputResult.projectOutput.tableName

class WorkItemShutdown(WorkItem):
    def __init__(self):
        super(WorkItemShutdown, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiShutdown

class WorkItemUdfAdd(WorkItem):
    def __init__(self, moduleName, source, userName=None, userIdUnique=None):
        super(WorkItemUdfAdd, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiUdfAdd
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.udfAddUpdateInput = UdfModuleSrcT()
        self.workItem.input.udfAddUpdateInput.type = UdfTypeT.UdfTypePython
        self.workItem.input.udfAddUpdateInput.moduleName = moduleName
        self.workItem.input.udfAddUpdateInput.source = source

    def output(self, executeOutput):
        return executeOutput.output.outputResult.udfAddUpdateOutput

class WorkItemUdfUpdate(WorkItem):
    def __init__(self, moduleName, source):
        super(WorkItemUdfUpdate, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiUdfUpdate
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.udfAddUpdateInput = UdfModuleSrcT()
        self.workItem.input.udfAddUpdateInput.type = UdfTypeT.UdfTypePython
        self.workItem.input.udfAddUpdateInput.moduleName = moduleName
        self.workItem.input.udfAddUpdateInput.source = source

    def output(self, executeOutput):
        return executeOutput.output.outputResult.udfAddUpdateOutput

class WorkItemUdfDelete(WorkItem):
    def __init__(self, moduleName):
        super(WorkItemUdfDelete, self).__init__()
        self.workItem.api = XcalarApisT.XcalarApiUdfDelete
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.udfDeleteInput = XcalarApiUdfDeleteInputT()
        self.workItem.input.udfDeleteInput.moduleName = moduleName

class WorkItemUdfGet(WorkItem):
    def __init__(self, moduleName, userName=None, userIdUnique=None):
        super(WorkItemUdfGet, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiUdfGet
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.udfGetInput = XcalarApiUdfGetInputT()
        self.workItem.input.udfGetInput.moduleName = moduleName

    def output(self, getOutput):
        return getOutput.output.outputResult.udfGetOutput

class WorkItemListXdfs(WorkItem):
    def __init__(self,
                 fnNamePattern,
                 categoryPattern,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemListXdfs, self).__init__(userName, userIdUnique)

        self.workItem.api = XcalarApisT.XcalarApiListXdfs
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.listXdfsInput = XcalarApiListXdfsInputT()
        self.workItem.input.listXdfsInput.fnNamePattern = fnNamePattern
        self.workItem.input.listXdfsInput.categoryPattern = categoryPattern

    def output(self, getOutput):
        return getOutput.output.outputResult.listXdfsOutput

class WorkItemGetStats(WorkItem):
    def __init__(self, nodeId=0, userName=None, userIdUnique=None):
        super(WorkItemGetStats, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetStat
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.statInput = XcalarApiStatInputT()
        self.workItem.input.statInput.nodeId = nodeId

    def output(self, rsOutput):
        return rsOutput.output.outputResult.statOutput.stats

class WorkItemGetStatGroupIdMap(WorkItem):
    def __init__(self, nodeId=0, userName=None, userIdUnique=None):
        super(WorkItemGetStatGroupIdMap, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetStatGroupIdMap
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.statInput = XcalarApiStatInputT()
        self.workItem.input.statInput.nodeId = nodeId

    def output(self, rsOutput):
        return rsOutput.output.outputResult.statGroupIdMapOutput

class WorkItemExport(WorkItem):
    def __init__(self,
                 tableName,
                 driverName,
                 driverParams,
                 columns,
                 destDagNodeName="",
                 userName=None,
                 userIdUnique=None):
        super(WorkItemExport, self).__init__(userName, userIdUnique)
        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiExport
        self.workItem.input.exportInput = XcalarApiExportInputT()
        self.workItem.input.exportInput.source = tableName
        self.workItem.input.exportInput.driverName = driverName
        self.workItem.input.exportInput.driverParams = driverParams
        self.workItem.input.exportInput.columns = columns
        self.workItem.input.exportInput.dest = destDagNodeName

class WorkItemImportRetina(WorkItem):
    def __init__(self,
                 retinaName,
                 overwrite,
                 retinaTar=None,
                 retinaJsonStr=None,
                 userName=None,
                 userIdUnique=None,
                 udfUserName=None,
                 udfSessionName=None):
        super(WorkItemImportRetina, self).__init__(userName, userIdUnique)

        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiImportRetina

        self.workItem.input.importRetinaInput = XcalarApiImportRetinaInputT()
        self.workItem.input.importRetinaInput.retinaName = retinaName
        self.workItem.input.importRetinaInput.overwriteExistingUdf = overwrite

        if retinaTar:
            encodedRetina = base64.b64encode(retinaTar).decode("ascii")
            self.workItem.input.importRetinaInput.retinaCount = len(
                encodedRetina)
            self.workItem.input.importRetinaInput.retina = encodedRetina
            self.workItem.input.importRetinaInput.loadRetinaJson = False

        if retinaJsonStr:
            self.workItem.input.importRetinaInput.loadRetinaJson = True
            self.workItem.input.importRetinaInput.retinaJson = retinaJsonStr

        self.workItem.input.importRetinaInput.udfUserName = udfUserName
        self.workItem.input.importRetinaInput.udfSessionName = udfSessionName

    def output(self, executeOutput):
        return executeOutput.output.outputResult.importRetinaOutput

class WorkItemExportRetina(WorkItem):
    def __init__(self, retinaName, userName=None, userIdUnique=None):
        super(WorkItemExportRetina, self).__init__(userName, userIdUnique)

        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiExportRetina

        self.workItem.input.exportRetinaInput = XcalarApiExportRetinaInputT()
        self.workItem.input.exportRetinaInput.retinaName = retinaName

    def output(self, executeOutput):
        retinaStr = executeOutput.output.outputResult.exportRetinaOutput.retina

        return base64.b64decode(retinaStr)

class WorkItemMakeRetina(WorkItem):
    def __init__(self,
                 retinaName,
                 dstTables,
                 srcTables,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemMakeRetina, self).__init__(userName, userIdUnique)

        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiMakeRetina

        self.workItem.input.makeRetinaInput = XcalarApiMakeRetinaInputT()
        self.workItem.input.makeRetinaInput.retinaName = retinaName
        self.workItem.input.makeRetinaInput.numTables = len(dstTables)
        self.workItem.input.makeRetinaInput.tableArray = dstTables
        self.workItem.input.makeRetinaInput.numSrcTables = len(srcTables)
        self.workItem.input.makeRetinaInput.srcTables = srcTables

class WorkItemListRetina(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemListRetina, self).__init__(userName, userIdUnique)

        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiListRetinas
        self.workItem.input.listRetinasInput = XcalarApiListRetinasInputT()
        self.workItem.input.listRetinasInput.namePattern = "*"

    def output(self, executeOutput):
        return executeOutput.output.outputResult.listRetinasOutput

class WorkItemDeleteRetina(WorkItem):
    def __init__(self, retinaName, userName=None, userIdUnique=None):
        super(WorkItemDeleteRetina, self).__init__(userName, userIdUnique)

        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiDeleteRetina

        self.workItem.input.deleteRetinaInput = XcalarApiDeleteRetinaInputT()
        self.workItem.input.deleteRetinaInput.delRetInput = retinaName

class WorkItemAppSet(WorkItem):
    def __init__(self,
                 name,
                 hostType,
                 execStr,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemAppSet, self).__init__(userName, userIdUnique)
        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiAppSet

        self.workItem.input.appSetInput = XcalarApiAppSetInputT()
        self.workItem.input.appSetInput.name = name
        self.workItem.input.appSetInput.hostType = hostType
        self.workItem.input.appSetInput.execStr = execStr

class WorkItemAppRun(WorkItem):
    def __init__(self, name, isGlobal, inStr, userName=None,
                 userIdUnique=None):
        super(WorkItemAppRun, self).__init__(userName, userIdUnique)
        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiAppRun

        self.workItem.input.appRunInput = XcalarApiAppRunInputT()
        self.workItem.input.appRunInput.name = name
        self.workItem.input.appRunInput.isGlobal = isGlobal
        self.workItem.input.appRunInput.inStr = inStr

class WorkItemAppReap(WorkItem):
    def __init__(self, groupId, cancel=False, userName=None,
                 userIdUnique=None):
        super(WorkItemAppReap, self).__init__(userName, userIdUnique)
        self.workItem.input = XcalarApiInputT()
        self.workItem.api = XcalarApisT.XcalarApiAppReap

        self.workItem.input.appReapInput = XcalarApiAppReapInputT()
        self.workItem.input.appReapInput.appGroupId = groupId
        self.workItem.input.appReapInput.cancel = cancel

class WorkItemRuntimeSetParam(WorkItem):
    def __init__(self, runtimeSetParams, userName=None, userIdUnique=None):
        super(WorkItemRuntimeSetParam, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiRuntimeSetParam
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.runtimeSetParamInput = XcalarApiRuntimeSetParamInputT(
        )
        self.workItem.input.runtimeSetParamInput = runtimeSetParams

class WorkItemRuntimeGetParam(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemRuntimeGetParam, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiRuntimeGetParam

    def output(self, executeOutput):
        return executeOutput.output.outputResult.runtimeGetParamOutput

class WorkItemSetConfigParam(WorkItem):
    def __init__(self, paramName, paramValue, userName=None,
                 userIdUnique=None):
        super(WorkItemSetConfigParam, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSetConfigParam
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.setConfigParamInput = XcalarApiSetConfigParamInputT(
        )
        self.workItem.input.setConfigParamInput.paramName = paramName
        self.workItem.input.setConfigParamInput.paramValue = paramValue

class WorkItemGetConfigParams(WorkItem):
    def __init__(self, userName=None, userIdUnique=None):
        super(WorkItemGetConfigParams, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetConfigParams

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getConfigParamsOutput

class WorkItemGetRetina(WorkItem):
    def __init__(self, retinaName, userName=None, userIdUnique=None):
        super(WorkItemGetRetina, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetRetina
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.getRetinaInput = XcalarApiGetRetinaInputT()
        self.workItem.input.getRetinaInput.retInput = retinaName

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getRetinaOutput

class WorkItemGetRetinaJson(WorkItem):
    def __init__(self, retinaName, userName=None, userIdUnique=None):
        super(WorkItemGetRetinaJson, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiGetRetinaJson
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.getRetinaJsonInput = XcalarApiGetRetinaJsonInputT()
        self.workItem.input.getRetinaJsonInput.retinaName = retinaName

    def output(self, executeOutput):
        return executeOutput.output.outputResult.getRetinaJsonOutput

class WorkItemUpdateRetina(WorkItem):
    def __init__(self,
                 retinaName,
                 retinaJson,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemUpdateRetina, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiUpdateRetina

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.updateRetinaInput = XcalarApiUpdateRetinaInputT()
        self.workItem.input.updateRetinaInput.retinaName = retinaName
        self.workItem.input.updateRetinaInput.retinaJson = retinaJson

class WorkItemUpdateTable(WorkItem):
    def __init__(self,
                 srcTableNames,
                 dstTableNames,
                 times=[],
                 dropSrc=False,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemUpdateTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiUpdate

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.updateInput = XcalarApiUpdateInputT()
        self.workItem.input.updateInput.updates = []

        if (isinstance(srcTableNames, list)):
            for ii in range(0, len(srcTableNames)):
                update = XcalarApiUpdateTableInputT()
                update.source = srcTableNames[ii]
                update.dest = dstTableNames[ii]

                if len(times) > 0:
                    update.unixTS = times[ii]
                else:
                    update.unixTS = 0
                update.dropSrc = dropSrc
                self.workItem.input.updateInput.updates.append(update)
        else:
            update = XcalarApiUpdateTableInputT()
            update.source = srcTableNames
            update.dest = dstTableNames

            if isinstance(times, list):
                update.unixTS = 0
            else:
                update.unixTS = times
            update.dropSrc = dropSrc

            self.workItem.input.updateInput.updates.append(update)

        def output(self, executeOutput):
            return executeOutput.output.outputResult.updateOutput.batchId

class WorkItemPublishTable(WorkItem):
    def __init__(self,
                 srcTableName,
                 dstTableName,
                 time=0,
                 dropSrc=False,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemPublishTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiPublish

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.publishInput = XcalarApiPublishInputT()
        self.workItem.input.publishInput.source = srcTableName
        self.workItem.input.publishInput.dest = dstTableName
        self.workItem.input.publishInput.unixTS = time
        self.workItem.input.publishInput.dropSrc = dropSrc

class WorkItemSelectTable(WorkItem):
    def __init__(self,
                 srcTableName,
                 dstTableName,
                 maxBatchId=-1,
                 minBatchId=-1,
                 filterString="",
                 columns=[],
                 limitRows=0,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemSelectTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSelect

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.selectInput = XcalarApiSelectInputT()
        self.workItem.input.selectInput.source = srcTableName
        self.workItem.input.selectInput.dest = dstTableName
        self.workItem.input.selectInput.minBatchId = minBatchId
        self.workItem.input.selectInput.maxBatchId = maxBatchId
        self.workItem.input.selectInput.filterString = filterString
        self.workItem.input.selectInput.columns = columns
        self.workItem.input.selectInput.limitRows = limitRows

    def output(self, executeOutput):
        return executeOutput.output.outputResult.selectOutput.tableName

class WorkItemUnpublishTable(WorkItem):
    def __init__(self,
                 srcTableName,
                 inactivateOnly=False,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemUnpublishTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiUnpublish

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.unpublishInput = XcalarApiUnpublishInputT()
        self.workItem.input.unpublishInput.source = srcTableName
        self.workItem.input.unpublishInput.inactivateOnly = inactivateOnly

class WorkItemRestoreTable(WorkItem):
    def __init__(self, srcTableName, userName=None, userIdUnique=None):
        super(WorkItemRestoreTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiRestoreTable

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.restoreTableInput = XcalarApiRestoreTableInputT()
        self.workItem.input.restoreTableInput.publishedTableName = srcTableName

    def output(self, executeOutput):
        return executeOutput.output.outputResult.restoreTableOutput

class WorkItemCoalesceTable(WorkItem):
    def __init__(self, srcTableName, userName=None, userIdUnique=None):
        super(WorkItemCoalesceTable, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiCoalesce

        self.workItem.input = XcalarApiInputT()
        self.workItem.input.coalesceInput = XcalarApiCoalesceInputT()
        self.workItem.input.coalesceInput.source = srcTableName

class WorkItemSynthesize(WorkItem):
    def __init__(self,
                 src,
                 dstTable,
                 columns,
                 userName=None,
                 userIdUnique=None):
        super(WorkItemSynthesize, self).__init__(userName, userIdUnique)
        self.workItem.api = XcalarApisT.XcalarApiSynthesize
        self.workItem.input = XcalarApiInputT()
        self.workItem.input.synthesizeInput = XcalarApiSynthesizeInputT()
        self.workItem.input.synthesizeInput.source = src
        self.workItem.input.synthesizeInput.dest = dstTable
        self.workItem.input.synthesizeInput.columns = columns
        self.workItem.input.synthesizeInput.sameSession = True

    def output(self, executeOutput):
        return executeOutput.output.outputResult.synthesizeOutput


class Node(object):
    def __init__(self, X, rot_op, border, e, left, right, node_type = ''):
        self.e = e
        self.size = len(X)
        self.X = X # to be removed
        self.rot_op = rot_op
        self.border = border  #border
        self.left = left
        self.right = right
        self.ntype = node_type


import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(0,5,11)
y = x ** 2

print(x)
print(y)

'''
#Functional
plt.plot(x,y,'r-')
plt.xlabel('XLabel')
plt.ylabel('YLabel')
plt.title('Title of the Graph')
'''

'''
plt.subplot(1,2,1)
plt.plot(x,y,'r-')

plt.subplot(1,2,2)
plt.plot(y,x,'b-')
'''

#OO
'''
fig = plt.figure()

axes1 = fig.add_axes([0.1,0.1,0.8,0.8])
axes1.plot(x,y,'b-')
axes1.set_xlabel("lxlabel")
axes1.set_ylabel("lylabel")
axes1.set_title("Larger")

axes2 = fig.add_axes([0.2,0.5,0.4,0.3])
axes2.plot(y,x,'r-')
axes2.set_xlabel("sxlabel")
axes2.set_ylabel("sylabel")
axes2.set_title("Smaller")
'''

fig,axes = plt.subplots(nrows=1,ncols=2,figsize=(2,8))

axes[0].plot(x,y)
axes[0].set_title("x col1")

axes[1].plot(y,x)
axes[1].set_title("y col2")

plt.show()