Incorporating LyA 1B changes into AMTL

py/LSS/SV3/altmtltools.py

@@ -19,22 +19,37 @@
 import astropy.io.fits as pf
 from astropy.table import Table,join,vstack
 
+from desiutil.log import get_logger
+log = get_logger()
+
 #import memory_profiler
 #from memory_profiler import profile
 
+import importlib
+from pkg_resources import packaging
+
+try:
+    desitarget_version = importlib.metadata.version('desitarget')
+    log.info('Running with desitarget/{}'.format(desitarget_version))
+except importlib.metadata.PackageNotFoundError: #this occurs when on main branch
+    desitarget_version = '9999.9.9' #dummy value to ensure we import update_lya_1b on main branch 
+    log.info('Running with desitarget/main'.format(desitarget_version))
+
 import desitarget
 from desitarget import io, mtl
 from desitarget.cuts import random_fraction_of_trues
 from desitarget.mtl import get_mtl_dir, get_mtl_tile_file_name,get_mtl_ledger_format
 from desitarget.mtl import get_zcat_dir, get_ztile_file_name, tiles_to_be_processed
 from desitarget.mtl import make_zcat,survey_data_model,update_ledger, get_utc_date
 
+#we can only import this function on desitarget versions > 3.4.2
+if packaging.version.parse(desitarget_version) >= packaging.version.parse('3.4.2'):
+    from desitarget.mtl import update_lya_1b
+
 from desitarget.targets import initial_priority_numobs, decode_targetid
 from desitarget.targetmask import obsconditions, obsmask
 from desitarget.targetmask import desi_mask, bgs_mask, mws_mask, zwarn_mask
 
-from desiutil.log import get_logger
-
 import fitsio
 import LSS.common_tools as common
 
@@ -63,11 +78,11 @@
 from datetime import datetime, timedelta
 import glob
 
+# LGN 20250909, This will allow us to determine which tiles need to run under pre 3.4.2 desitarget version
+startdate_1b = '2025-07-21T23:36:04+00:00'
 
 pr = cProfile.Profile()
 
-log = get_logger()
-
 #os.environ['DESIMODEL'] = '/global/common/software/desi/cori/desiconda/current/code/desimodel/master'
 #os.environ['DESIMODEL'] = '/global/common/software/desi/perlmutter/desiconda/current/code/desimodel/main'
 
@@ -533,7 +548,7 @@ def updateTileTracker(altmtldir, endDate, survey = 'main', obscon = 'DARK'):
 def makeTileTrackerFN(dirName, survey, obscon):
     return dirName + '/{0}survey-{1}obscon-TileTracker.ecsv'.format(survey, obscon.upper())
 def makeTileTracker(altmtldir, survey = 'main', obscon = 'DARK', startDate = None,
-    endDate = None, overwrite = True, update_only = False):
+    endDate = None, overwrite = True, update_only = False, lya1b = True):
     """Create action file which orders all actions to do with AMTL in order 
     in which real survey did them.
 
@@ -550,6 +565,16 @@ def makeTileTracker(altmtldir, survey = 'main', obscon = 'DARK', startDate = Non
         Used to look up the correct ledger, in combination with `obscon`.
         Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
         for the main survey and different iterations of SV, respectively.
+    update_only : :class:`bool`, optional, defaults to False
+        Used when only actions since the startdate are needed, for example
+        in the updateTileTracker function. 
+    lya1b : :class:`bool`, optional, defaults to True
+        Used to determine if an action should be created to mimic the lya1b
+        numobs increase. (see: https://github.com/desihub/desitarget/pull/845/
+        for details.) Only runs for obscon = dark and if there are actions
+        at dates > 2025-07-21. Should be set to false in only extremely specific
+        scenarios, when one is not trying to mimic real survey decisions.
+
 
 
     Returns
@@ -585,8 +610,8 @@ def makeTileTracker(altmtldir, survey = 'main', obscon = 'DARK', startDate = Non
 
     TSS = Table.read(TSSFN)
 
+
     MTLDTFN = '/global/cfs/cdirs/desi/survey/ops/surveyops/trunk/mtl/mtl-done-tiles.ecsv'
-
     MTLDT = Table.read(MTLDTFN)
 
     #tiles-specstatus file filtered to only matching obscon and surveySURVEY FAPRGRM
@@ -616,6 +641,8 @@ def makeTileTracker(altmtldir, survey = 'main', obscon = 'DARK', startDate = Non
     TileIDs = []
     TypeOfActions = []
     TimesOfActions = []
+    #times for fa actions come from the fiberassign-{tile}.fits file
+    #times for update actions come from the mtl done tiles file
     doneFlag = []
     archiveDates = []
 
@@ -676,6 +703,21 @@ def makeTileTracker(altmtldir, survey = 'main', obscon = 'DARK', startDate = Non
                 doneFlag.append(False)
             archiveDates.append(update['ARCHIVEDATE'])
             reprocFlag = True
+
+
+    #LGN 07/29/25: adding new special action for the LyA QSO NUMOBS increase
+    #LGN This runs for all dark time surveys with actions at times later than 2025-07-21
+    #LGN unless the lya1b flag is set to false. Only change this flag if you are certain
+    #LGN you don't want to mimic real survey decisions.
+    if (lya1b) and (obscon.lower() == 'dark') and (max(TimesOfActions) > startdate_1b):
+        log.info('Adding QSO NUMOBS Increase Action')
+        TileIDs.append(-1)
+        TypeOfActions.append('lya1b')
+        TimesOfActions.append(startdate_1b)
+        doneFlag.append(False)
+        archiveDates.append(20250721)
+
+
     ActionList = [TileIDs, TypeOfActions, TimesOfActions, doneFlag, archiveDates]
     t = Table(ActionList,
            names=('TILEID', 'ACTIONTYPE', 'ACTIONTIME', 'DONEFLAG', 'ARCHIVEDATE'),
@@ -1381,27 +1423,49 @@ def update_alt_ledger(altmtldir,althpdirname, altmtltilefn,  actions, survey = '
         msg = "Update state for {} targets".format(ntargs)
         msg += " (the zcats also contain {} skies with +ve TARGETIDs)".format(nsky)
         log.info(msg)
+
+        # setting up update info
         didUpdateHappen = False
+        if obscon.lower() == 'dark1b' or obscon.lower() == 'bright1b':
+            log.info('setting 1B/ext flag for update')
+            is_1b = True
+
+            # LGN 20251104 Adding an edge case for calling update ledger on 1b tiles observed before lya1b changes
+            # LGN 20251104 The ext keyword does not exist for the update_ledger function until desitarget 3.4.2 
+            if t['ACTIONTIME'] < startdate_1b:
+                passext = False
+            else:
+                passext = True
+        else:
+            is_1b = False
+
         # ADM update the appropriate ledger.
-        if mock:
-
-            if targets is None:
-                raise ValueError('If processing mocks, you MUST specify a target file')
-            log.info('update loc a')
-            update_ledger(althpdirname, altZCat, obscon=obscon.upper(),
-                      numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv')#, targets = targets)
-            didUpdateHappen = True
-        elif targets is None:
-            log.info('update loc b')
-            update_ledger(althpdirname, altZCat, obscon=obscon.upper(),
-                      numobs_from_ledger=numobs_from_ledger)
-            didUpdateHappen = True
+        # LGN 20250909 Revising this section to update for 1B changes and simplifying
+        if mock and targets is None: #Is this part still necessary??
+            raise ValueError('If processing mocks, you MUST specify a target file')
+
+        if is_1b:
+            #getting abbreviated obscon/path for non 1B survey (i.e. dark1b -> dark)
+            althpdirname_short = os.path.join(os.path.dirname(althpdirname), os.path.basename(althpdirname).replace('1b', ''))
+            obscon_short = obscon.replace('1B','')
+
+            #CHECK WITH AURELIO ABOUT TARGETS KEYWORD FOR MOCKS
+
+            if passext:
+                #Updating the non-1b ledger
+                update_ledger(althpdirname_short, altZCat, obscon=obscon_short.upper(),numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv', ext=False, targets = targets)
+                #Updating the 1b ledger
+                update_ledger(althpdirname, altZCat, obscon=obscon.upper(),numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv', ext=True, targets = targets)
+            else:
+                #Updating the non-1b ledger
+                update_ledger(althpdirname_short, altZCat, obscon=obscon_short.upper(),numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv', targets = targets)
+                #Updating the 1b ledger
+                update_ledger(althpdirname, altZCat, obscon=obscon.upper(),numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv', targets = targets)                
         else:
-            log.info('update loc c')
-            update_ledger(althpdirname, altZCat, obscon=obscon.upper(),
-                      numobs_from_ledger=numobs_from_ledger, targets = targets)
-            didUpdateHappen = True
-        assert(didUpdateHappen)
+            #Updating ledger (No ext keyword)
+            #Need clarification on mocks using target file in update_ledger()
+            update_ledger(althpdirname, altZCat, obscon=obscon.upper(),numobs_from_ledger=numobs_from_ledger, tabform='ascii.ecsv', targets = targets)
+
         if verbose or debug:
             log.info('if main, should sleep 1 second')
         #thisUTCDate = get_utc_date(survey=survey)
@@ -1548,6 +1612,12 @@ def loop_alt_ledger(obscon, survey='sv3', zcatdir=None, mtldir=None,
         iterloop = range(ndirs)
     ### JL - End of directory/loop variable construction ###
 
+    # LGN 20250916 - Reading mtl done tiles here to check if correct desitarget version is loaded
+    MTLDTFN = '/global/cfs/cdirs/desi/survey/ops/surveyops/trunk/mtl/mtl-done-tiles.ecsv'
+    MTLDT = Table.read(MTLDTFN)
+
+    prelya1b_tiles = MTLDT[MTLDT['TIMESTAMP'] < startdate_1b]['TILEID']
+
 
 
     if quickRestart:
@@ -1595,6 +1665,21 @@ def loop_alt_ledger(obscon, survey='sv3', zcatdir=None, mtldir=None,
 
         for action in actionList:
 
+            # LGN 20250909. Adding checks here to ensure desitarget version is correct for pre/post 1b tiles
+            '''
+            if action['TILEID'] in prelya1b_tiles and packaging.version.parse(desitarget_version) >= packaging.version.parse('3.4.2'):
+                log.info('You are running a pre DESI1b tile using a DESI1b version of desitarget')
+                log.info('This will cause unexpected behavior with LyA QSO target assignments')
+                log.info('Please module swap to a version of desitarget before version 3.4.2')
+                raise ValueError('Incorrect DESITARGET version is loaded')
+            elif action['TILEID'] not in prelya1b_tiles and packaging.version.parse(desitarget_version) < packaging.version.parse('3.4.2'):
+                log.info('You are running a DESI1b tile using a pre DESI1b version of desitarget')
+                log.info('This will cause unexpected behavior with LyA QSO target assignments')
+                log.info('Please module swap to a version of desitarget >= version 3.4.2')
+                raise ValueError('Incorrect DESITARGET version is loaded')
+            ''';
+
+
             if action['ACTIONTYPE'] == 'fa':
                 OrigFAs, AltFAs, AltFAs2, TSs, fadates, tiles = do_fiberassignment(altmtldir, [action], survey = survey, obscon = obscon ,verbose = verbose, debug = debug, getosubp = getosubp, redoFA = redoFA, mock = mock, reproducing = reproducing)
                 assert(len(OrigFAs))
@@ -1610,9 +1695,16 @@ def loop_alt_ledger(obscon, survey='sv3', zcatdir=None, mtldir=None,
                 retval = reprocess_alt_ledger(altmtldir, action, obscon=obscon, survey = survey)
                 if debug or verbose:
                     log.info(f'retval = {retval}')
+
+            #LGN 07/29/25: Adding new LyA1B case
+            elif action['ACTIONTYPE'] == 'lya1b':
+                #run update on the realization
+                update_lya_1b(obscon=obscon, mtldir=altmtldir, timestamp=action['ACTIONTIME'], donefile=False)
+                #record succesful update in ledger
+                retval = write_amtl_tile_tracker(altmtldir, [action], obscon = obscon, survey = survey)
 
             else:
-                raise ValueError('actiontype must be `fa`, `update`, or `reproc`.')
+                raise ValueError('actiontype must be `fa`, `update`, `reproc` or `lya1b`.')
             #retval = write_amtl_tile_tracker(altmtldir, None, None, today, obscon = obscon, survey = survey, mode = 'endofday')
             #log.info('write_amtl_tile_tracker retval = {0}'.format(retval))
 
@@ -2384,7 +2476,9 @@ def write_amtl_tile_tracker(dirname, tiles, obscon = 'dark', survey = 'main'):
         tileid = t['TILEID']
         #reprocFlag = t['REPROCFLAG']
         actionType = t['ACTIONTYPE']
-        cond = (TileTracker['TILEID'] == tileid) & (TileTracker['ACTIONTYPE'] == actionType)
+        #LGN 20251103 - Adding actiontime as an additional condition due to the existance of tiles with multiple reproc actions
+        actionTime = t['ACTIONTIME']
+        cond = (TileTracker['TILEID'] == tileid) & (TileTracker['ACTIONTYPE'] == actionType) & (TileTracker['ACTIONTIME'] == actionTime)
         log.info('for tile {0}, number of matching tiles = {1}'.format(tileid, np.sum(cond)))
         #debugTrap = np.copy(TileTracker[dateKey])
         TileTracker['DONEFLAG'][cond] = True

py/LSS/SV3/fatools.py

@@ -388,18 +388,16 @@ def get_fba_fromnewmtl(tileid,mtldir=None,getosubp=False,outdir=None,faver=None,
                     log.info('targver must be 1.0.0 (or at least not 1.1.1) and reproducing must be True')
                     log.info(f'targver  = {targver}')
                     log.info(f'reproducing = {reproducing}')
+                if not os.path.exists(outdir):
+                    log.info('running makedirs. making {0}'.format(outdir))
+                    os.makedirs(outdir)
 
                 shutil.copyfile(indir+ts+'-targ.fits', tarfn)
 
         else:
             log.critical('invalid input directory. must contain either sv3, main, or holding')
             raise ValueError('indir must contain either sv3, main, or holding')
 
-
-    #LGN 06/11/25: Moving this outside of loop to work with DARK1B program
-    if not os.path.exists(outdir):
-        log.info('running makedirs. making {0}'.format(outdir))
-        os.makedirs(outdir)
 
     if getosubp:
         if tileid == 315:
@@ -582,17 +580,44 @@ def altcreate_mtl(
         log.critical(tiles)
         raise ValueError('When processing tile 315, code should strip out processing of all other tiles. ')
     else:
-
-        d = io.read_targets_in_tiles(
-            mtldir,
-            tiles,
-            quick=False,
-            mtl=True,
-            unique=True,
-            isodate=mtltime,
-            verbose=verbose,
-            tabform='ascii.ecsv'
-        )
+        # LGN: Adding handling for bright1b/dark1b tiles
+        if 'dark1b' or 'bright1b' in mtldir:
+            log.info('Running with maketwostyle=True')
+            is_ext = True
+            # LGN Formatting the path to bright or dark ledgers
+            # LGN Passing list of directories to read_targets_in_tiles
+            mtldir_short = os.path.join(
+                os.path.dirname(mtldir), 
+                os.path.basename(mtldir).replace('1b', '')
+            )
+            mtldirs = [mtldir_short, mtldir]
+
+            d = io.read_targets_in_tiles(
+                mtldirs,
+                tiles,
+                quick=False,
+                mtl=True,
+                unique=True,
+                isodate=mtltime,
+                verbose=verbose,
+                tabform='ascii.ecsv',
+                maketwostyle = is_ext
+            )
+
+        else:
+            is_ext = False
+
+            d = io.read_targets_in_tiles(
+                mtldir,
+                tiles,
+                quick=False,
+                mtl=True,
+                unique=True,
+                isodate=mtltime,
+                verbose=verbose,
+                tabform='ascii.ecsv',
+                maketwostyle = is_ext
+            )
 
 
     try: