From f09bdaed4a5bc97965130366faf7472132662ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eirik=20Gjerl=C3=B8w?= Date: Tue, 19 May 2026 11:51:32 +0200 Subject: [PATCH 1/4] Add solar-centric pixels to HDF files --- .../AKARI/akari_commander_data_adapter.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/commander3/todscripts/AKARI/akari_commander_data_adapter.py b/commander3/todscripts/AKARI/akari_commander_data_adapter.py index d38122028..c4b9feabd 100644 --- a/commander3/todscripts/AKARI/akari_commander_data_adapter.py +++ b/commander3/todscripts/AKARI/akari_commander_data_adapter.py @@ -429,22 +429,31 @@ def _process_akari_tod_reader_data(self): curr_data[f'{det}/pixel_flag'], curr_data['frame_flag'], curr_data['status_flag'], gads_flags[det], mode) - c = SkyCoord(ra=detlons[det]*u.deg, dec=detlats[det]*u.deg, frame='icrs') + c = SkyCoord(ra=detlons[det]*u.deg, dec=detlats[det]*u.deg, frame='icrs', distance=1*u.AU) todreader_data[band][det]['pix'] = healpy.ang2pix(self.nside, c.galactic.l.value, c.galactic.b.value, lonlat=True) + theta = 0.5 * np.pi - c.hcrs.dec.radian + phi = c.hcrs.ra.radian + todreader_data[band][det]['pix_solarcentric'] = healpy.ang2pix( + self.nside, theta, phi) ra = curr_data['ra'] dec = curr_data['dec'] - c = SkyCoord(ra=ra*u.deg, dec=dec*u.deg, frame='icrs') + c = SkyCoord(ra=ra*u.deg, dec=dec*u.deg, frame='icrs', distance=1*u.AU) todreader_data[band]['pix'] = healpy.ang2pix(self.nside, c.galactic.l.value, c.galactic.b.value, lonlat=True) + theta = 0.5 * np.pi - c.hcrs.dec.radian + phi = c.hcrs.ra.radian + todreader_data[band]['pix_solarcentric'] = healpy.ang2pix( + self.nside, theta, phi) + vecs = c.galactic.cartesian.get_xyz().value.T spin_axis = self.todreaders[band]._ring_spin_axis(vecs) @@ -537,7 +546,8 @@ def get_chunk_data(self, band: str, det:str): tod = self.all_chunk_data[band][det]['tod'] psi_arr = np.zeros_like(self.all_chunk_data[band][det]['pix'], dtype=int) return (tod, - self.all_chunk_data[band][det]['pix'], + np.array([self.all_chunk_data[band][det]['pix'], + self.all_chunk_data[band][det]['pix_solarcentric']]), psi_arr, self.all_chunk_data[band][det]['flag']) From 8ebd260fae8162a89d6a0f68bee1bb8ad2b096b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eirik=20Gjerl=C3=B8w?= Date: Tue, 19 May 2026 12:34:20 +0200 Subject: [PATCH 2/4] Enable tod and todz compression in the same file --- .../AKARI/akari_commander_data_adapter.py | 35 +++++++------------ commander3/todscripts/AKARI/fits2hdf.py | 6 ++-- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/commander3/todscripts/AKARI/akari_commander_data_adapter.py b/commander3/todscripts/AKARI/akari_commander_data_adapter.py index c4b9feabd..a50dad0b1 100644 --- a/commander3/todscripts/AKARI/akari_commander_data_adapter.py +++ b/commander3/todscripts/AKARI/akari_commander_data_adapter.py @@ -264,17 +264,13 @@ def fits2output_formatter(file, start_index, end_index, band, # an okay assumption for now, but I just note it here. start_det_ind = start_det_inds[band] end_det_ind = start_det_ind + num_detectors[band] - if should_compress: - tot_adu = file[1]['DET'].read()[start_index:end_index, start_det_ind:end_det_ind] - else: - tot_flux = file[1]['FLUX'].read()[start_index:end_index, start_det_ind:end_det_ind] + tot_adu = file[1]['DET'].read()[start_index:end_index, start_det_ind:end_det_ind] + tot_flux = file[1]['FLUX'].read()[start_index:end_index, start_det_ind:end_det_ind] tot_pixflag = file[1]['PIX_FLAG'].read()[start_index:end_index, start_det_ind:end_det_ind, :].astype(bool) for local_det_idx, detname in enumerate(band_dets[band]): - if should_compress: - out_data[f'{detname}/todz'] = tot_adu[:, local_det_idx] - else: - out_data[f'{detname}/tod'] = tot_flux[:, local_det_idx] + out_data[f'{detname}/tod'] = tot_flux[:, local_det_idx] + out_data[f'{detname}/todz'] = tot_adu[:, local_det_idx] out_data[f'{detname}/pixel_flag'] = tot_pixflag[:, local_det_idx, :] out_data['status_flag'] = file[1]['STATUS'].read()[start_index:end_index].astype(bool) # NB! This is necessary because we interpret the shtop flag inversely internally in commander @@ -341,8 +337,6 @@ def __init__(self, akari_fits_dir, nside, bands=BANDS, self.nchunks[band] = len(self.chunk_file_map[band]) self.reference_time = reference_time - self.should_compress = True - def _calculate_chunk_files(self): """ @@ -410,8 +404,7 @@ def _process_akari_tod_reader_data(self): # within a file rather than at the beginning of the file. start_idx = self.todreaders[band].get_file_index_range(files[0])[0] end_idx = self.todreaders[band].get_file_index_range(files[-1])[1] - curr_data = self.todreaders[band].get_data(start_idx, end_idx, - should_compress=self.should_compress) + curr_data = self.todreaders[band].get_data(start_idx, end_idx) mode = curr_data['packet_id'] if len(files) > 1: raise NotImplementedError("""We currently don't support loading gads flags for several @@ -421,10 +414,8 @@ def _process_akari_tod_reader_data(self): # print(gads_flags) for detidx, det in enumerate(self.band_dets[band]): todreader_data[band][det] = {} - if self.should_compress: - todreader_data[band][det]['todz'] = curr_data[f'{det}/todz'] - else: - todreader_data[band][det]['tod'] = curr_data[f'{det}/tod'] + todreader_data[band][det]['tod'] = curr_data[f'{det}/tod'] + todreader_data[band][det]['todz'] = curr_data[f'{det}/todz'] todreader_data[band][det]['flag'] = self._process_akari_flags( curr_data[f'{det}/pixel_flag'], curr_data['frame_flag'], @@ -482,7 +473,8 @@ def _process_akari_tod_reader_data(self): return todreader_data - def _process_akari_flags(self, pixflag_array, frameflag_array, statusflag_array, gadsflag_array, + def _process_akari_flags(self, pixflag_array, frameflag_array, + statusflag_array, gadsflag_array, mode, pixflagmap=PIX_FLAG_MAP, frameflagmap=FRAME_FLAG_MAP, statusflagmap=STATUS_FLAG_MAP, gadsflagmap=GADS_FLAG_MAP, desired_flags=DESIRED_FLAGS): @@ -540,12 +532,11 @@ def inner(self, *args): @get_chunk def get_chunk_data(self, band: str, det:str): # The third entry is just zeroes - i.e. currently we're not operating with a 'psi'. - if self.should_compress: - tod = self.all_chunk_data[band][det]['todz'] - else: - tod = self.all_chunk_data[band][det]['tod'] + tod = self.all_chunk_data[band][det]['tod'] + todz = self.all_chunk_data[band][det]['todz'] psi_arr = np.zeros_like(self.all_chunk_data[band][det]['pix'], dtype=int) return (tod, + todz, np.array([self.all_chunk_data[band][det]['pix'], self.all_chunk_data[band][det]['pix_solarcentric']]), psi_arr, @@ -579,8 +570,6 @@ def get_chunk_end_earthpos(self): def get_chunk_satvel(self): return [0]*3 - def get_should_compress_tods(self): - return self.should_compress def get_gain(self, band: str, detector: str): return 1 diff --git a/commander3/todscripts/AKARI/fits2hdf.py b/commander3/todscripts/AKARI/fits2hdf.py index 8d771d5c7..a4db50efc 100755 --- a/commander3/todscripts/AKARI/fits2hdf.py +++ b/commander3/todscripts/AKARI/fits2hdf.py @@ -20,8 +20,10 @@ def run_single_band_write(band): akari_comm_data_adapter = akari_commander_data_adapter.AKARICommanderDataAdapter( AKARI_FITS_DIR, NSIDE, bands=[band]) comm_todwriter = CommanderHDFWriter(akari_comm_data_adapter) - comm_todwriter.write_hdf_files(OUTPATH, overwrite=True, num_processes=NUM_SEGMENT_PROCESSES, - bands=[band]) + comm_todwriter.write_hdf_files(OUTPATH, overwrite=True, + num_processes=NUM_SEGMENT_PROCESSES, + bands=[band], + include_solarcentric_pixpointing=True) def main(): for band in BANDS: From b5285b407b523a7017f8b39d21fa46adf622ec97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eirik=20Gjerl=C3=B8w?= Date: Tue, 19 May 2026 12:36:54 +0200 Subject: [PATCH 3/4] Enable extending the reset flag to 4 samples --- .../AKARI/akari_commander_data_adapter.py | 14 +++++++++++--- commander3/todscripts/AKARI/fits2hdf.py | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/commander3/todscripts/AKARI/akari_commander_data_adapter.py b/commander3/todscripts/AKARI/akari_commander_data_adapter.py index a50dad0b1..9a28ef45d 100644 --- a/commander3/todscripts/AKARI/akari_commander_data_adapter.py +++ b/commander3/todscripts/AKARI/akari_commander_data_adapter.py @@ -296,11 +296,13 @@ class AKARICommanderDataAdapter(CommanderDataAdapter): """ def __init__(self, akari_fits_dir, nside, bands=BANDS, num_detectors=NUM_DETECTORS, band_dets=BAND_DETS, - reference_time=REFERENCE_TIME): + reference_time=REFERENCE_TIME, + extend_reset_flag=False): self.bands = bands self.num_detectors = num_detectors self.band_dets = band_dets self.nside = nside + self.extend_reset_flag = extend_reset_flag self.fsamp = { @@ -419,7 +421,8 @@ def _process_akari_tod_reader_data(self): todreader_data[band][det]['flag'] = self._process_akari_flags( curr_data[f'{det}/pixel_flag'], curr_data['frame_flag'], - curr_data['status_flag'], gads_flags[det], mode) + curr_data['status_flag'], gads_flags[det], mode, + extend_reset_flag=self.extend_reset_flag) c = SkyCoord(ra=detlons[det]*u.deg, dec=detlats[det]*u.deg, frame='icrs', distance=1*u.AU) todreader_data[band][det]['pix'] = healpy.ang2pix(self.nside, c.galactic.l.value, @@ -477,7 +480,7 @@ def _process_akari_flags(self, pixflag_array, frameflag_array, statusflag_array, gadsflag_array, mode, pixflagmap=PIX_FLAG_MAP, frameflagmap=FRAME_FLAG_MAP, statusflagmap=STATUS_FLAG_MAP, gadsflagmap=GADS_FLAG_MAP, - desired_flags=DESIRED_FLAGS): + desired_flags=DESIRED_FLAGS, extend_reset_flag=False): """ Internal function that takes the flags coming from the fits files and turns them into the bitmasks needed for the HDF file""" @@ -497,6 +500,11 @@ def _process_akari_flags(self, pixflag_array, frameflag_array, for flag_name, target_bit in desired_flags[flagtype].items(): flag_idx = flagmap[flag_name] curr_mask = flag_arr[:, flag_idx] + if extend_reset_flag and flagtype == 'pix_flags' and flag_name == 'reset': + # If this option is set, we want to extend the reset flag + # to cover four samples rather than one. + for i in range(1, 4): + curr_mask[i:] = curr_mask[i:] | curr_mask[:-i] outflags[curr_mask] += 2 ** int(target_bit) modeflags = desired_flags['mode'] # These numbers are the modes of the pixflag_array field, listed in the AKARI fits data user diff --git a/commander3/todscripts/AKARI/fits2hdf.py b/commander3/todscripts/AKARI/fits2hdf.py index a4db50efc..abcf64a02 100755 --- a/commander3/todscripts/AKARI/fits2hdf.py +++ b/commander3/todscripts/AKARI/fits2hdf.py @@ -18,7 +18,8 @@ def run_single_band_write(band): akari_comm_data_adapter = akari_commander_data_adapter.AKARICommanderDataAdapter( - AKARI_FITS_DIR, NSIDE, bands=[band]) + AKARI_FITS_DIR, NSIDE, bands=[band], + extend_reset_flag=EXTEND_RESET_FLAG) comm_todwriter = CommanderHDFWriter(akari_comm_data_adapter) comm_todwriter.write_hdf_files(OUTPATH, overwrite=True, num_processes=NUM_SEGMENT_PROCESSES, From c92915ef30b4d525445c37b19d2dcc562360c170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eirik=20Gjerl=C3=B8w?= Date: Thu, 28 May 2026 11:30:06 +0200 Subject: [PATCH 4/4] Update fits2hdf to include previous changes --- commander3/todscripts/AKARI/fits2hdf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/commander3/todscripts/AKARI/fits2hdf.py b/commander3/todscripts/AKARI/fits2hdf.py index abcf64a02..199bd4049 100755 --- a/commander3/todscripts/AKARI/fits2hdf.py +++ b/commander3/todscripts/AKARI/fits2hdf.py @@ -15,6 +15,7 @@ # instances, one for each band #BANDS = ['065', '090', '140', '160'] BANDS = ['065'] +EXTEND_RESET_FLAG = True def run_single_band_write(band): akari_comm_data_adapter = akari_commander_data_adapter.AKARICommanderDataAdapter(