From 843644cb3bb9f1c5c1a5e4ddaeec9052df382538 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 2 Jun 2025 16:56:46 -0500 Subject: [PATCH 01/24] NOT READY: [tree] Fix handling of poorly cluster file in the retain previous cluster's basket code. Without this all the baskets of a poorly clustered branch are retaining until the deletion of the TTree object --- tree/tree/src/TBranch.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx index e0cfeb01c36b2..6647cf67c44e7 100644 --- a/tree/tree/src/TBranch.cxx +++ b/tree/tree/src/TBranch.cxx @@ -1992,7 +1992,7 @@ TBasket *TBranch::GetFreshCluster(TBuffer* user_buffer) // basket, just iterate backwards until the correct basket is reached. This should // be fast as long as the number of baskets per cluster is small Int_t basketToUnload = fReadBasket; - while (fBasketEntry[basketToUnload] != entryToUnload) { + while (fBasketEntry[basketToUnload] > entryToUnload) { basketToUnload--; if (basketToUnload < 0) { return CreateOrReuseBasket(); From ad482eea4de2a857ef2ed7ef84c54214c126f2ac Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 13:26:29 -0500 Subject: [PATCH 02/24] io: Improve ByteCountStack assert --- io/io/src/TBufferFile.cxx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 833e141ead3bf..51bf8e1325005 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -436,7 +436,10 @@ Long64_t TBufferFile::CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const T // The position is above 4GB but was cached using a 32 bit variable. startpos = fByteCountStack.back().locator; // See below - R__ASSERT((fByteCountStack.back().cl == nullptr || clss == fByteCountStack.back().cl) + R__ASSERT((fByteCountStack.back().cl == nullptr || + (clss == nullptr && (classname == nullptr || classname[0] == '\0')) || + (clss && clss == fByteCountStack.back().cl) || + (classname && strcmp(classname, fByteCountStack.back().cl->GetName()) == 0)) && "Class on the byte count position stack does not match the passed class"); } else { // This assert allows to reject cases that used to be valid (missing or From 954b4f01812cd002df53ac4feff8b61e1a3f956b Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 17:07:37 -0500 Subject: [PATCH 03/24] io: TBufferClass fix handling of large bytecount. In TBufferFile::ReadClass properly set the byte count returned to kOverflowCount when the byte count in the stream was not updated (was too large). --- io/io/src/TBufferFile.cxx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 51bf8e1325005..dcfe405d7ab6f 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -3014,6 +3014,14 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) // return bytecount in objTag if (objTag) { *objTag = (bcnt & ~kByteCountMask); + if (*objTag == 0) { + // The byte count was stored but is zero, this means the data + // did not fit and thus we stored it in 'fByteCounts' instead. + // Mark this case by setting startpos to kOverflowCount. + *objTag = kOverflowCount; + // We do not have access to the caller's "cntpos" and can not + // update to kOverflowPosition (unlike the same code in ReadVersion). + } if (cl) fByteCountStack.back().alt = cl; } From 380beb0a0f5924c2b8b14cefd1dc9d555dc012df Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 16 Feb 2026 12:58:57 -0600 Subject: [PATCH 04/24] WIP: for debugging ByteCountStack - Consider REVERTing --- io/io/src/TBufferFile.cxx | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index dcfe405d7ab6f..3c144d86e699f 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2657,6 +2657,8 @@ void TBufferFile::SkipObjectAny(Long64_t start, UInt_t count) /// real beginning of the object in memory. You will need to use a /// dynamic_cast later if you need to retrieve it. + + void *TBufferFile::ReadObjectAny(const TClass *clCast) { R__ASSERT(IsReading()); @@ -2664,6 +2666,20 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) // make sure fMap is initialized InitMap(); + struct CaptureAndCheck { + using ByteCountStack_t = TBufferFile::ByteCountStack_t; + size_t fCurrent; + ByteCountStack_t *fByteCountStack; + + CaptureAndCheck(ByteCountStack_t *stack) : fCurrent(stack->size()), fByteCountStack(stack) {} + ~CaptureAndCheck() { + if (fCurrent != fByteCountStack->size()) { + ::Fatal("CaptureAndCheck", "Byte count stack was not properly cleaned up. Current size is %zu but should be %zu", fByteCountStack->size(), fCurrent); + } + } + }; + CaptureAndCheck checker(&(this->fByteCountStack)); + // before reading object save start position ULong64_t startpos = static_cast(fBufCur-fBuffer); ULong64_t cntpos = startpos <= kMaxCountPosition ? startpos : kOverflowPosition; @@ -3851,6 +3867,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass else version = ReadVersion(&R__s, &R__c, cl); + size_t current = fByteCountStack.size(); Bool_t v2file = kFALSE; TFile *file = (TFile*)GetParent(); if (file && file->GetVersion() < 30000) { @@ -3869,6 +3886,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass Error("ReadClassBuffer", "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", onFileClass->GetName(), version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, onFileClass); return 0; } @@ -3893,6 +3911,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (version < -1 || version >= infocapacity) { Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %d", cl->GetName(), version, Length()); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3951,11 +3970,13 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass // When the object was written the class was version zero, so // there is no StreamerInfo to be found. // Check that the buffer position corresponds to the byte count. + --current; CheckByteCount(R__s, R__c, cl); return 0; } else { Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3967,6 +3988,9 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass ApplySequence(*(sinfo->GetReadObjectWiseActions()), (char*)pointer ); if (sinfo->TStreamerInfo::IsRecovered()) R__c=0; // 'TStreamerInfo::' avoids going via a virtual function. + if (current != fByteCountStack.size()) + Fatal("ReadClassBuffer", "We are out of sync at level %lld vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); + --current; // Check that the buffer position corresponds to the byte count. CheckByteCount(R__s, R__c, cl); @@ -4035,8 +4059,11 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter).PrintDebug(*this,obj); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); } } else { @@ -4045,7 +4072,10 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); } } From 6d9499566899fa3949312a4c7c0b7fb4c66e98f5 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:11:37 -0500 Subject: [PATCH 05/24] io: Improve ByteCountStack error message --- io/io/src/TBufferFile.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 3c144d86e699f..5ef3c3cf89b0e 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -4075,7 +4075,7 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se auto current = fByteCountStack.size(); (*iter)(*this,obj); if (current != fByteCountStack.size()) - Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); + Fatal("ApplySequence", "ByteCountStack is out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } From fc5c15cc42084e2ee252e648d86ca1febf113e3b Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 9 Feb 2026 22:09:30 +0100 Subject: [PATCH 06/24] io/meta: Ensure NewArray* takes a 64 bit integer. It is staying signed for historical reason since we do not plan to support collection larger than std::numeric_limit::max() as onfile we reserve the sign bit as a marker. --- core/cont/inc/TVirtualCollectionProxy.h | 8 ++++---- core/meta/inc/TClass.h | 8 ++++---- core/meta/src/TClass.cxx | 12 ++++++++---- io/io/inc/TEmulatedCollectionProxy.h | 8 ++++---- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 12d6e412285d8..316f75d2e563b 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -123,19 +123,19 @@ class TVirtualCollectionProxy { } /// Construct an array of `nElements` container objects and return the base address of the array - virtual void *NewArray(Int_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } + virtual void *NewArray(Long64_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual void *NewArray(Int_t nElements, void *arena) const + virtual void *NewArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements, arena); } /// Construct an array of `nElements` container objects and return the base address of the array - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements, void *arena) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements, arena); } diff --git a/core/meta/inc/TClass.h b/core/meta/inc/TClass.h index d1e298cddcbe9..e64f03fd1f5e9 100644 --- a/core/meta/inc/TClass.h +++ b/core/meta/inc/TClass.h @@ -541,12 +541,12 @@ friend class TStreamerInfo; void Move(void *arenaFrom, void *arenaTo) const; void *New(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; void *New(void *arena, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; ObjectPtr NewObject(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; ObjectPtr NewObject(void *arena, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; virtual void PostLoadCheck(); Long_t Property() const override; Int_t ReadBuffer(TBuffer &b, void *pointer, Int_t version, UInt_t start, UInt_t count); diff --git a/core/meta/src/TClass.cxx b/core/meta/src/TClass.cxx index 137f09b0bcd8c..c4ac0cdc5b1f2 100644 --- a/core/meta/src/TClass.cxx +++ b/core/meta/src/TClass.cxx @@ -5245,8 +5245,9 @@ TClass::ObjectPtr TClass::NewObject(void *arena, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5261,8 +5262,9 @@ void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { @@ -5347,8 +5349,9 @@ TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstruct /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, arena, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5362,8 +5365,9 @@ void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) c /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { diff --git a/io/io/inc/TEmulatedCollectionProxy.h b/io/io/inc/TEmulatedCollectionProxy.h index c179f81f46fdc..1e6ccf53f717a 100644 --- a/io/io/inc/TEmulatedCollectionProxy.h +++ b/io/io/inc/TEmulatedCollectionProxy.h @@ -71,16 +71,16 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { TClass::ObjectPtr NewObject(void* memory) const override { return {new(memory) Cont_t, nullptr}; } // Virtual array constructor - void* NewArray(Int_t nElements) const override { return new Cont_t[nElements]; } + void* NewArray(Long64_t nElements) const override { return new Cont_t[nElements]; } // Virtual in-place constructor - void* NewArray(Int_t nElements, void* memory) const override { return new(memory) Cont_t[nElements]; } + void* NewArray(Long64_t nElements, void* memory) const override { return new(memory) Cont_t[nElements]; } // Virtual array constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements) const override { return {new Cont_t[nElements], nullptr}; } + TClass::ObjectPtr NewObjectArray(Long64_t nElements) const override { return {new Cont_t[nElements], nullptr}; } // Virtual in-place constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements, void* memory) const override { return {new(memory) Cont_t[nElements], nullptr}; } + TClass::ObjectPtr NewObjectArray(Long64_t nElements, void* memory) const override { return {new(memory) Cont_t[nElements], nullptr}; } // Virtual destructor void Destructor(void* p, Bool_t dtorOnly = kFALSE) const override; From 78b24e88547c432284767cc58e1ff52fe00cc665 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 12 Feb 2026 23:27:35 +0100 Subject: [PATCH 07/24] WIP io: Lift size limit on TBufferFile .. is this temporary? --- core/base/inc/TBuffer.h | 8 +++++--- core/base/src/TBuffer.cxx | 15 ++++++++------- io/io/src/TBufferFile.cxx | 2 +- io/io/src/TBufferIO.cxx | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index 5490b90f5d58c..5976c7e92dc3a 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -45,10 +45,12 @@ class TBuffer : public TObject { protected: typedef std::vector CacheList_t; + // FIXME: decide if this is signed or unsigned. + using Size_t = std::size_t; Bool_t fMode; //Read or write mode Int_t fVersion; //Buffer format version - Int_t fBufSize; //Size of buffer + Size_t fBufSize; //Size of buffer char *fBuffer; //Buffer used to store objects char *fBufCur; //Current position in buffer char *fBufMax; //End of buffer @@ -105,9 +107,9 @@ class TBuffer : public TObject { TObject *GetParent() const; char *Buffer() const { return fBuffer; } char *GetCurrent() const { return fBufCur; } - Int_t BufferSize() const { return fBufSize; } + Size_t BufferSize() const { return fBufSize; } void DetachBuffer() { fBuffer = nullptr; } - Int_t Length() const { return (Int_t)(fBufCur - fBuffer); } + Size_t Length() const { return (Size_t)(fBufCur - fBuffer); } void Expand(Long64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize void AutoExpand(Long64_t size_needed); // expand buffer to newsize Bool_t ByteSwapBuffer(Long64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 0a27db1c284c4..316a53f5ee098 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -20,7 +20,8 @@ Buffer base class used for serializing objects. #include "TProcessID.h" constexpr Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) -constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size. +constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). +constexpr ULong64_t kMaxLargeBufferSize = 0x7FFFFFFFFFFFFFFE; // largest possible size. @@ -102,8 +103,8 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize) TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) { - if (bufsize > kMaxBufferSize) - Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); + if (bufsize > kMaxLargeBufferSize) + Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%llx.", bufsize, kMaxLargeBufferSize); fBufSize = bufsize; fMode = mode; fVersion = 0; @@ -229,11 +230,11 @@ void TBuffer::Expand(Long64_t newsize, Bool_t copy) } const Int_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; - if ( newsize > kMaxBufferSize - kExtraSpace) { - if (l < kMaxBufferSize) { - newsize = kMaxBufferSize - extraspace; + if ( newsize > kMaxLargeBufferSize - kExtraSpace) { + if (l < kMaxLargeBufferSize) { + newsize = kMaxLargeBufferSize - extraspace; } else { - Fatal("Expand","Requested size (%lld) is too large (max is %d).", newsize, kMaxBufferSize); + Fatal("Expand","Requested size (%lld) is too large (max is %lld).", newsize, kMaxBufferSize); } } if ( (fMode&kWrite)!=0 ) { diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 5ef3c3cf89b0e..faf9675a438f6 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -357,7 +357,7 @@ void TBufferFile::SetByteCount(ULong64_t cntpos, Bool_t packInVersion) assert( (cntpos == kOverflowPosition || (cntpos < kOverflowPosition && (sizeof(UInt_t) + cntpos) < static_cast(fBufCur - fBuffer))) && (fBufCur >= fBuffer) - && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() + && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() && "Byte count position is after the end of the buffer"); // We can either make this unconditional or we could split the routine diff --git a/io/io/src/TBufferIO.cxx b/io/io/src/TBufferIO.cxx index 30df90184fee7..3289114185343 100644 --- a/io/io/src/TBufferIO.cxx +++ b/io/io/src/TBufferIO.cxx @@ -195,7 +195,7 @@ void TBufferIO::MapObject(const TObject *obj, ULong64_t offset) void TBufferIO::MapObject(const void *obj, const TClass *cl, ULong64_t offset) { - R__ASSERT(offset <= kMaxUInt); + R__ASSERT(offset <= kMaxLong64); if (IsWriting()) { if (!fMap) InitMap(); From 150a1b3652f9697831cfd046b6f5a0d9d4deafe4 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 3 Apr 2026 14:07:57 -0500 Subject: [PATCH 08/24] WIP: Update type to ULong64_t --- core/base/inc/TBuffer.h | 14 +++++------ core/base/src/TBuffer.cxx | 35 ++++++++++++++------------- io/io/src/TStreamerInfoReadBuffer.cxx | 8 +++--- tree/tree/inc/TBasket.h | 12 ++++----- tree/tree/inc/TBasketSQL.h | 6 ++--- tree/tree/inc/TBranch.h | 2 +- tree/tree/inc/TTree.h | 2 +- tree/tree/src/TBasket.cxx | 27 +++++++++++++-------- tree/tree/src/TBasketSQL.cxx | 6 ++--- tree/tree/src/TBranch.cxx | 2 +- tree/tree/src/TTree.cxx | 2 +- 11 files changed, 62 insertions(+), 54 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index 5976c7e92dc3a..dc86b60239ff5 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -90,8 +90,8 @@ class TBuffer : public TObject { enum { kInitialSize = 1024, kMinimalSize = 128 }; TBuffer(EMode mode); - TBuffer(EMode mode, Long64_t bufsize); - TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + TBuffer(EMode mode, ULong64_t bufsize); + TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); virtual ~TBuffer(); Int_t GetBufferVersion() const { return fVersion; } @@ -99,10 +99,10 @@ class TBuffer : public TObject { Bool_t IsWriting() const { return (fMode & kWrite) != 0; } void SetReadMode(); void SetWriteMode(); - void SetBuffer(void *buf, Long64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + void SetBuffer(void *buf, ULong64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); ReAllocCharFun_t GetReAllocFunc() const; void SetReAllocFunc(ReAllocCharFun_t reallocfunc = nullptr); - void SetBufferOffset(Long64_t offset = 0) { fBufCur = fBuffer+offset; } + void SetBufferOffset(ULong64_t offset = 0) { fBufCur = fBuffer+offset; } void SetParent(TObject *parent); TObject *GetParent() const; char *Buffer() const { return fBuffer; } @@ -110,9 +110,9 @@ class TBuffer : public TObject { Size_t BufferSize() const { return fBufSize; } void DetachBuffer() { fBuffer = nullptr; } Size_t Length() const { return (Size_t)(fBufCur - fBuffer); } - void Expand(Long64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize - void AutoExpand(Long64_t size_needed); // expand buffer to newsize - Bool_t ByteSwapBuffer(Long64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer + void Expand(ULong64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize + void AutoExpand(ULong64_t size_needed); // expand buffer to newsize + Bool_t ByteSwapBuffer(ULong64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer virtual Bool_t CheckObject(const TObject *obj) = 0; virtual Bool_t CheckObject(const void *obj, const TClass *ptrClass) = 0; diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 316a53f5ee098..6f7f55a9dd6d0 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -19,8 +19,8 @@ Buffer base class used for serializing objects. #include "TClass.h" #include "TProcessID.h" -constexpr Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) -constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). +constexpr UInt_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) +constexpr UInt_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). constexpr ULong64_t kMaxLargeBufferSize = 0x7FFFFFFFFFFFFFFE; // largest possible size. @@ -70,7 +70,7 @@ TBuffer::TBuffer(EMode mode) /// Create an I/O buffer object. Mode should be either TBuffer::kRead or /// TBuffer::kWrite. -TBuffer::TBuffer(EMode mode, Long64_t bufsize) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize) { if (bufsize > kMaxBufferSize) Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); @@ -101,7 +101,7 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) { if (bufsize > kMaxLargeBufferSize) Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%llx.", bufsize, kMaxLargeBufferSize); @@ -129,7 +129,7 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocC SetReAllocFunc( reallocfunc ); - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { + if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < kMinimalSize) { Expand( kMinimalSize ); } } @@ -155,13 +155,13 @@ TBuffer::~TBuffer() /// If the size_needed is larger than the current size, the policy /// is to expand to double the current size or the size_needed which ever is largest. -void TBuffer::AutoExpand(Long64_t size_needed) +void TBuffer::AutoExpand(ULong64_t size_needed) { if (size_needed > kMaxBufferSize) { Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%x.", size_needed, kMaxBufferSize); } if (size_needed > fBufSize) { - Long64_t doubling = 2LLU * fBufSize; + ULong64_t doubling = 2LLU * fBufSize; if (doubling > kMaxBufferSize) doubling = kMaxBufferSize; if (size_needed > doubling) { @@ -184,7 +184,7 @@ void TBuffer::AutoExpand(Long64_t size_needed) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) +void TBuffer::SetBuffer(void *buf, ULong64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) { if (newsiz > kMaxBufferSize) Fatal("SetBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", newsiz, kMaxBufferSize); @@ -222,22 +222,23 @@ void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun /// In order to avoid losing data, if the current length is greater than /// the requested size, we only shrink down to the current length. -void TBuffer::Expand(Long64_t newsize, Bool_t copy) +void TBuffer::Expand(ULong64_t newsize, Bool_t copy) { - Int_t l = Length(); - if ( (Long64_t(l) > newsize) && copy ) { + ULong64_t l = Length(); + if ( l > newsize && copy ) { newsize = l; } - const Int_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; + const ULong64_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; if ( newsize > kMaxLargeBufferSize - kExtraSpace) { if (l < kMaxLargeBufferSize) { newsize = kMaxLargeBufferSize - extraspace; } else { - Fatal("Expand","Requested size (%lld) is too large (max is %lld).", newsize, kMaxBufferSize); + Fatal("Expand","Requested size (%llu) is too large (max is %llu).", newsize, kMaxLargeBufferSize); } } if ( (fMode&kWrite)!=0 ) { + // FIXME-LARGE: The size can overflow and the signature of fReAllocFunc is not large enough to handle the size of the buffer. fBuffer = fReAllocFunc(fBuffer, newsize+kExtraSpace, copy ? fBufSize+kExtraSpace : 0); } else { @@ -391,13 +392,13 @@ TVirtualArray *TBuffer::PopDataCache() /// Byte-swap N primitive-elements in the buffer. /// Bulk API relies on this function. -Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) +Bool_t TBuffer::ByteSwapBuffer(ULong64_t n, EDataType type) { char *input_buf = GetCurrent(); if ((type == EDataType::kShort_t) || (type == EDataType::kUShort_t)) { #ifdef R__BYTESWAP Short_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -406,7 +407,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kFloat_t) || (type == EDataType::kInt_t) || (type == EDataType::kUInt_t)) { #ifdef R__BYTESWAP Float_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -415,7 +416,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kDouble_t) || (type == EDataType::kLong64_t) || (type == EDataType::kULong64_t)) { #ifdef R__BYTESWAP Double_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 611abf936bacb..0754648b25a38 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -89,7 +89,7 @@ TStreamerElement *TStreamerInfo::GetCurrentElement() Char_t isArray; \ b >> isArray; \ Int_t *l = (Int_t*)(arr[index]+imethod); \ - if (*l < 0 || *l > b.BufferSize()) continue; \ + if (*l < 0 || static_cast(*l) > b.BufferSize()) continue; \ name **f = (name**)(arr[index]+ioffset); \ int j; \ if (isArray) for(j=0;jfLength;j++) { \ @@ -486,7 +486,7 @@ Int_t TStreamerInfo::ReadBufferSkip(TBuffer &b, const T &arr, const TCompInfo *c DOLOOP { \ b >> isArray; \ Int_t *l = (Int_t*)(arr[k]+imethod); \ - if (*l>0 && *l < b.BufferSize()) { \ + if (*l>0 && static_cast(*l) < b.BufferSize()) { \ readbuf = new name[*l]; \ switch(newtype) { \ case TStreamerInfo::kBool: ConvCBasicPointerTo(Bool_t,ReadArrayFunc); \ @@ -798,7 +798,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%zu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -811,7 +811,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d\n", + " %s, bufpos=%zu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], ioffset); } diff --git a/tree/tree/inc/TBasket.h b/tree/tree/inc/TBasket.h index 66a737ac2ec2c..e926496513c76 100644 --- a/tree/tree/inc/TBasket.h +++ b/tree/tree/inc/TBasket.h @@ -39,11 +39,11 @@ friend class TBranch; TBasket& operator=(const TBasket&); ///< TBasket objects are not copiable. // Internal corner cases for ReadBasketBuffers - Int_t ReadBasketBuffersUnzip(char*, Int_t, bool, TFile*); + Int_t ReadBasketBuffersUnzip(char*, ULong64_t, bool, TFile*); Int_t ReadBasketBuffersUncompressedCase(); // Helper for managing the compressed buffer. - void InitializeCompressedBuffer(Int_t len, TFile* file); + void InitializeCompressedBuffer(ULong64_t len, TFile* file); // Handles special logic around deleting / reseting the entry offset pointer. void ResetEntryOffset(); @@ -130,9 +130,9 @@ friend class TBranch; Int_t GetNevBufSize() const {return fNevBufSize;} Int_t GetLast() const {return fLast;} virtual void MoveEntries(Int_t dentries); - virtual void PrepareBasket(Long64_t /* entry */) {}; - Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - Int_t ReadBasketBytes(Long64_t pos, TFile *file); + virtual void PrepareBasket(ULong64_t /* entry */) {}; + Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + Int_t ReadBasketBytes(ULong64_t pos, TFile *file); virtual void WriteReset(); // Time spent reseting basket sizes (typically, at event cluster boundaries), in microseconds @@ -142,7 +142,7 @@ friend class TBranch; // Count of resets performed of basket size. bool GetResetAllocationCount() const { return fResetAllocation; } - Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree = nullptr); + Int_t LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree = nullptr); Long64_t CopyTo(TFile *to); void SetBranch(TBranch *branch) { fBranch = branch; } diff --git a/tree/tree/inc/TBasketSQL.h b/tree/tree/inc/TBasketSQL.h index efe9caab8261c..ba7925e8d33e7 100644 --- a/tree/tree/inc/TBasketSQL.h +++ b/tree/tree/inc/TBasketSQL.h @@ -45,9 +45,9 @@ class TBasketSQL : public TBasket TBranch *branch, TSQLResult **rs, TString *insert_query, std::vector *vc, TSQLRow **row); ~TBasketSQL() override; - void PrepareBasket(Long64_t entry) override; - virtual Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - virtual Int_t ReadBasketBytes(Long64_t pos, TFile *file); + void PrepareBasket(ULong64_t entry) override; + virtual Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + virtual Int_t ReadBasketBytes(ULong64_t pos, TFile *file); TSQLResult * GetResultSet() { return *fResultPtr;} void CreateBuffer(const char *name, TString title, std::vector * vc, TBranch *branch, TSQLResult ** rs); diff --git a/tree/tree/inc/TBranch.h b/tree/tree/inc/TBranch.h index f91d1e8380495..64be8447bdc58 100644 --- a/tree/tree/inc/TBranch.h +++ b/tree/tree/inc/TBranch.h @@ -263,7 +263,7 @@ class TBranch : public TNamed, public TAttFill { virtual bool GetMakeClass() const; TBranch *GetMother() const; TBranch *GetSubBranch(const TBranch *br) const; - TBuffer *GetTransientBuffer(Int_t size); + TBuffer *GetTransientBuffer(ULong64_t size); bool IsAutoDelete() const; bool IsFolder() const override; virtual void KeepCircular(Long64_t maxEntries); diff --git a/tree/tree/inc/TTree.h b/tree/tree/inc/TTree.h index 4250c4789da0b..64988c1ff1a82 100644 --- a/tree/tree/inc/TTree.h +++ b/tree/tree/inc/TTree.h @@ -599,7 +599,7 @@ class TTree : public TNamed, public TAttLine, public TAttFill, public TAttMarker TTreeFormula *GetSelect() { return GetPlayer()->GetSelect(); } virtual Long64_t GetSelectedRows() { return GetPlayer()->GetSelectedRows(); } virtual Int_t GetTimerInterval() const { return fTimerInterval; } - TBuffer* GetTransientBuffer(Int_t size); + TBuffer* GetTransientBuffer(ULong64_t size); virtual Long64_t GetTotBytes() const { return fTotBytes; } virtual TTree *GetTree() const { return const_cast(this); } virtual TVirtualIndex *GetTreeIndex() const { return fTreeIndex; } diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx index a77855a498986..d907be30bdfe5 100644 --- a/tree/tree/src/TBasket.cxx +++ b/tree/tree/src/TBasket.cxx @@ -241,7 +241,7 @@ Int_t TBasket::GetEntryPointer(Int_t entry) /// This function is called by TTreeCloner. /// The function returns 0 in case of success, 1 in case of error. -Int_t TBasket::LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree) +Int_t TBasket::LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree) { if (fBufferRef) { // Reuse the buffer if it exist. @@ -376,7 +376,7 @@ Int_t TBasket::ReadBasketBuffersUncompressedCase() //////////////////////////////////////////////////////////////////////////////// /// We always create the TBuffer for the basket but it hold the buffer from the cache. -Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, TFile* file) +Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, ULong64_t size, bool mustFree, TFile* file) { if (fBufferRef) { fBufferRef->SetBuffer(buffer, size, mustFree); @@ -406,15 +406,19 @@ Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, T //////////////////////////////////////////////////////////////////////////////// /// Initialize a buffer for reading if it is not already initialized -static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file) +static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, ULong64_t len, TFile* file) { TBuffer* result; if (R__likely(bufferRef)) { bufferRef->SetReadMode(); - Int_t curBufferSize = bufferRef->BufferSize(); + ULong64_t curBufferSize = bufferRef->BufferSize(); if (curBufferSize < len) { // Experience shows that giving 5% "wiggle-room" decreases churn. - bufferRef->Expand(Int_t(len*1.05)); + double newSizeD = len * 1.05; + auto newSize = newSizeD < (double)std::numeric_limits::max() + ? static_cast(newSizeD) + : std::numeric_limits::max(); + bufferRef->Expand(newSize); } bufferRef->Reset(); result = bufferRef; @@ -428,7 +432,7 @@ static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t l //////////////////////////////////////////////////////////////////////////////// /// Initialize the compressed buffer; either from the TTree or create a local one. -void inline TBasket::InitializeCompressedBuffer(Int_t len, TFile* file) +void inline TBasket::InitializeCompressedBuffer(ULong64_t len, TFile* file) { bool compressedBufferExists = fCompressedBufferRef != nullptr; fCompressedBufferRef = R__InitializeReadBasketBuffer(fCompressedBufferRef, len, file); @@ -460,7 +464,7 @@ void TBasket::ResetEntryOffset() /// There is a lot of code duplication but it was necessary to assure /// the expected behavior when there is no cache. -Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) +Int_t TBasket::ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file) { if(!fBranch->GetDirectory()) { return -1; @@ -580,7 +584,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) // Note that in previous versions we didn't allocate buffers until we verified // the zip headers; this is no longer beforehand as the buffer lifetime is scoped // to the TBranch. - uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen; + uncompressedBufferLen = len > static_cast(fObjlen+fKeylen) ? len : fObjlen+fKeylen; fBufferRef = R__InitializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file); rawUncompressedBuffer = fBufferRef->Buffer(); fBuffer = rawUncompressedBuffer; @@ -672,7 +676,10 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) if (R__unlikely(!fEntryOffset)) { fEntryOffset = new Int_t[fNevBuf+1]; fEntryOffset[0] = fKeylen; - Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen); + Warning("ReadBasketBuffers", + "basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%llu, len=%llu, fNbytes=%d, fObjlen=%d, trying to " + "repair", + GetName(), fNevBuf, pos, len, fNbytes, fObjlen); return 0; } if (fIOBits & static_cast(TBasket::EIOBits::kGenerateOffsetMap)) { @@ -704,7 +711,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) /// Read first bytes of a logical record starting at position pos /// return record length (first 4 bytes of record). -Int_t TBasket::ReadBasketBytes(Long64_t pos, TFile *file) +Int_t TBasket::ReadBasketBytes(ULong64_t pos, TFile *file) { const Int_t len = 128; char buffer[len]; diff --git a/tree/tree/src/TBasketSQL.cxx b/tree/tree/src/TBasketSQL.cxx index 5c0edf3639030..6bb0ed64a33b2 100644 --- a/tree/tree/src/TBasketSQL.cxx +++ b/tree/tree/src/TBasketSQL.cxx @@ -111,7 +111,7 @@ void TBasketSQL::CreateBuffer(const char *name, TString title, //////////////////////////////////////////////////////////////////////////////// /// Prepare the basket for the next entry. -void TBasketSQL::PrepareBasket(Long64_t entry) +void TBasketSQL::PrepareBasket(ULong64_t entry) { ((TBufferSQL*)fBufferRef)->ResetOffset(); ((TTreeSQL*)fBranch->GetTree())->PrepEntry(entry); @@ -121,7 +121,7 @@ void TBasketSQL::PrepareBasket(Long64_t entry) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBytes. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) +Int_t TBasketSQL::ReadBasketBytes(ULong64_t , TFile *) { Error("ReadBasketBytes","This member function should not be called!"); return 0; @@ -130,7 +130,7 @@ Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBuffers. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBuffers(Long64_t , Int_t, TFile *) +Int_t TBasketSQL::ReadBasketBuffers(ULong64_t , ULong64_t, TFile *) { Error("ReadBasketBuffers","This member function should not be called!"); return 0; diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx index 6647cf67c44e7..a57d0789d09c8 100644 --- a/tree/tree/src/TBranch.cxx +++ b/tree/tree/src/TBranch.cxx @@ -523,7 +523,7 @@ TBranch::~TBranch() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TBranch for reading/writing baskets. -TBuffer* TBranch::GetTransientBuffer(Int_t size) +TBuffer* TBranch::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) { diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx index 0893cc95f2080..a98038375c61f 100644 --- a/tree/tree/src/TTree.cxx +++ b/tree/tree/src/TTree.cxx @@ -1062,7 +1062,7 @@ TTree::~TTree() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TTree for reading/writing baskets. -TBuffer* TTree::GetTransientBuffer(Int_t size) +TBuffer* TTree::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) { From 69c0ce16de8c9fd0015c3ff9f9c08d11142d3351 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 13:03:45 -0500 Subject: [PATCH 09/24] WIP-io: Read/WriteFastArray Float to Long64_t --- core/base/inc/TBuffer.h | 2 +- io/io/inc/TBufferFile.h | 2 +- io/io/inc/TBufferJSON.h | 2 +- io/io/src/TBufferFile.cxx | 16 ++++++++-------- io/io/src/TBufferJSON.cxx | 2 +- io/sql/inc/TBufferSQL2.h | 2 +- io/sql/src/TBufferSQL2.cxx | 2 +- io/xml/inc/TBufferXML.h | 2 +- io/xml/src/TBufferXML.cxx | 2 +- tree/tree/inc/TBufferSQL.h | 2 +- tree/tree/src/TBufferSQL.cxx | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index dc86b60239ff5..1ae6ed901e84f 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -322,7 +322,7 @@ class TBuffer : public TObject { virtual void ReadFastArray(ULong_t *l, Int_t n) = 0; virtual void ReadFastArray(Long64_t *l, Int_t n) = 0; virtual void ReadFastArray(ULong64_t *l, Int_t n) = 0; - virtual void ReadFastArray(Float_t *f, Int_t n) = 0; + virtual void ReadFastArray(Float_t *f, Long64_t n) = 0; virtual void ReadFastArray(Double_t *d, Int_t n) = 0; virtual void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) = 0; virtual void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) = 0; diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index 3ef653218dd32..8f2b285b4788b 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -189,7 +189,7 @@ class TBufferFile : public TBufferIO { void ReadFastArray(ULong_t *l, Int_t n) override; void ReadFastArray(Long64_t *l, Int_t n) override; void ReadFastArray(ULong64_t *l, Int_t n) override; - void ReadFastArray(Float_t *f, Int_t n) override; + void ReadFastArray(Float_t *f, Long64_t n) override; void ReadFastArray(Double_t *d, Int_t n) override; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) override; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) override; diff --git a/io/io/inc/TBufferJSON.h b/io/io/inc/TBufferJSON.h index a9b6fbaf9da0a..4fd36806315ec 100644 --- a/io/io/inc/TBufferJSON.h +++ b/io/io/inc/TBufferJSON.h @@ -160,7 +160,7 @@ class TBufferJSON final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, const TClass *onFileClass = nullptr) final; diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index faf9675a438f6..aee480c87d50e 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -1526,9 +1526,9 @@ void TBufferFile::ReadFastArray(Long64_t *ll, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of n floats from the I/O buffer. -void TBufferFile::ReadFastArray(Float_t *f, Int_t n) +void TBufferFile::ReadFastArray(Float_t *f, Long64_t n) { - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (l <= 0 || l > fBufSize) return; #ifdef R__BYTESWAP @@ -1536,7 +1536,7 @@ void TBufferFile::ReadFastArray(Float_t *f, Int_t n) bswapcpy32(f, fBufCur, n); fBufCur += sizeof(Float_t)*n; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) frombuf(fBufCur, &f[i]); # endif #else @@ -2329,15 +2329,15 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) { if (n == 0) return; - constexpr Int_t dataWidth = static_cast(sizeof(Float_t)); - const Int_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; + constexpr Long64_t dataWidth = static_cast(sizeof(Float_t)); + const Long64_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; if (n < 0 || n > maxElements) { - Fatal("WriteFastArray", "Not enough space left in the buffer (1GB limit). %lld elements is greater than the max left of %d", n, maxElements); + Fatal("WriteFastArray", "Not enough space left in the buffer. %lld elements is greater than the max left of %lld", n, maxElements); return; // In case the user re-routes the error handler to not die when Fatal is called } - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (fBufCur + l > fBufMax) AutoExpand(fBufSize+l); #ifdef R__BYTESWAP @@ -2345,7 +2345,7 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) bswapcpy32(fBufCur, f, n); fBufCur += l; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) tobuf(fBufCur, f[i]); # endif #else diff --git a/io/io/src/TBufferJSON.cxx b/io/io/src/TBufferJSON.cxx index a440ede2b88a2..ee045577e3db8 100644 --- a/io/io/src/TBufferJSON.cxx +++ b/io/io/src/TBufferJSON.cxx @@ -3011,7 +3011,7 @@ void TBufferJSON::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// read array of Float_t from buffer -void TBufferJSON::ReadFastArray(Float_t *f, Int_t n) +void TBufferJSON::ReadFastArray(Float_t *f, Long64_t n) { JsonReadFastArray(f, n); } diff --git a/io/sql/inc/TBufferSQL2.h b/io/sql/inc/TBufferSQL2.h index 8db572a85b362..1088ce6f20161 100644 --- a/io/sql/inc/TBufferSQL2.h +++ b/io/sql/inc/TBufferSQL2.h @@ -198,7 +198,7 @@ class TBufferSQL2 final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/sql/src/TBufferSQL2.cxx b/io/sql/src/TBufferSQL2.cxx index 184e147c1084e..15c3719587469 100644 --- a/io/sql/src/TBufferSQL2.cxx +++ b/io/sql/src/TBufferSQL2.cxx @@ -1252,7 +1252,7 @@ void TBufferSQL2::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferSQL2::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL2::ReadFastArray(Float_t *f, Long64_t n) { SqlReadFastArray(f, n); } diff --git a/io/xml/inc/TBufferXML.h b/io/xml/inc/TBufferXML.h index b25aff793a838..db9202f2f983c 100644 --- a/io/xml/inc/TBufferXML.h +++ b/io/xml/inc/TBufferXML.h @@ -126,7 +126,7 @@ class TBufferXML final : public TBufferText, public TXMLSetup { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/xml/src/TBufferXML.cxx b/io/xml/src/TBufferXML.cxx index 1843137f5760c..1f956042fb796 100644 --- a/io/xml/src/TBufferXML.cxx +++ b/io/xml/src/TBufferXML.cxx @@ -1779,7 +1779,7 @@ void TBufferXML::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferXML::ReadFastArray(Float_t *f, Int_t n) +void TBufferXML::ReadFastArray(Float_t *f, Long64_t n) { XmlReadFastArray(f, n); } diff --git a/tree/tree/inc/TBufferSQL.h b/tree/tree/inc/TBufferSQL.h index 72908333e689c..d988488b7bd49 100644 --- a/tree/tree/inc/TBufferSQL.h +++ b/tree/tree/inc/TBufferSQL.h @@ -116,7 +116,7 @@ class TBufferSQL final : public TBufferFile { void ReadFastArray(ULong_t *, Int_t ) final; void ReadFastArray(Long64_t *, Int_t ) final; void ReadFastArray(ULong64_t *, Int_t ) final; - void ReadFastArray(Float_t *, Int_t ) final; + void ReadFastArray(Float_t *, Long64_t ) final; void ReadFastArray(Double_t *, Int_t ) final; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele=nullptr) final; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele=nullptr) final; diff --git a/tree/tree/src/TBufferSQL.cxx b/tree/tree/src/TBufferSQL.cxx index ba4d09306ef5c..eae27a605a42b 100644 --- a/tree/tree/src/TBufferSQL.cxx +++ b/tree/tree/src/TBufferSQL.cxx @@ -850,9 +850,9 @@ void TBufferSQL::ReadFastArray(ULong64_t *ull, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// ReadFastArray SQL implementation. -void TBufferSQL::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL::ReadFastArray(Float_t *f, Long64_t n) { - for(int i=0; iGetField(*fIter)); ++fIter; } From efae37b0f2466dcc1216c2685765f9f1336a2b46 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 15:48:38 -0500 Subject: [PATCH 10/24] io: Relax TBuffer::AutoExpand to 64bits --- core/base/src/TBuffer.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 6f7f55a9dd6d0..1e5441307e70b 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -157,13 +157,13 @@ TBuffer::~TBuffer() void TBuffer::AutoExpand(ULong64_t size_needed) { - if (size_needed > kMaxBufferSize) { - Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%x.", size_needed, kMaxBufferSize); + if (size_needed > kMaxLargeBufferSize) { + Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%llx.", size_needed, kMaxLargeBufferSize); } if (size_needed > fBufSize) { ULong64_t doubling = 2LLU * fBufSize; - if (doubling > kMaxBufferSize) - doubling = kMaxBufferSize; + if (doubling > kMaxLargeBufferSize) + doubling = kMaxLargeBufferSize; if (size_needed > doubling) { Expand(size_needed); } else { From 2b97bf91ecfb1816c7ac0fd3840ece98a4396877 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:03:19 -0500 Subject: [PATCH 11/24] io: 64bits TVirtualCollectionProxy --- core/cont/inc/TVirtualCollectionProxy.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 316f75d2e563b..98d61c8b01c55 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -71,6 +71,8 @@ class TVirtualCollectionProxy { kCustomAlloc = BIT(5) ///< The collection has a custom allocator. }; + using size_type = UInt_t; + /// RAII helper class that ensures that `PushProxy()` / `PopProxy()` are called when entering / leaving a C++ context class TPushPop { public: @@ -155,7 +157,7 @@ class TVirtualCollectionProxy { } /// Return the `sizeof()` of the collection object - virtual UInt_t Sizeof() const = 0; + virtual size_type Sizeof() const = 0; /// Set the address of the container being proxied and keep track of the previous one virtual void PushProxy(void *objectstart) = 0; @@ -174,19 +176,19 @@ class TVirtualCollectionProxy { virtual EDataType GetType() const = 0; /// Return the address of the value at index `idx` - virtual void *At(UInt_t idx) = 0; + virtual void *At(size_type idx) = 0; /// Clear the container virtual void Clear(const char *opt = "") = 0; /// Return the current number of elements in the container - virtual UInt_t Size() const = 0; + virtual size_type Size() const = 0; /// Allocates space for storing at least `n` elements. This function returns a pointer to the actual object on /// which insertions should take place. For associative collections, this function returns a pointer to a temporary /// buffer known as the staging area. If the insertion happened in a staging area (i.e. the returned pointer != /// proxied object), `Commit()` should be called on the value returned by this function. - virtual void* Allocate(UInt_t n, Bool_t forceDelete) = 0; + virtual void* Allocate(size_type n, Bool_t forceDelete) = 0; /// Commits pending elements in a staging area (see Allocate() for more information). virtual void Commit(void*) = 0; @@ -196,7 +198,7 @@ class TVirtualCollectionProxy { virtual void Insert(const void *data, void *container, size_t size) = 0; /// Return the address of the value at index `idx` - char *operator[](UInt_t idx) const { return (char *)(const_cast(this))->At(idx); } + char *operator[](size_type idx) const { return (char *)(const_cast(this))->At(idx); } // Functions related to member-wise actions virtual TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *oldClass, Int_t version) = 0; From ebe288f3238e1c2c247542cfc880a40a36b0ec09 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:03:52 -0500 Subject: [PATCH 12/24] io: Read TStreamerInfo version to 11, 64bits collection size in old code --- io/io/src/TStreamerInfoReadBuffer.cxx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 0754648b25a38..63c521b993b1e 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -1303,7 +1303,17 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, TVirtualCollectionProxy::TPushPop helper( newProxy, obj ); Int_t nobjects; b >> nobjects; - void* env = newProxy->Allocate(nobjects,true); + ULong64_t nobjects64; + if (std::signbit(nobjects) && vers >= 11) { + nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; + UInt_t nobjectsLow; + b >> nobjectsLow; + nobjects64 |= nobjectsLow; + } else { + nobjects64 = nobjects; + } + R__ASSERT(nobjects64 == nobjects && "Allocate is not yet 64 bit enabled"); + void* env = newProxy->Allocate(nobjects64, true); if (!nobjects && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. From d7fecbec389341d087f9550652af95de64e128c6 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:07:34 -0500 Subject: [PATCH 13/24] io: (for) TStreamerInfo version to 11, 64bits collection size --- io/io/src/TStreamerInfoActions.cxx | 12 ++--- io/io/src/TStreamerInfoCollectionUtils.h | 67 ++++++++++++++++++++++++ io/io/src/TStreamerInfoReadBuffer.cxx | 17 ++---- 3 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 io/io/src/TStreamerInfoCollectionUtils.h diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index 35f315029fce5..57093b5cd8f99 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -19,6 +19,7 @@ #include "TVirtualArray.h" #include "TBufferFile.h" #include "TBufferText.h" +#include "TStreamerInfoCollectionUtils.h" #include "TMemberStreamer.h" #include "TClassEdit.h" #include "TVirtualCollectionIterators.h" @@ -2234,21 +2235,20 @@ namespace TStreamerInfoActions TConfigSTL *config = (TConfigSTL*)conf; UInt_t start, count; - /* Version_t vers = */ buf.ReadVersion(&start, &count, config->fOldClass); + Version_t vers = buf.ReadVersion(&start, &count, config->fOldClass); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues; - buf.ReadInt(nvalues); - vec->resize(nvalues); + ULong64_t nvalues64 = TStreamerInfoUtils::ReadCollectionSize(buf, vers); + vec->resize(nvalues64); #ifdef R__VISUAL_CPLUSPLUS - if (nvalues <= 0) { + if (nvalues64 == 0) { buf.CheckByteCount(start,count,config->fTypeName); return 0; } #endif T *begin = vec->data(); - buf.ReadFastArray(begin, nvalues); + buf.ReadFastArray(begin, nvalues64); buf.CheckByteCount(start,count,config->fTypeName); return 0; diff --git a/io/io/src/TStreamerInfoCollectionUtils.h b/io/io/src/TStreamerInfoCollectionUtils.h new file mode 100644 index 0000000000000..be6cad82c80d6 --- /dev/null +++ b/io/io/src/TStreamerInfoCollectionUtils.h @@ -0,0 +1,67 @@ +// @(#)root/io:$Id$ +// Author: Philippe Canal 04/2026 + +/************************************************************************* + * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TStreamerInfoCollectionUtils +#define ROOT_TStreamerInfoCollectionUtils + +#include "TBuffer.h" +#include "RtypesCore.h" +#include // std::signbit +#include // std::numeric_limits + +namespace TStreamerInfoUtils { + +/// Read the number-of-objects count from the buffer using the encoding introduced +/// in TStreamerInfo version 11: when the high bit of the 32-bit word is set the +/// remaining 31 bits form the high part of a 64-bit count, and the following +/// UInt_t carries the low 32 bits. +/// +/// \param b The input buffer (operator>> must be available for Int_t/UInt_t). +/// \param vers The TStreamerInfo version that was used when the data was written. +/// \return The number of objects encoded in the stream. +inline ULong64_t ReadCollectionSize(TBuffer &b, Version_t vers) +{ + Int_t nobjects; + b >> nobjects; + ULong64_t nobjects64; + if (std::signbit(nobjects) && vers >= 11) { + nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; + UInt_t nobjectsLow; + b >> nobjectsLow; + nobjects64 |= nobjectsLow; + } else { + nobjects64 = nobjects; + } + return nobjects64; +} + +/// Write the number-of-objects count to the buffer using the encoding introduced +/// in TStreamerInfo version 11: counts that fit in a non-negative Int_t are written +/// as a single Int_t; larger counts set the high bit of the first Int_t (carrying +/// bits 32–62 of the count) and follow with a UInt_t carrying the low 32 bits. +/// +/// \param b The output buffer (WriteInt/WriteUInt must be available). +/// \param n The number of objects to encode. +inline void WriteCollectionSize(TBuffer &b, ULong64_t n) +{ + if (n <= static_cast(std::numeric_limits::max())) { + b.WriteInt(static_cast(n)); + } else { + // Encode high 31 bits with the sign bit set, then the low 32 bits. + Int_t high = static_cast((n >> 32) | 0x80000000u); + b.WriteInt(high); + b.WriteUInt(static_cast(n & 0xffffffffu)); + } +} + +} // namespace TStreamerInfoUtils + +#endif // ROOT_TStreamerInfoCollectionUtils diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 63c521b993b1e..4042c0a99a80a 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -13,6 +13,7 @@ #include "TFile.h" #include "TClass.h" #include "TBufferFile.h" +#include "TStreamerInfoCollectionUtils.h" #include "TClonesArray.h" #include "TError.h" #include "TRef.h" @@ -1301,20 +1302,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, for(; obj> nobjects; - ULong64_t nobjects64; - if (std::signbit(nobjects) && vers >= 11) { - nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; - UInt_t nobjectsLow; - b >> nobjectsLow; - nobjects64 |= nobjectsLow; - } else { - nobjects64 = nobjects; - } - R__ASSERT(nobjects64 == nobjects && "Allocate is not yet 64 bit enabled"); + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + R__ASSERT(nobjects64 <= (ULong64_t)std::numeric_limits::max() && "Allocate is not yet 64 bit enabled"); void* env = newProxy->Allocate(nobjects64, true); - if (!nobjects && (vers>=7)) { + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { From a0cc53a9d2948ea5abe9a82b4770095b62b8270e Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 18:08:26 -0500 Subject: [PATCH 14/24] WIP-io: Extent TVirtualCollection to 64bits. NOTE: this might be backward incompatible --- core/cont/inc/TVirtualCollectionProxy.h | 2 +- io/io/inc/TEmulatedCollectionProxy.h | 10 +++++----- io/io/inc/TEmulatedMapProxy.h | 4 ++-- io/io/inc/TGenCollectionProxy.h | 10 +++++----- io/io/src/TEmulatedCollectionProxy.cxx | 8 ++++---- io/io/src/TEmulatedMapProxy.cxx | 4 ++-- io/io/src/TGenCollectionProxy.cxx | 20 ++++++++++---------- tree/ntuple/test/CustomStruct.hxx | 8 ++++---- tree/ntuple/test/SimpleCollectionProxy.hxx | 8 ++++---- tree/ntuple/test/ntuple_emulated.cxx | 8 ++++---- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 98d61c8b01c55..ec3d3e7a76cce 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -71,7 +71,7 @@ class TVirtualCollectionProxy { kCustomAlloc = BIT(5) ///< The collection has a custom allocator. }; - using size_type = UInt_t; + using size_type = ULong64_t; /// RAII helper class that ensures that `PushProxy()` / `PopProxy()` are called when entering / leaving a C++ context class TPushPop { diff --git a/io/io/inc/TEmulatedCollectionProxy.h b/io/io/inc/TEmulatedCollectionProxy.h index 1e6ccf53f717a..981f91edb95c1 100644 --- a/io/io/inc/TEmulatedCollectionProxy.h +++ b/io/io/inc/TEmulatedCollectionProxy.h @@ -89,22 +89,22 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { void DeleteArray(void* p, Bool_t dtorOnly = kFALSE) const override; // TVirtualCollectionProxy overload: Return the sizeof the collection object. - UInt_t Sizeof() const override { return sizeof(Cont_t); } + size_type Sizeof() const override { return sizeof(Cont_t); } // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container void Clear(const char *opt = "") override; // Resize the container - void Resize(UInt_t n, Bool_t force_delete) override; + void Resize(size_type n, Bool_t force_delete) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Block commit of containees void Commit(void* env) override; diff --git a/io/io/inc/TEmulatedMapProxy.h b/io/io/inc/TEmulatedMapProxy.h index 48e990d85e86d..232a7b1bbd27f 100644 --- a/io/io/inc/TEmulatedMapProxy.h +++ b/io/io/inc/TEmulatedMapProxy.h @@ -38,10 +38,10 @@ class TEmulatedMapProxy : public TEmulatedCollectionProxy { ~TEmulatedMapProxy() override; // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Read portion of the streamer void ReadBuffer(TBuffer &buff, void *pObj) override; diff --git a/io/io/inc/TGenCollectionProxy.h b/io/io/inc/TGenCollectionProxy.h index 310f8a1020bb0..8f1abae20c08b 100644 --- a/io/io/inc/TGenCollectionProxy.h +++ b/io/io/inc/TGenCollectionProxy.h @@ -372,7 +372,7 @@ class TGenCollectionProxy ULong_t GetIncrement() const override; // Return the sizeof the collection object. - UInt_t Sizeof() const override; + size_type Sizeof() const override; // Push new proxy environment. void PushProxy(void *objstart) override; @@ -390,19 +390,19 @@ class TGenCollectionProxy EDataType GetType() const override; // Return the address of the value at index 'idx'. - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container. void Clear(const char *opt = "") override; // Resize the container. - virtual void Resize(UInt_t n, Bool_t force_delete); + virtual void Resize(size_type n, Bool_t force_delete); // Return the current size of the container. - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees. - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Insert data into the container where data is a C-style array of the actual type contained in the collection // of the given size. For associative container (map, etc.), the data type is the pair. diff --git a/io/io/src/TEmulatedCollectionProxy.cxx b/io/io/src/TEmulatedCollectionProxy.cxx index 8b84e00c17b61..54b48b7ad4af1 100644 --- a/io/io/src/TEmulatedCollectionProxy.cxx +++ b/io/io/src/TEmulatedCollectionProxy.cxx @@ -246,7 +246,7 @@ Bool_t TEmulatedCollectionProxy::IsValid() const return (0 != fCreateEnv.call); } -UInt_t TEmulatedCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedCollectionProxy::Size() const { // Return the current size of the container @@ -447,7 +447,7 @@ void TEmulatedCollectionProxy::Expand(UInt_t nCurr, UInt_t left) } } -void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) +void TEmulatedCollectionProxy::Resize(TVirtualCollectionProxy::size_type left, Bool_t force) { // Resize the container @@ -468,7 +468,7 @@ void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) Fatal("TEmulatedCollectionProxy","Resize> Logic error - no proxy object set."); } -void* TEmulatedCollectionProxy::At(UInt_t idx) +void* TEmulatedCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx' if ( fEnv && fEnv->fObject ) { @@ -483,7 +483,7 @@ void* TEmulatedCollectionProxy::At(UInt_t idx) return 0; } -void* TEmulatedCollectionProxy::Allocate(UInt_t n, Bool_t forceDelete) +void* TEmulatedCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t forceDelete) { // Allocate the necessary space. diff --git a/io/io/src/TEmulatedMapProxy.cxx b/io/io/src/TEmulatedMapProxy.cxx index 548f7a8bef4c9..3dd2dbe810d27 100644 --- a/io/io/src/TEmulatedMapProxy.cxx +++ b/io/io/src/TEmulatedMapProxy.cxx @@ -66,7 +66,7 @@ TVirtualCollectionProxy* TEmulatedMapProxy::Generate() const return new TEmulatedMapProxy(*this); } -void* TEmulatedMapProxy::At(UInt_t idx) +void* TEmulatedMapProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx'. if ( fEnv && fEnv->fObject ) { @@ -77,7 +77,7 @@ void* TEmulatedMapProxy::At(UInt_t idx) return 0; } -UInt_t TEmulatedMapProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedMapProxy::Size() const { // Return the current size of the container. if ( fEnv && fEnv->fObject ) { diff --git a/io/io/src/TGenCollectionProxy.cxx b/io/io/src/TGenCollectionProxy.cxx index 3defdece6e7fd..49598d89d9053 100644 --- a/io/io/src/TGenCollectionProxy.cxx +++ b/io/io/src/TGenCollectionProxy.cxx @@ -47,7 +47,7 @@ class TGenVectorProxy : public TGenCollectionProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { fEnv->fIdx = idx; @@ -97,7 +97,7 @@ class TGenVectorBoolProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -143,7 +143,7 @@ class TGenBitsetProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -198,7 +198,7 @@ class TGenListProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { switch( idx ) { @@ -240,7 +240,7 @@ class TGenSetProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { if ( fEnv->fUseTemp ) { @@ -1015,7 +1015,7 @@ ULong_t TGenCollectionProxy::GetIncrement() const { //////////////////////////////////////////////////////////////////////////////// /// Return the sizeof the collection object. -UInt_t TGenCollectionProxy::Sizeof() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Sizeof() const { return fClass->Size(); } @@ -1065,7 +1065,7 @@ EDataType TGenCollectionProxy::GetType() const //////////////////////////////////////////////////////////////////////////////// /// Return the address of the value at index 'idx' -void* TGenCollectionProxy::At(UInt_t idx) +void* TGenCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { if ( fEnv && fEnv->fObject ) { switch (fSTL_type) { @@ -1154,7 +1154,7 @@ void TGenCollectionProxy::Clear(const char* opt) //////////////////////////////////////////////////////////////////////////////// /// Return the current size of the container -UInt_t TGenCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Size() const { if ( fEnv && fEnv->fObject ) { if (fEnv->fUseTemp) { @@ -1170,7 +1170,7 @@ UInt_t TGenCollectionProxy::Size() const //////////////////////////////////////////////////////////////////////////////// /// Resize the container -void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) +void TGenCollectionProxy::Resize(TVirtualCollectionProxy::size_type n, Bool_t force) { if ( fEnv && fEnv->fObject ) { if ( force && fPointers ) { @@ -1193,7 +1193,7 @@ void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) /// For associative collection, this returns a TStaging object that /// need to be deleted manually __or__ returned by calling Commit(TStaging*) -void* TGenCollectionProxy::Allocate(UInt_t n, Bool_t /* forceDelete */ ) +void* TGenCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t /* forceDelete */ ) { if ( fEnv && fEnv->fObject ) { switch ( fSTL_type ) { diff --git a/tree/ntuple/test/CustomStruct.hxx b/tree/ntuple/test/CustomStruct.hxx index c3d6e633f356b..55f6f8a2857b9 100644 --- a/tree/ntuple/test/CustomStruct.hxx +++ b/tree/ntuple/test/CustomStruct.hxx @@ -359,16 +359,16 @@ public: TVirtualCollectionProxy *Generate() const final { return new CyclicCollectionProxy(); } Int_t GetCollectionType() const final { return 0; } ULong_t GetIncrement() const final { return 0; } - UInt_t Sizeof() const final { return 0; } + size_type Sizeof() const final { return 0; } bool HasPointers() const final { return false; } TClass *GetValueClass() const final; EDataType GetType() const final { return EDataType::kOther_t; } void PushProxy(void *) final {} void PopProxy() final {} - void *At(UInt_t) final { return nullptr; } + void *At(size_type) final { return nullptr; } void Clear(const char * = "") final {} - UInt_t Size() const final { return 0; } - void *Allocate(UInt_t, bool) final { return nullptr; } + size_type Size() const final { return 0; } + void *Allocate(size_type, bool) final { return nullptr; } void Commit(void *) final {} void Insert(const void *, void *, size_t) final {} TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *, Int_t) final { return nullptr; } diff --git a/tree/ntuple/test/SimpleCollectionProxy.hxx b/tree/ntuple/test/SimpleCollectionProxy.hxx index 6d152f76c8482..bbf2ca6c4cdd1 100644 --- a/tree/ntuple/test/SimpleCollectionProxy.hxx +++ b/tree/ntuple/test/SimpleCollectionProxy.hxx @@ -55,7 +55,7 @@ public: } Int_t GetCollectionType() const override { return CollectionKind; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -80,10 +80,10 @@ public: void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; diff --git a/tree/ntuple/test/ntuple_emulated.cxx b/tree/ntuple/test/ntuple_emulated.cxx index 013a8f1fd30bf..31d6d4b938399 100644 --- a/tree/ntuple/test/ntuple_emulated.cxx +++ b/tree/ntuple/test/ntuple_emulated.cxx @@ -566,7 +566,7 @@ TEST(RNTupleEmulated, CollectionProxy) } Int_t GetCollectionType() const override { return ROOT::kSTLvector; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -589,10 +589,10 @@ TEST(RNTupleEmulated, CollectionProxy) void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; From 425e636eac689fd888dd488fda8eb96c15e2c644 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 15:48:57 -0500 Subject: [PATCH 15/24] io: TStreamerInfo version to 11, 64bits collection size --- io/io/inc/TStreamerInfo.h | 2 +- io/io/src/TStreamerInfoActions.cxx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/io/io/inc/TStreamerInfo.h b/io/io/inc/TStreamerInfo.h index a604a3cc196ea..3e8c5837ab134 100644 --- a/io/io/inc/TStreamerInfo.h +++ b/io/io/inc/TStreamerInfo.h @@ -251,7 +251,7 @@ class TStreamerInfo : public TVirtualStreamerInfo { Int_t WriteBufferAux (TBuffer &b, const T &arr, TCompInfo *const*const compinfo, Int_t first, Int_t last, Int_t narr,Int_t eoffset,Int_t mode); //WARNING this class version must be the same as TVirtualStreamerInfo - ClassDefOverride(TStreamerInfo, 10) // Streamer information for one class version + ClassDefOverride(TStreamerInfo, 11) // Streamer information for one class version }; diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index 57093b5cd8f99..ee763e005d308 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -2263,8 +2263,8 @@ namespace TStreamerInfoActions UInt_t start = buf.WriteVersion(config->fInfo->IsA(), kTRUE); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues = vec->size(); - buf.WriteInt(nvalues); + auto nvalues = vec->size(); + TStreamerInfoUtils::WriteCollectionSize(buf, nvalues); T *begin = vec->data(); buf.WriteFastArray(begin, nvalues); From 09c719b96ab46ef6cc16585f97433032709e8644 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 8 Apr 2026 10:31:02 -0500 Subject: [PATCH 16/24] To Review: io: Address warning after signature changes --- io/io/src/TBufferFile.cxx | 70 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index aee480c87d50e..1235a8cf79e28 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -859,7 +859,7 @@ Int_t TBufferFile::ReadArray(Bool_t *&b) Int_t n; *this >> n; - if (n <= 0 || n > fBufSize) return 0; + if (n <= 0 || static_cast(n) > fBufSize) return 0; if (!b) b = new Bool_t[n]; @@ -888,7 +888,7 @@ Int_t TBufferFile::ReadArray(Char_t *&c) *this >> n; Int_t l = sizeof(Char_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!c) c = new Char_t[n]; @@ -911,7 +911,7 @@ Int_t TBufferFile::ReadArray(Short_t *&h) *this >> n; Int_t l = sizeof(Short_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!h) h = new Short_t[n]; @@ -944,7 +944,7 @@ Int_t TBufferFile::ReadArray(Int_t *&ii) *this >> n; Int_t l = sizeof(Int_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ii) ii = new Int_t[n]; @@ -977,7 +977,7 @@ Int_t TBufferFile::ReadArray(Long_t *&ll) *this >> n; Int_t l = sizeof(Long_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ll) ll = new Long_t[n]; @@ -1003,7 +1003,7 @@ Int_t TBufferFile::ReadArray(Long64_t *&ll) *this >> n; Int_t l = sizeof(Long64_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ll) ll = new Long64_t[n]; @@ -1031,7 +1031,7 @@ Int_t TBufferFile::ReadArray(Float_t *&f) *this >> n; Int_t l = sizeof(Float_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!f) f = new Float_t[n]; @@ -1064,7 +1064,7 @@ Int_t TBufferFile::ReadArray(Double_t *&d) *this >> n; Int_t l = sizeof(Double_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!d) d = new Double_t[n]; @@ -1092,7 +1092,7 @@ Int_t TBufferFile::ReadArrayFloat16(Float_t *&f, TStreamerElement *ele) Int_t n; *this >> n; - if (n <= 0 || 3*n > fBufSize) return 0; + if (n <= 0 || static_cast(3*n) > fBufSize) return 0; if (!f) f = new Float_t[n]; @@ -1114,7 +1114,7 @@ Int_t TBufferFile::ReadArrayDouble32(Double_t *&d, TStreamerElement *ele) Int_t n; *this >> n; - if (n <= 0 || 3*n > fBufSize) return 0; + if (n <= 0 || static_cast(3*n) > fBufSize) return 0; if (!d) d = new Double_t[n]; @@ -1134,7 +1134,7 @@ Int_t TBufferFile::ReadStaticArray(Bool_t *b) Int_t n; *this >> n; - if (n <= 0 || n > fBufSize) return 0; + if (n <= 0 || static_cast(n) > fBufSize) return 0; if (!b) return 0; @@ -1162,7 +1162,7 @@ Int_t TBufferFile::ReadStaticArray(Char_t *c) *this >> n; Int_t l = sizeof(Char_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!c) return 0; @@ -1184,7 +1184,7 @@ Int_t TBufferFile::ReadStaticArray(Short_t *h) *this >> n; Int_t l = sizeof(Short_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!h) return 0; @@ -1216,7 +1216,7 @@ Int_t TBufferFile::ReadStaticArray(Int_t *ii) *this >> n; Int_t l = sizeof(Int_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ii) return 0; @@ -1248,7 +1248,7 @@ Int_t TBufferFile::ReadStaticArray(Long_t *ll) *this >> n; Int_t l = sizeof(Long_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ll) return 0; @@ -1273,7 +1273,7 @@ Int_t TBufferFile::ReadStaticArray(Long64_t *ll) *this >> n; Int_t l = sizeof(Long64_t)*n; - if (l <= 0 || l > fBufSize) return 0; + if (l <= 0 || static_cast(l) > fBufSize) return 0; if (!ll) return 0; @@ -1300,7 +1300,7 @@ Int_t TBufferFile::ReadStaticArray(Float_t *f) *this >> n; Int_t l = sizeof(Float_t)*n; - if (n <= 0 || l > fBufSize) return 0; + if (n <= 0 || static_cast(l) > fBufSize) return 0; if (!f) return 0; @@ -1332,7 +1332,7 @@ Int_t TBufferFile::ReadStaticArray(Double_t *d) *this >> n; Int_t l = sizeof(Double_t)*n; - if (n <= 0 || l > fBufSize) return 0; + if (n <= 0 || static_cast(l) > fBufSize) return 0; if (!d) return 0; @@ -1359,7 +1359,7 @@ Int_t TBufferFile::ReadStaticArrayFloat16(Float_t *f, TStreamerElement *ele) Int_t n; *this >> n; - if (n <= 0 || 3*n > fBufSize) return 0; + if (n <= 0 || static_cast(3*n) > fBufSize) return 0; if (!f) return 0; @@ -1380,7 +1380,7 @@ Int_t TBufferFile::ReadStaticArrayDouble32(Double_t *d, TStreamerElement *ele) Int_t n; *this >> n; - if (n <= 0 || 3*n > fBufSize) return 0; + if (n <= 0 || static_cast(3*n) > fBufSize) return 0; if (!d) return 0; @@ -1394,7 +1394,7 @@ Int_t TBufferFile::ReadStaticArrayDouble32(Double_t *d, TStreamerElement *ele) void TBufferFile::ReadFastArray(Bool_t *b, Int_t n) { - if (n <= 0 || n > fBufSize) return; + if (n <= 0 || static_cast(n) > fBufSize) return; if (sizeof(Bool_t) > 1) { for (int i = 0; i < n; i++) @@ -1411,7 +1411,7 @@ void TBufferFile::ReadFastArray(Bool_t *b, Int_t n) void TBufferFile::ReadFastArray(Char_t *c, Int_t n) { - if (n <= 0 || n > fBufSize) return; + if (n <= 0 || static_cast(n) > fBufSize) return; Int_t l = sizeof(Char_t)*n; memcpy(c, fBufCur, l); @@ -1432,7 +1432,7 @@ void TBufferFile::ReadFastArrayString(Char_t *c, Int_t n) *this >> len; } if (len) { - if (len <= 0 || len > fBufSize) return; + if (len <= 0 || static_cast(len) > fBufSize) return; Int_t blen = len; if (len >= n) len = n-1; @@ -1452,7 +1452,7 @@ void TBufferFile::ReadFastArrayString(Char_t *c, Int_t n) void TBufferFile::ReadFastArray(Short_t *h, Int_t n) { Int_t l = sizeof(Short_t)*n; - if (n <= 0 || l > fBufSize) return; + if (n <= 0 || static_cast(l) > fBufSize) return; #ifdef R__BYTESWAP # ifdef USE_BSWAPCPY @@ -1474,7 +1474,7 @@ void TBufferFile::ReadFastArray(Short_t *h, Int_t n) void TBufferFile::ReadFastArray(Int_t *ii, Int_t n) { Int_t l = sizeof(Int_t)*n; - if (l <= 0 || l > fBufSize) return; + if (l <= 0 || static_cast(l) > fBufSize) return; #ifdef R__BYTESWAP # ifdef USE_BSWAPCPY @@ -1496,7 +1496,7 @@ void TBufferFile::ReadFastArray(Int_t *ii, Int_t n) void TBufferFile::ReadFastArray(Long_t *ll, Int_t n) { Int_t l = sizeof(Long_t)*n; - if (l <= 0 || l > fBufSize) return; + if (l <= 0 || static_cast(l) > fBufSize) return; TFile *file = (TFile*)fParent; if (file && file->GetVersion() < 30006) { @@ -1512,7 +1512,7 @@ void TBufferFile::ReadFastArray(Long_t *ll, Int_t n) void TBufferFile::ReadFastArray(Long64_t *ll, Int_t n) { Int_t l = sizeof(Long64_t)*n; - if (l <= 0 || l > fBufSize) return; + if (l <= 0 || static_cast(l) > fBufSize) return; #ifdef R__BYTESWAP for (int i = 0; i < n; i++) @@ -1529,7 +1529,7 @@ void TBufferFile::ReadFastArray(Long64_t *ll, Int_t n) void TBufferFile::ReadFastArray(Float_t *f, Long64_t n) { auto l = sizeof(Float_t)*n; - if (l <= 0 || l > fBufSize) return; + if (l <= 0 || static_cast(l) > fBufSize) return; #ifdef R__BYTESWAP # ifdef USE_BSWAPCPY @@ -1551,7 +1551,7 @@ void TBufferFile::ReadFastArray(Float_t *f, Long64_t n) void TBufferFile::ReadFastArray(Double_t *d, Int_t n) { Int_t l = sizeof(Double_t)*n; - if (l <= 0 || l > fBufSize) return; + if (l <= 0 || static_cast(l) > fBufSize) return; #ifdef R__BYTESWAP for (int i = 0; i < n; i++) @@ -1568,7 +1568,7 @@ void TBufferFile::ReadFastArray(Double_t *d, Int_t n) void TBufferFile::ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *elem) { - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; // The parameter should be const, however we have not yet decided how to // transition the signature since the function is virtual. This ensures @@ -1614,7 +1614,7 @@ void TBufferFile::ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *el void TBufferFile::ReadFastArrayWithFactor(Float_t *ptr, Int_t n, Double_t factor, Double_t minvalue) { - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; //a range was specified. We read an integer and convert it back to a float for (int j=0;j < n; j++) { @@ -1628,7 +1628,7 @@ void TBufferFile::ReadFastArrayWithFactor(Float_t *ptr, Int_t n, Double_t factor void TBufferFile::ReadFastArrayWithNbits(Float_t *ptr, Int_t n, Int_t nbits) { - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; if (!nbits) nbits = 12; //we read the exponent and the truncated mantissa of the float @@ -1661,7 +1661,7 @@ void TBufferFile::ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement * // that the function does not inadvertently use non-const parts of // TStreamerElement. const TStreamerElement *ele = elem; - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; if (ele && ele->GetFactor() != 0) { //a range was specified. We read an integer and convert it back to a double. @@ -1709,7 +1709,7 @@ void TBufferFile::ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement * void TBufferFile::ReadFastArrayWithFactor(Double_t *d, Int_t n, Double_t factor, Double_t minvalue) { - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; //a range was specified. We read an integer and convert it back to a double. for (int j=0;j < n; j++) { @@ -1723,7 +1723,7 @@ void TBufferFile::ReadFastArrayWithFactor(Double_t *d, Int_t n, Double_t factor, void TBufferFile::ReadFastArrayWithNbits(Double_t *d, Int_t n, Int_t nbits) { - if (n <= 0 || 3*n > fBufSize) return; + if (n <= 0 || static_cast(3*n) > fBufSize) return; if (!nbits) { //we read a float and convert it to double From c5ce2a63f0500a22324084236ef1769582c1dafd Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 8 Apr 2026 10:31:20 -0500 Subject: [PATCH 17/24] io: Address warning after signature changes --- io/io/src/TBufferFile.cxx | 16 ++++++++-------- io/io/src/TBufferText.cxx | 12 ++++++------ io/io/src/TStreamerInfoActions.cxx | 6 +++--- io/io/src/TStreamerInfoWriteBuffer.cxx | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 1235a8cf79e28..3cd57d531712c 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -3774,7 +3774,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); CheckByteCount(start, count, onFileClass); return 0; @@ -3790,7 +3790,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length() ); CheckByteCount(start, count, cl); return 0; @@ -3821,7 +3821,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio CheckByteCount(start, count, cl); return 0; } else { - Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); CheckByteCount(start, count, cl); return 0; @@ -3884,7 +3884,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); --current; CheckByteCount(R__s, R__c, onFileClass); @@ -3909,7 +3909,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass Int_t infocapacity = infos->Capacity(); if (infocapacity) { if (version < -1 || version >= infocapacity) { - Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); --current; CheckByteCount(R__s, R__c, cl); @@ -3974,7 +3974,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass CheckByteCount(R__s, R__c, cl); return 0; } else { - Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); --current; CheckByteCount(R__s, R__c, cl); @@ -3989,7 +3989,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (sinfo->TStreamerInfo::IsRecovered()) R__c=0; // 'TStreamerInfo::' avoids going via a virtual function. if (current != fByteCountStack.size()) - Fatal("ReadClassBuffer", "We are out of sync at level %lld vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); + Fatal("ReadClassBuffer", "We are out of sync at level %zu vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); --current; // Check that the buffer position corresponds to the byte count. CheckByteCount(R__s, R__c, cl); @@ -4063,7 +4063,7 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se (*iter).PrintDebug(*this,obj); (*iter)(*this,obj); if (current != fByteCountStack.size()) - Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); + Fatal("ApplySequence", "We are out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } else { diff --git a/io/io/src/TBufferText.cxx b/io/io/src/TBufferText.cxx index 76e625f8ead24..4494acab8177d 100644 --- a/io/io/src/TBufferText.cxx +++ b/io/io/src/TBufferText.cxx @@ -249,7 +249,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(start, count, onFileClass); return 0; @@ -265,7 +265,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(start, count, cl); return 0; @@ -294,7 +294,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(start, count, cl); return 0; @@ -352,7 +352,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, onFileClass); return 0; @@ -377,7 +377,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (infocapacity) { if (version < -1 || version >= infocapacity) { Error("ReadClassBuffer", - "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(R__s, R__c, cl); return 0; @@ -440,7 +440,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, cl); return 0; diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index ee763e005d308..2ad2384357c05 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -127,7 +127,7 @@ namespace TStreamerInfoActions aElement->GetSequenceType(sequenceType); printf("StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d (%s)\n", + " %s, bufpos=%lu, arr=%p, offset=%d (%s)\n", info->GetClass()->GetName(), aElement->GetName(), fElemId, fCompInfo->fType, aElement->ClassName(), buf.Length(), addr, fOffset, sequenceType.Data()); } @@ -1275,7 +1275,7 @@ namespace TStreamerInfoActions { if (gDebug > 1) { TStreamerInfo *info = (TStreamerInfo*)fInfo; - printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%d, arr=%p, offset=%d, onfileObject=%p\n", + printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%lu, arr=%p, offset=%d, onfileObject=%p\n", info->GetClass()->GetName(), fOnfileObject ? "Push" : "Pop", buffer.Length(), object, fOffset, fOnfileObject); } @@ -1351,7 +1351,7 @@ namespace TStreamerInfoActions TStreamerInfo *info = (TStreamerInfo*)fInfo; TStreamerElement *aElement = fCompInfo->fElem; fprintf(stdout,"StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", info->GetClass()->GetName(),aElement->GetName(),fElemId,fCompInfo->fType, aElement->ClassName(),b.Length(),addr, 0,b.PeekDataCache() ? b.PeekDataCache()->GetObjectAt(0) : 0); } diff --git a/io/io/src/TStreamerInfoWriteBuffer.cxx b/io/io/src/TStreamerInfoWriteBuffer.cxx index 7a62f21ae346b..1b09138fb45ce 100644 --- a/io/io/src/TStreamerInfoWriteBuffer.cxx +++ b/io/io/src/TStreamerInfoWriteBuffer.cxx @@ -151,7 +151,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -161,7 +161,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, not a write rule, skipping.\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, not a write rule, skipping.\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], eoffset); } @@ -175,7 +175,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d, %s, " - "bufpos=%d, arr=%p, offset=%d\n", + "bufpos=%lu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType,aElement->ClassName(), b.Length(),arr[0],ioffset); } From 16870429dd65dac30ee1af7308585113341970d8 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 15:52:01 -0500 Subject: [PATCH 18/24] WIP-io: TBuffer ctor and SetBuffer 64bits: --- core/base/src/TBuffer.cxx | 32 ++++++++++++++------------- roottest/root/io/buffer/ownership.ref | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 1e5441307e70b..586067282e145 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -112,12 +112,20 @@ TBuffer::TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt, ReAlloc SetBit(kIsOwner); + if (buf && !adopt) + ResetBit(kIsOwner); + + SetReAllocFunc( reallocfunc ); + if (buf) { fBuffer = (char *)buf; if ( (fMode&kWrite)!=0 ) { - fBufSize -= kExtraSpace; + if (fBufSize < kExtraSpace) { + Expand( kMinimalSize ); + } else { + fBufSize -= kExtraSpace; + } } - if (!adopt) ResetBit(kIsOwner); } else { if (fBufSize < kMinimalSize) { fBufSize = kMinimalSize; @@ -126,12 +134,6 @@ TBuffer::TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt, ReAlloc } fBufCur = fBuffer; fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < kMinimalSize) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// @@ -196,22 +198,22 @@ void TBuffer::SetBuffer(void *buf, ULong64_t newsiz, Bool_t adopt, ReAllocCharFu else ResetBit(kIsOwner); + SetReAllocFunc( reallocfunc ); + fBuffer = (char *)buf; fBufCur = fBuffer; if (newsiz > 0) { if ( (fMode&kWrite)!=0 ) { - fBufSize = newsiz - kExtraSpace; + if (newsiz >= kExtraSpace) { + fBufSize = newsiz - kExtraSpace; + } else { + Expand( kMinimalSize ); + } } else { fBufSize = newsiz; } } fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// diff --git a/roottest/root/io/buffer/ownership.ref b/roottest/root/io/buffer/ownership.ref index 5c633dbad1d39..e39e5dca5c19e 100644 --- a/roottest/root/io/buffer/ownership.ref +++ b/roottest/root/io/buffer/ownership.ref @@ -1,7 +1,7 @@ Processing runownership.C+... Fatal in : Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided. The expected error "Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided." was seen -Running custom allocator (increase from 3 to 136) +Running custom allocator (increase from 128 to 136) Running custom allocator (increase from 136 to 264) The test has completed successfully (int) 0 From f45e0568b6b7917850200088c58a036dc6c413ac Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 11 Feb 2026 13:53:16 +0100 Subject: [PATCH 19/24] io: test long range Read/WriteObjectAny --- io/io/test/testByteCount.cxx | 103 ++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/io/io/test/testByteCount.cxx b/io/io/test/testByteCount.cxx index 07a6966786e56..a4a464f366ea3 100644 --- a/io/io/test/testByteCount.cxx +++ b/io/io/test/testByteCount.cxx @@ -5,7 +5,7 @@ #include -int testByteCount() +int unittestByteCount() { int errors = 0; unsigned int expectedByteCounts = 1; @@ -124,3 +124,104 @@ int testByteCount() std::cerr << "The end.\n"; return errors; } + +struct LargeByteCountsFixture { + std::size_t fSize{0}; + void resize(size_t size) { + fSize = size; + } + void Streamer(TBuffer &b) { + // Bare minimum to trigger the large byte count mechanism, + // we don't care about the content of the data. + if (b.IsReading()) { + b >> fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } else { + b << fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } + } +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class LargeByteCountsFixture-; +#endif + +int readAndCheck(const std::string &msg, TBufferFile &b, size_t expected_size) +{ + // Same as b >> ptr; + auto obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeByteCountsFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back the object\n"; + return 1; + } else { + auto* readFixture = static_cast(obj); + if (readFixture->fSize != expected_size ) { + std::cerr << msg << ": The size of the data vectors do not match the original ones\n"; + delete readFixture; + return 1; + } + delete readFixture; + } + return 0; +} + + +char *DoNothingAllocator(char* input, size_t, size_t) +{ + // We 'could' check that the requested memory in under what we + // preallocated. + return input; +} + +int testReadWriteObjectAny() +{ + int errors = 0; + + // TBufferFile currently reject size larger than 2GB. + // SetBufferOffset does not check against the size, + // so we can provide and use a larger buffer. + std::vector databuffer{}; + databuffer.reserve(8 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 8 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); + + LargeByteCountsFixture fixture; + fixture.resize(100); // Small object, should be written with the regular byte count mechanism. + + auto startPos = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + + errors += readAndCheck("Small object written in regular section", b, 100); + + // Large object, should be written with the large byte count mechanism + b.SetWriteMode(); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in regular section", b, 1024 * 1024 * 256); + + // Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in long range section", b, 1024 * 1024 * 256); + + return errors; +} + + +int testByteCount() +{ + int res = unittestByteCount(); + res += testReadWriteObjectAny(); + return res; +} \ No newline at end of file From 8dc3a4625b109c79f0b3d18cdad73412b34522f8 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 16:53:46 -0500 Subject: [PATCH 20/24] io: Add test for ByteCount for 4GB+ objects --- io/io/test/testByteCount.cxx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/io/io/test/testByteCount.cxx b/io/io/test/testByteCount.cxx index a4a464f366ea3..8c9e71a6c97e0 100644 --- a/io/io/test/testByteCount.cxx +++ b/io/io/test/testByteCount.cxx @@ -182,8 +182,8 @@ int testReadWriteObjectAny() // SetBufferOffset does not check against the size, // so we can provide and use a larger buffer. std::vector databuffer{}; - databuffer.reserve(8 * 1024 * 1024 * 1024ll); - TBufferFile b(TBuffer::kWrite, 8 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); + databuffer.reserve(9 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 9 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); LargeByteCountsFixture fixture; fixture.resize(100); // Small object, should be written with the regular byte count mechanism. @@ -215,6 +215,17 @@ int testReadWriteObjectAny() b.SetBufferOffset(startPos); errors += readAndCheck("Large object written in long range section", b, 1024 * 1024 * 256); + // Very Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 200); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 1024 + 8ll); // a bit more than 4GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Very Large object written in long range section", b, 1024 * 1024 * 1024 + 8); + return errors; } From 0ccb8358a6fda89781713aeaf23453132be3493d Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:03:22 -0500 Subject: [PATCH 21/24] net: Keep TMessage to 32 bits (no real enforcement yet) --- net/net/src/TMessage.cxx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/net/net/src/TMessage.cxx b/net/net/src/TMessage.cxx index f36e8b296804c..394d403ae4b55 100644 --- a/net/net/src/TMessage.cxx +++ b/net/net/src/TMessage.cxx @@ -392,10 +392,16 @@ Int_t TMessage::Compress() fCompPos = fBufCur; bufcur = fBufComp; + if (CompLength() > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Compressed buffer length too large (%d)", CompLength()); + } tobuf(bufcur, (UInt_t)(CompLength() - sizeof(UInt_t))); Int_t what = fWhat | kMESS_ZIP; tobuf(bufcur, what); - tobuf(bufcur, Length()); // original uncompressed buffer length + if (Length() > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Original buffer length too large (%lld)", Length()); + } + tobuf(bufcur, (UInt_t)Length()); // original uncompressed buffer length return 0; } From 138f23d057084d1387dced58a19eff97f9838850 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:45:53 -0500 Subject: [PATCH 22/24] io: old StreamerInfo read/write 64 bit collections --- io/io/src/TStreamerInfoReadBuffer.cxx | 10 +++++----- io/io/src/TStreamerInfoWriteBuffer.cxx | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 4042c0a99a80a..65b23432aa4d6 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -1192,10 +1192,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, cont = contp[j]; } TVirtualCollectionProxy::TPushPop helper( newProxy, cont ); - Int_t nobjects; - b >> nobjects; - env = newProxy->Allocate(nobjects,true); - if (!nobjects && (vers>=7)) { + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + + env = newProxy->Allocate(nobjects64,true); + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { @@ -1303,7 +1303,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, for(; obj::max() && "Allocate is not yet 64 bit enabled"); + void* env = newProxy->Allocate(nobjects64, true); if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, diff --git a/io/io/src/TStreamerInfoWriteBuffer.cxx b/io/io/src/TStreamerInfoWriteBuffer.cxx index 1b09138fb45ce..9c37917f8b1fe 100644 --- a/io/io/src/TStreamerInfoWriteBuffer.cxx +++ b/io/io/src/TStreamerInfoWriteBuffer.cxx @@ -17,6 +17,7 @@ #include "TStreamer.h" #include "TStreamerElement.h" #include "TStreamerInfo.h" +#include "TStreamerInfoCollectionUtils.h" #include "TVirtualCollectionProxy.h" #include "TRefTable.h" #include "TFile.h" @@ -523,9 +524,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(int j=0;jfLength;++j) { char *cont = contp[j]; TVirtualCollectionProxy::TPushPop helper( proxy, cont ); - Int_t nobjects = cont ? proxy->Size() : 0; - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = cont ? proxy->Size() : 0; + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; @@ -587,9 +588,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(Int_t j=0; jSize(); - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = proxy->Size(); + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; From 47f8daac218178a71a9b372c16fdbdd47b6d4efe Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 2 Dec 2025 11:45:50 +0100 Subject: [PATCH 23/24] io: add unit test infrastructure for streaming large objects Uses RStreamerField as a test bed for (de-)serializing large objects with TBufferFile. --- tree/ntuple/test/CMakeLists.txt | 5 ++ tree/ntuple/test/StreamerBeyond.cxx | 1 + tree/ntuple/test/StreamerBeyond.hxx | 14 +++++ tree/ntuple/test/StreamerBeyondLinkDef.h | 5 ++ tree/ntuple/test/rfield_streamer_beyond.cxx | 70 +++++++++++++++++++++ 5 files changed, 95 insertions(+) create mode 100644 tree/ntuple/test/StreamerBeyond.cxx create mode 100644 tree/ntuple/test/StreamerBeyond.hxx create mode 100644 tree/ntuple/test/StreamerBeyondLinkDef.h create mode 100644 tree/ntuple/test/rfield_streamer_beyond.cxx diff --git a/tree/ntuple/test/CMakeLists.txt b/tree/ntuple/test/CMakeLists.txt index 0301a5480b3bf..c1a2c471da24d 100644 --- a/tree/ntuple/test/CMakeLists.txt +++ b/tree/ntuple/test/CMakeLists.txt @@ -114,6 +114,11 @@ ROOT_GENERATE_DICTIONARY(StreamerFieldDict ${CMAKE_CURRENT_SOURCE_DIR}/StreamerF MODULE rfield_streamer LINKDEF StreamerFieldLinkDef.h OPTIONS -inlineInputHeader DEPENDENCIES RIO) +ROOT_ADD_GTEST(rfield_streamer_beyond rfield_streamer_beyond.cxx StreamerBeyond.cxx LIBRARIES ROOTNTuple) +ROOT_GENERATE_DICTIONARY(StreamerBeyondDict ${CMAKE_CURRENT_SOURCE_DIR}/StreamerBeyond.hxx + MODULE rfield_streamer_beyond LINKDEF StreamerBeyondLinkDef.h OPTIONS -inlineInputHeader + DEPENDENCIES RIO) + if(MSVC) set(command ${CMAKE_COMMAND} -E env "ROOTIGNOREPREFIX=1" $) else() diff --git a/tree/ntuple/test/StreamerBeyond.cxx b/tree/ntuple/test/StreamerBeyond.cxx new file mode 100644 index 0000000000000..dbe871afe35c7 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyond.cxx @@ -0,0 +1 @@ +#include "StreamerBeyond.hxx" diff --git a/tree/ntuple/test/StreamerBeyond.hxx b/tree/ntuple/test/StreamerBeyond.hxx new file mode 100644 index 0000000000000..eca5facb34bd6 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyond.hxx @@ -0,0 +1,14 @@ +#ifndef ROOT_RNTuple_Test_StreamerBeyond +#define ROOT_RNTuple_Test_StreamerBeyond + +#include + +#include +#include + +struct StreamerBeyond { + std::vector fOne; + std::vector fTwo; +}; + +#endif diff --git a/tree/ntuple/test/StreamerBeyondLinkDef.h b/tree/ntuple/test/StreamerBeyondLinkDef.h new file mode 100644 index 0000000000000..605c9f1a3bc66 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyondLinkDef.h @@ -0,0 +1,5 @@ +#ifdef __CLING__ + +#pragma link C++ options=rntupleStreamerMode(true) struct StreamerBeyond+; + +#endif diff --git a/tree/ntuple/test/rfield_streamer_beyond.cxx b/tree/ntuple/test/rfield_streamer_beyond.cxx new file mode 100644 index 0000000000000..a655aab12a7aa --- /dev/null +++ b/tree/ntuple/test/rfield_streamer_beyond.cxx @@ -0,0 +1,70 @@ +#include +#include +#include +#include + +#include +#include +#include + +#include "StreamerBeyond.hxx" +#include "gtest/gtest.h" + +namespace { + +class FileRaii { +private: + std::string fPath; + bool fPreserveFile = false; + +public: + explicit FileRaii(const std::string &path) : fPath(path) {} + FileRaii(FileRaii &&) = default; + FileRaii(const FileRaii &) = delete; + FileRaii &operator=(FileRaii &&) = default; + FileRaii &operator=(const FileRaii &) = delete; + ~FileRaii() + { + if (!fPreserveFile) + std::remove(fPath.c_str()); + } + std::string GetPath() const { return fPath; } + + // Useful if you want to keep a test file after the test has finished running + // for debugging purposes. Should only be used locally and never pushed. + void PreserveFile() { fPreserveFile = true; } +}; + +} // anonymous namespace + +TEST(RField, StreamerBeyond) +{ + FileRaii fileGuard("test_ntuple_rfield_streamer_beyond.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + auto f = ROOT::RFieldBase::Create("f", "StreamerBeyond").Unwrap(); + EXPECT_TRUE(dynamic_cast(f.get())); + model->AddField(std::move(f)); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto ptr = writer->GetModel().GetDefaultEntry().GetPtr("f"); + ptr->fOne = std::vector(100000000, -1); + ptr->fTwo = std::vector(100000000, -2); + + writer->Fill(); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + ASSERT_EQ(1u, reader->GetNEntries()); + StreamerBeyond sb; + auto view = reader->GetView("f", &sb, "StreamerBeyond"); + + view(0); + + auto ptr = view.GetValue().GetPtr(); + EXPECT_EQ(100000000u, ptr->fOne.size()); + EXPECT_EQ(-1, ptr->fOne.at(1000)); + EXPECT_EQ(100000000u, ptr->fTwo.size()); + EXPECT_EQ(-2, ptr->fTwo.at(2000)); +} From c364f50b158db2aaeb2eee62056e375304224c16 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 11:57:14 -0500 Subject: [PATCH 24/24] io: test very large collections --- io/io/test/CMakeLists.txt | 8 + io/io/test/testLargeCollection.cxx | 442 +++++++++++++++++++++++++++++ io/io/test/testLargeCollection.ref | 8 + 3 files changed, 458 insertions(+) create mode 100644 io/io/test/testLargeCollection.cxx create mode 100644 io/io/test/testLargeCollection.ref diff --git a/io/io/test/CMakeLists.txt b/io/io/test/CMakeLists.txt index 1e31fc3d9f98f..bb52a5aa010d1 100644 --- a/io/io/test/CMakeLists.txt +++ b/io/io/test/CMakeLists.txt @@ -62,4 +62,12 @@ if(NOT _32BIT) MACRO testByteCount.cxx+ OUTREF testByteCount.ref FIXTURES_REQUIRED io-io-tobj-fixture) + + ROOTTEST_COMPILE_MACRO(testLargeCollection.cxx + FIXTURES_SETUP io-io-testcoll-fixture) + + ROOTTEST_ADD_TEST(testLargeCollections + MACRO testLargeCollection.cxx+ + OUTREF testLargeCollection.ref + FIXTURES_REQUIRED io-io-testcoll-fixture) endif() diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx new file mode 100644 index 0000000000000..95a8c4488e6ed --- /dev/null +++ b/io/io/test/testLargeCollection.cxx @@ -0,0 +1,442 @@ +/* Test large collections + + Need to test: + - direct store in a TBufferFile + - store as part of an object + - those 2 things both for numerical type and structs. + - at least one nested test. + - (maybe not) as part of a split TTree. +*/ + +#include "TBufferFile.h" +#include "TClass.h" + +#include +#include +#include + +// ----------------------------------------------------------------------- +// Helper: a non-trivial struct so we exercise the object-array path +// ----------------------------------------------------------------------- +struct DataPoint { + float x{0}, y{0}, z{0}; + bool operator==(const DataPoint &o) const { return x == o.x && y == o.y && z == o.z; } +}; + +// ----------------------------------------------------------------------- +// Helper: an object that owns a large numerical vector and a large struct +// vector, with a minimal hand-written Streamer so we control the layout. +// ----------------------------------------------------------------------- +struct LargeCollectionFixture { + std::vector fFloats; + std::vector fPoints; + +#if 0 + void Streamer(TBuffer &b) + { + if (b.IsReading()) { + Long64_t nf = 0, np = 0; + b >> nf; + fFloats.resize(nf); + b.ReadFastArray(fFloats.data(), nf); + b >> np; + fPoints.resize(np); + b.ReadFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } else { + Long64_t nf = fFloats.size(), np = fPoints.size(); + b << nf; + b.WriteFastArray(fFloats.data(), nf); + b << np; + b.WriteFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } + } +#endif +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class DataPoint + ; +#pragma link C++ class LargeCollectionFixture + ; +#endif + +// ----------------------------------------------------------------------- +// A do-nothing reallocator: we pre-allocate the buffer ourselves. +// ----------------------------------------------------------------------- +static char *DoNothingAllocator(char *input, size_t, size_t) +{ + return input; +} + +// ----------------------------------------------------------------------- +// 1. Direct store of a numerical array in a TBufferFile +// ----------------------------------------------------------------------- +int testDirectNumerical() +{ + int errors = 0; + + // Use a pre-allocated 6 GB region so TBuffer never has to realloc. + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + // --- small array (stays in the <2 GB region) --- + const Long64_t smallN = 1000; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = float(i) * 0.1f; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), smallN); + + // --- large array (crosses the 2 GB boundary) --- + // 512 M floats = 2 GB of payload + const Long64_t largeN = 512 * 1024 * 1024ll; + std::vector orig_large(largeN); + for (Long64_t i = 0; i < largeN; ++i) + orig_large[i] = float(i % 65536); + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + b.WriteFastArray(orig_large.data(), largeN); + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectNumerical: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (got != orig_small) { + std::cerr << "testDirectNumerical: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectNumerical: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (got != orig_large) { + std::cerr << "testDirectNumerical: large array content mismatch\n"; + int nprinted = 0; + for (Long64_t i = 0; i < n && nprinted < 10; ++i) { + if (got[i] != orig_large[i]) { + std::cerr << " [" << i << "] expected " << orig_large[i] << " got " << got[i] << '\n'; + ++nprinted; + } + } + ++errors; + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2. Direct store of a struct array in a TBufferFile +// ----------------------------------------------------------------------- +int testDirectStruct() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *pointClass = TClass::GetClass(typeid(DataPoint)); + + // --- small struct array --- + const Long64_t smallN = 500; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), pointClass, smallN); + + // --- large struct array (crosses 2 GB) --- + // ~170 M Points * 12 bytes = ~2 GB + const Long64_t largeN = 170 * 1024 * 1024ll; + std::vector orig_large(largeN); + for (Long64_t i = 0; i < largeN; ++i) + orig_large[i] = {float(i % 1000), float((i + 1) % 1000), float((i + 2) % 1000)}; + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + b.WriteFastArray(orig_large.data(), pointClass, largeN); + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectStruct: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), pointClass, n); + if (got != orig_small) { + std::cerr << "testDirectStruct: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectStruct: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), pointClass, n); + if (got != orig_large) { + std::cerr << "testDirectStruct: large array content mismatch\n"; + ++errors; + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2b. Direct store of std::vector via StreamObject +// ----------------------------------------------------------------------- +int testDirectVector() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *floatVecClass = TClass::GetClass("vector"); + auto *pointVecClass = TClass::GetClass("vector"); + + // --- small std::vector --- + std::vector orig_small_f(1000); + for (int i = 0; i < 1000; ++i) + orig_small_f[i] = float(i) * 0.5f; + + auto startSmallF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_f, floatVecClass); + + // --- large std::vector (512 M entries = 2 GB) --- + std::vector orig_large_f(512 * 1024 * 1024ll); + for (size_t i = 0; i < orig_large_f.size(); ++i) + orig_large_f[i] = float(i % 65536); + + auto startLargeF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_large_f, floatVecClass); + + // --- small std::vector --- + std::vector orig_small_p(500); + for (int i = 0; i < 500; ++i) + orig_small_p[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmallP = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_p, pointVecClass); + + // --- read back small float vector --- + b.SetReadMode(); + b.SetBufferOffset(startSmallF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (got != orig_small_f) { + std::cerr << "testDirectVector: small float vector content mismatch\n"; + ++errors; + } + } + + // --- read back large float vector --- + b.SetBufferOffset(startLargeF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (got != orig_large_f) { + std::cerr << "testDirectVector: large float vector content mismatch\n"; + ++errors; + } + } + + // --- read back small DataPoint vector --- + b.SetBufferOffset(startSmallP); + { + std::vector got; + b.StreamObject(&got, pointVecClass); + if (got != orig_small_p) { + std::cerr << "testDirectVector: small DataPoint vector content mismatch\n"; + ++errors; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 3. Store as part of an object (numerical + struct members) +// ----------------------------------------------------------------------- +static int readAndCheckFixture(const std::string &msg, TBufferFile &b, Long64_t startPos, size_t expectedFloats, + size_t expectedPoints) +{ + b.SetReadMode(); + b.SetBufferOffset(startPos); + auto *obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back object\n"; + return 1; + } + auto *f = static_cast(obj); + int errors = 0; + if (f->fFloats.size() != expectedFloats) { + std::cerr << msg << ": fFloats size mismatch: got " << f->fFloats.size() << " expected " << expectedFloats + << '\n'; + ++errors; + } + if (f->fPoints.size() != expectedPoints) { + std::cerr << msg << ": fPoints size mismatch: got " << f->fPoints.size() << " expected " << expectedPoints + << '\n'; + ++errors; + } + delete f; + return errors; +} + +int testAsPartOfObject() +{ + int errors = 0; + + std::vector raw; + raw.reserve(10 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 10 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + LargeCollectionFixture fixture; + + // --- small object (well within 2 GB region) --- + fixture.fFloats.assign(1000, 1.0f); + fixture.fPoints.assign(500, {1.f, 2.f, 3.f}); + auto startSmall = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("small object", b, startSmall, 1000, 500); + + // --- large object (floats cross 2 GB, written in regular section) --- + b.SetWriteMode(); + fixture.fFloats.assign(2 * 1024 * 1024 * 1024ll, 2.0f); // 2 G of floats (8GB) + fixture.fPoints.assign(100, {4.f, 5.f, 6.f}); + auto startLarge = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in regular section", b, startLarge, 2 * 1024 * 1024 * 1024ll, 100); + + // --- large object written past the 4 GB mark --- + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + fixture.fFloats.assign(256 * 1024 * 1024ll, 3.0f); // 1 GB of floats + fixture.fPoints.assign(200, {7.f, 8.f, 9.f}); + auto startFar = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in long-range section", b, startFar, 256 * 1024 * 1024ll, 200); + + return errors; +} + +// ----------------------------------------------------------------------- +// 4. Nested: a vector of LargeCollectionFixture objects +// ----------------------------------------------------------------------- +int testNested() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *fixtureClass = TClass::GetClass(typeid(LargeCollectionFixture)); + const Long64_t nObjects = 3; + std::vector objs(nObjects); + // Give each fixture a different size so we can verify round-trip. + objs[0].fFloats.assign(100, 1.0f); + objs[0].fPoints.assign(10, {1, 1, 1}); + objs[1].fFloats.assign(200, 2.0f); + objs[1].fPoints.assign(20, {2, 2, 2}); + objs[2].fFloats.assign(300, 3.0f); + objs[2].fPoints.assign(30, {3, 3, 3}); + + auto startPos = b.GetCurrent() - b.Buffer(); + b << nObjects; + b.WriteFastArray(objs.data(), fixtureClass, nObjects); + + b.SetReadMode(); + b.SetBufferOffset(startPos); + { + Long64_t n = 0; + b >> n; + if (n != nObjects) { + std::cerr << "testNested: object count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), fixtureClass, n); + for (Long64_t i = 0; i < n; ++i) { + if (got[i].fFloats != objs[i].fFloats || got[i].fPoints != objs[i].fPoints) { + std::cerr << "testNested: content mismatch at index " << i << '\n'; + ++errors; + } + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// Entry point +// ----------------------------------------------------------------------- +int testLargeCollection() +{ + int errors = 0; + + std::cerr << "testDirectNumerical ...\n"; + errors += testDirectNumerical(); + + std::cerr << "testDirectStruct ...\n"; + errors += testDirectStruct(); + + std::cerr << "testDirectVector ...\n"; + errors += testDirectVector(); + + std::cerr << "testAsPartOfObject ...\n"; + errors += testAsPartOfObject(); + + std::cerr << "testNested ...\n"; + errors += testNested(); + + std::cerr << "Done. errors=" << errors << '\n'; + return errors; +} diff --git a/io/io/test/testLargeCollection.ref b/io/io/test/testLargeCollection.ref new file mode 100644 index 0000000000000..54a710a0334ea --- /dev/null +++ b/io/io/test/testLargeCollection.ref @@ -0,0 +1,8 @@ +Processing io/io/test/testLargeCollection.C... +testDirectNumerical ... +testDirectStruct ... +testDirectVector ... +testAsPartOfObject ... +testNested ... +Done. errors=0 +(int) 0