From 200df9e70503c03e56a670b78befff100e1bbb72 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 18 Apr 2025 06:48:48 +0200 Subject: [PATCH 01/14] create --- SWIPs/swip-39.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 SWIPs/swip-39.md diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md new file mode 100644 index 0000000..89b30dc --- /dev/null +++ b/SWIPs/swip-39.md @@ -0,0 +1,54 @@ +--- +SWIP: +title: +author: , FirstName (@GitHubUsername) and GitHubUsername (@GitHubUsername)> +discussions-to: +status: Draft +type: +category (*only required for Standard Track): +created: +requires (*optional): +replaces (*optional): +--- + + +This is the suggested template for new SWIPs. + +Note that a SWIP number will be assigned by an editor. When opening a pull request to submit your SWIP, please use an abbreviated title in the filename, `SWIP-draft_title_abbrev.md`. + +The title should be 44 characters or less. + +## Simple Summary + +If you can't explain it simply, you don't understand it well enough." Provide a simplified and layman-accessible explanation of the SWIP. + +## Abstract + +A short (~200 word) description of the technical issue being addressed. + +## Motivation + +The motivation is critical for SWIPs that want to change the Swarm protocol. It should clearly explain why the existing protocol specification is inadequate to address the problem that the SWIP solves. SWIP submissions without sufficient motivation may be rejected outright. + +## Specification + +The technical specification should describe the syntax and semantics of any new feature. The specification should be detailed enough to allow competing, interoperable implementations for the current Swarm platform and future client implementations. + +## Rationale + +The rationale fleshes out the specification by describing what motivated the design and why particular design decisions were made. It should describe alternate designs that were considered and related work, e.g. how the feature is supported in other languages. The rationale may also provide evidence of consensus within the community, and should discuss important objections or concerns raised during discussion. + +## Backwards Compatibility + +All SWIPs that introduce backwards incompatibilities must include a section describing these incompatibilities and their severity. The SWIP must explain how the author proposes to deal with these incompatibilities. SWIP submissions without a sufficient backwards compatibility treatise may be rejected outright. + +## Test Cases + +Test cases for an implementation are mandatory for SWIPs that are affecting changes to data and message formats. Other SWIPs can choose to include links to test cases if applicable. + +## Implementation + +The implementations must be completed before any SWIP is given status "Final", but it need not be completed before the SWIP is accepted. While there is merit to the approach of reaching consensus on the specification and rationale before writing code, the principle of "rough consensus and running code" is still useful when it comes to resolving many discussions of API details. + +## Copyright +Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). From c2bf1acf42827eef8b2591ed201208041d6eeca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 09:53:02 +0200 Subject: [PATCH 02/14] Update swip-39.md --- SWIPs/swip-39.md | 440 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 402 insertions(+), 38 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 89b30dc..cb51768 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -1,54 +1,418 @@ --- -SWIP: -title: -author: , FirstName (@GitHubUsername) and GitHubUsername (@GitHubUsername)> -discussions-to: +SWIP: 39 +title: Balanced Neighbourhood Registry aka Smart Neighbourhood Management +author: Viktor Trón (@zelig) +discussions-to: https://discord.gg/Q6BvSkCv status: Draft -type: -category (*only required for Standard Track): -created: -requires (*optional): -replaces (*optional): +type: Standards Track +category: Core +created: 2025-07-21 --- - -This is the suggested template for new SWIPs. +# Balanced Neighbourhood Registry aka Smart Neighbourhood Management -Note that a SWIP number will be assigned by an editor. When opening a pull request to submit your SWIP, please use an abbreviated title in the filename, `SWIP-draft_title_abbrev.md`. +## Abstract -The title should be 44 characters or less. +This SWIP introduces a systematic way for operators to enter a service network in such a way that their overlay addresses are balanced within the address space. Importantly, operators need to register their commitment to take part and are assigned a random neighbourhood (of depth $d=int(log_2(N))+1$). -## Simple Summary - -If you can't explain it simply, you don't understand it well enough." Provide a simplified and layman-accessible explanation of the SWIP. +## Motivation -## Abstract - -A short (~200 word) description of the technical issue being addressed. +There are multiple considerations that motivate such a scheme: +- **load-balancing**: any decentralised service network will be fair if tasks are distributed to nodes so that the workload assigned to each participant is roughly equal. Such load balancing is achieved if tasks are assigned based on uniform random label (i.e., the content hash of the descriptor)[^1] and nodes providing the service are balanced in the address space. -## Motivation - -The motivation is critical for SWIPs that want to change the Swarm protocol. It should clearly explain why the existing protocol specification is inadequate to address the problem that the SWIP solves. SWIP submissions without sufficient motivation may be rejected outright. +[^1]: and the average size (computation/storage requirement) of tasks over a typical period of payment has tolerable variance. + +- **arbitrary neighbourhood assigment**: the system needs to make sure that assignment of an overlay address to participant nodes is arbitrary. In particular, it is impractical (expensive) for any operator to attempt to place several nodes in the same storage neighbourhood (neighbourhood of depth $d=int(log_2(N))+1$). Note that, this scheme constitutes an effective solution to the problem of "one operator, one node in a neighbourhood". Taking the storage incentive system as an example, this will take care of the sybil issue[^2] without resorting to the rather weak incentive of additive stake as a proof of redundancy.[^3] + +[^2]: that operators run several nodes in one neighbourhood without truely replicating storage and yet get paid. + +[^3]: the idea is that if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. + +## Architecture + +The balanced neighbourhood assignment (associated with a service) is orchestrated by a smart contract which is deployed together with a [staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). The contract API provides 2 transactional endpoints relevant for entry: + +One registers a node by its ether address ($a_\xi$) in the *committers' list* $C$ that records nodes' commitment to partpicipate as a provider in the associated decentralised service network. + +The other one is called by the staking contract after the service network stake is deposited with a valid nonce, i.e., one that will put the node in the right neighbourhood. This call will place the node among the active node set for the service, and removes the entry from the committers list. This function includes a read-only call that takes as argument a node's ether address and returns a neighbourhood the node is assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. ## Specification - -The technical specification should describe the syntax and semantics of any new feature. The specification should be detailed enough to allow competing, interoperable implementations for the current Swarm platform and future client implementations. -## Rationale - -The rationale fleshes out the specification by describing what motivated the design and why particular design decisions were made. It should describe alternate designs that were considered and related work, e.g. how the feature is supported in other languages. The rationale may also provide evidence of consensus within the community, and should discuss important objections or concerns raised during discussion. +### Registration +An initially empty list (*committers' list*) of *entry struct* types holds the current committers. The struct holds information about the ether address of the node, the blockheight the address registered at. + + + +#### Deposit +In order for a node to get its address registered, an amount of $S_0$ (stake zero) must be deposited. + +#### Uniqueness +In order to prevent repeated trials, each node must be registered only once. + +After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_n=$) to the end of committers list. + +#### Validity +The entry is valid for a period of $B$ blocks after the registration[^55]. + +[^55]: $B$ is less than $256$, the number of blocks available from within the EVM). + +Since the blockheight values of the list items are monotonically increasing, entries at the beginning of the list expire first. By iterating upto the first valid entry, expired entries can be iterated on efficiently. + +### Expiry + +This function call iterates through all expired entries, burns their deposit, and, by setting the head of the list to the first valid item, removes them from the committer's list. + +This is called by the assign function (itself called by the staking contract) before the read only call checking if the resulting overlay address falls into the neighbourhood that the registrant was assigned to, i.e., the correctness of the nonce submitted from the perspective of the staking contract. + +### Assign + +The assign call is the second transactional endpoint called by the staking contract. It takes the provider's ether address and as well as the mined overlay as arguments. +After calling expiry, the validity of the registration is checked by finding the entry for the ether address in the committers' list. + +#### Initialisation + +Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remainding unassigned neighbouprhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered node's overlay addresses. +Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a^O_i$ at position $i$ copy $a^O_i$ to position $2\cdot i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: + +$$ +b_i=a^O_i[d/8] +$$ + +$$ +b_i\gg=7-(d\mod 8) +$$ + +$$ +b_i\,\&\hspace{-3pt}=\mathtt{0x1} +$$ + +In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1) depending on the subsequent bit in their overlay. + +$$ +\forall 0\leq i < 2^d, A_d[2\cdot i+b_i]=A_{d-1}[i] +$$ + +As we are filling the assignment list, we know that the whenever a neighbourhood is filled with an already existing node, its sister neighbourhood will be unassigned, therefore we can just record those in the remaining list. + +$$ +\forall 0\leq i < R_d[i]=2\cdot i+1-b_i +$$ + + +#### Random seed + +This internal read-only call takes as its single argument an ether address (a_\Xi) and returns a random $uint256$. +After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash it to yield what will serve as the random seed for this provider.[^4] + +$$ +\sigma:=\mathit{blockAtHeight}(\mathit{entry}(a_\Xi).\mathit{height}+1).\mathit{difficulty}() +$$ + +$$ +\varrho:=\mathit{uint256}(H(\sigma|a_\Xi)) +$$ + +[^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+D$ with $D$ being the validity period in blocks. + +[^4]: Even if on a POA chain, and no randao, this seed cannot be known to the registering provider and their colluding associates, but nonetheless should be deterministic once its set. + +#### Neighbourhood + +This component must be available as a public readonly endpoint taking a node's ($n$) ether address ($a^\Xi_n$) as a single argument. +The random nonce $\varrho_n$ is used to select a neighbourhood $nh$ for a provider from the remaining unassigned neighbourhood list of level $d$: + +$$ +i:=\varrho\mod len(R_d) +$$ + +$$ +nh=R\,[\,i\,] +$$ + +#### Checking the overlay + +The overlay (obtained by mining the nonce) is checked to fall in the correct neighbourhood r: +The check validates the address $a^O_n$ if and only if: + +$$ +r=a^O_n\gg(255-d) +$$ + +#### Assignment + +If the overlay check passes, + +- the nodes' overlay address is assigned to a neighbourhood of depth $d$. + +$$ +A_d[r]=a^O_n +$$ + +- $N$ is incremented +- the $i$-th item is removed from remaining open neighbourhood list $R_d$. +- if $R_d$ is now of zero length, the $d$ is incremented, and new assignment and remaining lists are initialised as per section 'initialisation' above. +- provider's entry is removed from the committers' list. + + +### Further endpoints + +A public read-only endpoint exists for querying neighbourhoods as well as nodes. Accessor for $d$ and $N$ will return the current neighbourhood depth and the current number of assigned neighbourhoods. A public accessor for $A_d$ will return for a neighbourhood (between $0$ and $2^d-1 inclusive) the overlay of the node assigned to that neighbourhood. Another endpoint will return for any overlay $o$ the closest node, so that the network service can find responsible nodes for any task with address in the space shared by overlays: + +$$ +g(a)=A_d[a\gg(255-d)] +$$ + +### Deregistration + +Only called from the Staking contract, deregister deletes the entry for the neighbourhood belonging to the given address, makes the neighbourhood available in $R$ + + +## Implementation notes + +### Changes to the staking contract + +### Changes to the bee client + +A new endpoint to bee client must be added to register a node that is not yet registered to be assigned a neighbourhood. Once the neighbourhood is known, the client can mine the nonce needed to place the overlay in the required neighbourhood. + +### Migration + +Since a new updated staking contract, a stake migration will be needed for the upgrade. Before the change, all the simplification of the staking contract is recommended, especially to allow fixed stake in order to realign redundancy +of storage and monetary incentive: with a fixed amount staked, total stake is linearly proportional to the number of nodes, and therefore comparisons across neighbourhoods can be made based on the number of nodes. In particular, the arbitrary balanced assignment makes sense in terms of incentives (expected revenue). + +### Putting a node in each neighbourhood. + +## Contract + +```sol +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.19; + +contract BalancedNeighbourhoodRegistry { + // -------------------- + // Configurable values + // -------------------- + uint256 public constant STAKE = 10,000,000,000,000,000 wei; // Stake required + uint256 public constant VALID_FOR = 128; // Validity window in blocks + + // -------------------- + // Structs and Storage + // -------------------- + struct Entry { + address committer; + uint256 height; + } + + Entry[] public committers; // List of committers (registered but as yet unassigned nodes) + mapping(address => bool) public hasCommitted; // Track if an address has committed + + uint256 public N; // Number of assigned nodes + uint256 public d = 1; // Neighbourhood depth + uint256 public currentPower = 2; // 2^d + + // Assignments: overlay address for neighbourhoods of depth d + mapping(uint256 => address) public A; + + // Remaining unassigned neighbourhoods of depth d + uint256[] public R; + + // Overlay lookup + mapping(address => bytes32) public overlays; + + // -------------------- + // Events + // -------------------- + event Registered(address indexed node, uint256 blockHeight); + event Assigned(address indexed node, uint256 neighbourhood, bytes32 overlay); + event DepthUpgraded(uint256 newDepth); + event Deregistered(address indexed node, uint256 neighbourhood); + + // -------------------- + // Registration endpoint + // -------------------- + function register() external payable { + require(msg.value == STAKE, "Invalid stake"); + require(!hasCommitted[msg.sender], "Already registered"); + + committers.push(Entry(msg.sender, block.number)); + hasCommitted[msg.sender] = true; + + emit Registered(msg.sender, block.number); + } + + // -------------------- + // Expire old entries + // -------------------- + function _expire() internal { + while (committers.length > 0) { + Entry storage e = committers[0]; + if (block.number <= e.height + VALID_FOR) { + break; + } + hasCommitted[e.committer] = false; + _removeCommitter(0); + // Burn logic: funds stay locked. + } + } + + // -------------------- + // Remove committer from the list + // -------------------- + // This function is used internally to remove a committer from the list. + // It shifts the elements to the left and pops the last element. + function _removeCommitter(uint index) internal { + if (index >= committers.length) return; + + for (uint i = index; i < committers.length - 1; i++) { + committers[i] = committers[i + 1]; + } + committers.pop(); + } + + // -------------------- + // Find entry for a committer in the committer list + // Returns the index of the entry if found, otherwise reverts. + // -------------------- + function _findEntryFor(address _a) internal view returns (uint) { + for (uint i = 0; i < committers.length; i++) { + if (committers[i].committer == _a) { + return ì; + } + } + revert("Not registered"); + } + + // -------------------- + // Randomness + // -------------------- + function _randomSeed(address _a) internal view returns (uint256) { + uint memory i = _findEntryFor(_a); + uint256 h = committers[i].height; + // Ensure the block number is valid as expire may not have been called + require(block.number > h + 1, "Too early"); + require(block.number <= h + VALID_FOR, "Registration expired"); + // Use blockhash to generate a random seed + bytes32 bh = blockhash(h + 1); + return uint256(keccak256(abi.encodePacked(bh, _a))); + } + + // -------------------- + // Public View: Neighbourhood + // -------------------- + function getNeighbourhood(address _a) public view returns (uint256) { + uint256 r = _randomSeed(_a); + require(R.length > 0, "No available neighbourhoods"); + return R[r % R.length]; + } + + // -------------------- + // Assign node overlay to neighbourhood + // -------------------- + function assign(address _a, bytes32 _overlay) external { + _expire(); + + // Check registration + uint256 nh = getNeighbourhood(_a); + uint256 overlayNh = uint256(_overlay) >> (256 - d); + + require(overlayNh == nh, "Overlay doesn't match neighbourhood"); + + A[nh] = _overlay; + overlays[_a] = _overlay; + N++; + + // Remove neighbourhood from R + _removeFromR(nh); + + // Check if R is empty + if (R.length == 0) { + _upgradeDepth(); + } + + hasCommitted[_a] = false; + _removeEntry(_a); + + emit Assigned(_a, nh, _overlay); + } + + function deregister(address a) external { + overlay = overlays[a]; + require(overlay != bytes32(0), "Not registered"); + overlays[a] = bytes32(0); + uint nh = uint256(overlay) >> (256 - d); + A[nh] = bytes32(0); + R.push(nh); + N--; + // return funds to the committer + payable(a).transfer(STAKE); + emit Deregistered(a, nh); + } + + // -------------------- + // Internal functions to manage R and committers + // -------------------- + function _removeFromR(uint256 nh) internal { + for (uint i = 0; i < R.length; i++) { + if (R[i] == nh) { + // replace the removed element with the last element and pop + R[i] = R[R.length - 1]; + R.pop(); + return; + } + } + } + + function _removeEntry(address _a) internal { + uint i = _findEntryFor(_a); + require(i < committers.length, "Entry not found"); + _removeCommitter(i); + } + + // -------------------- + // Expand A and R when needed + // -------------------- + function _upgradeDepth() internal { + currentPower = 2 ** d; + + R = new uint[](currentPower); + // A.length *= 2 // does this not work to extend to its double with zero values? + for (uint i = 0; i < currentPower; i++) { + // Ensure A has enough space for the new neighbourhoods); + A.push(bytes32(0)); + } + for (uint i = currentPower - 1; i < currentPower; i--) { + uint b = uint256(A[i]) >> (256 - d) % 2; + A[2*i+b] = A[i]; + j = 2*i+1-b; + if (2 * j < currentPower) { + A[j] = bytes32(0); // Clear the old address + } + R[i] = j + } + d++; + require(d <= 32, "Maximum depth exceeded"); + emit DepthUpgraded(d); + } + + // -------------------- + // Accessors + // -------------------- + function getDepth() external view returns (uint) { + return d; + } -## Backwards Compatibility - -All SWIPs that introduce backwards incompatibilities must include a section describing these incompatibilities and their severity. The SWIP must explain how the author proposes to deal with these incompatibilities. SWIP submissions without a sufficient backwards compatibility treatise may be rejected outright. + function getN() external view returns (uint256) { + return N; + } -## Test Cases - -Test cases for an implementation are mandatory for SWIPs that are affecting changes to data and message formats. Other SWIPs can choose to include links to test cases if applicable. + function getOverlayForNeighbourhood(uint256 nh) external view returns (bytes32) { + return A[nh]; + } -## Implementation - -The implementations must be completed before any SWIP is given status "Final", but it need not be completed before the SWIP is accepted. While there is merit to the approach of reaching consensus on the specification and rationale before writing code, the principle of "rough consensus and running code" is still useful when it comes to resolving many discussions of API details. + function getClosestNode(bytes32 key) external view returns (bytes32) { + uint256 prefix = uint256(key) >> (256 - d); + return A[prefix]; + } +} +``` -## Copyright -Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). From 7cca5af21453781f33847b5992f53b0f2cdfd6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:44:21 +0200 Subject: [PATCH 03/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index cb51768..dae8395 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -32,7 +32,7 @@ There are multiple considerations that motivate such a scheme: The balanced neighbourhood assignment (associated with a service) is orchestrated by a smart contract which is deployed together with a [staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). The contract API provides 2 transactional endpoints relevant for entry: -One registers a node by its ether address ($a_\xi$) in the *committers' list* $C$ that records nodes' commitment to partpicipate as a provider in the associated decentralised service network. +One registers a node by its ether address ($a_\xi$) in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the associated decentralised service network. The other one is called by the staking contract after the service network stake is deposited with a valid nonce, i.e., one that will put the node in the right neighbourhood. This call will place the node among the active node set for the service, and removes the entry from the committers list. This function includes a read-only call that takes as argument a node's ether address and returns a neighbourhood the node is assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. From 955db1561d1ed0791928d2da4e544f4a487f5d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:44:44 +0200 Subject: [PATCH 04/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index dae8395..57a4dd3 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -71,7 +71,7 @@ After calling expiry, the validity of the registration is checked by finding the #### Initialisation -Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remainding unassigned neighbouprhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered node's overlay addresses. +Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered node's overlay addresses. Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a^O_i$ at position $i$ copy $a^O_i$ to position $2\cdot i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: $$ From d4cd473f9154f99a181fd92638e35e22941df811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:45:03 +0200 Subject: [PATCH 05/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 57a4dd3..2c4a0db 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -276,7 +276,7 @@ contract BalancedNeighbourhoodRegistry { function _findEntryFor(address _a) internal view returns (uint) { for (uint i = 0; i < committers.length; i++) { if (committers[i].committer == _a) { - return ì; + return i; } } revert("Not registered"); From 537a60fbf52166c1c00d94eb1f39a4f37d5318a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:45:40 +0200 Subject: [PATCH 06/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 2c4a0db..77a37da 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -286,7 +286,7 @@ contract BalancedNeighbourhoodRegistry { // Randomness // -------------------- function _randomSeed(address _a) internal view returns (uint256) { - uint memory i = _findEntryFor(_a); + uint i = _findEntryFor(_a); uint256 h = committers[i].height; // Ensure the block number is valid as expire may not have been called require(block.number > h + 1, "Too early"); From 5781c8b1b5e67f4a1e41ed94da641fcccf2dd732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:47:08 +0200 Subject: [PATCH 07/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 77a37da..b7e6c0d 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -383,7 +383,7 @@ contract BalancedNeighbourhoodRegistry { for (uint i = currentPower - 1; i < currentPower; i--) { uint b = uint256(A[i]) >> (256 - d) % 2; A[2*i+b] = A[i]; - j = 2*i+1-b; + uint256 j = 2*i+1-b; if (2 * j < currentPower) { A[j] = bytes32(0); // Clear the old address } From a71cad1dfc36b16155aa74c7ded551cbe7d26295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:47:24 +0200 Subject: [PATCH 08/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index b7e6c0d..3022bb0 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -387,7 +387,7 @@ contract BalancedNeighbourhoodRegistry { if (2 * j < currentPower) { A[j] = bytes32(0); // Clear the old address } - R[i] = j + R[i] = j; } d++; require(d <= 32, "Maximum depth exceeded"); From fff7410af0d6e0235bb812c7ae4f42a35f5b0933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 20:50:29 +0200 Subject: [PATCH 09/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 3022bb0..73903d5 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -211,7 +211,7 @@ contract BalancedNeighbourhoodRegistry { uint256 public currentPower = 2; // 2^d // Assignments: overlay address for neighbourhoods of depth d - mapping(uint256 => address) public A; + mapping(uint256 => bytes32) public A; // Remaining unassigned neighbourhoods of depth d uint256[] public R; From c0edaffea28a96ddab0ecc2e940f70b96965b5ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 21 Jul 2025 22:38:59 +0200 Subject: [PATCH 10/14] Update swip-39.md --- SWIPs/swip-39.md | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 73903d5..f4316e8 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -32,7 +32,7 @@ There are multiple considerations that motivate such a scheme: The balanced neighbourhood assignment (associated with a service) is orchestrated by a smart contract which is deployed together with a [staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). The contract API provides 2 transactional endpoints relevant for entry: -One registers a node by its ether address ($a_\xi$) in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the associated decentralised service network. +One registers a node by its ether address ($a^\Xi$) in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the associated decentralised service network. The other one is called by the staking contract after the service network stake is deposited with a valid nonce, i.e., one that will put the node in the right neighbourhood. This call will place the node among the active node set for the service, and removes the entry from the committers list. This function includes a read-only call that takes as argument a node's ether address and returns a neighbourhood the node is assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. @@ -49,7 +49,7 @@ In order for a node to get its address registered, an amount of $S_0$ (stake ze #### Uniqueness In order to prevent repeated trials, each node must be registered only once. -After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_n=$) to the end of committers list. +After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_n=$) to the end of committers list. #### Validity The entry is valid for a period of $B$ blocks after the registration[^55]. @@ -71,7 +71,7 @@ After calling expiry, the validity of the registration is checked by finding the #### Initialisation -Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered node's overlay addresses. +Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered nodes' overlay addresses. Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a^O_i$ at position $i$ copy $a^O_i$ to position $2\cdot i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: $$ @@ -82,11 +82,12 @@ $$ b_i\gg=7-(d\mod 8) $$ + $$ -b_i\,\&\hspace{-3pt}=\mathtt{0x1} +b_i{\land\hspace{-3pt}=}1 $$ -In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1) depending on the subsequent bit in their overlay. +In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1$) depending on the subsequent bit in their overlay. $$ \forall 0\leq i < 2^d, A_d[2\cdot i+b_i]=A_{d-1}[i] @@ -95,21 +96,21 @@ $$ As we are filling the assignment list, we know that the whenever a neighbourhood is filled with an already existing node, its sister neighbourhood will be unassigned, therefore we can just record those in the remaining list. $$ -\forall 0\leq i < R_d[i]=2\cdot i+1-b_i +\forall 0\leq i < len(R_d), R_d[i]=2\cdot i+1-b_i $$ #### Random seed -This internal read-only call takes as its single argument an ether address (a_\Xi) and returns a random $uint256$. +This internal read-only call takes as its single argument an ether address ($a^\Xi$) and returns a random $uint256$. After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash it to yield what will serve as the random seed for this provider.[^4] $$ -\sigma:=\mathit{blockAtHeight}(\mathit{entry}(a_\Xi).\mathit{height}+1).\mathit{difficulty}() +\sigma:=\mathit{blockAtHeight}(\mathit{entry}(a^\Xi).\mathit{height}+1).\mathit{difficulty}() $$ $$ -\varrho:=\mathit{uint256}(H(\sigma|a_\Xi)) +\varrho:=\mathit{uint256}(H(\sigma|a^\Xi)) $$ [^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+D$ with $D$ being the validity period in blocks. @@ -126,7 +127,7 @@ i:=\varrho\mod len(R_d) $$ $$ -nh=R\,[\,i\,] +nh=R[i] $$ #### Checking the overlay @@ -211,21 +212,18 @@ contract BalancedNeighbourhoodRegistry { uint256 public currentPower = 2; // 2^d // Assignments: overlay address for neighbourhoods of depth d - mapping(uint256 => bytes32) public A; + bytes32[] public A; // Remaining unassigned neighbourhoods of depth d uint256[] public R; - // Overlay lookup - mapping(address => bytes32) public overlays; - // -------------------- // Events // -------------------- event Registered(address indexed node, uint256 blockHeight); event Assigned(address indexed node, uint256 neighbourhood, bytes32 overlay); event DepthUpgraded(uint256 newDepth); - event Deregistered(address indexed node, uint256 neighbourhood); + event Unassigned(address indexed node, uint256 neighbourhood); // -------------------- // Registration endpoint @@ -318,7 +316,6 @@ contract BalancedNeighbourhoodRegistry { require(overlayNh == nh, "Overlay doesn't match neighbourhood"); A[nh] = _overlay; - overlays[_a] = _overlay; N++; // Remove neighbourhood from R @@ -335,17 +332,17 @@ contract BalancedNeighbourhoodRegistry { emit Assigned(_a, nh, _overlay); } - function deregister(address a) external { - overlay = overlays[a]; - require(overlay != bytes32(0), "Not registered"); - overlays[a] = bytes32(0); - uint nh = uint256(overlay) >> (256 - d); + //---------------------- + // unregister + //---------------------- + function unregister(address _a, bytes32 _overlay) external { + uint nh = uint256(_overlay) >> (256 - d); A[nh] = bytes32(0); R.push(nh); N--; // return funds to the committer - payable(a).transfer(STAKE); - emit Deregistered(a, nh); + payable(_a).transfer(STAKE); + emit Unassigned(_a, nh); } // -------------------- From f97b73d7644718614c529b3c9a8c6d2d74e078ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Fri, 25 Jul 2025 01:36:52 +0200 Subject: [PATCH 11/14] Update SWIPs/swip-39.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- SWIPs/swip-39.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index f4316e8..c7fdb28 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -371,9 +371,9 @@ contract BalancedNeighbourhoodRegistry { function _upgradeDepth() internal { currentPower = 2 ** d; - R = new uint[](currentPower); - // A.length *= 2 // does this not work to extend to its double with zero values? + delete R; for (uint i = 0; i < currentPower; i++) { + R.push(0); // Ensure A has enough space for the new neighbourhoods); A.push(bytes32(0)); } From 77f3efcf1af50b1d193cb216c761f5fe4e0a4d9e Mon Sep 17 00:00:00 2001 From: significance Date: Mon, 11 Aug 2025 11:46:23 +0100 Subject: [PATCH 12/14] trivial changes, standardised some orthography, mathematical expression (#75) --- SWIPs/swip-39.md | 91 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index c7fdb28..d48749f 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -13,28 +13,55 @@ created: 2025-07-21 ## Abstract -This SWIP introduces a systematic way for operators to enter a service network in such a way that their overlay addresses are balanced within the address space. Importantly, operators need to register their commitment to take part and are assigned a random neighbourhood (of depth $d=int(log_2(N))+1$). +This SWIP introduces a systematic way for operators to enter the network in such a way that their overlay addresses are balanced within the address space. Importantly, operators must register their commitment to take part and on doing so will be assigned a random neighbourhood from the queue of those next to be occupied. Since a joining node operator is unable to position themselves at a specific point in the network without a significant time or financial penalty, attack vectors that rely on this are rendered infeasible. ## Motivation There are multiple considerations that motivate such a scheme: -- **load-balancing**: any decentralised service network will be fair if tasks are distributed to nodes so that the workload assigned to each participant is roughly equal. Such load balancing is achieved if tasks are assigned based on uniform random label (i.e., the content hash of the descriptor)[^1] and nodes providing the service are balanced in the address space. -[^1]: and the average size (computation/storage requirement) of tasks over a typical period of payment has tolerable variance. +- **load-balancing**: a decentralised service network will be fair if tasks are distributed to nodes so that the workload assigned to each participant is roughly equal. Presuming load balancing is achieved if tasks[^1] are assigned based on uniform random label (i.e., the content hash of the descriptor) and nodes providing the service are balanced in the address space. -- **arbitrary neighbourhood assigment**: the system needs to make sure that assignment of an overlay address to participant nodes is arbitrary. In particular, it is impractical (expensive) for any operator to attempt to place several nodes in the same storage neighbourhood (neighbourhood of depth $d=int(log_2(N))+1$). Note that, this scheme constitutes an effective solution to the problem of "one operator, one node in a neighbourhood". Taking the storage incentive system as an example, this will take care of the sybil issue[^2] without resorting to the rather weak incentive of additive stake as a proof of redundancy.[^3] +[^1]: it is assumed that average resource utilisation (network/computation/storage requirements) of tasks over a typical period of payment remains within a tolerable variance. -[^2]: that operators run several nodes in one neighbourhood without truely replicating storage and yet get paid. +- **arbitrary neighbourhood assigment**: the system needs to make sure that assignment of an overlay address to participant nodes is arbitrary. In particular, it is impractical (expensive) for any operator to attempt to place several nodes in the same storage neighbourhood without truely replicating storage and yet get paid. Note that, this scheme constitutes an effective solution to the problem of "one operator, one node in a neighbourhood". Taking the storage incentive system as an example, this will mitigate this sybil attack, without resorting to the rather weak incentive of additive stake as a proof of redundancy.[^2] -[^3]: the idea is that if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. +[^2]: the idea is that if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. ↩ + +- **extensible and user friendly approach**: the current approach taken by swarm is based on the idea is that, if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. While this has proven to be an effective means to secure the protocol, it has also led to some confusion and it's complexity means it is difficult to develop. The proposed system is intended to be much simpler to reason with and for operators to use and understand. ## Architecture -The balanced neighbourhood assignment (associated with a service) is orchestrated by a smart contract which is deployed together with a [staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). The contract API provides 2 transactional endpoints relevant for entry: +The network $S$, comprises the set of all network provider nodes $\{a_0, \ldots, a_n\}$, with cardinality N. + +Each node $a$ has an associated ether address $a_i^\Xi$ and will determine a 32 byte overlay address $a_i^\theta$. + +$ +N = |S|, \quad S = \{a_0, \ldots, a_n\} \quad a_i^{\theta} \in \mathbb{O}_{32}\,,\ a_i^\Xi \in \mathbb{E} +$ + +Then the current depth of the node assignment tree is defined as: + +$ +d_c = \lfloor \log_2(N) \rfloor + 1, \quad d_c \in \mathbb{N} +$ + +The balanced node assignments are orchestrated by a smart contract which will maintain the active node list, their corresponding neighbourhoods, the overlay addresses they have determined, and manage the ingress/egress processes as nodes joini and exit the set of active network service providers. + +This contract is deployed together with a staking contract similar to the [swarm storage incentive staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). This contract will retain the total stake treasury, as well as enabling a node operator to deposit, withdrawal and maintain their stake. Concerns should be strictly separated to improve security of locked funds and upgradability of both contracts. + +The node assignment contract is composed of several transactional endpoints: + +__Commit :__ Initially, a node's ether address $a^\Xi$ is registered in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the service network, depositing their application fee ${\$_a}$, which is non-refundable. + +__Get Assigned Overlay :__ -One registers a node by its ether address ($a^\Xi$) in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the associated decentralised service network. + This function includes a read-only call that takes as argument a node's ether address $a^\Xi$ and returns the neighbourhood that the node is currently assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. -The other one is called by the staking contract after the service network stake is deposited with a valid nonce, i.e., one that will put the node in the right neighbourhood. This call will place the node among the active node set for the service, and removes the entry from the committers list. This function includes a read-only call that takes as argument a node's ether address and returns a neighbourhood the node is assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. +__Assign Overlay:__ One is called by the staking contract, after the service network stake has been deposited with a valid nonce $\vartheta$, i.e., one that, when it is used as a parameter with the Swarm Overlay address calculation $\mathcal{O}$, will produce an overlay address $a^\theta$ from the nodes corresponding ether address $a^\Xi$ which is in the correct neighbourhood. + +$\vartheta \in \mathbb{Z}_{\geq 0} , \quad \mathcal{O}(a,\vartheta) \equiv a^\theta $ + +This call will place the node among the active node set for the service, and removes the entry from the *committers list*. ## Specification @@ -43,24 +70,27 @@ An initially empty list (*committers' list*) of *entry struct* types holds the c + #### Deposit -In order for a node to get its address registered, an amount of $S_0$ (stake zero) must be deposited. +In order for a node to get its address registered, an amount of ${\$_a}$ must be deposited which is non-refundable. #### Uniqueness In order to prevent repeated trials, each node must be registered only once. -After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_n=$) to the end of committers list. +After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_i\ = \ $) to the end of committers list. #### Validity -The entry is valid for a period of $B$ blocks after the registration[^55]. +The entry is valid for a period of $B, \quad B < 256 $ blocks after the registration. -[^55]: $B$ is less than $256$, the number of blocks available from within the EVM). +$B$ must be less than $256$, the number of blocks for which the blockhash is available from within the EVM. ((unless the blockhash is recorded)) -Since the blockheight values of the list items are monotonically increasing, entries at the beginning of the list expire first. By iterating upto the first valid entry, expired entries can be iterated on efficiently. +Since the blockheight values of the list items are monotonically increasing, entries at the beginning of the list expire first. By iterating upto the first valid entry, expired entries can be iterated on efficiently. + +>> SIG//NOTE should have time limit on commiting to assigned neighbourhood overlay before providingn nonce. this queing has some unintended consequences in terms of enabling squatting or blocking. the economic disincentives must be calculated and parameters/constants and/or slashing formula created i.e. how much to probably block one neighbourhood then subsequently provoke a split causing data loss. ### Expiry -This function call iterates through all expired entries, burns their deposit, and, by setting the head of the list to the first valid item, removes them from the committer's list. +This function call iterates through all expired entries, burns their deposit, and, by setting the head of the list to the first valid item, removes them from the committer's list. This is called by the assign function (itself called by the staking contract) before the read only call checking if the resulting overlay address falls into the neighbourhood that the registrant was assigned to, i.e., the correctness of the nonce submitted from the perspective of the staking contract. @@ -71,11 +101,14 @@ After calling expiry, the validity of the registration is checked by finding the #### Initialisation -Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, and let $d$ be its exponent ($d=int(log_2(N))+1$). Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $len(R)=\overline{N}-N$). Whenever $len(R)$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the registered nodes' overlay addresses. -Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a^O_i$ at position $i$ copy $a^O_i$ to position $2\cdot i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: +Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, where $d$ and $N$ are defined as before. Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $|R|=\overline{N}-N$). + +Whenever the number of slots at this depth, $|R|$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the currently registered nodes' overlay addresses. + +Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a_i^\theta$ at position $i$ copy $a_i^\theta$ to position $2i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: $$ -b_i=a^O_i[d/8] +b_i=a^\theta_i[d/8] $$ $$ @@ -90,37 +123,38 @@ $$ In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1$) depending on the subsequent bit in their overlay. $$ -\forall 0\leq i < 2^d, A_d[2\cdot i+b_i]=A_{d-1}[i] +\forall\ 0\leq i < 2^d,\quad A_d[2i+b_i]=A_{d-1}[i] $$ As we are filling the assignment list, we know that the whenever a neighbourhood is filled with an already existing node, its sister neighbourhood will be unassigned, therefore we can just record those in the remaining list. $$ -\forall 0\leq i < len(R_d), R_d[i]=2\cdot i+1-b_i +\forall\ 0\leq i < |R_d|,\quad R_d[i]=2i+1-b_i $$ #### Random seed -This internal read-only call takes as its single argument an ether address ($a^\Xi$) and returns a random $uint256$. -After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash it to yield what will serve as the random seed for this provider.[^4] +This internal read-only call takes as its single argument an ether address ($a^\Xi$) and returns a (?) random $\rho \in \mathbb{Z}_\text{uint256}$. +After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash using $H$ it to yield what will serve as the random seed for this provider.[^4] $$ -\sigma:=\mathit{blockAtHeight}(\mathit{entry}(a^\Xi).\mathit{height}+1).\mathit{difficulty}() +\sigma=\mathit{blockAtHeight}(\mathit{entry}(a^\Xi).\mathit{height}+1).\mathit{difficulty}() $$ $$ -\varrho:=\mathit{uint256}(H(\sigma|a^\Xi)) +\varrho=\mathit{uint256}(H(\sigma|a^\Xi)) $$ -[^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+D$ with $D$ being the validity period in blocks. +[^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+B$ with $B$ being the validity period in blocks. [^4]: Even if on a POA chain, and no randao, this seed cannot be known to the registering provider and their colluding associates, but nonetheless should be deterministic once its set. #### Neighbourhood -This component must be available as a public readonly endpoint taking a node's ($n$) ether address ($a^\Xi_n$) as a single argument. -The random nonce $\varrho_n$ is used to select a neighbourhood $nh$ for a provider from the remaining unassigned neighbourhood list of level $d$: +This component must be available as a public readonly endpoint taking a node's ($j\text{-th}$) ether address ($a^\Xi_j$) as a single argument. + +A random nonce $\varrho_n$ is used to select a neighbourhood $nh$ for a provider from the remaining unassigned neighbourhood list of level $d$: $$ i:=\varrho\mod len(R_d) @@ -154,6 +188,9 @@ $$ - if $R_d$ is now of zero length, the $d$ is incremented, and new assignment and remaining lists are initialised as per section 'initialisation' above. - provider's entry is removed from the committers' list. +### Economic Analysis + + ### Further endpoints From 54c1fd2096437011dceeffe92d53b78a883c8649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Sat, 21 Mar 2026 08:32:46 +0100 Subject: [PATCH 13/14] Revise SWIP-39 for clarity and detail enhancement (#87) Refine the SWIP-39 document to clarify the protocol mechanism for balanced neighbourhood registry and node assignment. Enhance sections on architecture, model, and data structure to improve understanding of the system's operational principles. --- SWIPs/swip-39.md | 479 +++++++++++++---------------------------------- 1 file changed, 135 insertions(+), 344 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index d48749f..930ff0c 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -11,9 +11,15 @@ created: 2025-07-21 # Balanced Neighbourhood Registry aka Smart Neighbourhood Management + ## Abstract -This SWIP introduces a systematic way for operators to enter the network in such a way that their overlay addresses are balanced within the address space. Importantly, operators must register their commitment to take part and on doing so will be assigned a random neighbourhood from the queue of those next to be occupied. Since a joining node operator is unable to position themselves at a specific point in the network without a significant time or financial penalty, attack vectors that rely on this are rendered infeasible. +This proposal defines a protocol mechanism that enforces uniform distribution of nodes across the Swarm address space while preventing adversarial positioning. Nodes commit to participation and are assigned neighbourhoods through delayed entropy derived from on-chain randomness. The assignment is therefore unpredictable at commit time and reproducible at validation time. Since a joining node operator is unable to position themselves at a specific point in the network without a significant time or financial penalty, attack vectors that rely on this are rendered infeasible. + + +The system maintains a structural invariant over the address space ensuring that no prefix of length $d-1$ is empty. This invariant is preserved through strictly local operations, namely assignment, deregistration, and rebalance, without requiring global restructuring. The design achieves fairness, bounded operational cost, and resistance to manipulation. + + ## Motivation @@ -29,424 +35,209 @@ There are multiple considerations that motivate such a scheme: - **extensible and user friendly approach**: the current approach taken by swarm is based on the idea is that, if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. While this has proven to be an effective means to secure the protocol, it has also led to some confusion and it's complexity means it is difficult to develop. The proposed system is intended to be much simpler to reason with and for operators to use and understand. -## Architecture +A decentralised service network assumes that workload is distributed according to a uniform random process over the address space. This assumption only holds if the distribution of nodes itself is approximately uniform. In practice, allowing operators to influence their placement leads to clustering and adversarial positioning, while global rebalancing mechanisms are impractical in a smart contract setting due to their cost and coordination requirements. -The network $S$, comprises the set of all network provider nodes $\{a_0, \ldots, a_n\}$, with cardinality N. +The present design replaces behavioural assumptions with structural guarantees. Node placement is determined externally by the protocol, and the system continuously enforces coverage through a local invariant that is preserved under all admissible transitions. -Each node $a$ has an associated ether address $a_i^\Xi$ and will determine a 32 byte overlay address $a_i^\theta$. +--- -$ -N = |S|, \quad S = \{a_0, \ldots, a_n\} \quad a_i^{\theta} \in \mathbb{O}_{32}\,,\ a_i^\Xi \in \mathbb{E} -$ +## Model and Notation -Then the current depth of the node assignment tree is defined as: +Let the set of active nodes be denoted by -$ -d_c = \lfloor \log_2(N) \rfloor + 1, \quad d_c \in \mathbb{N} -$ +$$ +S = \{a_0, \ldots, a_n\}, \quad N = |S|. +$$ -The balanced node assignments are orchestrated by a smart contract which will maintain the active node list, their corresponding neighbourhoods, the overlay addresses they have determined, and manage the ingress/egress processes as nodes joini and exit the set of active network service providers. +Each node is identified by an Ethereum address $a^\Xi$ and an overlay address $a^\theta \in \mathbb{O}_{32}$. -This contract is deployed together with a staking contract similar to the [swarm storage incentive staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). This contract will retain the total stake treasury, as well as enabling a node operator to deposit, withdrawal and maintain their stake. Concerns should be strictly separated to improve security of locked funds and upgradability of both contracts. +The system maintains a depth parameter $d \in \mathbb{N}$ such that -The node assignment contract is composed of several transactional endpoints: +$$ +2^{d-1} < N \le 2^d. +$$ -__Commit :__ Initially, a node's ether address $a^\Xi$ is registered in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the service network, depositing their application fee ${\$_a}$, which is non-refundable. +Neighbourhoods are represented as indices in a complete binary space: -__Get Assigned Overlay :__ +$$ +I : \{0, \ldots, 2^d - 1\} \to \mathbb{O}_{32} \cup \{0\}, +$$ - This function includes a read-only call that takes as argument a node's ether address $a^\Xi$ and returns the neighbourhood that the node is currently assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. +where $I[i] = 0$ denotes an empty slot. A reverse mapping -__Assign Overlay:__ One is called by the staking contract, after the service network stake has been deposited with a valid nonce $\vartheta$, i.e., one that, when it is used as a parameter with the Swarm Overlay address calculation $\mathcal{O}$, will produce an overlay address $a^\theta$ from the nodes corresponding ether address $a^\Xi$ which is in the correct neighbourhood. +$$ +J : a^\Xi \mapsto i +$$ -$\vartheta \in \mathbb{Z}_{\geq 0} , \quad \mathcal{O}(a,\vartheta) \equiv a^\theta $ +associates each node with its assigned index. -This call will place the node among the active node set for the service, and removes the entry from the *committers list*. +For each index $i$, define the pair -## Specification +$$ +\mathcal{P}_i = \{2i, 2i+1\}. +$$ -### Registration -An initially empty list (*committers' list*) of *entry struct* types holds the current committers. The struct holds information about the ether address of the node, the blockheight the address registered at. - +--- +## Structural Invariant +The system enforces the condition -#### Deposit -In order for a node to get its address registered, an amount of ${\$_a}$ must be deposited which is non-refundable. +$$ +\forall i, \quad I[2i] \neq 0 \;\lor\; I[2i+1] \neq 0, +$$ -#### Uniqueness -In order to prevent repeated trials, each node must be registered only once. +which ensures that every prefix of length $d-1$ contains at least one node. This invariant defines the admissible states of the system. -After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_i\ = \ $) to the end of committers list. +--- -#### Validity -The entry is valid for a period of $B, \quad B < 256 $ blocks after the registration. +## Data Structure -$B$ must be less than $256$, the number of blocks for which the blockhash is available from within the EVM. ((unless the blockhash is recorded)) +The assignment structure is implemented as an implicit complete binary trie over the index space. Each node $v$ of the trie corresponds to a contiguous interval of indices and maintains two quantities. -Since the blockheight values of the list items are monotonically increasing, entries at the beginning of the list expire first. By iterating upto the first valid entry, expired entries can be iterated on efficiently. +The first quantity $F(v)$ denotes the number of free slots in the subtree rooted at $v$. Formally, if $L(v)$ denotes the set of leaf indices under $v$, then ->> SIG//NOTE should have time limit on commiting to assigned neighbourhood overlay before providingn nonce. this queing has some unintended consequences in terms of enabling squatting or blocking. the economic disincentives must be calculated and parameters/constants and/or slashing formula created i.e. how much to probably block one neighbourhood then subsequently provoke a split causing data loss. +$$ +F(v) = |\{ i \in L(v) \mid I[i] = 0 \}|. +$$ -### Expiry +The second quantity $E(v)$ denotes the number of indices $i$ in the subtree such that both elements of $\mathcal{P}_i$ are occupied. These correspond to candidate donor pairs. -This function call iterates through all expired entries, burns their deposit, and, by setting the head of the list to the first valid item, removes them from the committer's list. +Both quantities satisfy recursive relations -This is called by the assign function (itself called by the staking contract) before the read only call checking if the resulting overlay address falls into the neighbourhood that the registrant was assigned to, i.e., the correctness of the nonce submitted from the perspective of the staking contract. +$$ +F(v) = F(v_L) + F(v_R), \quad E(v) = E(v_L) + E(v_R), +$$ -### Assign +where $v_L$ and $v_R$ denote the left and right children of $v$. Updates propagate along the path from a leaf to the root, resulting in logarithmic complexity. -The assign call is the second transactional endpoint called by the staking contract. It takes the provider's ether address and as well as the mined overlay as arguments. -After calling expiry, the validity of the registration is checked by finding the entry for the ether address in the committers' list. +--- -#### Initialisation +## Data Structure Illustration + +```mermaid +graph TD + R((root)) + R --> A + R --> B + A --> A1 + A --> A2 + B --> B1 + B --> B2 +``` -Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, where $d$ and $N$ are defined as before. Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $|R|=\overline{N}-N$). +Each leaf corresponds to an index. Internal nodes aggregate subtree quantities $F$ and $E$. -Whenever the number of slots at this depth, $|R|$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the currently registered nodes' overlay addresses. +--- -Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a_i^\theta$ at position $i$ copy $a_i^\theta$ to position $2i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: +## Entropy -$$ -b_i=a^\theta_i[d/8] -$$ +A node that registers at block height $h$ derives its randomness from $$ -b_i\gg=7-(d\mod 8) +\rho = H(\text{blockhash}(h+1) \parallel a^\Xi \parallel h), $$ +which is not known at the time of registration. The validity window $B < 256$ ensures that the referenced blockhash remains accessible. -$$ -b_i{\land\hspace{-3pt}=}1 -$$ - -In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1$) depending on the subsequent bit in their overlay. +--- -$$ -\forall\ 0\leq i < 2^d,\quad A_d[2i+b_i]=A_{d-1}[i] -$$ +## Assignment -As we are filling the assignment list, we know that the whenever a neighbourhood is filled with an already existing node, its sister neighbourhood will be unassigned, therefore we can just record those in the remaining list. +Let $M = 2^d - N$ denote the number of free slots. A node computes $$ -\forall\ 0\leq i < |R_d|,\quad R_d[i]=2i+1-b_i +k = \rho \bmod M. $$ +The assigned index is determined by descending the trie. At a node $v$, let $F(v_L)$ denote the number of free slots in the left subtree. If $k < F(v_L)$, the traversal continues to the left child. Otherwise, the traversal continues to the right child with updated rank $k \leftarrow k - F(v_L)$. This procedure terminates at a leaf index $i$ such that $I[i] = 0$. -#### Random seed +--- -This internal read-only call takes as its single argument an ether address ($a^\Xi$) and returns a (?) random $\rho \in \mathbb{Z}_\text{uint256}$. -After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash using $H$ it to yield what will serve as the random seed for this provider.[^4] +## Deregistration and Rebalancing -$$ -\sigma=\mathit{blockAtHeight}(\mathit{entry}(a^\Xi).\mathit{height}+1).\mathit{difficulty}() -$$ +Consider a node occupying index $r$. Let $i = \lfloor r/2 \rfloor$ be the corresponding pair index. -$$ -\varrho=\mathit{uint256}(H(\sigma|a^\Xi)) -$$ +If the sibling index in $\mathcal{P}_i$ is occupied, removal proceeds directly and the invariant remains satisfied. -[^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+B$ with $B$ being the validity period in blocks. +If removal would leave $\mathcal{P}_i$ empty, a rebalance is required. A donor pair is selected using the same rank-based traversal over $E(v)$. From the selected pair, one of the two nodes is chosen and removed. The donor node is reinserted into the commit queue and assigned to the empty pair. -[^4]: Even if on a POA chain, and no randao, this seed cannot be known to the registering provider and their colluding associates, but nonetheless should be deterministic once its set. +The original node is removed only after the donor successfully completes reassignment, ensuring that the invariant is never violated. -#### Neighbourhood +--- -This component must be available as a public readonly endpoint taking a node's ($j\text{-th}$) ether address ($a^\Xi_j$) as a single argument. +## Depth Reduction -A random nonce $\varrho_n$ is used to select a neighbourhood $nh$ for a provider from the remaining unassigned neighbourhood list of level $d$: +If $N = 2^{d-1}$, every pair contains exactly one node and no donor exists. In this case, the representation is reduced by setting $d \leftarrow d-1$ and mapping each pair to a single index. This transformation is deterministic because no pair contains more than one node. -$$ -i:=\varrho\mod len(R_d) -$$ +--- -$$ -nh=R[i] -$$ +## Interface -#### Checking the overlay +| Function | Description | +|----------|------------| +| Register(a) | Records commitment and blockheight | +| GetPrefix(a) | Returns assigned index | +| Validate(a,o) | Verifies overlay matches assigned index | +| Insert(a,o) | Inserts node and updates structure | +| Deregister(a) | Initiates removal | +| Remove(i) | Removes node from index | +| Rebalance(i) | Repairs invariant | -The overlay (obtained by mining the nonce) is checked to fall in the correct neighbourhood r: -The check validates the address $a^O_n$ if and only if: +--- -$$ -r=a^O_n\gg(255-d) -$$ +## Sequence Diagram + +```mermaid +sequenceDiagram + participant Client + participant Registry + participant Trie + + Client->>Registry: Register(a) + Registry-->>Client: prefix + Client->>Registry: Validate(a,o) + Registry->>Trie: update(F,E) + Client->>Registry: Deregister(a) + Registry->>Trie: check invariant + Registry->>Trie: select donor + Registry->>Trie: update +``` -#### Assignment +--- -If the overlay check passes, +## Economic Considerations -- the nodes' overlay address is assigned to a neighbourhood of depth $d$. +Fairness of the system requires that each node is associated with a fixed stake. Under this assumption, total stake is proportional to the number of nodes, $$ -A_d[r]=a^O_n +\text{Total stake} \propto N, $$ -- $N$ is incremented -- the $i$-th item is removed from remaining open neighbourhood list $R_d$. -- if $R_d$ is now of zero length, the $d$ is incremented, and new assignment and remaining lists are initialised as per section 'initialisation' above. -- provider's entry is removed from the committers' list. +and no operator gains advantage by concentrating stake into fewer nodes. This aligns economic incentives with the structural invariant. -### Economic Analysis +--- +## Impact +The system achieves uniform distribution of nodes, resistance to adversarial placement, and predictable operational costs. Rebalancing introduces a dependency on donor behaviour; however, the resulting delay is bounded and practically absorbed by the off-boarding process, which already operates as a queue. Consequently, rebalance latency does not introduce observable instability. -### Further endpoints +--- -A public read-only endpoint exists for querying neighbourhoods as well as nodes. Accessor for $d$ and $N$ will return the current neighbourhood depth and the current number of assigned neighbourhoods. A public accessor for $A_d$ will return for a neighbourhood (between $0$ and $2^d-1 inclusive) the overlay of the node assigned to that neighbourhood. Another endpoint will return for any overlay $o$ the closest node, so that the network service can find responsible nodes for any task with address in the space shared by overlays: +## Security Analysis -$$ -g(a)=A_d[a\gg(255-d)] -$$ +The commit-and-delay entropy mechanism prevents nodes from predicting or influencing their placement. Structural constraints prevent clustering, making Sybil attacks costly and ineffective. Validator influence is limited to a single block and does not provide sufficient control to bias assignment. -### Deregistration +Rebalancing cannot be blocked as long as $N > 2^{d-1}$, since a donor must exist. If a donor fails to complete reassignment, expiry ensures that another donor is selected, guaranteeing progress. -Only called from the Staking contract, deregister deletes the entry for the neighbourhood belonging to the given address, makes the neighbourhood available in $R$ +--- +## Testing -## Implementation notes - -### Changes to the staking contract - -### Changes to the bee client - -A new endpoint to bee client must be added to register a node that is not yet registered to be assigned a neighbourhood. Once the neighbourhood is known, the client can mine the nonce needed to place the overlay in the required neighbourhood. - -### Migration - -Since a new updated staking contract, a stake migration will be needed for the upgrade. Before the change, all the simplification of the staking contract is recommended, especially to allow fixed stake in order to realign redundancy -of storage and monetary incentive: with a fixed amount staked, total stake is linearly proportional to the number of nodes, and therefore comparisons across neighbourhoods can be made based on the number of nodes. In particular, the arbitrary balanced assignment makes sense in terms of incentives (expected revenue). - -### Putting a node in each neighbourhood. - -## Contract - -```sol -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.19; - -contract BalancedNeighbourhoodRegistry { - // -------------------- - // Configurable values - // -------------------- - uint256 public constant STAKE = 10,000,000,000,000,000 wei; // Stake required - uint256 public constant VALID_FOR = 128; // Validity window in blocks - - // -------------------- - // Structs and Storage - // -------------------- - struct Entry { - address committer; - uint256 height; - } - - Entry[] public committers; // List of committers (registered but as yet unassigned nodes) - mapping(address => bool) public hasCommitted; // Track if an address has committed - - uint256 public N; // Number of assigned nodes - uint256 public d = 1; // Neighbourhood depth - uint256 public currentPower = 2; // 2^d - - // Assignments: overlay address for neighbourhoods of depth d - bytes32[] public A; - - // Remaining unassigned neighbourhoods of depth d - uint256[] public R; - - // -------------------- - // Events - // -------------------- - event Registered(address indexed node, uint256 blockHeight); - event Assigned(address indexed node, uint256 neighbourhood, bytes32 overlay); - event DepthUpgraded(uint256 newDepth); - event Unassigned(address indexed node, uint256 neighbourhood); - - // -------------------- - // Registration endpoint - // -------------------- - function register() external payable { - require(msg.value == STAKE, "Invalid stake"); - require(!hasCommitted[msg.sender], "Already registered"); - - committers.push(Entry(msg.sender, block.number)); - hasCommitted[msg.sender] = true; - - emit Registered(msg.sender, block.number); - } - - // -------------------- - // Expire old entries - // -------------------- - function _expire() internal { - while (committers.length > 0) { - Entry storage e = committers[0]; - if (block.number <= e.height + VALID_FOR) { - break; - } - hasCommitted[e.committer] = false; - _removeCommitter(0); - // Burn logic: funds stay locked. - } - } - - // -------------------- - // Remove committer from the list - // -------------------- - // This function is used internally to remove a committer from the list. - // It shifts the elements to the left and pops the last element. - function _removeCommitter(uint index) internal { - if (index >= committers.length) return; - - for (uint i = index; i < committers.length - 1; i++) { - committers[i] = committers[i + 1]; - } - committers.pop(); - } - - // -------------------- - // Find entry for a committer in the committer list - // Returns the index of the entry if found, otherwise reverts. - // -------------------- - function _findEntryFor(address _a) internal view returns (uint) { - for (uint i = 0; i < committers.length; i++) { - if (committers[i].committer == _a) { - return i; - } - } - revert("Not registered"); - } - - // -------------------- - // Randomness - // -------------------- - function _randomSeed(address _a) internal view returns (uint256) { - uint i = _findEntryFor(_a); - uint256 h = committers[i].height; - // Ensure the block number is valid as expire may not have been called - require(block.number > h + 1, "Too early"); - require(block.number <= h + VALID_FOR, "Registration expired"); - // Use blockhash to generate a random seed - bytes32 bh = blockhash(h + 1); - return uint256(keccak256(abi.encodePacked(bh, _a))); - } - - // -------------------- - // Public View: Neighbourhood - // -------------------- - function getNeighbourhood(address _a) public view returns (uint256) { - uint256 r = _randomSeed(_a); - require(R.length > 0, "No available neighbourhoods"); - return R[r % R.length]; - } - - // -------------------- - // Assign node overlay to neighbourhood - // -------------------- - function assign(address _a, bytes32 _overlay) external { - _expire(); - - // Check registration - uint256 nh = getNeighbourhood(_a); - uint256 overlayNh = uint256(_overlay) >> (256 - d); - - require(overlayNh == nh, "Overlay doesn't match neighbourhood"); - - A[nh] = _overlay; - N++; - - // Remove neighbourhood from R - _removeFromR(nh); - - // Check if R is empty - if (R.length == 0) { - _upgradeDepth(); - } - - hasCommitted[_a] = false; - _removeEntry(_a); - - emit Assigned(_a, nh, _overlay); - } - - //---------------------- - // unregister - //---------------------- - function unregister(address _a, bytes32 _overlay) external { - uint nh = uint256(_overlay) >> (256 - d); - A[nh] = bytes32(0); - R.push(nh); - N--; - // return funds to the committer - payable(_a).transfer(STAKE); - emit Unassigned(_a, nh); - } - - // -------------------- - // Internal functions to manage R and committers - // -------------------- - function _removeFromR(uint256 nh) internal { - for (uint i = 0; i < R.length; i++) { - if (R[i] == nh) { - // replace the removed element with the last element and pop - R[i] = R[R.length - 1]; - R.pop(); - return; - } - } - } - - function _removeEntry(address _a) internal { - uint i = _findEntryFor(_a); - require(i < committers.length, "Entry not found"); - _removeCommitter(i); - } - - // -------------------- - // Expand A and R when needed - // -------------------- - function _upgradeDepth() internal { - currentPower = 2 ** d; - - delete R; - for (uint i = 0; i < currentPower; i++) { - R.push(0); - // Ensure A has enough space for the new neighbourhoods); - A.push(bytes32(0)); - } - for (uint i = currentPower - 1; i < currentPower; i--) { - uint b = uint256(A[i]) >> (256 - d) % 2; - A[2*i+b] = A[i]; - uint256 j = 2*i+1-b; - if (2 * j < currentPower) { - A[j] = bytes32(0); // Clear the old address - } - R[i] = j; - } - d++; - require(d <= 32, "Maximum depth exceeded"); - emit DepthUpgraded(d); - } - - // -------------------- - // Accessors - // -------------------- - function getDepth() external view returns (uint) { - return d; - } - - function getN() external view returns (uint256) { - return N; - } - - function getOverlayForNeighbourhood(uint256 nh) external view returns (bytes32) { - return A[nh]; - } - - function getClosestNode(bytes32 key) external view returns (bytes32) { - uint256 prefix = uint256(key) >> (256 - d); - return A[prefix]; - } -} -``` +Correctness requires verifying that the invariant is preserved under all state transitions. Simulation should confirm that assignment remains uniform, that rebalancing converges, and that depth transitions occur at the correct thresholds. + +--- + +## Conclusion +This proposal replaces global restructuring with local invariant enforcement over a compact data structure. It achieves fairness, efficiency, and robustness through deterministic rules and probabilistic assignment, making it suitable for large-scale decentralised operation. From ce60560c6f967a52ac84b2f13127a4a2206cfd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Sat, 21 Mar 2026 08:51:04 +0100 Subject: [PATCH 14/14] Revert "Revise SWIP-39 for clarity and detail enhancement (#87)" (#89) This reverts commit 54c1fd2096437011dceeffe92d53b78a883c8649. --- SWIPs/swip-39.md | 479 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 344 insertions(+), 135 deletions(-) diff --git a/SWIPs/swip-39.md b/SWIPs/swip-39.md index 930ff0c..d48749f 100644 --- a/SWIPs/swip-39.md +++ b/SWIPs/swip-39.md @@ -11,15 +11,9 @@ created: 2025-07-21 # Balanced Neighbourhood Registry aka Smart Neighbourhood Management - ## Abstract -This proposal defines a protocol mechanism that enforces uniform distribution of nodes across the Swarm address space while preventing adversarial positioning. Nodes commit to participation and are assigned neighbourhoods through delayed entropy derived from on-chain randomness. The assignment is therefore unpredictable at commit time and reproducible at validation time. Since a joining node operator is unable to position themselves at a specific point in the network without a significant time or financial penalty, attack vectors that rely on this are rendered infeasible. - - -The system maintains a structural invariant over the address space ensuring that no prefix of length $d-1$ is empty. This invariant is preserved through strictly local operations, namely assignment, deregistration, and rebalance, without requiring global restructuring. The design achieves fairness, bounded operational cost, and resistance to manipulation. - - +This SWIP introduces a systematic way for operators to enter the network in such a way that their overlay addresses are balanced within the address space. Importantly, operators must register their commitment to take part and on doing so will be assigned a random neighbourhood from the queue of those next to be occupied. Since a joining node operator is unable to position themselves at a specific point in the network without a significant time or financial penalty, attack vectors that rely on this are rendered infeasible. ## Motivation @@ -35,209 +29,424 @@ There are multiple considerations that motivate such a scheme: - **extensible and user friendly approach**: the current approach taken by swarm is based on the idea is that, if stake is variable and earnings are linearly proportional to earnings then, mutatis mutandis, it is always more economical for one operator to run just one node with all the stake than several nodes due to the added operational costs. While this has proven to be an effective means to secure the protocol, it has also led to some confusion and it's complexity means it is difficult to develop. The proposed system is intended to be much simpler to reason with and for operators to use and understand. -A decentralised service network assumes that workload is distributed according to a uniform random process over the address space. This assumption only holds if the distribution of nodes itself is approximately uniform. In practice, allowing operators to influence their placement leads to clustering and adversarial positioning, while global rebalancing mechanisms are impractical in a smart contract setting due to their cost and coordination requirements. +## Architecture -The present design replaces behavioural assumptions with structural guarantees. Node placement is determined externally by the protocol, and the system continuously enforces coverage through a local invariant that is preserved under all admissible transitions. +The network $S$, comprises the set of all network provider nodes $\{a_0, \ldots, a_n\}$, with cardinality N. ---- +Each node $a$ has an associated ether address $a_i^\Xi$ and will determine a 32 byte overlay address $a_i^\theta$. -## Model and Notation +$ +N = |S|, \quad S = \{a_0, \ldots, a_n\} \quad a_i^{\theta} \in \mathbb{O}_{32}\,,\ a_i^\Xi \in \mathbb{E} +$ -Let the set of active nodes be denoted by +Then the current depth of the node assignment tree is defined as: -$$ -S = \{a_0, \ldots, a_n\}, \quad N = |S|. -$$ +$ +d_c = \lfloor \log_2(N) \rfloor + 1, \quad d_c \in \mathbb{N} +$ -Each node is identified by an Ethereum address $a^\Xi$ and an overlay address $a^\theta \in \mathbb{O}_{32}$. +The balanced node assignments are orchestrated by a smart contract which will maintain the active node list, their corresponding neighbourhoods, the overlay addresses they have determined, and manage the ingress/egress processes as nodes joini and exit the set of active network service providers. -The system maintains a depth parameter $d \in \mathbb{N}$ such that +This contract is deployed together with a staking contract similar to the [swarm storage incentive staking contract](https://github.com/ethersphere/storage-incentives/blob/master/src/Staking.sol). This contract will retain the total stake treasury, as well as enabling a node operator to deposit, withdrawal and maintain their stake. Concerns should be strictly separated to improve security of locked funds and upgradability of both contracts. -$$ -2^{d-1} < N \le 2^d. -$$ +The node assignment contract is composed of several transactional endpoints: -Neighbourhoods are represented as indices in a complete binary space: +__Commit :__ Initially, a node's ether address $a^\Xi$ is registered in the *committers' list* $C$ that records nodes' commitment to participate as a provider in the service network, depositing their application fee ${\$_a}$, which is non-refundable. -$$ -I : \{0, \ldots, 2^d - 1\} \to \mathbb{O}_{32} \cup \{0\}, -$$ +__Get Assigned Overlay :__ -where $I[i] = 0$ denotes an empty slot. A reverse mapping + This function includes a read-only call that takes as argument a node's ether address $a^\Xi$ and returns the neighbourhood that the node is currently assigned to. This call is public so that the client can enquire about the neighbourhood they are assigned to -- so that they can mine an overlay address into it, ie., find a nonce that is needed to generate the overlay address. -$$ -J : a^\Xi \mapsto i -$$ +__Assign Overlay:__ One is called by the staking contract, after the service network stake has been deposited with a valid nonce $\vartheta$, i.e., one that, when it is used as a parameter with the Swarm Overlay address calculation $\mathcal{O}$, will produce an overlay address $a^\theta$ from the nodes corresponding ether address $a^\Xi$ which is in the correct neighbourhood. -associates each node with its assigned index. +$\vartheta \in \mathbb{Z}_{\geq 0} , \quad \mathcal{O}(a,\vartheta) \equiv a^\theta $ -For each index $i$, define the pair +This call will place the node among the active node set for the service, and removes the entry from the *committers list*. -$$ -\mathcal{P}_i = \{2i, 2i+1\}. -$$ +## Specification ---- +### Registration +An initially empty list (*committers' list*) of *entry struct* types holds the current committers. The struct holds information about the ether address of the node, the blockheight the address registered at. + -## Structural Invariant -The system enforces the condition -$$ -\forall i, \quad I[2i] \neq 0 \;\lor\; I[2i+1] \neq 0, -$$ +#### Deposit +In order for a node to get its address registered, an amount of ${\$_a}$ must be deposited which is non-refundable. -which ensures that every prefix of length $d-1$ contains at least one node. This invariant defines the admissible states of the system. +#### Uniqueness +In order to prevent repeated trials, each node must be registered only once. ---- +After checking the deposit amount and the uniqueness check on the ether address, the current blockheight is recorded with the address by pushing the entry struct ($e_i\ = \ $) to the end of committers list. -## Data Structure +#### Validity +The entry is valid for a period of $B, \quad B < 256 $ blocks after the registration. -The assignment structure is implemented as an implicit complete binary trie over the index space. Each node $v$ of the trie corresponds to a contiguous interval of indices and maintains two quantities. +$B$ must be less than $256$, the number of blocks for which the blockhash is available from within the EVM. ((unless the blockhash is recorded)) -The first quantity $F(v)$ denotes the number of free slots in the subtree rooted at $v$. Formally, if $L(v)$ denotes the set of leaf indices under $v$, then +Since the blockheight values of the list items are monotonically increasing, entries at the beginning of the list expire first. By iterating upto the first valid entry, expired entries can be iterated on efficiently. -$$ -F(v) = |\{ i \in L(v) \mid I[i] = 0 \}|. -$$ +>> SIG//NOTE should have time limit on commiting to assigned neighbourhood overlay before providingn nonce. this queing has some unintended consequences in terms of enabling squatting or blocking. the economic disincentives must be calculated and parameters/constants and/or slashing formula created i.e. how much to probably block one neighbourhood then subsequently provoke a split causing data loss. -The second quantity $E(v)$ denotes the number of indices $i$ in the subtree such that both elements of $\mathcal{P}_i$ are occupied. These correspond to candidate donor pairs. +### Expiry -Both quantities satisfy recursive relations +This function call iterates through all expired entries, burns their deposit, and, by setting the head of the list to the first valid item, removes them from the committer's list. -$$ -F(v) = F(v_L) + F(v_R), \quad E(v) = E(v_L) + E(v_R), -$$ +This is called by the assign function (itself called by the staking contract) before the read only call checking if the resulting overlay address falls into the neighbourhood that the registrant was assigned to, i.e., the correctness of the nonce submitted from the perspective of the staking contract. -where $v_L$ and $v_R$ denote the left and right children of $v$. Updates propagate along the path from a leaf to the root, resulting in logarithmic complexity. +### Assign ---- +The assign call is the second transactional endpoint called by the staking contract. It takes the provider's ether address and as well as the mined overlay as arguments. +After calling expiry, the validity of the registration is checked by finding the entry for the ether address in the committers' list. -## Data Structure Illustration - -```mermaid -graph TD - R((root)) - R --> A - R --> B - A --> A1 - A --> A2 - B --> B1 - B --> B2 -``` +#### Initialisation -Each leaf corresponds to an index. Internal nodes aggregate subtree quantities $F$ and $E$. +Let $\overline{N}=2^d$ be the lowest power of 2 that is greater than $N$, the number of already assigned registrants, where $d$ and $N$ are defined as before. Let $R$ be the array of the remaining unassigned neighbourhoods of this level (i.e., $|R|=\overline{N}-N$). ---- +Whenever the number of slots at this depth, $|R|$ drops to $0$, $d$ is incremented and $\overline{N}=2^d$ adjusts. At any point in time, a $uint256$ array of length $\overline{N}$ is maintained, called the *assignments list* $A$ holding the currently registered nodes' overlay addresses. -## Entropy +Whenever $d$ changes, new arrays for assigments $A_d$ and remainders $R_d$ are created (both twice the size of the previous one $A_{d-1}$ and $R_{d-1}$). We iterate through the current array and for each overlay address $a_i^\theta$ at position $i$ copy $a_i^\theta$ to position $2i +b_i$ where $b_i=1$ if the $d$-th bit of the address is set: -A node that registers at block height $h$ derives its randomness from +$$ +b_i=a^\theta_i[d/8] +$$ $$ -\rho = H(\text{blockhash}(h+1) \parallel a^\Xi \parallel h), +b_i\gg=7-(d\mod 8) $$ -which is not known at the time of registration. The validity window $B < 256$ ensures that the referenced blockhash remains accessible. ---- +$$ +b_i{\land\hspace{-3pt}=}1 +$$ -## Assignment +In fact, $A_d$ stands for nodes of a binary tree on level $d$ and the value at each position is the overlay address filling that neighbourhood. When $d$ is incremented, we will need to fill some of the neighbourhoods with the existing nodes. Each node will fill the left child node (when $b_i=0$) or the right one ($b_i=1$) depending on the subsequent bit in their overlay. + +$$ +\forall\ 0\leq i < 2^d,\quad A_d[2i+b_i]=A_{d-1}[i] +$$ -Let $M = 2^d - N$ denote the number of free slots. A node computes +As we are filling the assignment list, we know that the whenever a neighbourhood is filled with an already existing node, its sister neighbourhood will be unassigned, therefore we can just record those in the remaining list. $$ -k = \rho \bmod M. +\forall\ 0\leq i < |R_d|,\quad R_d[i]=2i+1-b_i $$ -The assigned index is determined by descending the trie. At a node $v$, let $F(v_L)$ denote the number of free slots in the left subtree. If $k < F(v_L)$, the traversal continues to the left child. Otherwise, the traversal continues to the right child with updated rank $k \leftarrow k - F(v_L)$. This procedure terminates at a leaf index $i$ such that $I[i] = 0$. ---- +#### Random seed -## Deregistration and Rebalancing +This internal read-only call takes as its single argument an ether address ($a^\Xi$) and returns a (?) random $\rho \in \mathbb{Z}_\text{uint256}$. +After checking if the ether address is a valid registrant by finding the corresponding first (unique) $entry$ struct,[^33] the `difficulty` (randao) is called on the block subsequent to the blockheight registered. Append to it the ether address and hash using $H$ it to yield what will serve as the random seed for this provider.[^4] -Consider a node occupying index $r$. Let $i = \lfloor r/2 \rfloor$ be the corresponding pair index. +$$ +\sigma=\mathit{blockAtHeight}(\mathit{entry}(a^\Xi).\mathit{height}+1).\mathit{difficulty}() +$$ -If the sibling index in $\mathcal{P}_i$ is occupied, removal proceeds directly and the invariant remains satisfied. +$$ +\varrho=\mathit{uint256}(H(\sigma|a^\Xi)) +$$ -If removal would leave $\mathcal{P}_i$ empty, a rebalance is required. A donor pair is selected using the same rank-based traversal over $E(v)$. From the selected pair, one of the two nodes is chosen and removed. The donor node is reinserted into the commit queue and assigned to the empty pair. +[^33]: Since expiry is not necessarily called when the random seed it called, the blockheight needs to be checked: 1) if block after $h$ (the one in which the registration happened) is available, 2) that the height is not greater than $h+B$ with $B$ being the validity period in blocks. -The original node is removed only after the donor successfully completes reassignment, ensuring that the invariant is never violated. +[^4]: Even if on a POA chain, and no randao, this seed cannot be known to the registering provider and their colluding associates, but nonetheless should be deterministic once its set. ---- +#### Neighbourhood -## Depth Reduction +This component must be available as a public readonly endpoint taking a node's ($j\text{-th}$) ether address ($a^\Xi_j$) as a single argument. -If $N = 2^{d-1}$, every pair contains exactly one node and no donor exists. In this case, the representation is reduced by setting $d \leftarrow d-1$ and mapping each pair to a single index. This transformation is deterministic because no pair contains more than one node. +A random nonce $\varrho_n$ is used to select a neighbourhood $nh$ for a provider from the remaining unassigned neighbourhood list of level $d$: ---- +$$ +i:=\varrho\mod len(R_d) +$$ -## Interface +$$ +nh=R[i] +$$ -| Function | Description | -|----------|------------| -| Register(a) | Records commitment and blockheight | -| GetPrefix(a) | Returns assigned index | -| Validate(a,o) | Verifies overlay matches assigned index | -| Insert(a,o) | Inserts node and updates structure | -| Deregister(a) | Initiates removal | -| Remove(i) | Removes node from index | -| Rebalance(i) | Repairs invariant | +#### Checking the overlay ---- +The overlay (obtained by mining the nonce) is checked to fall in the correct neighbourhood r: +The check validates the address $a^O_n$ if and only if: -## Sequence Diagram - -```mermaid -sequenceDiagram - participant Client - participant Registry - participant Trie - - Client->>Registry: Register(a) - Registry-->>Client: prefix - Client->>Registry: Validate(a,o) - Registry->>Trie: update(F,E) - Client->>Registry: Deregister(a) - Registry->>Trie: check invariant - Registry->>Trie: select donor - Registry->>Trie: update -``` +$$ +r=a^O_n\gg(255-d) +$$ ---- +#### Assignment -## Economic Considerations +If the overlay check passes, -Fairness of the system requires that each node is associated with a fixed stake. Under this assumption, total stake is proportional to the number of nodes, +- the nodes' overlay address is assigned to a neighbourhood of depth $d$. $$ -\text{Total stake} \propto N, +A_d[r]=a^O_n $$ -and no operator gains advantage by concentrating stake into fewer nodes. This aligns economic incentives with the structural invariant. +- $N$ is incremented +- the $i$-th item is removed from remaining open neighbourhood list $R_d$. +- if $R_d$ is now of zero length, the $d$ is incremented, and new assignment and remaining lists are initialised as per section 'initialisation' above. +- provider's entry is removed from the committers' list. ---- +### Economic Analysis -## Impact -The system achieves uniform distribution of nodes, resistance to adversarial placement, and predictable operational costs. Rebalancing introduces a dependency on donor behaviour; however, the resulting delay is bounded and practically absorbed by the off-boarding process, which already operates as a queue. Consequently, rebalance latency does not introduce observable instability. ---- +### Further endpoints -## Security Analysis +A public read-only endpoint exists for querying neighbourhoods as well as nodes. Accessor for $d$ and $N$ will return the current neighbourhood depth and the current number of assigned neighbourhoods. A public accessor for $A_d$ will return for a neighbourhood (between $0$ and $2^d-1 inclusive) the overlay of the node assigned to that neighbourhood. Another endpoint will return for any overlay $o$ the closest node, so that the network service can find responsible nodes for any task with address in the space shared by overlays: -The commit-and-delay entropy mechanism prevents nodes from predicting or influencing their placement. Structural constraints prevent clustering, making Sybil attacks costly and ineffective. Validator influence is limited to a single block and does not provide sufficient control to bias assignment. - -Rebalancing cannot be blocked as long as $N > 2^{d-1}$, since a donor must exist. If a donor fails to complete reassignment, expiry ensures that another donor is selected, guaranteeing progress. - ---- +$$ +g(a)=A_d[a\gg(255-d)] +$$ -## Testing +### Deregistration -Correctness requires verifying that the invariant is preserved under all state transitions. Simulation should confirm that assignment remains uniform, that rebalancing converges, and that depth transitions occur at the correct thresholds. +Only called from the Staking contract, deregister deletes the entry for the neighbourhood belonging to the given address, makes the neighbourhood available in $R$ ---- -## Conclusion +## Implementation notes + +### Changes to the staking contract + +### Changes to the bee client + +A new endpoint to bee client must be added to register a node that is not yet registered to be assigned a neighbourhood. Once the neighbourhood is known, the client can mine the nonce needed to place the overlay in the required neighbourhood. + +### Migration + +Since a new updated staking contract, a stake migration will be needed for the upgrade. Before the change, all the simplification of the staking contract is recommended, especially to allow fixed stake in order to realign redundancy +of storage and monetary incentive: with a fixed amount staked, total stake is linearly proportional to the number of nodes, and therefore comparisons across neighbourhoods can be made based on the number of nodes. In particular, the arbitrary balanced assignment makes sense in terms of incentives (expected revenue). + +### Putting a node in each neighbourhood. + +## Contract + +```sol +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.19; + +contract BalancedNeighbourhoodRegistry { + // -------------------- + // Configurable values + // -------------------- + uint256 public constant STAKE = 10,000,000,000,000,000 wei; // Stake required + uint256 public constant VALID_FOR = 128; // Validity window in blocks + + // -------------------- + // Structs and Storage + // -------------------- + struct Entry { + address committer; + uint256 height; + } + + Entry[] public committers; // List of committers (registered but as yet unassigned nodes) + mapping(address => bool) public hasCommitted; // Track if an address has committed + + uint256 public N; // Number of assigned nodes + uint256 public d = 1; // Neighbourhood depth + uint256 public currentPower = 2; // 2^d + + // Assignments: overlay address for neighbourhoods of depth d + bytes32[] public A; + + // Remaining unassigned neighbourhoods of depth d + uint256[] public R; + + // -------------------- + // Events + // -------------------- + event Registered(address indexed node, uint256 blockHeight); + event Assigned(address indexed node, uint256 neighbourhood, bytes32 overlay); + event DepthUpgraded(uint256 newDepth); + event Unassigned(address indexed node, uint256 neighbourhood); + + // -------------------- + // Registration endpoint + // -------------------- + function register() external payable { + require(msg.value == STAKE, "Invalid stake"); + require(!hasCommitted[msg.sender], "Already registered"); + + committers.push(Entry(msg.sender, block.number)); + hasCommitted[msg.sender] = true; + + emit Registered(msg.sender, block.number); + } + + // -------------------- + // Expire old entries + // -------------------- + function _expire() internal { + while (committers.length > 0) { + Entry storage e = committers[0]; + if (block.number <= e.height + VALID_FOR) { + break; + } + hasCommitted[e.committer] = false; + _removeCommitter(0); + // Burn logic: funds stay locked. + } + } + + // -------------------- + // Remove committer from the list + // -------------------- + // This function is used internally to remove a committer from the list. + // It shifts the elements to the left and pops the last element. + function _removeCommitter(uint index) internal { + if (index >= committers.length) return; + + for (uint i = index; i < committers.length - 1; i++) { + committers[i] = committers[i + 1]; + } + committers.pop(); + } + + // -------------------- + // Find entry for a committer in the committer list + // Returns the index of the entry if found, otherwise reverts. + // -------------------- + function _findEntryFor(address _a) internal view returns (uint) { + for (uint i = 0; i < committers.length; i++) { + if (committers[i].committer == _a) { + return i; + } + } + revert("Not registered"); + } + + // -------------------- + // Randomness + // -------------------- + function _randomSeed(address _a) internal view returns (uint256) { + uint i = _findEntryFor(_a); + uint256 h = committers[i].height; + // Ensure the block number is valid as expire may not have been called + require(block.number > h + 1, "Too early"); + require(block.number <= h + VALID_FOR, "Registration expired"); + // Use blockhash to generate a random seed + bytes32 bh = blockhash(h + 1); + return uint256(keccak256(abi.encodePacked(bh, _a))); + } + + // -------------------- + // Public View: Neighbourhood + // -------------------- + function getNeighbourhood(address _a) public view returns (uint256) { + uint256 r = _randomSeed(_a); + require(R.length > 0, "No available neighbourhoods"); + return R[r % R.length]; + } + + // -------------------- + // Assign node overlay to neighbourhood + // -------------------- + function assign(address _a, bytes32 _overlay) external { + _expire(); + + // Check registration + uint256 nh = getNeighbourhood(_a); + uint256 overlayNh = uint256(_overlay) >> (256 - d); + + require(overlayNh == nh, "Overlay doesn't match neighbourhood"); + + A[nh] = _overlay; + N++; + + // Remove neighbourhood from R + _removeFromR(nh); + + // Check if R is empty + if (R.length == 0) { + _upgradeDepth(); + } + + hasCommitted[_a] = false; + _removeEntry(_a); + + emit Assigned(_a, nh, _overlay); + } + + //---------------------- + // unregister + //---------------------- + function unregister(address _a, bytes32 _overlay) external { + uint nh = uint256(_overlay) >> (256 - d); + A[nh] = bytes32(0); + R.push(nh); + N--; + // return funds to the committer + payable(_a).transfer(STAKE); + emit Unassigned(_a, nh); + } + + // -------------------- + // Internal functions to manage R and committers + // -------------------- + function _removeFromR(uint256 nh) internal { + for (uint i = 0; i < R.length; i++) { + if (R[i] == nh) { + // replace the removed element with the last element and pop + R[i] = R[R.length - 1]; + R.pop(); + return; + } + } + } + + function _removeEntry(address _a) internal { + uint i = _findEntryFor(_a); + require(i < committers.length, "Entry not found"); + _removeCommitter(i); + } + + // -------------------- + // Expand A and R when needed + // -------------------- + function _upgradeDepth() internal { + currentPower = 2 ** d; + + delete R; + for (uint i = 0; i < currentPower; i++) { + R.push(0); + // Ensure A has enough space for the new neighbourhoods); + A.push(bytes32(0)); + } + for (uint i = currentPower - 1; i < currentPower; i--) { + uint b = uint256(A[i]) >> (256 - d) % 2; + A[2*i+b] = A[i]; + uint256 j = 2*i+1-b; + if (2 * j < currentPower) { + A[j] = bytes32(0); // Clear the old address + } + R[i] = j; + } + d++; + require(d <= 32, "Maximum depth exceeded"); + emit DepthUpgraded(d); + } + + // -------------------- + // Accessors + // -------------------- + function getDepth() external view returns (uint) { + return d; + } + + function getN() external view returns (uint256) { + return N; + } + + function getOverlayForNeighbourhood(uint256 nh) external view returns (bytes32) { + return A[nh]; + } + + function getClosestNode(bytes32 key) external view returns (bytes32) { + uint256 prefix = uint256(key) >> (256 - d); + return A[prefix]; + } +} +``` -This proposal replaces global restructuring with local invariant enforcement over a compact data structure. It achieves fairness, efficiency, and robustness through deterministic rules and probabilistic assignment, making it suitable for large-scale decentralised operation.