Verify the node is running
Validate if the process is running
pgrep rbbc
Check the status of the process
journalctl -u redbelly.service
Check the status of the Redbelly service
sudo systemctl status redbelly.service
Check the Redbelly logs
sudo tail -f /var/log/redbelly/rbn_logs/rbbc_logs.log
Check the error logs
sudo tail -f /var/log/redbelly/rbn_logs/rbbc_logs_error.log
Monitoring with a Script
As a node operator, you can use a script to check the state of your node. The following script enables basic local node monitoring though a local HTTP server.
To enable the execution of the script, edit the Redbelly service definition file using nano or vi, for example:
nano /etc/systemd/system/redbelly.service
Then, add the following options to the Redbelly service command line:
--statusserver.addr=127.0.0.1 --statusserver.port=6539
Where 127.0.0.1 is your local HTTP server, and 6539 is the default status server port, which can be changed.
Finally, execute the following commands to apply these changes:
sudo systemctl stop redbelly.service
sudo systemctl daemon-reload
sudo systemctl start redbelly.service
Make sure that you're running Python 3.9 and pip, and have installed the dependencies:
pip install python-dateutil requests
import argparse
from datetime import datetime, timezone, timedelta
import requests
import time
from dateutil import parser
from decimal import Decimal
ether_in_wei = Decimal("1000000000000000000")
blockChanges: list[tuple[int, datetime]] = []
def clear_screen():
print("\033[2J\033[H", end='', flush=True)
def get_block_ps(blockChange: list[tuple[int, datetime]]) -> float:
(startBlock, startTime) = blockChange[0]
(endBlock, endTime) = blockChange[-1]
timeChange = endTime.replace(tzinfo=timezone.utc) - startTime.replace(tzinfo=timezone.utc)
if timeChange.total_seconds() == 0:
return 0
return float(endBlock - startBlock) / timeChange.total_seconds()
def run_loop(address: str, min_addresss_bal: int, refresh_seconds: int):
try:
while True:
result = monitor(address, min_addresss_bal)
clear_screen()
print(result)
time.sleep(refresh_seconds)
except KeyboardInterrupt:
print("\nExiting")
except BaseException as e:
print(f"\nGot error when connecting to {address}, be sure status server is enabled in SEVM with flags --statusserver.addr=127.0.0.1 --statusserver.port=6539")
print(e)
exit(1)
def monitor(address: str, min_addresss_bal: int) -> str:
url = address + "/status"
parts = []
parts.append(f"Monitoring url {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
isRecoveryComplete = bool(data["isRecoveryComplete"])
lastCommittedBlockAt = datetime.fromtimestamp(0) if data["lastCommittedBlockAt"] == "" else parser.parse(data["lastCommittedBlockAt"])
timeSinceLastBlock = datetime.now(timezone.utc)-lastCommittedBlockAt.replace(tzinfo=timezone.utc)
currentBlock = int(data["currentBlock"])
lastBlockFromGovernors = int(data["lastBlockFromGovernors"])
lastSyncedWithGovernorNodes = datetime.fromtimestamp(0) if data["lastSyncedWithGovernorNodes"] == "" else parser.parse(data["lastSyncedWithGovernorNodes"])
timeSinceLastSyncWithGovs = datetime.now(timezone.utc)-lastSyncedWithGovernorNodes.replace(tzinfo=timezone.utc)
blocksBehind = lastBlockFromGovernors - currentBlock
blockChanges.append((currentBlock, lastCommittedBlockAt))
if len(blockChanges) > 10:
blockChanges.pop(0)
currentSuperblock = int(data["currentSuperblock"])
lastSyncedWithBootnodes = datetime.fromtimestamp(0) if data["lastSyncedWithBootnodes"] == "" else parser.parse(data["lastSyncedWithBootnodes"])
lastSuperblockFromBootnodes = int(data["lastSuperblockFromBootnodes"])
timeSinceLastSyncWithBootnodes = datetime.now(timezone.utc)-lastSyncedWithBootnodes.replace(tzinfo=timezone.utc)
superblocksBehind = lastSuperblockFromBootnodes - currentSuperblock
certificateDnsNames: list[str] = data["certificateDnsNames"]
certificatesValidUpto = datetime.fromtimestamp(0) if data["certificatesValidUpto"] == "" else parser.parse(data["certificatesValidUpto"])
certificateValidDuration = certificatesValidUpto.replace(tzinfo=timezone.utc)-datetime.now(timezone.utc)
signingAddress = str(data["signingAddress"])
signingAddressBalance = Decimal(data["signingAddressBalance"]) / ether_in_wei
version = str(data["version"])
if isRecoveryComplete:
parts.append("Node has completed initial sync")
else:
parts.append("Node is still running inital sync")
parts.append(f"Current block: {currentBlock}")
parts.append(f" Block processed at {lastCommittedBlockAt.astimezone()}, i.e. {timeSinceLastBlock} ago")
if timeSinceLastBlock > timedelta(minutes=5):
parts.append(f" \033[31mWARNING\033[0m: node has not processed a block in more than 5 minutes, may be out of sync with network")
parts.append(f"Block process rate: {get_block_ps(blockChanges)} blocks per second")
if blocksBehind > 100:
parts.append(f" \033[31mWARNING\033[0m: node is {blocksBehind} blocks behind the governor")
parts.append(f"Last block seen from governors: {lastBlockFromGovernors}")
parts.append(f" Time of last block sync with governors: {lastSyncedWithGovernorNodes.astimezone()}, i.e. {timeSinceLastSyncWithGovs} ago")
if timeSinceLastSyncWithGovs > timedelta(minutes=1):
parts.append(f" \033[31mWARNING\033[0m: node has not synced block number with governors in more than 1 minute, node may be out of sync with network")
parts.append(f"\nCurrent superblock: {currentSuperblock}")
if superblocksBehind > 100:
parts.append(f" \033[31mWARNING\033[0m: node is {superblocksBehind} superblocks behind the bootnodes")
parts.append(f"Last superblock seen from bootnodes: {lastSuperblockFromBootnodes}")
parts.append(f" Time of last superblock sync with bootnodes: {lastSyncedWithBootnodes.astimezone()}, i.e. {timeSinceLastSyncWithBootnodes} ago")
if timeSinceLastSyncWithGovs > timedelta(minutes=2):
parts.append(f" \033[31mWARNING\033[0m: node has not synced superblock number with bootnodes in more than 2 minutes, node may be out of sync with network")
parts.append(f"\nSigning address {signingAddress} balance: {signingAddressBalance} RBNT")
if min_addresss_bal > signingAddressBalance:
parts.append(f"\033[31mWARNING\033[0m: signing address balance is less than the minimum needed of {min_addresss_bal} RBNT")
parts.append(f"\nCertificate DNS names: {certificateDnsNames}")
parts.append(f" Certificate valid until {certificatesValidUpto.astimezone()}, i.e. {certificateValidDuration}")
if certificateValidDuration <= timedelta():
parts.append(f"\033[31mWARNING\033[0m: Certificate has expired")
if certificateValidDuration <= timedelta(days=7):
parts.append(f"\033[31mWARNING\033[0m: Certificate will expire soon")
parts.append(f"\nBinary version: {version}")
return "\n".join(parts)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Watch the stats of a local Redbelly node")
parser.add_argument("-a", "--address", type=str, default="http://localhost:6539", help="Address of the Redbelly node's status server")
parser.add_argument("-mb", "--minBalance", type=int, default=10, help="Minimum signing address balance in RBNT before warning")
parser.add_argument("-r", "--refreshSeconds", type=int, default=5, help="Frequency to refresh values")
args = parser.parse_args()
return args
if __name__ == "__main__":
args = parse_args()
run_loop(args.address, args.minBalance, args.refreshSeconds)
A note on Governance
Important!
Please note that as a candidate node, your node will not be able to submit transactions to be written to the network as this responsibility is reserved to the governor nodes.
As we run regular network reconfigurations in the future, your node might be randomly promoted to governorship for a pre-determined period of time (usually 2 weeks).
Until your node is promoted to governorship, you will need to use a currently available RPC endpoint, should you wish to submit a transaction to the network via a wallet such as Metamask. Learn more about Redbelly Network Governance.