tbc: store tx byte location in tx index, bump DB to v6

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Breaking Changes
 
+- `BlockHashByTxId` now returns `(*chainhash.Hash, wire.TxLoc, error)`;
+  callers that only need the hash use `bh, _, err :=`
+  ([#1052](https://github.com/hemilabs/heminetwork/pull/1052)).
 - Rename `TBC_BLOCKHEADER_CACHE_SIZE` environment variable to
   `TBC_HEADER_CACHE_SIZE`
   ([#1034](https://github.com/hemilabs/heminetwork/pull/1034)).
@@ -27,6 +30,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add `BlockRawByHash` to DB interface and `lazyBlock` type for zero-copy
+  per-tx block access without full deserialization
+  ([#1051](https://github.com/hemilabs/heminetwork/pull/1051)).
+- Store tx byte location (`TxLoc`) in tx index `'t'` entry values for
+  O(1) tx lookup; DB version 5 → 6
+  ([#1052](https://github.com/hemilabs/heminetwork/pull/1052)).
 - Add generic `lru` package with cost-based LRU cache (`lru.Cache[K,V]`)
   ([#1034](https://github.com/hemilabs/heminetwork/pull/1034)).
 - Add utxo read LRU cache (`TBC_UTXO_READ_CACHE_SIZE`) to reduce LevelDB
@@ -57,6 +66,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- `BlockTxUpdate` uses stack-allocated reusable buffers instead of slicing
+  loop variables, avoiding potential data integrity issues
+  ([#1052](https://github.com/hemilabs/heminetwork/pull/1052),
+  [#1050](https://github.com/hemilabs/heminetwork/issues/1050)).
 - Replace block and header caches in level package with generic `lru.Cache[K,V]`
   ([#1034](https://github.com/hemilabs/heminetwork/pull/1034)).
 

cmd/hemictl/hemictl.go

@@ -619,7 +619,7 @@ func tbcdb(pctx context.Context, flags []string) error {
 			return fmt.Errorf("chainhash: %w", err)
 		}
 
-		bh, err := s.BlockHashByTxId(ctx, *chtxid)
+		bh, _, err := s.BlockHashByTxId(ctx, *chtxid)
 		if err != nil {
 			return fmt.Errorf("block by txid: %w", err)
 		}

database/tbcd/database.go

@@ -112,6 +112,7 @@ type Database interface {
 	BlockInsert(ctx context.Context, b *btcutil.Block) (int64, error)
 	// BlocksInsert(ctx context.Context, bs []*btcutil.Block) (int64, error)
 	BlockByHash(ctx context.Context, hash chainhash.Hash) (*btcutil.Block, error)
+	BlockRawByHash(ctx context.Context, hash chainhash.Hash) ([]byte, error)
 	BlockExistsByHash(ctx context.Context, hash chainhash.Hash) (bool, error)
 	BlockCacheStats() CacheStats
 
@@ -120,7 +121,7 @@ type Database interface {
 	BlockHeaderByTxIndex(ctx context.Context) (*BlockHeader, error)
 	BlockUtxoUpdate(ctx context.Context, direction int, utxos map[Outpoint]CacheOutput, utxoIndexHash chainhash.Hash) error
 	BlockTxUpdate(ctx context.Context, direction int, txs map[TxKey]*TxValue, txIndexHash chainhash.Hash) error
-	BlockHashByTxId(ctx context.Context, txId chainhash.Hash) (*chainhash.Hash, error)
+	BlockHashByTxId(ctx context.Context, txId chainhash.Hash) (*chainhash.Hash, wire.TxLoc, error)
 	SpentOutputsByTxId(ctx context.Context, txId chainhash.Hash) ([]SpentInfo, error)
 	// ScriptHash returns the sha256 of PkScript for the provided outpoint.
 	BalanceByScriptHash(ctx context.Context, sh ScriptHash) (uint64, error)
@@ -480,6 +481,20 @@ func NewTxMapping(txId, blockHash *chainhash.Hash) (txKey TxKey) {
 	return txKey
 }
 
+// NewTxMappingWithLoc returns a TxKey and TxValue that maps a tx id to a block
+// hash with the tx byte location (offset + length) within the raw block. This
+// allows callers to jump directly to the tx's bytes without scanning the block.
+func NewTxMappingWithLoc(txId, blockHash *chainhash.Hash, loc wire.TxLoc) (txKey TxKey, txValue TxValue) {
+	txKey[0] = 't'
+	copy(txKey[1:33], txId[:])
+	copy(txKey[33:], blockHash[:])
+
+	binary.BigEndian.PutUint32(txValue[0:4], uint32(loc.TxStart))
+	binary.BigEndian.PutUint32(txValue[4:8], uint32(loc.TxLen))
+
+	return txKey, txValue
+}
+
 func TxIdBlockHashFromTxKey(txKey TxKey) (*chainhash.Hash, *chainhash.Hash, error) {
 	if txKey[0] != 't' {
 		return nil, nil, fmt.Errorf("invalid magic 0x%02x", txKey[0])

database/tbcd/level/level.go

@@ -48,7 +48,7 @@ import (
 //	UTXOs
 
 const (
-	ldbVersion = 5
+	ldbVersion = 6
 
 	logLevel = "INFO"
 	verbose  = false
@@ -312,6 +312,10 @@ func New(ctx context.Context, cfg *Config) (*ldb, error) {
 			// Upgrade to v5: wipe witness-stripped block
 			// bodies and rebuild blocksmissing from headers.
 			err = l.v5(ctx)
+		case 5:
+			// Upgrade to v6: wipe tx index so it rebuilds
+			// with TxLoc values in 't' entries.
+			err = l.v6(ctx)
 		default:
 			if ldbVersion == dbVersion {
 				if Welcome {
@@ -1589,6 +1593,37 @@ func (l *ldb) BlockByHash(ctx context.Context, hash chainhash.Hash) (*btcutil.Bl
 	return b, nil
 }
 
+func (l *ldb) BlockRawByHash(ctx context.Context, hash chainhash.Hash) ([]byte, error) {
+	log.Tracef("BlockRawByHash")
+	defer log.Tracef("BlockRawByHash exit")
+
+	// get from cache
+	var (
+		eb  []byte
+		err error
+	)
+	if l.cfg.blockCacheSize > 0 {
+		eb, _ = l.blockCache.Get(hash)
+	}
+
+	// get from db
+	if eb == nil {
+		bDB := l.rawPool[level.BlocksDB]
+		eb, err = bDB.Get(hash[:])
+		if err != nil {
+			if errors.Is(err, leveldb.ErrNotFound) {
+				return nil, database.BlockNotFoundError{Hash: hash}
+			}
+			return nil, fmt.Errorf("block raw get: %w", err)
+		}
+		if l.cfg.blockCacheSize > 0 {
+			l.blockCache.Put(hash, eb)
+		}
+	}
+
+	return eb, nil
+}
+
 func (l *ldb) BlockExistsByHash(ctx context.Context, hash chainhash.Hash) (bool, error) {
 	log.Tracef("BlockExistsByHash")
 	defer log.Tracef("BlockExistsByHash exit")
@@ -1611,36 +1646,38 @@ func (l *ldb) BlockExistsByHash(ctx context.Context, hash chainhash.Hash) (bool,
 	return ok, nil
 }
 
-func (l *ldb) BlockHashByTxId(ctx context.Context, txId chainhash.Hash) (*chainhash.Hash, error) {
+func (l *ldb) BlockHashByTxId(ctx context.Context, txId chainhash.Hash) (*chainhash.Hash, wire.TxLoc, error) {
 	log.Tracef("BlockHashByTxId")
 	defer log.Tracef("BlockHashByTxId exit")
 
-	blocks := make([]*chainhash.Hash, 0, 2)
 	txDB := l.pool[level.TransactionsDB]
-	var txid [33]byte
-	txid[0] = 't'
-	copy(txid[1:], txId[:])
-	it := txDB.NewIterator(util.BytesPrefix(txid[:]), nil)
+	var prefix [33]byte
+	prefix[0] = 't'
+	copy(prefix[1:], txId[:])
+	it := txDB.NewIterator(util.BytesPrefix(prefix[:]), nil)
 	defer it.Release()
+
+	var found bool
+	var blockHash chainhash.Hash
+	var loc wire.TxLoc
 	for it.Next() {
-		block, err := chainhash.NewHash(it.Key()[33:])
-		if err != nil {
-			return nil, err
+		if found {
+			panic(fmt.Sprintf("multiple blocks for tx %v", txId))
 		}
-		blocks = append(blocks, block)
+		copy(blockHash[:], it.Key()[33:])
+		if v := it.Value(); len(v) >= 8 {
+			loc.TxStart = int(binary.BigEndian.Uint32(v[0:4]))
+			loc.TxLen = int(binary.BigEndian.Uint32(v[4:8]))
+		}
+		found = true
 	}
 	if err := it.Error(); err != nil {
-		return nil, fmt.Errorf("blocks by id iterator: %w", err)
+		return nil, wire.TxLoc{}, fmt.Errorf("blocks by id iterator: %w", err)
 	}
-	switch len(blocks) {
-	case 0:
-		return nil, database.NotFoundError(fmt.Sprintf("tx not found: %v", txId))
-	case 1:
-		return blocks[0], nil
-	default:
-		panic(fmt.Sprintf("invalid blocks count %v: %v",
-			len(blocks), spew.Sdump(blocks)))
+	if !found {
+		return nil, wire.TxLoc{}, database.NotFoundError(fmt.Sprintf("tx not found: %v", txId))
 	}
+	return &blockHash, loc, nil
 }
 
 func (l *ldb) SpentOutputsByTxId(ctx context.Context, txId chainhash.Hash) ([]tbcd.SpentInfo, error) {
@@ -1886,32 +1923,39 @@ func (l *ldb) BlockTxUpdate(ctx context.Context, direction int, txs map[tbcd.TxK
 	}
 	defer txsDiscard()
 
-	block := make([]byte, 33)
-	block[0] = 'b'
 	var blk []byte
 	bm := make(map[string]struct{}, len(txs))
 	defer clear(bm)
 
 	txsBatch := new(leveldb.Batch)
+	var keyBuf [69]byte
+	var valBuf [36]byte
+	var blkBuf [33]byte
+	blkBuf[0] = 'b'
 	for k, v := range txs {
-		// cache is being emptied so we can slice it here.
 		var key, value []byte
 		switch k[0] {
 		case 't':
-			key = k[0:65]
-			value = nil
+			copy(keyBuf[:], k[0:65])
+			key = keyBuf[:65]
+			if v != nil {
+				copy(valBuf[:], v[0:8])
+				value = valBuf[:8]
+			}
 
 			// insert block hash to determine if it was indexed later
 			if _, ok := bm[string(k[33:65])]; !ok {
 				bm[string(k[33:65])] = struct{}{}
-				copy(block[1:], k[33:65])
-				blk = block
+				copy(blkBuf[1:], k[33:65])
+				blk = blkBuf[:]
 			} else {
 				blk = nil
 			}
 		case 's':
-			key = k[:]
-			value = v[:]
+			copy(keyBuf[:], k[:])
+			key = keyBuf[:]
+			copy(valBuf[:], v[:])
+			value = valBuf[:]
 
 			// don't insert block
 			blk = nil