diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff7f40a22..8004a043b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Changelog for NeoFS Node - Storage nodes do not calculate homomorphic hashes for objects (#3847) - Optimized GET/HEAD request forwarding (#3877) - Optimized netmap caching in node (#3966) +- Store in metabase associated object ID in bytes instead of Base58 (#3971) ### Removed - `policer.max_workers` configuration (#3920) diff --git a/pkg/core/object/metadata.go b/pkg/core/object/metadata.go index 7ab1af3ce8..2906f5aa87 100644 --- a/pkg/core/object/metadata.go +++ b/pkg/core/object/metadata.go @@ -591,7 +591,7 @@ func combineValues(attr string, dbVal []byte, fltVal string) ([]byte, []byte, er // - decoded filter byte is always 21 while the DB one is always 53 // so we'd get false mismatch. To avoid this, we have to decode each DB val. dbVal = []byte(base58.Encode(dbVal)) - case object.FilterFirstSplitObject, object.FilterParentID: + case object.FilterFirstSplitObject, object.FilterParentID, object.AttributeAssociatedObject: if len(dbVal) != oid.Size { return nil, nil, fmt.Errorf("invalid OID len %d != %d", len(dbVal), oid.Size) } @@ -660,7 +660,7 @@ func RestoreIntAttribute(b []byte) (string, error) { func restoreAttributeValue(attr string, stored []byte) (string, error) { switch attr { - case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID: + case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID, object.AttributeAssociatedObject: return base58.Encode(stored), nil //nolint:staticcheck // this is not DB's responsibility to force API rules, DB still may have these values inside case object.FilterPayloadHomomorphicHash: @@ -715,7 +715,7 @@ func PreprocessSearchQuery(fs object.SearchFilters, attrs []string, cursor strin switch attr := fs[0].Header(); attr { default: primValDB = []byte(primVal) - case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID: + case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID, object.AttributeAssociatedObject: var err error if primValDB, err = base58.Decode(primVal); err != nil { return nil, nil, fmt.Errorf("%w: decode %q attribute value from Base58: %w", errInvalidPrimaryFilter, attr, err) diff --git a/pkg/local_object_storage/engine/container_test.go b/pkg/local_object_storage/engine/container_test.go index ad9d3f31db..05a176e98e 100644 --- a/pkg/local_object_storage/engine/container_test.go +++ b/pkg/local_object_storage/engine/container_test.go @@ -13,7 +13,7 @@ import ( apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" + cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" "github.com/stretchr/testify/require" ) @@ -43,14 +43,14 @@ func TestStorageEngine_ContainerCleanUp(t *testing.T) { ) require.NoError(t, err) } - o1 := objecttest.Object() - o2 := objecttest.Object() + o1 := generateObjectWithCID(cidtest.ID()) + o2 := generateObjectWithCID(cidtest.ID()) o2.SetPayload(make([]byte, errSmallSize+1)) - err := e.Put(&o1, nil) + err := e.Put(o1, nil) require.NoError(t, err) - err = e.Put(&o2, nil) + err = e.Put(o2, nil) require.NoError(t, err) require.NoError(t, e.Init()) diff --git a/pkg/local_object_storage/engine/gc_test.go b/pkg/local_object_storage/engine/gc_test.go index c1442ff70c..6b8b6217cc 100644 --- a/pkg/local_object_storage/engine/gc_test.go +++ b/pkg/local_object_storage/engine/gc_test.go @@ -204,9 +204,8 @@ func TestGC(t *testing.T) { objAddr := obj.Address() lockObj := generateObjectWithCID(cnr) - lockObj.SetType(object.TypeLock) - addExpirationAttribute(lockObj, es.CurrentEpoch()+1) // lock expires after object lockObj.AssociateLocked(obj.GetID()) + addExpirationAttribute(lockObj, es.CurrentEpoch()+1) // lock expires after object require.NoError(t, e.Put(lockObj, nil)) tickEpoch(es, e) @@ -225,7 +224,7 @@ func TestGC(t *testing.T) { // t.Run("expired tombstone removed", func(t *testing.T) { tomb := generateObjectWithCID(cnr) - tomb.SetType(object.TypeTombstone) + tomb.AssociateDeleted(oidtest.ID()) addExpirationAttribute(tomb, es.CurrentEpoch()) require.NoError(t, e.Put(tomb, nil)) @@ -259,7 +258,6 @@ func TestGC(t *testing.T) { obj = generateObjectWithCID(cnr) require.NoError(t, e.Put(obj, nil)) tomb = generateObjectWithCID(cnr) - tomb.SetType(object.TypeTombstone) tomb.AssociateDeleted(obj.GetID()) addExpirationAttribute(tomb, es.CurrentEpoch()) require.NoError(t, e.Put(tomb, nil)) @@ -278,7 +276,6 @@ func TestGC(t *testing.T) { objAddr = obj.Address() tomb = generateObjectWithCID(cnr) - tomb.SetType(object.TypeTombstone) tomb.AssociateDeleted(obj.GetID()) addExpirationAttribute(tomb, es.CurrentEpoch()+1) // tombstone expires after object require.NoError(t, e.Put(tomb, nil)) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index a26e11d837..966030cdf2 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -404,10 +404,8 @@ func TestSplitObjectLockExpiration(t *testing.T) { t.Run("split object lock with expiring locker", func(t *testing.T) { lockObj := generateObjectWithCID(cnr) - lockObj.SetType(object.TypeLock) - addExpirationAttribute(lockObj, es.CurrentEpoch()+1) - lockObj.AssociateLocked(parentID) + addExpirationAttribute(lockObj, es.CurrentEpoch()+1) require.NoError(t, e.Put(lockObj, nil)) @@ -461,7 +459,6 @@ func TestSimpleLockExpiration(t *testing.T) { objAddr := obj.Address() lock := generateObjectWithCID(cnr) - lock.SetType(object.TypeLock) lock.AssociateLocked(objID) addExpirationAttribute(lock, es.CurrentEpoch()+1) diff --git a/pkg/local_object_storage/engine/put_test.go b/pkg/local_object_storage/engine/put_test.go index a75ab06e66..92ddf2779a 100644 --- a/pkg/local_object_storage/engine/put_test.go +++ b/pkg/local_object_storage/engine/put_test.go @@ -13,7 +13,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/stretchr/testify/require" @@ -23,11 +22,11 @@ import ( func TestStorageEngine_PutBinary(t *testing.T) { addr := oidtest.Address() - obj := objecttest.Object() + obj := *generateObjectWithCID(cidtest.ID()) obj.SetContainerID(addr.Container()) obj.SetID(addr.Object()) - obj2 := objecttest.Object() + obj2 := *generateObjectWithCID(cidtest.ID()) require.NotEqual(t, obj, obj2) obj2.SetContainerID(addr.Container()) obj2.SetID(addr.Object()) @@ -104,6 +103,13 @@ func testPutLock(t *testing.T, shardNum int) { obj := obj obj.SetType(typ) + switch typ { + case object.TypeTombstone: + obj.AssociateDeleted(oidtest.ID()) + case object.TypeLock: + obj.AssociateLocked(oidtest.ID()) + default: + } require.NoError(t, s.Put(&obj, nil)) @@ -282,7 +288,7 @@ func testPutTombstone(t *testing.T, shardNum int) { }}, {name: "target is lock", preset: func(t *testing.T, s *StorageEngine) { obj := obj - obj.SetType(object.TypeLock) + obj.AssociateLocked(oidtest.ID()) require.NoError(t, s.Put(&obj, nil)) }, assertPutErr: func(t *testing.T, err error) { require.ErrorIs(t, err, meta.ErrLockObjectRemoval) diff --git a/pkg/local_object_storage/engine/revive_test.go b/pkg/local_object_storage/engine/revive_test.go index 94f1e3deeb..ac7a6fd21c 100644 --- a/pkg/local_object_storage/engine/revive_test.go +++ b/pkg/local_object_storage/engine/revive_test.go @@ -21,8 +21,7 @@ func TestStorageEngine_ReviveObject(t *testing.T) { addr := obj.Address() ts := generateObjectWithCID(cnr) - ts.SetType(object.TypeTombstone) - addAttribute(ts, object.AttributeAssociatedObject, obj.GetID().EncodeToString()) + ts.AssociateDeleted(obj.GetID()) addAttribute(ts, object.AttributeExpirationEpoch, "0") tsAddr := ts.Address() diff --git a/pkg/local_object_storage/metabase/VERSION.md b/pkg/local_object_storage/metabase/VERSION.md index 060be99e8d..7b45477428 100644 --- a/pkg/local_object_storage/metabase/VERSION.md +++ b/pkg/local_object_storage/metabase/VERSION.md @@ -41,6 +41,9 @@ The lowest not used bucket index: 20. Object counters are updated (resynced) to fix potential issues caused by GC mark double counting. Homomorphic hash field indexes have been deleted. +`__NEOFS__ASSOCIATE` attribute value is rewritten from base58 string to raw +32-byte object ID in metadata indexes. + ## Version 10 Object counters are stored inside metadata bucket per container now. New ROOT, diff --git a/pkg/local_object_storage/metabase/bench_test.go b/pkg/local_object_storage/metabase/bench_test.go index bd64b95aef..3af73d6232 100644 --- a/pkg/local_object_storage/metabase/bench_test.go +++ b/pkg/local_object_storage/metabase/bench_test.go @@ -9,47 +9,126 @@ import ( meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/stretchr/testify/require" ) func BenchmarkSearch(b *testing.B) { - const objCount = 1000 - db := newDB(b, meta.WithMaxBatchDelay(time.Microsecond)) // 1000 puts shouldn't delay us much. - cid := cidtest.ID() + b.Run("user attribute", func(b *testing.B) { + const objCount = 1000 + db := newDB(b, meta.WithMaxBatchDelay(time.Microsecond)) // 1000 puts shouldn't delay us much. + cid := cidtest.ID() + + for i := range objCount { + var ( + attrfp object.Attribute + attrts object.Attribute + ) + attrfp.SetKey(object.AttributeFilePath) + attrfp.SetValue("path" + strconv.Itoa(i)) + attrts.SetKey(object.AttributeTimestamp) + attrts.SetValue(strconv.Itoa(1748028502 + i)) + obj := generateObjectWithCID(b, cid) + obj.SetAttributes(attrfp, attrts) + require.NoError(b, db.Put(obj)) + } + + fs := object.SearchFilters{} + fs.AddFilter("FilePath", "path100", object.MatchStringEqual) + fs.AddFilter("versioning-state", "", object.MatchNotPresent) + + var attrs = []string{object.AttributeFilePath, object.AttributeTimestamp, "versioning-state", + "delete-marker", "metatype", "objversion", object.AttributeExpirationEpoch, "lock-meta"} + + sfs, curs, err := objectcore.PreprocessSearchQuery(fs, attrs, "") + if err != nil { + b.Fatal(err) + } + + for b.Loop() { + res, _, err := db.Search(cid, sfs, attrs, curs, 1000) + if err != nil { + b.Fatal(err) + } + if len(res) != 1 { + b.Fatalf("failed to search") + } + } + }) + + b.Run("associated object with attribute", func(b *testing.B) { + const objCount = 1000 + db := newDB(b, meta.WithMaxBatchDelay(time.Microsecond)) + cid := cidtest.ID() + targets := make([]oid.ID, 0, objCount) + + for range objCount { + target := generateObjectWithCID(b, cid) + require.NoError(b, db.Put(target)) + targets = append(targets, target.GetID()) + } - for i := range objCount { - var ( - attrfp object.Attribute - attrts object.Attribute - ) - attrfp.SetKey(object.AttributeFilePath) - attrfp.SetValue("path" + strconv.Itoa(i)) - attrts.SetKey(object.AttributeTimestamp) - attrts.SetValue(strconv.Itoa(1748028502 + i)) - obj := generateObjectWithCID(b, cid) - obj.SetAttributes(attrfp, attrts) - require.NoError(b, db.Put(obj)) - } - - fs := object.SearchFilters{} - fs.AddFilter("FilePath", "path100", object.MatchStringEqual) - fs.AddFilter("versioning-state", "", object.MatchNotPresent) - - var attrs = []string{object.AttributeFilePath, object.AttributeTimestamp, "versioning-state", - "delete-marker", "metatype", "objversion", object.AttributeExpirationEpoch, "lock-meta"} - - sfs, curs, err := objectcore.PreprocessSearchQuery(fs, attrs, "") - if err != nil { - b.Fatal(err) - } - - for b.Loop() { - res, _, err := db.Search(cid, sfs, attrs, curs, 1000) + for i := range objCount { + lock := generateObjectWithCID(b, cid) + lock.AssociateLocked(targets[i]) + require.NoError(b, db.Put(lock)) + } + + fs := object.SearchFilters{} + fs.AddFilter(object.AttributeAssociatedObject, targets[objCount/2].EncodeToString(), object.MatchStringEqual) + + sfs, curs, err := objectcore.PreprocessSearchQuery(fs, []string{object.AttributeAssociatedObject}, "") if err != nil { b.Fatal(err) } - if len(res) != 1 { - b.Fatalf("failed to search") + + for b.Loop() { + res, _, err := db.Search(cid, sfs, []string{object.AttributeAssociatedObject}, curs, 1000) + if err != nil { + b.Fatal(err) + } + if len(res) != 1 { + b.Fatalf("failed to search") + } + } + }) +} + +func BenchmarkIsLocked(b *testing.B) { + db := newDB(b, meta.WithMaxBatchDelay(time.Microsecond)) + cid := cidtest.ID() + + lockedObj := generateObjectWithCID(b, cid) + require.NoError(b, db.Put(lockedObj)) + + lock := generateObjectWithCID(b, cid) + lock.AssociateLocked(lockedObj.GetID()) + require.NoError(b, db.Put(lock)) + + unlockedObj := generateObjectWithCID(b, cid) + require.NoError(b, db.Put(unlockedObj)) + + b.Run("locked", func(b *testing.B) { + for b.Loop() { + locked, err := db.IsLocked(lockedObj.Address()) + if err != nil { + b.Fatal(err) + } + if !locked { + b.Fatal("expected object to be locked") + } + } + }) + + b.Run("unlocked", func(b *testing.B) { + for b.Loop() { + locked, err := db.IsLocked(unlockedObj.Address()) + if err != nil { + b.Fatal(err) + } + if locked { + b.Fatal("expected object to be unlocked") + } } - } + }) } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index eddcb9feae..eabf06c4f5 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -92,6 +92,13 @@ func TestDB_ContainersCount(t *testing.T) { for range upload.amount { obj := generateObject(t) obj.SetType(upload.typ) + switch upload.typ { + case object.TypeTombstone: + obj.AssociateDeleted(oidtest.ID()) + case object.TypeLock: + obj.AssociateLocked(oidtest.ID()) + default: + } err := putBig(db, obj) require.NoError(t, err) @@ -198,9 +205,8 @@ func TestDB_DeleteContainer(t *testing.T) { require.NoError(t, err) // put a big one - o2 := objecttest.Object() - o2.SetContainerID(cID) - err = putBig(db, &o2) + o2 := generateObjectWithCID(t, cID) + err = putBig(db, o2) require.NoError(t, err) // TS diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 64ea75b865..c40e22fdbb 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -48,7 +48,7 @@ func TestDB_Exists(t *testing.T) { t.Run("tombstone object", func(t *testing.T) { ts := generateObject(t) - ts.SetType(object.TypeTombstone) + ts.AssociateDeleted(oidtest.ID()) err := putBig(db, ts) require.NoError(t, err) @@ -60,7 +60,7 @@ func TestDB_Exists(t *testing.T) { t.Run("lock object", func(t *testing.T) { lock := generateObject(t) - lock.SetType(object.TypeLock) + lock.AssociateLocked(oidtest.ID()) err := putBig(db, lock) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index fd583c5452..c8313ee6e4 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -162,6 +162,12 @@ func get(metaCursor *bbolt.Cursor, addr oid.Address, checkStatus, raw bool, curr obj.SetParentID(id) case object.FilterPhysical, object.FilterRoot: // Not real attributes, ignored. + case object.AttributeAssociatedObject: + id, err := oid.DecodeBytes(attrVal) + if err != nil { + return nil, fmt.Errorf("invalid associated object in meta of %s/%s: %w", cnr, objID, err) + } + attrs = append(attrs, object.NewAttribute(string(attrKey), id.EncodeToString())) default: attrs = append(attrs, object.NewAttribute(string(attrKey), string(attrVal))) } diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index b65742a688..333cec102c 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -41,27 +41,25 @@ func TestDB_Get(t *testing.T) { }) t.Run("put tombstone object", func(t *testing.T) { - raw.SetType(object.TypeTombstone) - raw.SetID(oidtest.ID()) + tombObj := generateTypedObject(t, object.TypeTombstone, 10) - err := putBig(db, raw) + err := putBig(db, tombObj) require.NoError(t, err) - newObj, err := metaGet(db, raw.Address(), false) + newObj, err := metaGet(db, tombObj.Address(), false) require.NoError(t, err) - require.Equal(t, raw.CutPayload(), newObj) + require.Equal(t, tombObj.CutPayload(), newObj) }) t.Run("put lock object", func(t *testing.T) { - raw.SetType(object.TypeLock) - raw.SetID(oidtest.ID()) + lockObj := generateTypedObject(t, object.TypeLock, 10) - err := putBig(db, raw) + err := putBig(db, lockObj) require.NoError(t, err) - newObj, err := metaGet(db, raw.Address(), false) + newObj, err := metaGet(db, lockObj.Address(), false) require.NoError(t, err) - require.Equal(t, raw.CutPayload(), newObj) + require.Equal(t, lockObj.CutPayload(), newObj) }) t.Run("put virtual object", func(t *testing.T) { @@ -253,7 +251,7 @@ func TestDB_GetContainer(t *testing.T) { // TS o8 := generateObjectWithCID(t, cID) // 5 - o8.SetType(object.TypeTombstone) + o8.AssociateDeleted(oidtest.ID()) err = metaPut(db, o8) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 0e1451d72d..15a25c811e 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -8,6 +8,7 @@ import ( cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -65,6 +66,13 @@ func TestDB_IterateExpired(t *testing.T) { func putWithExpiration(t *testing.T, db *meta.DB, typ object.Type, expiresAt uint64) oid.Address { obj := generateObject(t) obj.SetType(typ) + switch typ { + case object.TypeTombstone: + obj.AssociateDeleted(oidtest.ID()) + case object.TypeLock: + obj.AssociateLocked(oidtest.ID()) + default: + } addAttribute(obj, object.AttributeExpirationEpoch, strconv.FormatUint(expiresAt, 10)) require.NoError(t, putBig(db, obj)) diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index b76cb44451..b81050f29c 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -148,14 +148,14 @@ func TestLisObjectsWithCursor(t *testing.T) { // add one tombstone obj = generateObjectWithCID(t, containerID) - obj.SetType(object.TypeTombstone) + obj.AssociateDeleted(oidtest.ID()) err = putBig(db, obj) require.NoError(t, err) expected = append(expected, objectcore.AddressWithAttributes{Address: obj.Address(), Type: object.TypeTombstone}) // add one lock obj = generateObjectWithCID(t, containerID) - obj.SetType(object.TypeLock) + obj.AssociateLocked(oidtest.ID()) err = putBig(db, obj) require.NoError(t, err) expected = append(expected, objectcore.AddressWithAttributes{Address: obj.Address(), Type: object.TypeLock}) diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 152abc5544..9c8292ef88 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -11,9 +11,7 @@ import ( // zero currEpoch skips expiration check. Returns associated object ID if it's // present. func associatedWithTypedObject(currEpoch uint64, metaCursor *bbolt.Cursor, idObj oid.ID, typ object.Type) (bool, oid.ID) { - var idStr = idObj.EncodeToString() - - for associateID := range iterAttrVal(metaCursor, object.AttributeAssociatedObject, []byte(idStr)) { + for associateID := range iterAttrVal(metaCursor, object.AttributeAssociatedObject, idObj[:]) { var cur = metaCursor.Bucket().Cursor() if isObjectType(cur, associateID, typ) { diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 1be758385c..f17bd63c38 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -12,7 +12,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/stretchr/testify/require" @@ -37,9 +36,9 @@ func TestDB_IsLocked(t *testing.T) { // existing but not locked obj - anotherObj := objecttest.Object() + anotherObj := generateObject(t) - err = db.Put(&anotherObj) + err = db.Put(anotherObj) require.NoError(t, err) locked, err = db.IsLocked(anotherObj.Address()) @@ -52,28 +51,29 @@ func TestDB_IsLocked(t *testing.T) { cnr := cidtest.ID() - o := objecttest.Object() + o := generateObject(t) o.SetContainerID(cnr) - o.SetType(object.TypeRegular) - l := objecttest.Object() + l := generateObject(t) + l.SetType(object.TypeLock) l.SetContainerID(cnr) l.SetAttributes(object.NewAttribute(object.AttributeExpirationEpoch, fmt.Sprintf("%d", currEpoch))) l.AssociateLocked(o.GetID()) - err := putBig(db, &o) + err := putBig(db, o) require.NoError(t, err) - err = putBig(db, &l) + err = putBig(db, l) require.NoError(t, err) - ts := objecttest.Object() + ts := generateObject(t) + ts.SetType(object.TypeTombstone) ts.SetContainerID(cnr) ts.AssociateDeleted(o.GetID()) - err = putBig(db, &ts) + err = putBig(db, ts) require.ErrorIs(t, err, apistatus.ErrObjectLocked) es.e = currEpoch + 1 - err = putBig(db, &ts) + err = putBig(db, ts) require.NoError(t, err) }) @@ -105,10 +105,11 @@ func TestDB_Lock_Expired(t *testing.T) { require.ErrorIs(t, err, meta.ErrObjectIsExpired) // lock the obj - l := objecttest.Object() + l := generateObject(t) + l.SetType(object.TypeLock) l.SetContainerID(addr.Container()) l.AssociateLocked(addr.Object()) - require.NoError(t, metaPut(db, &l)) + require.NoError(t, metaPut(db, l)) // object is expired but locked, thus, must be available _, err = metaGet(db, addr, false) @@ -120,10 +121,10 @@ func TestDB_Lock_Expired(t *testing.T) { func putObjAndLockIt(t *testing.T, db *meta.DB) (object.Object, object.Object) { cnr := cidtest.ID() - o := objecttest.Object() + o := *generateObject(t) o.SetContainerID(cnr) - o.SetType(object.TypeRegular) - l := objecttest.Object() + l := *generateObject(t) + l.SetType(object.TypeLock) l.SetContainerID(cnr) l.AssociateLocked(o.GetID()) diff --git a/pkg/local_object_storage/metabase/metadata.go b/pkg/local_object_storage/metabase/metadata.go index 72802a9762..c7c262d539 100644 --- a/pkg/local_object_storage/metabase/metadata.go +++ b/pkg/local_object_storage/metabase/metadata.go @@ -126,10 +126,19 @@ func PutMetadataForObject(tx *bbolt.Tx, hdr object.Object, phy bool) error { attrs := hdr.Attributes() for i := range attrs { ak, av := attrs[i].Key(), attrs[i].Value() - if n, isInt := parseInt(av); isInt { - err = putIntAttribute(metaBkt, &keyBuf, id, ak, av, &n) - } else { - err = putPlainAttribute(metaBkt, &keyBuf, id, ak, av) + switch ak { + case object.AttributeAssociatedObject: + var associated oid.ID + if err = associated.DecodeString(av); err != nil { + return fmt.Errorf("decode %q attribute value as object ID: %w", ak, err) + } + err = putPlainAttribute(metaBkt, &keyBuf, id, ak, associated[:]) + default: + if n, isInt := parseInt(av); isInt { + err = putIntAttribute(metaBkt, &keyBuf, id, ak, av, &n) + } else { + err = putPlainAttribute(metaBkt, &keyBuf, id, ak, av) + } } if err != nil { return err diff --git a/pkg/local_object_storage/metabase/metadata_test.go b/pkg/local_object_storage/metabase/metadata_test.go index 2962416dcc..1c0a08744c 100644 --- a/pkg/local_object_storage/metabase/metadata_test.go +++ b/pkg/local_object_storage/metabase/metadata_test.go @@ -108,10 +108,12 @@ func TestPutMetadata(t *testing.T) { 127, 251, 248, 253, 176, 145, 101, 69, 75, 12, 97, 27, 19} pldHmmHash := checksum.NewTillichZemor(pldHmmHashBytes) //nolint:staticcheck // this is a test and such objects are possible splitID := []byte{240, 204, 35, 185, 222, 70, 69, 124, 160, 224, 208, 185, 9, 114, 37, 109} + associatedID := oidtest.ID() var attrs []object.Attribute addAttr := func(k, v string) { attrs = append(attrs, object.NewAttribute(k, v)) } addAttr("attr_1", "val_1") addAttr("attr_2", "val_2") + addAttr(object.AttributeAssociatedObject, associatedID.EncodeToString()) addAttr("num_negative_overflow", "-115792089237316195423570985008687907853269984665640564039457584007913129639936") addAttr("num_negative_min", "-115792089237316195423570985008687907853269984665640564039457584007913129639935") addAttr("num_negative_min64", "-9223372036854775808") @@ -170,6 +172,7 @@ func TestPutMetadata(t *testing.T) { assertAttr(t, mb, id, "$Object:payloadHash", pldHashBytes[:]) assertAttr(t, mb, id, "$Object:split.parent", parentID[:]) assertAttr(t, mb, id, "$Object:split.first", firstID[:]) + assertAttr(t, mb, id, object.AttributeAssociatedObject, associatedID[:]) assertIntAttr(t, mb, id, "$Object:creationEpoch", "7311064694303989735", []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 118, 30, 154, 145, 227, 159, 231}) assertIntAttr(t, mb, id, "$Object:payloadLength", "2091724451450177666", []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 26de7fceef..e36e59d670 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -198,7 +198,7 @@ func handleLinkObject(diff *CountersDiff) error { func handleObjectWithAssociation(metaBkt *bbolt.Bucket, diff *CountersDiff, currEpoch uint64, obj object.Object) error { target := obj.AssociatedObject() if target.IsZero() { - return nil + return fmt.Errorf("%s object has zero associated object", obj.Type()) } cID := obj.GetContainerID() oID := obj.GetID() diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index c5610ba146..4de1d79ba3 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -222,6 +222,24 @@ func TestDB_Put_InGarbageMarkedContainer(t *testing.T) { require.Equal(t, []string{"GC MARKED"}, st.State) } +func TestDB_Put_InvalidAssociatedObject(t *testing.T) { + db := newDB(t) + + lock := generateObject(t) + lock.SetType(object.TypeLock) + lock.SetAttributes(object.NewAttribute(object.AttributeAssociatedObject, "not-an-oid")) + + err := db.Put(lock) + require.EqualError(t, err, "LOCK object has zero associated object") + + tomb := generateObject(t) + tomb.SetType(object.TypeTombstone) + tomb.SetAttributes(object.NewAttribute(object.AttributeAssociatedObject, "not-an-oid")) + + err = db.Put(tomb) + require.EqualError(t, err, "TOMBSTONE object has zero associated object") +} + func assertObjectAvailability(t *testing.T, db *meta.DB, addr oid.Address, obj object.Object) { t.Run("get", func(t *testing.T) { res, err := db.Get(addr, false) @@ -299,6 +317,13 @@ func TestDB_Put_Lock(t *testing.T) { obj := obj obj.SetType(typ) + switch typ { + case object.TypeTombstone: + obj.AssociateDeleted(oidtest.ID()) + case object.TypeLock: + obj.AssociateLocked(oidtest.ID()) + default: + } require.NoError(t, db.Put(&obj)) @@ -425,7 +450,7 @@ func TestDB_Put_Tombstone(t *testing.T) { }}, {name: "target is lock", preset: func(t *testing.T, db *meta.DB) { obj := obj - obj.SetType(object.TypeLock) + obj.AssociateLocked(oidtest.ID()) require.NoError(t, db.Put(&obj)) }, assertPutErr: func(t *testing.T, err error) { require.ErrorIs(t, err, meta.ErrLockObjectRemoval) @@ -692,7 +717,7 @@ func TestDB_PutBatch(t *testing.T) { obj2 := generateObjectWithCID(t, cnr) tombstone := generateObjectWithCID(t, cnr) - tombstone.SetType(object.TypeTombstone) + tombstone.AssociateDeleted(oidtest.ID()) require.NoError(t, db.Put(tombstone)) invalidLock := generateObjectWithCID(t, cnr) diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index ce7a2ff00d..d135ce38bf 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -153,7 +153,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) { require.NoError(t, err) ts := generateObjectWithCID(t, cnr) - ts.SetType(object.TypeTombstone) + ts.AssociateDeleted(oidtest.ID()) err = putBig(db, ts) require.NoError(t, err) @@ -165,7 +165,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) { require.NoError(t, err) lock := generateObjectWithCID(t, cnr) - lock.SetType(object.TypeLock) + lock.AssociateLocked(oidtest.ID()) err = putBig(db, lock) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 79a70ab6a8..2e16402d97 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -407,6 +407,11 @@ func migrateFrom10Version(db *DB) error { return fmt.Errorf("drop homomorphic indexes: %w", err) } + err = updateContainersInterruptable(db, []byte{metadataPrefix}, migrateAssociatedObjectValueToIDBytes) + if err != nil { + return fmt.Errorf("rewrite %q attribute values in metadata: %w", object.AttributeAssociatedObject, err) + } + return db.boltDB.Update(func(tx *bbolt.Tx) error { err := syncCounter(tx, true) if err != nil { @@ -445,6 +450,121 @@ func dropHomomorphicIndexes(_ *zap.Logger, _ *bbolt.Tx, b *bbolt.Bucket, _ cid.I return uint(len(keysToDrop)), nil, nil } +func migrateAssociatedObjectValueToIDBytes(l *zap.Logger, _ *bbolt.Tx, b *bbolt.Bucket, cnr cid.ID, afterKey []byte, rem uint) (uint, []byte, error) { + c := b.Cursor() + pref := append(append([]byte{metaPrefixAttrIDPlain}, object.AttributeAssociatedObject...), 0) + + k, _ := c.Seek(pref) + if afterKey != nil { + k, _ = c.Seek(afterKey) + if bytes.Equal(k, afterKey) { + k, _ = c.Next() + } + } + + type keyRewrite struct { + oldAttrID []byte + newAttrID []byte + oldIDAttr []byte + newIDAttr []byte + } + + var ( + scanned uint + nextKey []byte + buf keyBuffer + updates []keyRewrite + ) + + for ; k != nil && bytes.HasPrefix(k, pref); k, _ = c.Next() { + scanned++ + nextKey = slices.Clone(k) + + val, idRaw, err := splitAttributeValueObjectID(k[len(pref):]) + if err != nil { + l.Warn("skip malformed associated object metadata entry during migration", + zap.Stringer("container", cnr), + zap.String("key", hex.EncodeToString(k)), + zap.Error(err)) + continue + } + + if _, err = oid.DecodeBytes(val); err == nil { + if scanned == rem { + break + } + continue + } + + var associated oid.ID + if err = associated.DecodeString(string(val)); err != nil { + l.Warn("skip malformed associated object metadata entry during migration", + zap.Stringer("container", cnr), + zap.String("key", hex.EncodeToString(k)), + zap.Error(err)) + continue + } + + var id oid.ID + copy(id[:], idRaw) + + newAttrID, off := prepareMetaAttrIDKey(&buf, id, object.AttributeAssociatedObject, oid.Size, false) + copy(newAttrID[off:], associated[:]) + newAttrID = slices.Clone(newAttrID) + oldIDAttr := prepareMetaIDAttrKey(&buf, id, object.AttributeAssociatedObject, len(val)) + copy(oldIDAttr[len(oldIDAttr)-len(val):], val) + oldIDAttr = slices.Clone(oldIDAttr) + newIDAttr := prepareMetaIDAttrKey(&buf, id, object.AttributeAssociatedObject, oid.Size) + copy(newIDAttr[len(newIDAttr)-oid.Size:], associated[:]) + newIDAttr = slices.Clone(newIDAttr) + + updates = append(updates, keyRewrite{ + oldAttrID: slices.Clone(k), + newAttrID: newAttrID, + oldIDAttr: oldIDAttr, + newIDAttr: newIDAttr, + }) + + if scanned == rem { + break + } + } + + for i := range updates { + if err := b.Put(updates[i].newAttrID, nil); err != nil { + return 0, nil, fmt.Errorf("put migrated attribute-to-ID key %s: %w", hex.EncodeToString(updates[i].newAttrID), err) + } + if err := b.Put(updates[i].newIDAttr, nil); err != nil { + return 0, nil, fmt.Errorf("put migrated ID-to-attribute key %s: %w", hex.EncodeToString(updates[i].newIDAttr), err) + } + if err := b.Delete(updates[i].oldAttrID); err != nil { + return 0, nil, fmt.Errorf("delete migrated attribute-to-ID key %s: %w", hex.EncodeToString(updates[i].oldAttrID), err) + } + if err := b.Delete(updates[i].oldIDAttr); err != nil { + return 0, nil, fmt.Errorf("delete migrated ID-to-attribute key %s: %w", hex.EncodeToString(updates[i].oldIDAttr), err) + } + } + + if scanned < rem { + nextKey = nil + } + + return scanned, nextKey, nil +} + +func splitAttributeValueObjectID(b []byte) ([]byte, []byte, error) { + if len(b) < oid.Size+1 { + return nil, nil, fmt.Errorf("too short len %d", len(b)) + } + + valEnd := len(b) - oid.Size - 1 + if b[valEnd] != 0 { + return nil, nil, errors.New("wrong value-object delimiter") + } + + return b[:valEnd], b[valEnd+1:], nil +} + func moveGarbageToMeta(log *zap.Logger, tx *bbolt.Tx) error { var garbageBkt = tx.Bucket(garbageObjectsBucketName) diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index 36a46f1faa..fff7b1dc91 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -2,6 +2,8 @@ package meta import ( "bytes" + "crypto/rand" + "crypto/sha256" "encoding/binary" "errors" "fmt" @@ -134,6 +136,20 @@ func newDB(t testing.TB, opts ...Option) *DB { return bdb } +func generateTypedObject(cnr cid.ID, typ object.Type) object.Object { + data := make([]byte, 32) + _, _ = rand.Read(data) + + obj := object.New(cnr, usertest.ID()) + obj.SetID(oidtest.ID()) + obj.SetType(typ) + obj.SetPayload(data) + obj.SetPayloadSize(uint64(len(data))) + obj.SetPayloadChecksum(checksum.NewSHA256(sha256.Sum256(data))) + + return *obj +} + func TestSlicesCloneNil(t *testing.T) { // not stated in docs, but migrateContainersToMetaBucket relies on this require.Nil(t, slices.Clone([]byte(nil))) @@ -492,35 +508,20 @@ func TestMigrate8to9(t *testing.T) { func TestMigrate9To10(t *testing.T) { cID := cidtest.ID() - oTombstoned := objecttest.Object() - oTombstoned.ResetRelations() - oTombstoned.SetType(object.TypeRegular) - oTombstoned.SetContainerID(cID) + oTombstoned := generateTypedObject(cID, object.TypeRegular) oTombstoned.SetPayloadSize(11) - o := objecttest.Object() - o.ResetRelations() - o.SetType(object.TypeRegular) - o.SetContainerID(cID) + o := generateTypedObject(cID, object.TypeRegular) o.SetPayloadSize(22) - ts := objecttest.Object() - ts.ResetRelations() - ts.SetType(object.TypeTombstone) - ts.SetContainerID(cID) + ts := generateTypedObject(cID, object.TypeTombstone) ts.AssociateDeleted(oTombstoned.GetID()) ts.SetPayloadSize(33) - link := objecttest.Object() - link.ResetRelations() - link.SetType(object.TypeLink) - link.SetContainerID(cID) + link := generateTypedObject(cID, object.TypeLink) link.SetPayloadSize(44) - lock := objecttest.Object() - lock.ResetRelations() - lock.SetType(object.TypeLock) - lock.SetContainerID(cID) + lock := generateTypedObject(cID, object.TypeLock) lock.SetPayloadSize(55) // every object except tombstoned one @@ -656,6 +657,11 @@ func TestMigrate10To11(t *testing.T) { objs = append(objs, o) } + associatedTarget := oidtest.ID() + associatedObj := objecttest.Object() + associatedObj.SetContainerID(cID1) + associatedObj.AssociateLocked(associatedTarget) + err := db.boltDB.Update(func(tx *bbolt.Tx) error { bkt1, err := tx.CreateBucketIfNotExists(metaBucketKey(cID1)) require.NoError(t, err) @@ -669,6 +675,10 @@ func TestMigrate10To11(t *testing.T) { } } + if err = PutMetadataForObject(tx, associatedObj, true); err != nil { + return err + } + for i, o := range objs { var bkt *bbolt.Bucket if i < numOfTestObjs/2 { @@ -688,6 +698,13 @@ func TestMigrate10To11(t *testing.T) { } } + newAttrIDKey := makeAssociatedAttrIDKey(associatedObj.GetID(), associatedTarget[:]) + newIDAttrKey := makeAssociatedIDAttrKey(associatedObj.GetID(), associatedTarget[:]) + require.NoError(t, bkt1.Delete(newAttrIDKey)) + require.NoError(t, bkt1.Delete(newIDAttrKey)) + require.NoError(t, bkt1.Put(makeAssociatedAttrIDKey(associatedObj.GetID(), []byte(associatedTarget.EncodeToString())), nil)) + require.NoError(t, bkt1.Put(makeAssociatedIDAttrKey(associatedObj.GetID(), []byte(associatedTarget.EncodeToString())), nil)) + return nil }) require.NoError(t, err) @@ -720,6 +737,9 @@ func TestMigrate10To11(t *testing.T) { err = updateContainersInterruptable(db, []byte{metadataPrefix}, dropHomomorphicIndexes) require.NoError(t, err) + err = updateContainersInterruptable(db, []byte{metadataPrefix}, migrateAssociatedObjectValueToIDBytes) + require.NoError(t, err) + numOfFieldsAfter, err := countFields(db.boltDB) require.NoError(t, err) @@ -748,4 +768,48 @@ func TestMigrate10To11(t *testing.T) { return nil }) require.NoError(t, err) + + err = db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(metaBucketKey(cID1)) + c := b.Cursor() + + require.Equal(t, associatedTarget[:], getObjAttribute(c, associatedObj.GetID(), object.AttributeAssociatedObject)) + + var collected []oid.ID + for id := range iterAttrVal(c, object.AttributeAssociatedObject, associatedTarget[:]) { + collected = append(collected, id) + } + require.Equal(t, []oid.ID{associatedObj.GetID()}, collected) + + for id := range iterAttrVal(c, object.AttributeAssociatedObject, []byte(associatedTarget.EncodeToString())) { + t.Fatalf("unexpected legacy string index hit after migration: %s", id) + } + + return nil + }) + require.NoError(t, err) +} + +func makeAssociatedAttrIDKey(id oid.ID, value []byte) []byte { + res := make([]byte, 1+len(object.AttributeAssociatedObject)+1+len(value)+1+oid.Size) + res[0] = metaPrefixAttrIDPlain + off := 1 + copy(res[1:], object.AttributeAssociatedObject) + res[off] = 0 + off++ + off += copy(res[off:], value) + res[off] = 0 + off++ + copy(res[off:], id[:]) + return res +} + +func makeAssociatedIDAttrKey(id oid.ID, value []byte) []byte { + res := make([]byte, 1+oid.Size+len(object.AttributeAssociatedObject)+1+len(value)) + res[0] = metaPrefixIDAttr + off := 1 + copy(res[1:], id[:]) + off += copy(res[off:], object.AttributeAssociatedObject) + res[off] = 0 + off++ + copy(res[off:], value) + return res } diff --git a/pkg/local_object_storage/shard/container_test.go b/pkg/local_object_storage/shard/container_test.go index 2ac8c9c8df..b1c714df9b 100644 --- a/pkg/local_object_storage/shard/container_test.go +++ b/pkg/local_object_storage/shard/container_test.go @@ -5,7 +5,7 @@ import ( "testing" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" - "github.com/nspcc-dev/neofs-sdk-go/object" + oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -20,7 +20,7 @@ func TestShard_DeleteContainer(t *testing.T) { require.NoError(t, err) o3 := generateObjectWithCID(cID) - o3.SetType(object.TypeLock) + o3.AssociateLocked(oidtest.ID()) err = sh.Put(o3, nil) require.NoError(t, err) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index fe859c98c7..cb9e690305 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -2,6 +2,7 @@ package shard import ( "crypto/rand" + "crypto/sha256" "os" "path/filepath" "testing" @@ -11,13 +12,14 @@ import ( meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache" + "github.com/nspcc-dev/neofs-sdk-go/checksum" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" + usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) @@ -90,11 +92,11 @@ func TestResyncMetabaseCorrupted(t *testing.T) { require.NoError(t, sh.Open()) require.NoError(t, sh.Init()) - obj := objecttest.Object() - obj.SetType(object.TypeRegular) + obj := generateTypedObject(cidtest.ID(), object.TypeRegular) obj.SetPayload([]byte{0, 1, 2, 3, 4, 5}) + obj.SetPayloadSize(6) - err := sh.Put(&obj, nil) + err := sh.Put(obj, nil) require.NoError(t, err) require.NoError(t, sh.Close()) @@ -155,8 +157,7 @@ func TestResyncMetabase(t *testing.T) { locked[0] = oidtest.ID() cnrLocked := cidtest.ID() for i := range uint64(objNum) { - obj := objecttest.Object() - obj.SetType(object.TypeRegular) + obj := generateTypedObject(cidtest.ID(), object.TypeRegular) if i < objNum/2 { payload := make([]byte, 1024) @@ -174,13 +175,13 @@ func TestResyncMetabase(t *testing.T) { addr := obj.Address() mObjs[addr] = objAddr{ - obj: &obj, + obj: obj, addr: addr, } } tombedID := oidtest.ID() - tombObj := objecttest.Object() + tombObj := generateTypedObject(cidtest.ID(), object.TypeTombstone) tombObj.AssociateDeleted(tombedID) tombedAddress := oid.NewAddress(tombObj.GetContainerID(), tombedID) @@ -190,16 +191,15 @@ func TestResyncMetabase(t *testing.T) { require.NoError(t, err) } - err = sh.Put(&tombObj, nil) + err = sh.Put(tombObj, nil) require.NoError(t, err) // LOCK object handling for _, lockedObj := range locked { - lockObj := objecttest.Object() - lockObj.SetContainerID(cnrLocked) + lockObj := generateTypedObject(cnrLocked, object.TypeLock) lockObj.AssociateLocked(lockedObj) - err = sh.Put(&lockObj, nil) + err = sh.Put(lockObj, nil) require.NoError(t, err) } @@ -249,7 +249,7 @@ func TestResyncMetabase(t *testing.T) { } checkAllObjs(true) - checkObj(tombObj.Address(), &tombObj) + checkObj(tombObj.Address(), tombObj) checkTombMembers(true) checkLocked(t, cnrLocked, locked) @@ -302,7 +302,21 @@ func TestResyncMetabase(t *testing.T) { require.Equal(t, phyBefore, c.Phy) checkAllObjs(true) - checkObj(tombObj.Address(), &tombObj) + checkObj(tombObj.Address(), tombObj) checkTombMembers(true) checkLocked(t, cnrLocked, locked) } + +func generateTypedObject(cnr cid.ID, typ object.Type) *object.Object { + data := make([]byte, 32) + _, _ = rand.Read(data) + + obj := object.New(cnr, usertest.ID()) + obj.SetID(oidtest.ID()) + obj.SetType(typ) + obj.SetPayload(data) + obj.SetPayloadSize(uint64(len(data))) + obj.SetPayloadChecksum(checksum.NewSHA256(sha256.Sum256(data))) + + return obj +} diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index bc49ae16ee..c491c2765d 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -20,6 +20,7 @@ import ( neofscryptotest "github.com/nspcc-dev/neofs-sdk-go/crypto/test" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -219,6 +220,9 @@ func TestExpiration(t *testing.T) { expAttr.SetValue(strconv.FormatUint(exp, 10)) obj.SetAttributes(expAttr) obj.SetType(typ) + if typ == object.TypeTombstone { + obj.AssociateDeleted(oidtest.ID()) + } require.NoError(t, obj.SetIDWithSignature(neofscryptotest.Signer())) err := sh.Put(obj, nil) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index dbb09a3e07..c2da5a901c 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -156,6 +156,7 @@ func TestCounters(t *testing.T) { t.Run("inhume_TS", func(t *testing.T) { tombObj := objecttest.Object() + tombObj.ResetRelations() tombObj.SetPayload(nil) tombObj.SetPayloadSize(0) tombObj.SetContainerID(oo[0].GetContainerID()) diff --git a/pkg/local_object_storage/shard/put_test.go b/pkg/local_object_storage/shard/put_test.go index 76f58e834e..f96e0a30a0 100644 --- a/pkg/local_object_storage/shard/put_test.go +++ b/pkg/local_object_storage/shard/put_test.go @@ -13,7 +13,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/stretchr/testify/require" @@ -22,11 +21,11 @@ import ( func TestShard_PutBinary(t *testing.T) { addr := oidtest.Address() - obj := objecttest.Object() + obj := *generateTypedObject(cidtest.ID(), object.TypeRegular) obj.SetContainerID(addr.Container()) obj.SetID(addr.Object()) - obj2 := objecttest.Object() + obj2 := *generateTypedObject(cidtest.ID(), object.TypeRegular) require.NotEqual(t, obj, obj2) obj2.SetContainerID(addr.Container()) obj2.SetID(addr.Object()) @@ -90,6 +89,13 @@ func TestShard_Put_Lock(t *testing.T) { obj := obj obj.SetType(typ) + switch typ { + case object.TypeTombstone: + obj.AssociateDeleted(oidtest.ID()) + case object.TypeLock: + obj.AssociateLocked(oidtest.ID()) + default: + } require.NoError(t, sh.Put(&obj, nil)) @@ -276,7 +282,7 @@ func TestDB_Put_Tombstone(t *testing.T) { }}, {name: "target is lock", preset: func(t *testing.T, sh *shard.Shard) { obj := obj - obj.SetType(object.TypeLock) + obj.AssociateLocked(oidtest.ID()) require.NoError(t, sh.Put(&obj, nil)) }, assertPutErr: func(t *testing.T, err error) { require.ErrorIs(t, err, meta.ErrLockObjectRemoval) diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 4f1af5713a..cbdab74a00 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -91,6 +91,15 @@ func generateObject() *object.Object { return generateObjectWithCID(cidtest.ID()) } +func generateTypedObject(cnr cid.ID, typ object.Type) *object.Object { + data := make([]byte, 32) + _, _ = rand.Read(data) + obj := generateObjectWithPayload(cnr, data) + obj.SetType(typ) + obj.SetPayloadSize(uint64(len(data))) + return obj +} + func generateObjectWithCID(cnr cid.ID) *object.Object { data := make([]byte, 32) _, _ = rand.Read(data) diff --git a/pkg/util/meta/test/metatest.go b/pkg/util/meta/test/metatest.go index ec3e38a606..bbb06a8557 100644 --- a/pkg/util/meta/test/metatest.go +++ b/pkg/util/meta/test/metatest.go @@ -910,6 +910,7 @@ func TestSearchObjects(t *testing.T, db DB, testSplitID bool) { 10, 114, 174, 138, 120, 108, 165, 104, 36, 100, 129, 235, 160, 213, 96, 230, 190, 15, 196, 5, 252, 194, 205, 48, 236, 57, 117, 238, 170, 36, 251, 104, 62, 124, 1, 206, 131, 226, 221, 111, 73, 54, 235, 100, 49, 32, 252, 255, 92, 51, 30, 77, 180, 53})) + obj.AssociateDeleted(oid.ID{72, 252, 127, 110, 34, 30, 129, 55, 62, 213, 158, 181, 237, 90, 58, 114, 225, 28, 150, 17, 199, 42, 111, 215, 65, 133, 29, 14, 95, 18, 193, 9}) obj.SetParentID(oid.ID{146, 29, 179, 9, 47, 100, 26, 60, 219, 24, 253, 162, 255, 167, 39, 143, 234, 249, 77, 247, 52, 61, 3, 239, 110, 167, 61, 199, 138, 223, 198, 245}) obj.SetFirstID(oid.ID{35, 78, 81, 228, 188, 71, 53, 15, 64, 54, 230, 84, 94, 176, 193, 118, 225, 186, 208, 33, 175, 155, 154, 205, 116, 5, 247, 138, 65, 155, 210, 145}) obj.SetSplitID(&splitID)