diff options
Diffstat (limited to 'vendor/github.com/authzed/zed/pkg/backupformat')
4 files changed, 641 insertions, 0 deletions
diff --git a/vendor/github.com/authzed/zed/pkg/backupformat/decoder.go b/vendor/github.com/authzed/zed/pkg/backupformat/decoder.go new file mode 100644 index 0000000..8278424 --- /dev/null +++ b/vendor/github.com/authzed/zed/pkg/backupformat/decoder.go @@ -0,0 +1,120 @@ +package backupformat + +import ( + "errors" + "fmt" + "io" + + "github.com/hamba/avro/v2" + "github.com/hamba/avro/v2/ocf" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" +) + +func init() { + // This defaults to a 1MiB limit, but large schemas can exceed this size. + avro.DefaultConfig = avro.Config{ + MaxByteSliceSize: 1024 * 1024 * 100, // 100 MiB + }.Freeze() +} + +func NewDecoder(r io.Reader) (*Decoder, error) { + dec, err := ocf.NewDecoder(r) + if err != nil { + return nil, fmt.Errorf("unable to create ocf decoder: %w", err) + } + + md := dec.Metadata() + var zedToken *v1.ZedToken + + if token, ok := md[metadataKeyZT]; ok { + zedToken = &v1.ZedToken{ + Token: string(token), + } + } + + var schemaText string + if dec.HasNext() { + var decodedSchema any + if err := dec.Decode(&decodedSchema); err != nil { + return nil, fmt.Errorf("unable to decode schema object: %w", err) + } + + schema, ok := decodedSchema.(SchemaV1) + if !ok { + return nil, fmt.Errorf("received schema object of wrong type: %T", decodedSchema) + } + schemaText = schema.SchemaText + } else { + return nil, errors.New("avro stream contains no schema object") + } + + return &Decoder{ + dec, + schemaText, + zedToken, + }, nil +} + +type Decoder struct { + dec *ocf.Decoder + schema string + zedToken *v1.ZedToken +} + +func (d *Decoder) Schema() string { + return d.schema +} + +func (d *Decoder) ZedToken() *v1.ZedToken { + return d.zedToken +} + +func (d *Decoder) Close() error { + return nil +} + +func (d *Decoder) Next() (*v1.Relationship, error) { + if !d.dec.HasNext() { + return nil, nil + } + + var nextRelIFace any + if err := d.dec.Decode(&nextRelIFace); err != nil { + return nil, fmt.Errorf("unable to decode relationship from avro stream: %w", err) + } + + flat := nextRelIFace.(RelationshipV1) + + rel := &v1.Relationship{ + Resource: &v1.ObjectReference{ + ObjectType: flat.ObjectType, + ObjectId: flat.ObjectID, + }, + Relation: flat.Relation, + Subject: &v1.SubjectReference{ + Object: &v1.ObjectReference{ + ObjectType: flat.SubjectObjectType, + ObjectId: flat.SubjectObjectID, + }, + OptionalRelation: flat.SubjectRelation, + }, + } + + if flat.CaveatName != "" { + var deserializedCtxt structpb.Struct + + if err := proto.Unmarshal(flat.CaveatContext, &deserializedCtxt); err != nil { + return nil, fmt.Errorf("unable to deserialize caveat context: %w", err) + } + + rel.OptionalCaveat = &v1.ContextualizedCaveat{ + CaveatName: flat.CaveatName, + Context: &deserializedCtxt, + } + } + + return rel, nil +} diff --git a/vendor/github.com/authzed/zed/pkg/backupformat/encoder.go b/vendor/github.com/authzed/zed/pkg/backupformat/encoder.go new file mode 100644 index 0000000..f591ee7 --- /dev/null +++ b/vendor/github.com/authzed/zed/pkg/backupformat/encoder.go @@ -0,0 +1,91 @@ +package backupformat + +import ( + "errors" + "fmt" + "io" + + "github.com/hamba/avro/v2/ocf" + "google.golang.org/protobuf/proto" + + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" +) + +func NewEncoderForExisting(w io.Writer) (*Encoder, error) { + avroSchema, err := avroSchemaV1() + if err != nil { + return nil, fmt.Errorf("unable to create avro schema: %w", err) + } + + enc, err := ocf.NewEncoder(avroSchema, w, ocf.WithCodec(ocf.Snappy)) + if err != nil { + return nil, fmt.Errorf("unable to create encoder: %w", err) + } + + return &Encoder{enc}, nil +} + +func NewEncoder(w io.Writer, schema string, token *v1.ZedToken) (*Encoder, error) { + avroSchema, err := avroSchemaV1() + if err != nil { + return nil, fmt.Errorf("unable to create avro schema: %w", err) + } + + if token == nil { + return nil, errors.New("missing expected token") + } + + md := map[string][]byte{ + metadataKeyZT: []byte(token.Token), + } + + enc, err := ocf.NewEncoder(avroSchema, w, ocf.WithCodec(ocf.Snappy), ocf.WithMetadata(md)) + if err != nil { + return nil, fmt.Errorf("unable to create encoder: %w", err) + } + + if err := enc.Encode(SchemaV1{ + SchemaText: schema, + }); err != nil { + return nil, fmt.Errorf("unable to encode SpiceDB schema object: %w", err) + } + + return &Encoder{enc}, nil +} + +type Encoder struct { + enc *ocf.Encoder +} + +func (e *Encoder) Append(rel *v1.Relationship) error { + var toEncode RelationshipV1 + + toEncode.ObjectType = rel.Resource.ObjectType + toEncode.ObjectID = rel.Resource.ObjectId + toEncode.Relation = rel.Relation + toEncode.SubjectObjectType = rel.Subject.Object.ObjectType + toEncode.SubjectObjectID = rel.Subject.Object.ObjectId + toEncode.SubjectRelation = rel.Subject.OptionalRelation + if rel.OptionalCaveat != nil { + contextBytes, err := proto.Marshal(rel.OptionalCaveat.Context) + if err != nil { + return fmt.Errorf("error marshaling caveat context: %w", err) + } + + toEncode.CaveatName = rel.OptionalCaveat.CaveatName + toEncode.CaveatContext = contextBytes + } + + if err := e.enc.Encode(toEncode); err != nil { + return fmt.Errorf("unable to encode relationship: %w", err) + } + + return nil +} + +func (e *Encoder) Close() error { + if err := e.enc.Flush(); err != nil { + return fmt.Errorf("unable to flush encoder: %w", err) + } + return nil +} diff --git a/vendor/github.com/authzed/zed/pkg/backupformat/redaction.go b/vendor/github.com/authzed/zed/pkg/backupformat/redaction.go new file mode 100644 index 0000000..c89c03f --- /dev/null +++ b/vendor/github.com/authzed/zed/pkg/backupformat/redaction.go @@ -0,0 +1,329 @@ +package backupformat + +import ( + "fmt" + "io" + "strconv" + + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" + "github.com/authzed/spicedb/pkg/namespace" + core "github.com/authzed/spicedb/pkg/proto/core/v1" + "github.com/authzed/spicedb/pkg/schemadsl/compiler" + "github.com/authzed/spicedb/pkg/schemadsl/generator" + "github.com/authzed/spicedb/pkg/schemadsl/input" + "github.com/authzed/spicedb/pkg/spiceerrors" + "github.com/authzed/spicedb/pkg/tuple" +) + +// RedactionOptions are the options to use when redacting data. +type RedactionOptions struct { + // RedactDefinitions will redact the definition names. + RedactDefinitions bool + + // RedactRelations will redact the relation names. + RedactRelations bool + + // RedactObjectIDs will redact the object IDs. + RedactObjectIDs bool +} + +// RedactionMap is the map of original names to their redacted names. +type RedactionMap struct { + // Definitions is the map of original definition names to their redacted names. + Definitions map[string]string + + // Caveats is the map of original caveat names to their redacted names. + Caveats map[string]string + + // Relations is the map of original relation names to their redacted names. + Relations map[string]string + + // ObjectIDs is the map of original object IDs to their redacted names. + ObjectIDs map[string]string +} + +// Invert returns the inverted redaction map, with the redacted names as the keys. +func (rm RedactionMap) Invert() RedactionMap { + inverted := RedactionMap{ + Definitions: make(map[string]string), + Caveats: make(map[string]string), + Relations: make(map[string]string), + ObjectIDs: make(map[string]string), + } + + for k, v := range rm.Definitions { + inverted.Definitions[v] = k + } + + for k, v := range rm.Caveats { + inverted.Caveats[v] = k + } + + for k, v := range rm.Relations { + inverted.Relations[v] = k + } + + for k, v := range rm.ObjectIDs { + inverted.ObjectIDs[v] = k + } + + return inverted +} + +// NewRedactor creates a new redactor that will redact the data as it is written. +func NewRedactor(dec *Decoder, w io.Writer, opts RedactionOptions) (*Redactor, error) { + // Rewrite the schema to redact as requested. + redactedSchema, redactionMap, err := redactSchema(dec.Schema(), opts) + if err != nil { + return nil, err + } + + // Create a new encoder with the redacted schema. + token := dec.ZedToken() + encoder, err := NewEncoder(w, redactedSchema, token) + if err != nil { + return nil, err + } + + return &Redactor{dec, opts, encoder, redactionMap}, nil +} + +type Redactor struct { + dec *Decoder + opts RedactionOptions + enc *Encoder + redactionMap RedactionMap +} + +// Next redacts the next record and writes it to the writer. +func (r *Redactor) Next() error { + // Read the next record. + rel, err := r.dec.Next() + if err != nil { + return err + } + + if rel == nil { + return io.EOF + } + + // Redact the record. + redactedRel, err := redactRelationship(rel, &r.redactionMap, r.opts) + if err != nil { + return err + } + + // Write the redacted record. + return r.enc.Append(redactedRel) +} + +// RedactionMap returns the redaction map containing the original names and their redacted names. +func (r *Redactor) RedactionMap() RedactionMap { + return r.redactionMap +} + +func (r *Redactor) Close() error { + if err := r.enc.Close(); err != nil { + return err + } + + return r.dec.Close() +} + +func redactSchema(schema string, opts RedactionOptions) (string, RedactionMap, error) { + // Parse the schema. + compiled, err := compiler.Compile(compiler.InputSchema{ + Source: input.Source("schema"), + SchemaString: schema, + }, compiler.AllowUnprefixedObjectType()) + if err != nil { + return "", RedactionMap{}, err + } + + // Create a new schema with the redacted fields. + redactionMap := RedactionMap{ + Definitions: make(map[string]string), + Caveats: make(map[string]string), + Relations: make(map[string]string), + ObjectIDs: make(map[string]string), + } + + redactionCount := 0 + + // Redact namespace and caveat names. + if opts.RedactDefinitions { + for _, nsDef := range compiled.ObjectDefinitions { + if opts.RedactDefinitions { + redactionMap.Definitions[nsDef.Name] = "def" + strconv.Itoa(redactionCount) + redactionCount++ + nsDef.Name = redactionMap.Definitions[nsDef.Name] + } + + namespace.FilterUserDefinedMetadataInPlace(nsDef) + } + + if len(compiled.CaveatDefinitions) > 0 { + fmt.Println("WARNING: Caveat parameters and comments are not currently redacted.") + } + + for _, caveatDef := range compiled.CaveatDefinitions { + if opts.RedactDefinitions { + redactionMap.Caveats[caveatDef.Name] = "cav" + strconv.Itoa(redactionCount) + redactionCount++ + caveatDef.Name = redactionMap.Caveats[caveatDef.Name] + } + + // TODO: Redact caveat parameters. + // TODO: filter caveat metadata. + } + } + + // Redact relation names. + if opts.RedactRelations { + for _, nsDef := range compiled.ObjectDefinitions { + for _, relDef := range nsDef.Relation { + if existing, ok := redactionMap.Relations[relDef.Name]; ok { + relDef.Name = existing + continue + } + + redactionMap.Relations[relDef.Name] = "rel" + strconv.Itoa(redactionCount) + redactionCount++ + relDef.Name = redactionMap.Relations[relDef.Name] + } + } + } + + // Redact type information. + if opts.RedactDefinitions || opts.RedactRelations { + for _, nsDef := range compiled.ObjectDefinitions { + for _, relDef := range nsDef.Relation { + if relDef.TypeInformation != nil { + for _, allowedDirect := range relDef.TypeInformation.AllowedDirectRelations { + if opts.RedactDefinitions { + allowedDirect.Namespace = redactionMap.Definitions[allowedDirect.Namespace] + + if allowedDirect.RequiredCaveat != nil { + allowedDirect.RequiredCaveat.CaveatName = redactionMap.Caveats[allowedDirect.RequiredCaveat.CaveatName] + } + } + + if opts.RedactRelations { + switch t := allowedDirect.RelationOrWildcard.(type) { + case *core.AllowedRelation_Relation: + t.Relation = redactionMap.Relations[t.Relation] + } + } + } + } + } + } + } + + // Redact within userset rewrites. + if opts.RedactRelations { + for _, nsDef := range compiled.ObjectDefinitions { + for _, relDef := range nsDef.Relation { + if relDef.UsersetRewrite != nil { + err := redactUsersetRewrite(relDef.UsersetRewrite, &redactionMap) + if err != nil { + return "", RedactionMap{}, err + } + } + } + } + } + + // Generate the schema string. + generated, _, err := generator.GenerateSchema(compiled.OrderedDefinitions) + return generated, redactionMap, err +} + +func redactUsersetRewrite(usersetRewrite *core.UsersetRewrite, redactionMap *RedactionMap) error { + switch t := usersetRewrite.RewriteOperation.(type) { + case *core.UsersetRewrite_Union: + return redactRewriteChildren(t.Union.Child, redactionMap) + + case *core.UsersetRewrite_Intersection: + return redactRewriteChildren(t.Intersection.Child, redactionMap) + + case *core.UsersetRewrite_Exclusion: + return redactRewriteChildren(t.Exclusion.Child, redactionMap) + + default: + return spiceerrors.MustBugf("unknown userset rewrite type: %T", t) + } +} + +func redactRewriteChildren(children []*core.SetOperation_Child, redactionMap *RedactionMap) error { + for _, child := range children { + switch t := child.ChildType.(type) { + case *core.SetOperation_Child_ComputedUserset: + t.ComputedUserset.Relation = redactionMap.Relations[t.ComputedUserset.Relation] + + case *core.SetOperation_Child_UsersetRewrite: + err := redactUsersetRewrite(t.UsersetRewrite, redactionMap) + if err != nil { + return err + } + + case *core.SetOperation_Child_TupleToUserset: + t.TupleToUserset.Tupleset.Relation = redactionMap.Relations[t.TupleToUserset.Tupleset.Relation] + t.TupleToUserset.ComputedUserset.Relation = redactionMap.Relations[t.TupleToUserset.ComputedUserset.Relation] + + case *core.SetOperation_Child_XNil: + // nothing to do + + case *core.SetOperation_Child_XThis: + // nothing to do + + default: + return spiceerrors.MustBugf("unknown child type: %T", t) + } + } + + return nil +} + +func redactRelationship(rel *v1.Relationship, redactionMap *RedactionMap, opts RedactionOptions) (*v1.Relationship, error) { + redactedRel := rel.CloneVT() + + // Redact the resource. + if opts.RedactDefinitions { + redactedRel.Resource.ObjectType = redactionMap.Definitions[redactedRel.Resource.ObjectType] + redactedRel.Subject.Object.ObjectType = redactionMap.Definitions[redactedRel.Subject.Object.ObjectType] + + if rel.OptionalCaveat != nil { + redactedRel.OptionalCaveat.CaveatName = redactionMap.Caveats[redactedRel.OptionalCaveat.CaveatName] + } + } + + // Redact the relation. + if opts.RedactRelations { + redactedRel.Relation = redactionMap.Relations[redactedRel.Relation] + + if rel.Subject.OptionalRelation != "" { + redactedRel.Subject.OptionalRelation = redactionMap.Relations[redactedRel.Subject.OptionalRelation] + } + } + + // Redact the object IDs. + if opts.RedactObjectIDs { + redactionMap.ObjectIDs[tuple.PublicWildcard] = tuple.PublicWildcard // wilcards are not redacted + if _, ok := redactionMap.ObjectIDs[redactedRel.Resource.ObjectId]; !ok { + if redactedRel.Resource.ObjectId != tuple.PublicWildcard { + redactionMap.ObjectIDs[redactedRel.Resource.ObjectId] = "obj" + strconv.Itoa(len(redactionMap.ObjectIDs)) + } + } + + redactedRel.Resource.ObjectId = redactionMap.ObjectIDs[redactedRel.Resource.ObjectId] + + if _, ok := redactionMap.ObjectIDs[redactedRel.Subject.Object.ObjectId]; !ok { + redactionMap.ObjectIDs[redactedRel.Subject.Object.ObjectId] = "obj" + strconv.Itoa(len(redactionMap.ObjectIDs)) + } + + redactedRel.Subject.Object.ObjectId = redactionMap.ObjectIDs[redactedRel.Subject.Object.ObjectId] + } + + return redactedRel, nil +} diff --git a/vendor/github.com/authzed/zed/pkg/backupformat/schema.go b/vendor/github.com/authzed/zed/pkg/backupformat/schema.go new file mode 100644 index 0000000..64a8ab6 --- /dev/null +++ b/vendor/github.com/authzed/zed/pkg/backupformat/schema.go @@ -0,0 +1,101 @@ +package backupformat + +import ( + "errors" + "fmt" + "reflect" + + "github.com/hamba/avro/v2" +) + +func init() { + avro.DefaultConfig.Register(spiceDBBackupNamespace+"."+schemaV1SchemaName, SchemaV1{}) + avro.DefaultConfig.Register(spiceDBBackupNamespace+"."+relationshipV1SchemaName, RelationshipV1{}) +} + +type RelationshipV1 struct { + ObjectType string `avro:"object_type"` + ObjectID string `avro:"object_id"` + Relation string `avro:"relation"` + SubjectObjectType string `avro:"subject_object_type"` + SubjectObjectID string `avro:"subject_object_id"` + SubjectRelation string `avro:"subject_relation"` + CaveatName string `avro:"caveat_name"` + CaveatContext []byte `avro:"caveat_context"` +} + +type SchemaV1 struct { + SchemaText string `avro:"schema_text"` +} + +const ( + spiceDBBackupNamespace = "com.authzed.spicedb.backup" + + relationshipV1SchemaName = "relationship_v1" + schemaV1SchemaName = "schema_v1" + + metadataKeyZT = "com.authzed.spicedb.zedtoken.v1" +) + +func avroSchemaV1() (string, error) { + relationshipSchema, err := recordSchemaFromAvroStruct( + relationshipV1SchemaName, + spiceDBBackupNamespace, + RelationshipV1{}, + ) + if err != nil { + return "", fmt.Errorf("unable to create schema: %w", err) + } + + schemaSchema, err := recordSchemaFromAvroStruct( + schemaV1SchemaName, + spiceDBBackupNamespace, + SchemaV1{}, + ) + if err != nil { + return "", fmt.Errorf("unable to create avro SpiceDB schema schema: %w", err) + } + + unionSchema, err := avro.NewUnionSchema([]avro.Schema{relationshipSchema, schemaSchema}) + if err != nil { + return "", fmt.Errorf("unable to create avro union schema: %w", err) + } + + serialized, err := unionSchema.MarshalJSON() + return string(serialized), err +} + +func recordSchemaFromAvroStruct(name, namespace string, avroStruct any) (*avro.RecordSchema, error) { + v := reflect.TypeOf(avroStruct) + schemaFields := make([]*avro.Field, 0, v.NumField()) + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + fieldName := f.Tag.Get("avro") + if fieldName == "" { + return nil, fmt.Errorf("field `%s` missing avro struct tag", f.Name) + } + fieldGoType := f.Type + + var fieldType avro.Type + switch fieldGoType.Kind() { + case reflect.String: + fieldType = avro.String + case reflect.Slice: + if fieldGoType.Elem().Kind() != reflect.Uint8 { + return nil, errors.New("unable to build schema for slice, only byte slices are supported") + } + fieldType = avro.Bytes + default: + return nil, fmt.Errorf("unsupported struct kind: %s", fieldGoType) + } + + schemaField, err := avro.NewField(fieldName, avro.NewPrimitiveSchema(fieldType, nil)) + if err != nil { + return nil, fmt.Errorf("unable to create avro schema field: %w", err) + } + + schemaFields = append(schemaFields, schemaField) + } + + return avro.NewRecordSchema(name, namespace, schemaFields) +} |