Skip to content

Commit

Permalink
Merge pull request etcd-io#16079 from serathius/robustness-range-request
Browse files Browse the repository at this point in the history 
Robustness range request
  • Loading branch information
@serathius
serathius authored Jun 15, 2023
2 parents 899a51e + 6979318 commit 7d27e33
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 119 deletions.
83 changes: 52 additions & 31 deletions tests/robustness/model/describe.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,25 @@ func describeEtcdNonDeterministicResponse(request EtcdRequest, response EtcdNonD
}

func describeEtcdResponse(request EtcdRequest, response EtcdResponse) string {
if request.Type == Txn {
switch request.Type {
case Range:
return fmt.Sprintf("%s, rev: %d", describeRangeResponse(request.Range.RangeOptions, *response.Range), response.Revision)
case Txn:
return fmt.Sprintf("%s, rev: %d", describeTxnResponse(request.Txn, response.Txn), response.Revision)
case LeaseGrant, LeaseRevoke, Defragment:
if response.Revision == 0 {
return "ok"
}
return fmt.Sprintf("ok, rev: %d", response.Revision)
default:
return fmt.Sprintf("<! unknown request type: %q !>", request.Type)
}
if response.Revision == 0 {
return "ok"
}
return fmt.Sprintf("ok, rev: %d", response.Revision)
}

func describeEtcdRequest(request EtcdRequest) string {
switch request.Type {
case Range:
return describeRangeRequest(request.Range.Key, request.Range.RangeOptions)
case Txn:
onSuccess := describeEtcdOperations(request.Txn.OperationsOnSuccess)
if len(request.Txn.Conditions) != 0 {
Expand Down Expand Up @@ -100,51 +108,64 @@ func describeTxnResponse(request *TxnRequest, response *TxnResponse) string {

func describeEtcdOperation(op EtcdOperation) string {
switch op.Type {
case Range:
if op.WithPrefix {
if op.Limit != 0 {
return fmt.Sprintf("range(%q, limit=%d)", op.Key, op.Limit)
}
return fmt.Sprintf("range(%q)", op.Key)
}
return fmt.Sprintf("get(%q)", op.Key)
case Put:
case RangeOperation:
return describeRangeRequest(op.Key, op.RangeOptions)
case PutOperation:
if op.LeaseID != 0 {
return fmt.Sprintf("put(%q, %s, %d)", op.Key, describeValueOrHash(op.Value), op.LeaseID)
}
return fmt.Sprintf("put(%q, %s)", op.Key, describeValueOrHash(op.Value))
case Delete:
case DeleteOperation:
return fmt.Sprintf("delete(%q)", op.Key)
default:
return fmt.Sprintf("<! unknown op: %q !>", op.Type)
}
}

func describeRangeRequest(key string, opts RangeOptions) string {
kwargs := []string{}
if opts.Limit != 0 {
kwargs = append(kwargs, fmt.Sprintf("limit=%d", opts.Limit))
}
command := "get"
if opts.WithPrefix {
command = "range"
}
if len(kwargs) == 0 {
return fmt.Sprintf("%s(%q)", command, key)
}
return fmt.Sprintf("%s(%q, %s)", command, key, strings.Join(kwargs, ", "))
}

func describeEtcdOperationResponse(req EtcdOperation, resp EtcdOperationResult) string {
switch req.Type {
case Range:
if req.WithPrefix {
kvs := make([]string, len(resp.KVs))
for i, kv := range resp.KVs {
kvs[i] = describeValueOrHash(kv.Value)
}
return fmt.Sprintf("[%s], count: %d", strings.Join(kvs, ","), resp.Count)
} else {
if len(resp.KVs) == 0 {
return "nil"
} else {
return describeValueOrHash(resp.KVs[0].Value)
}
}
case Put:
case RangeOperation:
return describeRangeResponse(req.RangeOptions, resp.RangeResponse)
case PutOperation:
return fmt.Sprintf("ok")
case Delete:
case DeleteOperation:
return fmt.Sprintf("deleted: %d", resp.Deleted)
default:
return fmt.Sprintf("<! unknown op: %q !>", req.Type)
}
}

func describeRangeResponse(opts RangeOptions, response RangeResponse) string {
if opts.WithPrefix {
kvs := make([]string, len(response.KVs))
for i, kv := range response.KVs {
kvs[i] = describeValueOrHash(kv.Value)
}
return fmt.Sprintf("[%s], count: %d", strings.Join(kvs, ","), response.Count)
} else {
if len(response.KVs) == 0 {
return "nil"
} else {
return describeValueOrHash(response.KVs[0].Value)
}
}
}

func describeValueOrHash(value ValueOrHash) string {
if value.Hash != 0 {
return fmt.Sprintf("hash: %d", value.Hash)
Expand Down
10 changes: 5 additions & 5 deletions tests/robustness/model/describe_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,18 +95,18 @@ func TestModelDescribe(t *testing.T) {
expectDescribe: `if(mod_rev(key9)==9).then(put("key9", "99")) -> err: "failed"`,
},
{
req: txnRequest([]EtcdCondition{{Key: "key9b", ExpectedRevision: 9}}, []EtcdOperation{{Type: Put, Key: "key9b", Value: ValueOrHash{Value: "991"}}}, []EtcdOperation{{Type: Range, Key: "key9b"}}),
req: txnRequest([]EtcdCondition{{Key: "key9b", ExpectedRevision: 9}}, []EtcdOperation{{Type: PutOperation, Key: "key9b", PutOptions: PutOptions{Value: ValueOrHash{Value: "991"}}}}, []EtcdOperation{{Type: RangeOperation, Key: "key9b"}}),
resp: txnResponse([]EtcdOperationResult{{}}, true, 10),
expectDescribe: `if(mod_rev(key9b)==9).then(put("key9b", "991")).else(get("key9b")) -> success(ok), rev: 10`,
},
{
req: txnRequest([]EtcdCondition{{Key: "key9c", ExpectedRevision: 9}}, []EtcdOperation{{Type: Put, Key: "key9c", Value: ValueOrHash{Value: "992"}}}, []EtcdOperation{{Type: Range, Key: "key9c"}}),
resp: txnResponse([]EtcdOperationResult{{KVs: []KeyValue{{Key: "key9c", ValueRevision: ValueRevision{Value: ValueOrHash{Value: "993"}, ModRevision: 10}}}}}, false, 10),
req: txnRequest([]EtcdCondition{{Key: "key9c", ExpectedRevision: 9}}, []EtcdOperation{{Type: PutOperation, Key: "key9c", PutOptions: PutOptions{Value: ValueOrHash{Value: "992"}}}}, []EtcdOperation{{Type: RangeOperation, Key: "key9c"}}),
resp: txnResponse([]EtcdOperationResult{{RangeResponse: RangeResponse{KVs: []KeyValue{{Key: "key9c", ValueRevision: ValueRevision{Value: ValueOrHash{Value: "993"}, ModRevision: 10}}}}}}, false, 10),
expectDescribe: `if(mod_rev(key9c)==9).then(put("key9c", "992")).else(get("key9c")) -> failure("993"), rev: 10`,
},
{
req: txnRequest(nil, []EtcdOperation{{Type: Range, Key: "10"}, {Type: Put, Key: "11", Value: ValueOrHash{Value: "111"}}, {Type: Delete, Key: "12"}}, nil),
resp: txnResponse([]EtcdOperationResult{{KVs: []KeyValue{{ValueRevision: ValueRevision{Value: ValueOrHash{Value: "110"}}}}}, {}, {Deleted: 1}}, true, 10),
req: txnRequest(nil, []EtcdOperation{{Type: RangeOperation, Key: "10"}, {Type: PutOperation, Key: "11", PutOptions: PutOptions{Value: ValueOrHash{Value: "111"}}}, {Type: DeleteOperation, Key: "12"}}, nil),
resp: txnResponse([]EtcdOperationResult{{RangeResponse: RangeResponse{KVs: []KeyValue{{ValueRevision: ValueRevision{Value: ValueOrHash{Value: "110"}}}}}}, {}, {Deleted: 1}}, true, 10),
expectDescribe: `get("10"), put("11", "111"), delete("12") -> "110", ok, deleted: 1, rev: 10`,
},
{
Expand Down
127 changes: 87 additions & 40 deletions tests/robustness/model/deterministic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ func initState(request EtcdRequest, response EtcdResponse) etcdState {
state := emptyState()
state.Revision = response.Revision
switch request.Type {
case Range:
for _, kv := range response.Range.KVs {
state.KeyValues[kv.Key] = ValueRevision{
Value: kv.Value,
ModRevision: kv.ModRevision,
}
}
case Txn:
if response.Txn.Failure {
return state
Expand All @@ -83,19 +90,19 @@ func initState(request EtcdRequest, response EtcdResponse) etcdState {
for i, op := range request.Txn.OperationsOnSuccess {
opResp := response.Txn.Results[i]
switch op.Type {
case Range:
case RangeOperation:
for _, kv := range opResp.KVs {
state.KeyValues[kv.Key] = ValueRevision{
Value: kv.Value,
ModRevision: kv.ModRevision,
}
}
case Put:
case PutOperation:
state.KeyValues[op.Key] = ValueRevision{
Value: op.Value,
ModRevision: response.Revision,
}
case Delete:
case DeleteOperation:
default:
panic("Unknown operation")
}
Expand Down Expand Up @@ -131,6 +138,9 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
}
s.KeyValues = newKVs
switch request.Type {
case Range:
resp := s.getRange(request.Range.Key, request.Range.RangeOptions)
return s, EtcdResponse{Range: &resp, Revision: s.Revision}
case Txn:
failure := false
for _, cond := range request.Txn.Conditions {
Expand All @@ -147,36 +157,11 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
increaseRevision := false
for i, op := range operations {
switch op.Type {
case Range:
case RangeOperation:
opResp[i] = EtcdOperationResult{
KVs: []KeyValue{},
RangeResponse: s.getRange(op.Key, op.RangeOptions),
}
if op.WithPrefix {
var count int64
for k, v := range s.KeyValues {
if strings.HasPrefix(k, op.Key) {
opResp[i].KVs = append(opResp[i].KVs, KeyValue{Key: k, ValueRevision: v})
count += 1
}
}
sort.Slice(opResp[i].KVs, func(j, k int) bool {
return opResp[i].KVs[j].Key < opResp[i].KVs[k].Key
})
if op.Limit != 0 && count > op.Limit {
opResp[i].KVs = opResp[i].KVs[:op.Limit]
}
opResp[i].Count = count
} else {
value, ok := s.KeyValues[op.Key]
if ok {
opResp[i].KVs = append(opResp[i].KVs, KeyValue{
Key: op.Key,
ValueRevision: value,
})
opResp[i].Count = 1
}
}
case Put:
case PutOperation:
_, leaseExists := s.Leases[op.LeaseID]
if op.LeaseID != 0 && !leaseExists {
break
Expand All @@ -190,7 +175,7 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
if leaseExists {
s = attachToNewLease(s, op.LeaseID, op.Key)
}
case Delete:
case DeleteOperation:
if _, ok := s.KeyValues[op.Key]; ok {
delete(s.KeyValues, op.Key)
increaseRevision = true
Expand Down Expand Up @@ -238,6 +223,38 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
}
}

func (s etcdState) getRange(key string, options RangeOptions) RangeResponse {
response := RangeResponse{
KVs: []KeyValue{},
}
if options.WithPrefix {
var count int64
for k, v := range s.KeyValues {
if strings.HasPrefix(k, key) {
response.KVs = append(response.KVs, KeyValue{Key: k, ValueRevision: v})
count += 1
}
}
sort.Slice(response.KVs, func(j, k int) bool {
return response.KVs[j].Key < response.KVs[k].Key
})
if options.Limit != 0 && count > options.Limit {
response.KVs = response.KVs[:options.Limit]
}
response.Count = count
} else {
value, ok := s.KeyValues[key]
if ok {
response.KVs = append(response.KVs, KeyValue{
Key: key,
ValueRevision: value,
})
response.Count = 1
}
}
return response
}

func detachFromOldLease(s etcdState, key string) etcdState {
if oldLeaseId, ok := s.KeyLeases[key]; ok {
delete(s.Leases[oldLeaseId].Keys, key)
Expand All @@ -255,6 +272,7 @@ func attachToNewLease(s etcdState, leaseID int64, key string) etcdState {
type RequestType string

const (
Range RequestType = "range"
Txn RequestType = "txn"
LeaseGrant RequestType = "leaseGrant"
LeaseRevoke RequestType = "leaseRevoke"
Expand All @@ -265,10 +283,28 @@ type EtcdRequest struct {
Type RequestType
LeaseGrant *LeaseGrantRequest
LeaseRevoke *LeaseRevokeRequest
Range *RangeRequest
Txn *TxnRequest
Defragment *DefragmentRequest
}

type RangeRequest struct {
Key string
RangeOptions
// TODO: Implement stale read using revision
revision int64
}

type RangeOptions struct {
WithPrefix bool
Limit int64
}

type PutOptions struct {
Value ValueOrHash
LeaseID int64
}

type TxnRequest struct {
Conditions []EtcdCondition
OperationsOnSuccess []EtcdOperation
Expand All @@ -281,14 +317,20 @@ type EtcdCondition struct {
}

type EtcdOperation struct {
Type OperationType
Key string
WithPrefix bool
Limit int64
Value ValueOrHash
LeaseID int64
Type OperationType
Key string
RangeOptions
PutOptions
}

type OperationType string

const (
RangeOperation OperationType = "range-operation"
PutOperation OperationType = "put-operation"
DeleteOperation OperationType = "delete-operation"
)

type LeaseGrantRequest struct {
LeaseID int64
}
Expand All @@ -300,6 +342,7 @@ type DefragmentRequest struct{}
type EtcdResponse struct {
Revision int64
Txn *TxnResponse
Range *RangeResponse
LeaseGrant *LeaseGrantReponse
LeaseRevoke *LeaseRevokeResponse
Defragment *DefragmentResponse
Expand All @@ -310,15 +353,19 @@ type TxnResponse struct {
Results []EtcdOperationResult
}

type RangeResponse struct {
KVs []KeyValue
Count int64
}

type LeaseGrantReponse struct {
LeaseID int64
}
type LeaseRevokeResponse struct{}
type DefragmentResponse struct{}

type EtcdOperationResult struct {
KVs []KeyValue
Count int64
RangeResponse
Deleted int64
}

Expand Down
Loading

0 comments on commit 7d27e33

Please sign in to comment.