Skip to content

Commit 03e58d2

Browse files
ggicciggicci
authored
Merge pull request #167 from CovenantSQL/feature/observerPagination
Update observer api to support queries pagination
2 parents 803c1c2 + 0c55ba6 commit 03e58d2

File tree

4 files changed

+165
-30
lines changed

4 files changed

+165
-30
lines changed

‎cmd/cql-observer/api.go

Lines changed: 116 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,32 @@ type explorerAPI struct {
5757
service *Service
5858
}
5959

60+
type paginationOps struct {
61+
page int
62+
size int
63+
queryType types.QueryType
64+
}
65+
66+
func newPaginationFromReq(r *http.Request) (op *paginationOps) {
67+
op = &paginationOps{}
68+
op.page, _ = strconv.Atoi(r.URL.Query().Get("page"))
69+
op.size, _ = strconv.Atoi(r.URL.Query().Get("size"))
70+
if r.URL.Query().Get("type") == types.ReadQuery.String() {
71+
op.queryType = types.ReadQuery
72+
} else if r.URL.Query().Get("type") == types.WriteQuery.String() {
73+
op.queryType = types.WriteQuery
74+
} else {
75+
op.queryType = types.NumberOfQueryType
76+
}
77+
if op.page <= 0 {
78+
op.page = 1
79+
}
80+
if op.size <= 0 {
81+
op.size = 10
82+
}
83+
return
84+
}
85+
6086
func (a *explorerAPI) GetAck(rw http.ResponseWriter, r *http.Request) {
6187
vars := mux.Vars(r)
6288

@@ -175,7 +201,9 @@ func (a *explorerAPI) GetBlockV3(rw http.ResponseWriter, r *http.Request) {
175201
return
176202
}
177203

178-
sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw)
204+
op := newPaginationFromReq(r)
205+
206+
sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw)
179207
}
180208

181209
func (a *explorerAPI) GetBlockByCount(rw http.ResponseWriter, r *http.Request) {
@@ -239,7 +267,9 @@ func (a *explorerAPI) GetBlockByCountV3(rw http.ResponseWriter, r *http.Request)
239267
return
240268
}
241269

242-
sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw)
270+
op := newPaginationFromReq(r)
271+
272+
sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw)
243273
}
244274

245275
func (a *explorerAPI) GetBlockByHeight(rw http.ResponseWriter, r *http.Request) {
@@ -303,7 +333,9 @@ func (a *explorerAPI) GetBlockByHeightV3(rw http.ResponseWriter, r *http.Request
303333
return
304334
}
305335

306-
sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw)
336+
op := newPaginationFromReq(r)
337+
338+
sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw)
307339
}
308340

309341
func (a *explorerAPI) GetHighestBlock(rw http.ResponseWriter, r *http.Request) {
@@ -396,7 +428,9 @@ func (a *explorerAPI) GetHighestBlockV3(rw http.ResponseWriter, r *http.Request)
396428
return
397429
}
398430

399-
sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw)
431+
op := newPaginationFromReq(r)
432+
433+
sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw)
400434
}
401435

402436
func (a *explorerAPI) formatBlock(height int32, b *types.Block) (res map[string]interface{}) {
@@ -425,30 +459,96 @@ func (a *explorerAPI) formatBlockV2(count, height int32, b *types.Block) (res ma
425459
return
426460
}
427461

428-
func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block) (res map[string]interface{}) {
462+
func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block,
463+
pagination *paginationOps) (res map[string]interface{}) {
429464
res = a.formatBlockV2(count, height, b)
430465
blockRes := res["block"].(map[string]interface{})
431-
blockRes["acks"] = func() (acks []interface{}) {
432-
acks = make([]interface{}, 0, len(b.Acks))
433-
434-
for _, ack := range b.Acks {
435-
acks = append(acks, a.formatAck(ack)["ack"])
436-
}
437-
438-
return
439-
}()
440466
blockRes["queries"] = func() (tracks []interface{}) {
441-
tracks = make([]interface{}, 0, len(b.QueryTxs))
467+
tracks = make([]interface{}, 0, len(b.QueryTxs)+len(b.FailedReqs))
468+
469+
var (
470+
offset = (pagination.page - 1) * pagination.size
471+
end = pagination.page * pagination.size
472+
pos = 0
473+
)
442474

443475
for _, tx := range b.QueryTxs {
476+
if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) &&
477+
tx.Request.Header.QueryType != pagination.queryType {
478+
// count all
479+
continue
480+
}
481+
482+
if pos >= end {
483+
return
484+
}
485+
444486
t := a.formatRequest(tx.Request)
445487
t["response"] = a.formatResponseHeader(tx.Response)["response"]
446-
tracks = append(tracks, t)
488+
t["failed"] = false
489+
490+
if pos >= offset {
491+
tracks = append(tracks, t)
492+
}
493+
494+
pos++
495+
}
496+
497+
for _, req := range b.FailedReqs {
498+
if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) &&
499+
req.Header.QueryType != pagination.queryType {
500+
// count all
501+
continue
502+
}
503+
504+
if pos >= end {
505+
return
506+
}
507+
508+
t := a.formatRequest(req)
509+
t["failed"] = true
510+
511+
if pos >= offset {
512+
tracks = append(tracks, t)
513+
}
514+
515+
pos++
447516
}
448517

449518
return
450519
}()
451520

521+
if pagination != nil {
522+
blockRes["pagination"] = func() (res map[string]interface{}) {
523+
// pagination features
524+
res = map[string]interface{}{}
525+
res["page"] = pagination.page
526+
res["size"] = pagination.size
527+
528+
if pagination.queryType != types.ReadQuery && pagination.queryType != types.WriteQuery {
529+
res["total"] = len(b.QueryTxs) + len(b.FailedReqs)
530+
} else {
531+
var total int
532+
533+
for _, tx := range b.QueryTxs {
534+
if tx.Request.Header.QueryType == pagination.queryType {
535+
total++
536+
}
537+
}
538+
539+
for _, req := range b.FailedReqs {
540+
if req.Header.QueryType == pagination.queryType {
541+
total++
542+
}
543+
}
544+
545+
res["total"] = total
546+
}
547+
548+
return
549+
}()
550+
}
551+
452552
return
453553
}
454554

‎cmd/cql-observer/observation_test.go

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -422,20 +422,55 @@ func TestFullProcess(t *testing.T) {
422422
genesisHash := ensureSuccess(res.String("block", "hash")).(string)
423423

424424
// test get first containable block
425-
res, err = getJSON("v3/height/%v/1", dbID)
426-
So(err, ShouldBeNil)
427-
So(ensureSuccess(res.Interface("block")), ShouldNotBeNil)
428-
So(ensureSuccess(res.Int("block", "height")), ShouldEqual, 1)
429-
So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty)
430-
So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash)
431-
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty)
432-
blockHash := ensureSuccess(res.String("block", "hash")).(string)
433-
byHeightBlockResult := ensureSuccess(res.Interface())
425+
var (
426+
blockHash string
427+
byHeightBlockResult interface{}
428+
)
429+
430+
// access 5 blocks
431+
for i := 1; i <= 5; i++ {
432+
res, err = getJSON("v3/height/%v/%d", dbID, i)
433+
So(err, ShouldBeNil)
434+
So(ensureSuccess(res.Interface("block")), ShouldNotBeNil)
435+
So(ensureSuccess(res.Int("block", "height")), ShouldEqual, i)
436+
So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty)
437+
So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash)
438+
if len(ensureSuccess(res.ArrayOfObjects("block", "queries")).([]map[string]interface{})) == 0 {
439+
// got empty block
440+
log.WithField("block", res).Debugf("got empty block, try next index")
441+
continue
442+
}
443+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty)
444+
blockHash = ensureSuccess(res.String("block", "hash")).(string)
445+
byHeightBlockResult = ensureSuccess(res.Interface())
446+
break
447+
}
434448

435449
// test get block by hash
436-
res, err = getJSON("v3/block/%v/%v", dbID, blockHash)
450+
res, err = getJSON("v3/block/%v/%v?size=1000", dbID, blockHash)
451+
So(err, ShouldBeNil)
452+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldResemble,
453+
ensureSuccess(jsonq.NewQuery(byHeightBlockResult).ArrayOfObjects("block", "queries")))
454+
455+
// test get block by hash v3 with pagination
456+
res, err = getJSON("v3/block/%v/%v?page=10000&size=10", dbID, blockHash)
457+
So(err, ShouldBeNil)
458+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldBeEmpty)
459+
460+
// test get block with page size = 1
461+
res, err = getJSON("v3/block/%v/%v?page=1&size=1", dbID, blockHash)
462+
So(err, ShouldBeNil)
463+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1)
464+
465+
// test get block with page size = 2
466+
res, err = getJSON("v3/block/%v/%v?page=1&size=2", dbID, blockHash)
467+
So(err, ShouldBeNil)
468+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 2)
469+
470+
// test get block with page size = 1, page = 2
471+
res, err = getJSON("v3/block/%v/%v?page=2&size=1", dbID, blockHash)
437472
So(err, ShouldBeNil)
438-
So(ensureSuccess(res.Interface()), ShouldResemble, byHeightBlockResult)
473+
So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1)
439474

440475
// test get block by hash using v1 version, returns ack hashes as queries
441476
res, err = getJSON("v1/block/%v/%v", dbID, blockHash)

‎codecov.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ parsers:
2424

2525
comment:
2626
layout: "header, diff"
27-
behavior: new
27+
behavior: default
2828
require_changes: no
2929

3030
ignore:

‎types/request_type.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ const (
3636
ReadQuery QueryType = iota
3737
// WriteQuery defines a write query type.
3838
WriteQuery
39-
// NumberOfPerm defines the number of query type.
40-
NumberOfPerm
39+
// NumberOfQueryType defines the number of query type.
40+
NumberOfQueryType
4141
)
4242

4343
// NamedArg defines the named argument structure for database.

0 commit comments

Comments
 (0)