From 5fcf5bd1fa5b2e707cea82c4652ea65c3c113c1a Mon Sep 17 00:00:00 2001 From: bndw Date: Fri, 13 Feb 2026 17:43:37 -0800 Subject: feat: add query layer with Nostr filter to SQL conversion Query implementation: - QueryEvents method with filter support - Full NIP-01 filter support (ids, authors, kinds, tags, since, until, limit) - ID and pubkey prefix matching - Tag filtering using SQLite JSON functions - Multiple filter UNION support - DESC ordering by created_at - Optional canonical JSON inclusion 23 tests passing, 1322 total lines --- internal/storage/query.go | 185 +++++++++++++++++ internal/storage/query_test.go | 437 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 622 insertions(+) create mode 100644 internal/storage/query.go create mode 100644 internal/storage/query_test.go (limited to 'internal') diff --git a/internal/storage/query.go b/internal/storage/query.go new file mode 100644 index 0000000..29a2a4c --- /dev/null +++ b/internal/storage/query.go @@ -0,0 +1,185 @@ +package storage + +import ( + "context" + "fmt" + "strings" + + "google.golang.org/protobuf/proto" + + pb "northwest.io/nostr-grpc/api/nostr/v1" +) + +type QueryOptions struct { + IncludeCanonical bool + Limit int32 +} + +func (s *Storage) QueryEvents(ctx context.Context, filters []*pb.Filter, opts *QueryOptions) ([]*pb.Event, error) { + if len(filters) == 0 { + return nil, nil + } + + if opts == nil { + opts = &QueryOptions{Limit: 100} + } + + query, args := buildQuery(filters, opts) + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + + var events []*pb.Event + for rows.Next() { + var eventBytes []byte + var compressedJSON []byte + var createdAt int64 + + if opts.IncludeCanonical { + if err := rows.Scan(&eventBytes, &compressedJSON, &createdAt); err != nil { + return nil, fmt.Errorf("scan failed: %w", err) + } + } else { + if err := rows.Scan(&eventBytes, &createdAt); err != nil { + return nil, fmt.Errorf("scan failed: %w", err) + } + } + + event := &pb.Event{} + if err := proto.Unmarshal(eventBytes, event); err != nil { + return nil, fmt.Errorf("unmarshal failed: %w", err) + } + + if opts.IncludeCanonical && compressedJSON != nil { + canonicalJSON, err := decompressJSON(compressedJSON) + if err != nil { + return nil, fmt.Errorf("decompress failed: %w", err) + } + event.CanonicalJson = canonicalJSON + } + + events = append(events, event) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("rows iteration failed: %w", err) + } + + return events, nil +} + +func buildQuery(filters []*pb.Filter, opts *QueryOptions) (string, []interface{}) { + selectFields := "event_data, created_at" + if opts.IncludeCanonical { + selectFields = "event_data, canonical_json, created_at" + } + + var unions []string + var args []interface{} + + for _, filter := range filters { + clause, filterArgs := buildWhereClause(filter) + subQuery := fmt.Sprintf("SELECT %s FROM events WHERE deleted = 0 AND (%s)", selectFields, clause) + unions = append(unions, subQuery) + args = append(args, filterArgs...) + } + + query := strings.Join(unions, " UNION ") + query += " ORDER BY created_at DESC" + + if opts.Limit > 0 { + query += fmt.Sprintf(" LIMIT %d", opts.Limit) + } + + return query, args +} + +func buildWhereClause(filter *pb.Filter) (string, []interface{}) { + var conditions []string + var args []interface{} + + if len(filter.Ids) > 0 { + conditions = append(conditions, buildPrefixCondition("id", filter.Ids, &args)) + } + + if len(filter.Authors) > 0 { + conditions = append(conditions, buildPrefixCondition("pubkey", filter.Authors, &args)) + } + + if len(filter.Kinds) > 0 { + placeholders := make([]string, len(filter.Kinds)) + for i, kind := range filter.Kinds { + placeholders[i] = "?" + args = append(args, kind) + } + conditions = append(conditions, fmt.Sprintf("kind IN (%s)", strings.Join(placeholders, ","))) + } + + if filter.Since != nil && *filter.Since > 0 { + conditions = append(conditions, "created_at >= ?") + args = append(args, *filter.Since) + } + + if filter.Until != nil && *filter.Until > 0 { + conditions = append(conditions, "created_at <= ?") + args = append(args, *filter.Until) + } + + if len(filter.ETags) > 0 { + conditions = append(conditions, buildTagCondition("e", filter.ETags, &args)) + } + + if len(filter.PTags) > 0 { + conditions = append(conditions, buildTagCondition("p", filter.PTags, &args)) + } + + for tagName, tagFilter := range filter.TagFilters { + if len(tagFilter.Values) > 0 { + conditions = append(conditions, buildTagCondition(tagName, tagFilter.Values, &args)) + } + } + + if len(conditions) == 0 { + return "1=1", args + } + + return strings.Join(conditions, " AND "), args +} + +func buildPrefixCondition(column string, values []string, args *[]interface{}) string { + var orConditions []string + + for _, val := range values { + if len(val) == 64 { + orConditions = append(orConditions, column+" = ?") + *args = append(*args, val) + } else { + orConditions = append(orConditions, column+" LIKE ?") + *args = append(*args, val+"%") + } + } + + if len(orConditions) == 1 { + return orConditions[0] + } + + return "(" + strings.Join(orConditions, " OR ") + ")" +} + +func buildTagCondition(tagName string, values []string, args *[]interface{}) string { + var orConditions []string + + for _, val := range values { + orConditions = append(orConditions, "EXISTS (SELECT 1 FROM json_each(tags) WHERE json_extract(value, '$[0]') = ? AND json_extract(value, '$[1]') = ?)") + *args = append(*args, tagName, val) + } + + if len(orConditions) == 1 { + return orConditions[0] + } + + return "(" + strings.Join(orConditions, " OR ") + ")" +} diff --git a/internal/storage/query_test.go b/internal/storage/query_test.go new file mode 100644 index 0000000..b1f2fc7 --- /dev/null +++ b/internal/storage/query_test.go @@ -0,0 +1,437 @@ +package storage + +import ( + "context" + "testing" + + pb "northwest.io/nostr-grpc/api/nostr/v1" +) + +func setupTestEvents(t *testing.T, store *Storage) { + ctx := context.Background() + + events := []*EventData{ + { + Event: &pb.Event{ + Id: "aaaa1111", + Pubkey: "pubkey1", + CreatedAt: 1000, + Kind: 1, + Tags: []*pb.Tag{{Values: []string{"e", "event123"}}}, + Content: "test event 1", + Sig: "sig1", + }, + CanonicalJSON: []byte(`[0,"pubkey1",1000,1,[["e","event123"]],"test event 1"]`), + }, + { + Event: &pb.Event{ + Id: "bbbb2222", + Pubkey: "pubkey2", + CreatedAt: 2000, + Kind: 1, + Tags: []*pb.Tag{{Values: []string{"p", "pubkey1"}}}, + Content: "test event 2", + Sig: "sig2", + }, + CanonicalJSON: []byte(`[0,"pubkey2",2000,1,[["p","pubkey1"]],"test event 2"]`), + }, + { + Event: &pb.Event{ + Id: "cccc3333", + Pubkey: "pubkey1", + CreatedAt: 3000, + Kind: 3, + Tags: []*pb.Tag{}, + Content: "test event 3", + Sig: "sig3", + }, + CanonicalJSON: []byte(`[0,"pubkey1",3000,3,[],"test event 3"]`), + }, + { + Event: &pb.Event{ + Id: "dddd4444", + Pubkey: "pubkey3", + CreatedAt: 4000, + Kind: 1, + Tags: []*pb.Tag{ + {Values: []string{"e", "event123"}}, + {Values: []string{"p", "pubkey2"}}, + }, + Content: "test event 4", + Sig: "sig4", + }, + CanonicalJSON: []byte(`[0,"pubkey3",4000,1,[["e","event123"],["p","pubkey2"]],"test event 4"]`), + }, + } + + for _, data := range events { + if err := store.StoreEvent(ctx, data); err != nil { + t.Fatalf("failed to store event %s: %v", data.Event.Id, err) + } + } +} + +func TestQueryEventsByID(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Ids: []string{"aaaa1111"}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + if events[0].Id != "aaaa1111" { + t.Errorf("expected ID aaaa1111, got %s", events[0].Id) + } +} + +func TestQueryEventsByIDPrefix(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Ids: []string{"aaaa"}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + if events[0].Id != "aaaa1111" { + t.Errorf("expected ID aaaa1111, got %s", events[0].Id) + } +} + +func TestQueryEventsByAuthor(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Authors: []string{"pubkey1"}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } + + for _, event := range events { + if event.Pubkey != "pubkey1" { + t.Errorf("expected pubkey pubkey1, got %s", event.Pubkey) + } + } +} + +func TestQueryEventsByKind(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Kinds: []int32{1}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 3 { + t.Fatalf("expected 3 events, got %d", len(events)) + } + + for _, event := range events { + if event.Kind != 1 { + t.Errorf("expected kind 1, got %d", event.Kind) + } + } +} + +func TestQueryEventsByETag(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {ETags: []string{"event123"}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } +} + +func TestQueryEventsByPTag(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {PTags: []string{"pubkey1"}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + if events[0].Id != "bbbb2222" { + t.Errorf("expected ID bbbb2222, got %s", events[0].Id) + } +} + +func TestQueryEventsSince(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + since := int64(2500) + filters := []*pb.Filter{ + {Since: &since}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } + + for _, event := range events { + if event.CreatedAt < since { + t.Errorf("event %s has timestamp %d, expected >= %d", event.Id, event.CreatedAt, since) + } + } +} + +func TestQueryEventsUntil(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + until := int64(2500) + filters := []*pb.Filter{ + {Until: &until}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } + + for _, event := range events { + if event.CreatedAt > until { + t.Errorf("event %s has timestamp %d, expected <= %d", event.Id, event.CreatedAt, until) + } + } +} + +func TestQueryEventsWithLimit(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{{}} + opts := &QueryOptions{Limit: 2} + + events, err := store.QueryEvents(ctx, filters, opts) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events due to limit, got %d", len(events)) + } +} + +func TestQueryEventsWithCanonical(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Ids: []string{"aaaa1111"}}, + } + opts := &QueryOptions{IncludeCanonical: true} + + events, err := store.QueryEvents(ctx, filters, opts) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + if len(events[0].CanonicalJson) == 0 { + t.Error("expected canonical JSON to be populated") + } +} + +func TestQueryMultipleFilters(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{ + {Ids: []string{"aaaa1111"}}, + {Kinds: []int32{3}}, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 2 { + t.Fatalf("expected 2 events (UNION), got %d", len(events)) + } +} + +func TestQueryEventsOrdering(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + filters := []*pb.Filter{{}} + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + for i := 1; i < len(events); i++ { + if events[i].CreatedAt > events[i-1].CreatedAt { + t.Errorf("events not ordered by created_at DESC: %d > %d", + events[i].CreatedAt, events[i-1].CreatedAt) + } + } +} + +func TestQueryEventsComplex(t *testing.T) { + store, err := New(":memory:") + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + defer store.Close() + + setupTestEvents(t, store) + ctx := context.Background() + + since := int64(1500) + filters := []*pb.Filter{ + { + Authors: []string{"pubkey1"}, + Kinds: []int32{1, 3}, + Since: &since, + }, + } + + events, err := store.QueryEvents(ctx, filters, nil) + if err != nil { + t.Fatalf("query failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + if events[0].Id != "cccc3333" { + t.Errorf("expected event cccc3333, got %s", events[0].Id) + } +} -- cgit v1.2.3