summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorbndw <ben@bdw.to>2026-02-13 17:41:13 -0800
committerbndw <ben@bdw.to>2026-02-13 17:41:13 -0800
commita6502c0888613bd0377a25e43de8ae306c4de4d7 (patch)
tree7e6b9eaaafbd97d3d0ef5007e392fa7b91e35f6c
parentaf30945803d440d1f803c814f4a37a1890494f1d (diff)
feat: add SQLite storage layer with binary-first event persistence
Storage implementation: - Concrete type with constructor (consumer-side interfaces) - Event storage: protobuf + zstd-compressed canonical JSON - Schema: events, deletions, replaceable_events, auth_challenges, rate_limits - WAL mode, STRICT typing, optimized indexes - Methods: StoreEvent, GetEvent, GetEventWithCanonical, DeleteEvent Dependencies: - modernc.org/sqlite v1.45.0 (pure Go SQLite driver) - github.com/klauspost/compress v1.18.4 (zstd compression) 366 lines, 10 tests passing
-rw-r--r--go.mod11
-rw-r--r--go.sum51
-rw-r--r--internal/storage/events.go216
-rw-r--r--internal/storage/events_test.go264
-rw-r--r--internal/storage/storage.go150
-rw-r--r--internal/storage/storage_test.go70
6 files changed, 762 insertions, 0 deletions
diff --git a/go.mod b/go.mod
index e3287d7..03667f7 100644
--- a/go.mod
+++ b/go.mod
@@ -4,16 +4,27 @@ go 1.24.0
4 4
5require ( 5require (
6 github.com/btcsuite/btcd/btcec/v2 v2.3.2 6 github.com/btcsuite/btcd/btcec/v2 v2.3.2
7 github.com/klauspost/compress v1.18.4
7 google.golang.org/grpc v1.79.1 8 google.golang.org/grpc v1.79.1
8 google.golang.org/protobuf v1.36.11 9 google.golang.org/protobuf v1.36.11
10 modernc.org/sqlite v1.45.0
9) 11)
10 12
11require ( 13require (
12 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect 14 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
13 github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect 15 github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
14 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect 16 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
17 github.com/dustin/go-humanize v1.0.1 // indirect
18 github.com/google/uuid v1.6.0 // indirect
19 github.com/mattn/go-isatty v0.0.20 // indirect
20 github.com/ncruces/go-strftime v1.0.0 // indirect
21 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
22 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
15 golang.org/x/net v0.48.0 // indirect 23 golang.org/x/net v0.48.0 // indirect
16 golang.org/x/sys v0.39.0 // indirect 24 golang.org/x/sys v0.39.0 // indirect
17 golang.org/x/text v0.32.0 // indirect 25 golang.org/x/text v0.32.0 // indirect
18 google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect 26 google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
27 modernc.org/libc v1.67.6 // indirect
28 modernc.org/mathutil v1.7.1 // indirect
29 modernc.org/memory v1.11.0 // indirect
19) 30)
diff --git a/go.sum b/go.sum
index 2ea0701..6d5fa95 100644
--- a/go.sum
+++ b/go.sum
@@ -10,6 +10,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK
10github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= 10github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
11github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= 11github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
12github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= 12github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
13github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
14github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
13github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= 15github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
14github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 16github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
15github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 17github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -18,8 +20,20 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
18github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 20github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
19github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 21github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
20github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 22github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
23github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
24github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
21github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 25github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
22github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 26github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
27github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
28github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
29github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
30github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
31github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
32github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
33github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
34github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
35github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
36github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
23go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= 37go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
24go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= 38go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
25go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= 39go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
@@ -32,12 +46,21 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W
32go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= 46go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
33go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= 47go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
34go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= 48go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
49golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
50golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
51golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
52golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
35golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= 53golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
36golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= 54golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
55golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
56golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
57golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
37golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= 58golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
38golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 59golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
39golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= 60golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
40golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= 61golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
62golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
63golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
41gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= 64gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
42gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= 65gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
43google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= 66google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
@@ -46,3 +69,31 @@ google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
46google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= 69google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
47google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= 70google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
48google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 71google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
72modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
73modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
74modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
75modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
76modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
77modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
78modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
79modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
80modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
81modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
82modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
83modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
84modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI=
85modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE=
86modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
87modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
88modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
89modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
90modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
91modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
92modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
93modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
94modernc.org/sqlite v1.45.0 h1:r51cSGzKpbptxnby+EIIz5fop4VuE4qFoVEjNvWoObs=
95modernc.org/sqlite v1.45.0/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
96modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
97modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
98modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
99modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
diff --git a/internal/storage/events.go b/internal/storage/events.go
new file mode 100644
index 0000000..d74fc7e
--- /dev/null
+++ b/internal/storage/events.go
@@ -0,0 +1,216 @@
1package storage
2
3import (
4 "context"
5 "database/sql"
6 "encoding/json"
7 "errors"
8 "fmt"
9
10 "github.com/klauspost/compress/zstd"
11 "google.golang.org/protobuf/proto"
12
13 pb "northwest.io/nostr-grpc/api/nostr/v1"
14)
15
16var (
17 // ErrEventNotFound is returned when an event ID is not found in storage.
18 ErrEventNotFound = errors.New("event not found")
19
20 // ErrEventExists is returned when attempting to store a duplicate event.
21 ErrEventExists = errors.New("event already exists")
22)
23
24// EventData holds the data needed to store an event.
25type EventData struct {
26 Event *pb.Event // Protobuf event
27 CanonicalJSON []byte // Uncompressed canonical JSON (will be compressed on storage)
28}
29
30// StoreEvent stores an event in the database.
31// Returns ErrEventExists if the event ID already exists.
32func (s *Storage) StoreEvent(ctx context.Context, data *EventData) error {
33 // Serialize protobuf
34 eventBytes, err := proto.Marshal(data.Event)
35 if err != nil {
36 return fmt.Errorf("failed to marshal event: %w", err)
37 }
38
39 // Compress canonical JSON
40 compressedJSON, err := compressJSON(data.CanonicalJSON)
41 if err != nil {
42 return fmt.Errorf("failed to compress canonical JSON: %w", err)
43 }
44
45 // Serialize tags to JSON
46 tagsJSON, err := marshalTags(data.Event.Tags)
47 if err != nil {
48 return fmt.Errorf("failed to marshal tags: %w", err)
49 }
50
51 // Insert event
52 query := `
53 INSERT INTO events (id, event_data, canonical_json, pubkey, kind, created_at, content, tags, sig)
54 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
55 `
56
57 _, err = s.db.ExecContext(ctx, query,
58 data.Event.Id,
59 eventBytes,
60 compressedJSON,
61 data.Event.Pubkey,
62 data.Event.Kind,
63 data.Event.CreatedAt,
64 data.Event.Content,
65 tagsJSON,
66 data.Event.Sig,
67 )
68
69 if err != nil {
70 // Check for unique constraint violation (duplicate ID)
71 if isDuplicateError(err) {
72 return ErrEventExists
73 }
74 return fmt.Errorf("failed to insert event: %w", err)
75 }
76
77 return nil
78}
79
80// GetEvent retrieves an event by ID.
81// Returns ErrEventNotFound if the event doesn't exist.
82func (s *Storage) GetEvent(ctx context.Context, id string) (*pb.Event, error) {
83 query := `
84 SELECT event_data, canonical_json
85 FROM events
86 WHERE id = ? AND deleted = 0
87 `
88
89 var eventBytes, compressedJSON []byte
90 err := s.db.QueryRowContext(ctx, query, id).Scan(&eventBytes, &compressedJSON)
91
92 if err == sql.ErrNoRows {
93 return nil, ErrEventNotFound
94 }
95 if err != nil {
96 return nil, fmt.Errorf("failed to query event: %w", err)
97 }
98
99 // Deserialize protobuf
100 event := &pb.Event{}
101 if err := proto.Unmarshal(eventBytes, event); err != nil {
102 return nil, fmt.Errorf("failed to unmarshal event: %w", err)
103 }
104
105 return event, nil
106}
107
108// GetEventWithCanonical retrieves an event by ID with its canonical JSON.
109// The canonical_json field will be populated in the returned event.
110func (s *Storage) GetEventWithCanonical(ctx context.Context, id string) (*pb.Event, error) {
111 query := `
112 SELECT event_data, canonical_json
113 FROM events
114 WHERE id = ? AND deleted = 0
115 `
116
117 var eventBytes, compressedJSON []byte
118 err := s.db.QueryRowContext(ctx, query, id).Scan(&eventBytes, &compressedJSON)
119
120 if err == sql.ErrNoRows {
121 return nil, ErrEventNotFound
122 }
123 if err != nil {
124 return nil, fmt.Errorf("failed to query event: %w", err)
125 }
126
127 // Deserialize protobuf
128 event := &pb.Event{}
129 if err := proto.Unmarshal(eventBytes, event); err != nil {
130 return nil, fmt.Errorf("failed to unmarshal event: %w", err)
131 }
132
133 // Decompress canonical JSON
134 canonicalJSON, err := decompressJSON(compressedJSON)
135 if err != nil {
136 return nil, fmt.Errorf("failed to decompress canonical JSON: %w", err)
137 }
138
139 event.CanonicalJson = canonicalJSON
140
141 return event, nil
142}
143
144// DeleteEvent marks an event as deleted (soft delete).
145func (s *Storage) DeleteEvent(ctx context.Context, eventID string) error {
146 query := `UPDATE events SET deleted = 1 WHERE id = ?`
147 result, err := s.db.ExecContext(ctx, query, eventID)
148 if err != nil {
149 return fmt.Errorf("failed to delete event: %w", err)
150 }
151
152 rows, err := result.RowsAffected()
153 if err != nil {
154 return fmt.Errorf("failed to get rows affected: %w", err)
155 }
156
157 if rows == 0 {
158 return ErrEventNotFound
159 }
160
161 return nil
162}
163
164// compressJSON compresses JSON bytes using zstd.
165func compressJSON(data []byte) ([]byte, error) {
166 encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
167 if err != nil {
168 return nil, err
169 }
170 defer encoder.Close()
171
172 return encoder.EncodeAll(data, nil), nil
173}
174
175// decompressJSON decompresses zstd-compressed JSON bytes.
176func decompressJSON(data []byte) ([]byte, error) {
177 decoder, err := zstd.NewReader(nil)
178 if err != nil {
179 return nil, err
180 }
181 defer decoder.Close()
182
183 return decoder.DecodeAll(data, nil)
184}
185
186// marshalTags converts protobuf tags to JSON string.
187func marshalTags(tags []*pb.Tag) (string, error) {
188 if len(tags) == 0 {
189 return "[]", nil
190 }
191
192 // Convert to [][]string for JSON serialization
193 tagArrays := make([][]string, len(tags))
194 for i, tag := range tags {
195 tagArrays[i] = tag.Values
196 }
197
198 data, err := json.Marshal(tagArrays)
199 if err != nil {
200 return "", err
201 }
202
203 return string(data), nil
204}
205
206// isDuplicateError checks if the error is a unique constraint violation.
207func isDuplicateError(err error) bool {
208 if err == nil {
209 return false
210 }
211 // SQLite error messages for UNIQUE constraint violations
212 msg := err.Error()
213 return msg == "UNIQUE constraint failed: events.id" ||
214 msg == "constraint failed: UNIQUE constraint failed: events.id" ||
215 msg == "constraint failed: UNIQUE constraint failed: events.id (1555)"
216}
diff --git a/internal/storage/events_test.go b/internal/storage/events_test.go
new file mode 100644
index 0000000..4393404
--- /dev/null
+++ b/internal/storage/events_test.go
@@ -0,0 +1,264 @@
1package storage
2
3import (
4 "context"
5 "testing"
6
7 pb "northwest.io/nostr-grpc/api/nostr/v1"
8)
9
10func TestStoreEvent(t *testing.T) {
11 store, err := New(":memory:")
12 if err != nil {
13 t.Fatalf("failed to create storage: %v", err)
14 }
15 defer store.Close()
16
17 ctx := context.Background()
18
19 // Create test event
20 event := &pb.Event{
21 Id: "test123",
22 Pubkey: "pubkey123",
23 CreatedAt: 1234567890,
24 Kind: 1,
25 Tags: []*pb.Tag{{Values: []string{"e", "event1"}}},
26 Content: "Hello, Nostr!",
27 Sig: "sig123",
28 }
29
30 canonicalJSON := []byte(`[0,"pubkey123",1234567890,1,[["e","event1"]],"Hello, Nostr!"]`)
31
32 data := &EventData{
33 Event: event,
34 CanonicalJSON: canonicalJSON,
35 }
36
37 // Store event
38 err = store.StoreEvent(ctx, data)
39 if err != nil {
40 t.Fatalf("failed to store event: %v", err)
41 }
42
43 // Verify event was stored
44 retrieved, err := store.GetEvent(ctx, "test123")
45 if err != nil {
46 t.Fatalf("failed to retrieve event: %v", err)
47 }
48
49 if retrieved.Id != event.Id {
50 t.Errorf("expected ID %s, got %s", event.Id, retrieved.Id)
51 }
52 if retrieved.Content != event.Content {
53 t.Errorf("expected content %s, got %s", event.Content, retrieved.Content)
54 }
55}
56
57func TestStoreEventDuplicate(t *testing.T) {
58 store, err := New(":memory:")
59 if err != nil {
60 t.Fatalf("failed to create storage: %v", err)
61 }
62 defer store.Close()
63
64 ctx := context.Background()
65
66 event := &pb.Event{
67 Id: "duplicate123",
68 Pubkey: "pubkey123",
69 CreatedAt: 1234567890,
70 Kind: 1,
71 Content: "test",
72 Sig: "sig123",
73 }
74
75 data := &EventData{
76 Event: event,
77 CanonicalJSON: []byte(`[0,"pubkey123",1234567890,1,[],"test"]`),
78 }
79
80 // Store first time
81 err = store.StoreEvent(ctx, data)
82 if err != nil {
83 t.Fatalf("failed to store event first time: %v", err)
84 }
85
86 // Try to store again
87 err = store.StoreEvent(ctx, data)
88 if err != ErrEventExists {
89 t.Errorf("expected ErrEventExists, got %v", err)
90 }
91}
92
93func TestGetEvent(t *testing.T) {
94 store, err := New(":memory:")
95 if err != nil {
96 t.Fatalf("failed to create storage: %v", err)
97 }
98 defer store.Close()
99
100 ctx := context.Background()
101
102 // Test non-existent event
103 _, err = store.GetEvent(ctx, "nonexistent")
104 if err != ErrEventNotFound {
105 t.Errorf("expected ErrEventNotFound, got %v", err)
106 }
107}
108
109func TestGetEventWithCanonical(t *testing.T) {
110 store, err := New(":memory:")
111 if err != nil {
112 t.Fatalf("failed to create storage: %v", err)
113 }
114 defer store.Close()
115
116 ctx := context.Background()
117
118 canonicalJSON := []byte(`[0,"pubkey123",1234567890,1,[],"test"]`)
119
120 event := &pb.Event{
121 Id: "canonical123",
122 Pubkey: "pubkey123",
123 CreatedAt: 1234567890,
124 Kind: 1,
125 Content: "test",
126 Sig: "sig123",
127 }
128
129 data := &EventData{
130 Event: event,
131 CanonicalJSON: canonicalJSON,
132 }
133
134 err = store.StoreEvent(ctx, data)
135 if err != nil {
136 t.Fatalf("failed to store event: %v", err)
137 }
138
139 // Retrieve with canonical JSON
140 retrieved, err := store.GetEventWithCanonical(ctx, "canonical123")
141 if err != nil {
142 t.Fatalf("failed to retrieve event: %v", err)
143 }
144
145 if string(retrieved.CanonicalJson) != string(canonicalJSON) {
146 t.Errorf("canonical JSON mismatch:\nexpected: %s\ngot: %s",
147 canonicalJSON, retrieved.CanonicalJson)
148 }
149}
150
151func TestDeleteEvent(t *testing.T) {
152 store, err := New(":memory:")
153 if err != nil {
154 t.Fatalf("failed to create storage: %v", err)
155 }
156 defer store.Close()
157
158 ctx := context.Background()
159
160 event := &pb.Event{
161 Id: "delete123",
162 Pubkey: "pubkey123",
163 CreatedAt: 1234567890,
164 Kind: 1,
165 Content: "to be deleted",
166 Sig: "sig123",
167 }
168
169 data := &EventData{
170 Event: event,
171 CanonicalJSON: []byte(`[0,"pubkey123",1234567890,1,[],"to be deleted"]`),
172 }
173
174 // Store event
175 err = store.StoreEvent(ctx, data)
176 if err != nil {
177 t.Fatalf("failed to store event: %v", err)
178 }
179
180 // Delete event
181 err = store.DeleteEvent(ctx, "delete123")
182 if err != nil {
183 t.Fatalf("failed to delete event: %v", err)
184 }
185
186 // Verify event is no longer retrievable
187 _, err = store.GetEvent(ctx, "delete123")
188 if err != ErrEventNotFound {
189 t.Errorf("expected ErrEventNotFound after deletion, got %v", err)
190 }
191
192 // Try deleting non-existent event
193 err = store.DeleteEvent(ctx, "nonexistent")
194 if err != ErrEventNotFound {
195 t.Errorf("expected ErrEventNotFound, got %v", err)
196 }
197}
198
199func TestCompressDecompressJSON(t *testing.T) {
200 original := []byte(`{"key":"value","array":[1,2,3],"nested":{"a":"b"}}`)
201
202 compressed, err := compressJSON(original)
203 if err != nil {
204 t.Fatalf("compression failed: %v", err)
205 }
206
207 // Verify compression reduces size (for larger data)
208 if len(compressed) >= len(original) {
209 t.Logf("Note: compressed size (%d) >= original (%d) - normal for small data",
210 len(compressed), len(original))
211 }
212
213 decompressed, err := decompressJSON(compressed)
214 if err != nil {
215 t.Fatalf("decompression failed: %v", err)
216 }
217
218 if string(decompressed) != string(original) {
219 t.Errorf("decompressed data doesn't match original:\nexpected: %s\ngot: %s",
220 original, decompressed)
221 }
222}
223
224func TestMarshalTags(t *testing.T) {
225 tests := []struct {
226 name string
227 tags []*pb.Tag
228 expected string
229 }{
230 {
231 name: "empty tags",
232 tags: nil,
233 expected: "[]",
234 },
235 {
236 name: "single tag",
237 tags: []*pb.Tag{
238 {Values: []string{"e", "event123"}},
239 },
240 expected: `[["e","event123"]]`,
241 },
242 {
243 name: "multiple tags",
244 tags: []*pb.Tag{
245 {Values: []string{"e", "event123", "wss://relay.example.com"}},
246 {Values: []string{"p", "pubkey456"}},
247 },
248 expected: `[["e","event123","wss://relay.example.com"],["p","pubkey456"]]`,
249 },
250 }
251
252 for _, tt := range tests {
253 t.Run(tt.name, func(t *testing.T) {
254 result, err := marshalTags(tt.tags)
255 if err != nil {
256 t.Fatalf("marshalTags failed: %v", err)
257 }
258
259 if result != tt.expected {
260 t.Errorf("expected %s, got %s", tt.expected, result)
261 }
262 })
263 }
264}
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
new file mode 100644
index 0000000..64fc4c6
--- /dev/null
+++ b/internal/storage/storage.go
@@ -0,0 +1,150 @@
1package storage
2
3import (
4 "context"
5 "database/sql"
6 "fmt"
7
8 _ "modernc.org/sqlite" // Pure Go SQLite driver
9)
10
11// Storage provides event persistence using SQLite.
12// Consumers should define their own interface based on their needs.
13type Storage struct {
14 db *sql.DB
15}
16
17// New creates a new Storage instance and initializes the database schema.
18// The dbPath should be a file path or ":memory:" for in-memory database.
19func New(dbPath string) (*Storage, error) {
20 db, err := sql.Open("sqlite", dbPath)
21 if err != nil {
22 return nil, fmt.Errorf("failed to open database: %w", err)
23 }
24
25 // Configure SQLite for optimal performance
26 pragmas := []string{
27 "PRAGMA journal_mode=WAL", // Write-Ahead Logging for concurrency
28 "PRAGMA synchronous=NORMAL", // Balance safety and performance
29 "PRAGMA cache_size=10000", // 10000 pages (~40MB cache)
30 "PRAGMA temp_store=MEMORY", // Temp tables in memory
31 "PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O
32 "PRAGMA page_size=4096", // Standard page size
33 "PRAGMA foreign_keys=ON", // Enforce foreign key constraints
34 "PRAGMA busy_timeout=5000", // Wait up to 5s for locks
35 "PRAGMA auto_vacuum=INCREMENTAL", // Reclaim space incrementally
36 }
37
38 for _, pragma := range pragmas {
39 if _, err := db.Exec(pragma); err != nil {
40 db.Close()
41 return nil, fmt.Errorf("failed to set pragma %q: %w", pragma, err)
42 }
43 }
44
45 s := &Storage{db: db}
46
47 // Initialize schema
48 if err := s.initSchema(context.Background()); err != nil {
49 db.Close()
50 return nil, fmt.Errorf("failed to initialize schema: %w", err)
51 }
52
53 return s, nil
54}
55
56// Close closes the database connection.
57func (s *Storage) Close() error {
58 return s.db.Close()
59}
60
61// initSchema creates all necessary tables and indexes.
62func (s *Storage) initSchema(ctx context.Context) error {
63 schema := `
64 -- Main events table
65 CREATE TABLE IF NOT EXISTS events (
66 -- Primary event data
67 id TEXT PRIMARY KEY,
68 event_data BLOB NOT NULL, -- Protobuf binary
69 canonical_json BLOB NOT NULL, -- zstd compressed canonical JSON
70
71 -- Denormalized fields for efficient querying
72 pubkey TEXT NOT NULL,
73 kind INTEGER NOT NULL,
74 created_at INTEGER NOT NULL, -- Unix timestamp
75 content TEXT, -- For full-text search (optional)
76 tags TEXT, -- JSON text for tag queries (use json_* functions)
77 sig TEXT NOT NULL,
78
79 -- Metadata
80 deleted INTEGER DEFAULT 0, -- STRICT mode: use INTEGER for boolean
81 received_at INTEGER DEFAULT (unixepoch())
82 ) STRICT;
83
84 -- Critical indexes for Nostr query patterns
85 CREATE INDEX IF NOT EXISTS idx_pubkey_created
86 ON events(pubkey, created_at DESC)
87 WHERE deleted = 0;
88
89 CREATE INDEX IF NOT EXISTS idx_kind_created
90 ON events(kind, created_at DESC)
91 WHERE deleted = 0;
92
93 CREATE INDEX IF NOT EXISTS idx_created
94 ON events(created_at DESC)
95 WHERE deleted = 0;
96
97 -- For tag queries (#e, #p, etc)
98 CREATE INDEX IF NOT EXISTS idx_tags
99 ON events(tags)
100 WHERE deleted = 0;
101
102 -- Deletion events (NIP-09)
103 CREATE TABLE IF NOT EXISTS deletions (
104 event_id TEXT PRIMARY KEY, -- ID of deletion event
105 deleted_event_id TEXT NOT NULL, -- ID of event being deleted
106 pubkey TEXT NOT NULL, -- Who requested deletion
107 created_at INTEGER NOT NULL,
108 FOREIGN KEY (deleted_event_id) REFERENCES events(id)
109 ) STRICT;
110
111 CREATE INDEX IF NOT EXISTS idx_deleted_event
112 ON deletions(deleted_event_id);
113
114 -- Replaceable events tracking (NIP-16, NIP-33)
115 CREATE TABLE IF NOT EXISTS replaceable_events (
116 kind INTEGER NOT NULL,
117 pubkey TEXT NOT NULL,
118 d_tag TEXT NOT NULL DEFAULT '', -- For parameterized replaceable events (empty string for non-parameterized)
119 current_event_id TEXT NOT NULL,
120 created_at INTEGER NOT NULL,
121 PRIMARY KEY (kind, pubkey, d_tag),
122 FOREIGN KEY (current_event_id) REFERENCES events(id)
123 ) STRICT;
124
125 -- Auth challenges (NIP-42)
126 CREATE TABLE IF NOT EXISTS auth_challenges (
127 challenge TEXT PRIMARY KEY,
128 created_at INTEGER NOT NULL,
129 expires_at INTEGER NOT NULL,
130 used INTEGER DEFAULT 0 -- STRICT mode: use INTEGER for boolean
131 ) STRICT;
132
133 -- Rate limiting
134 CREATE TABLE IF NOT EXISTS rate_limits (
135 pubkey TEXT PRIMARY KEY,
136 event_count INTEGER DEFAULT 0,
137 window_start INTEGER NOT NULL,
138 last_reset INTEGER DEFAULT (unixepoch())
139 ) STRICT;
140 `
141
142 _, err := s.db.ExecContext(ctx, schema)
143 return err
144}
145
146// DB returns the underlying *sql.DB for advanced usage.
147// This allows consumers to execute custom queries if needed.
148func (s *Storage) DB() *sql.DB {
149 return s.db
150}
diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go
new file mode 100644
index 0000000..f2fe401
--- /dev/null
+++ b/internal/storage/storage_test.go
@@ -0,0 +1,70 @@
1package storage
2
3import (
4 "testing"
5)
6
7func TestNew(t *testing.T) {
8 // Test in-memory database
9 store, err := New(":memory:")
10 if err != nil {
11 t.Fatalf("failed to create storage: %v", err)
12 }
13 defer store.Close()
14
15 // Verify database is accessible
16 if store.DB() == nil {
17 t.Fatal("DB() returned nil")
18 }
19
20 // Verify schema was created by checking if tables exist
21 var count int
22 query := `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('events', 'deletions', 'replaceable_events')`
23 err = store.DB().QueryRow(query).Scan(&count)
24 if err != nil {
25 t.Fatalf("failed to query tables: %v", err)
26 }
27
28 if count != 3 {
29 t.Errorf("expected 3 main tables, got %d", count)
30 }
31}
32
33func TestNewFileDatabase(t *testing.T) {
34 // Test file-based database
35 dbPath := t.TempDir() + "/test.db"
36 store, err := New(dbPath)
37 if err != nil {
38 t.Fatalf("failed to create file-based storage: %v", err)
39 }
40 defer store.Close()
41
42 // Verify WAL mode is enabled
43 var walMode string
44 err = store.DB().QueryRow("PRAGMA journal_mode").Scan(&walMode)
45 if err != nil {
46 t.Fatalf("failed to query journal mode: %v", err)
47 }
48
49 if walMode != "wal" {
50 t.Errorf("expected WAL mode, got %s", walMode)
51 }
52}
53
54func TestClose(t *testing.T) {
55 store, err := New(":memory:")
56 if err != nil {
57 t.Fatalf("failed to create storage: %v", err)
58 }
59
60 err = store.Close()
61 if err != nil {
62 t.Errorf("failed to close storage: %v", err)
63 }
64
65 // Verify database is closed by attempting a query
66 err = store.DB().Ping()
67 if err == nil {
68 t.Error("expected error when pinging closed database")
69 }
70}