ysweet_format.go
1package ygo
2
3import (
4 "encoding/binary"
5 "errors"
6 "io"
7 "os"
8 "sort"
9)
10
11const (
12 ysweetV1Prefix = 0x00
13 ysweetKeyspaceOID = 0x00
14 ysweetKeyspaceDoc = 0x01
15 ysweetSubDoc = 0x00
16 ysweetSubStateVec = 0x01
17 ysweetSubUpdate = 0x02
18 ysweetSubMeta = 0x03
19 ysweetTerminator = 0x00
20)
21
22var (
23 ErrNoDocument = errors.New("document not found")
24 ErrNoOID = errors.New("document OID not found")
25 ErrInvalidData = errors.New("invalid ysweet data format")
26)
27
28type ysweetEntry struct {
29 key []byte
30 value []byte
31}
32
33func (e *ysweetEntry) Key() []byte {
34 return e.key
35}
36
37func (e *ysweetEntry) Value() []byte {
38 return e.value
39}
40
41type ysweetMap map[string][]byte
42
43func (m ysweetMap) RangeIterate(from, to []byte) []ysweetEntry {
44 var entries []ysweetEntry
45 fromStr := string(from)
46 toStr := string(to)
47 for k, v := range m {
48 if len(k) >= len(fromStr) && k[:len(fromStr)] >= fromStr {
49 if toStr == "" || (len(k) >= len(toStr) && k[:len(toStr)] < toStr) {
50 entries = append(entries, ysweetEntry{key: []byte(k), value: v})
51 }
52 }
53 }
54 sort.Slice(entries, func(i, j int) bool {
55 return string(entries[i].key) < string(entries[j].key)
56 })
57 return entries
58}
59
60func ysweetKeyOID(docName string) []byte {
61 key := []byte{ysweetV1Prefix, ysweetKeyspaceOID}
62 key = append(key, docName...)
63 key = append(key, ysweetTerminator)
64 return key
65}
66
67func ysweetKeyDoc(oid uint32) []byte {
68 key := []byte{ysweetV1Prefix, ysweetKeyspaceDoc}
69 key = append(key, uint32ToBytesBE(oid)...)
70 key = append(key, ysweetSubDoc)
71 return key
72}
73
74func ysweetKeyStateVector(oid uint32) []byte {
75 key := []byte{ysweetV1Prefix, ysweetKeyspaceDoc}
76 key = append(key, uint32ToBytesBE(oid)...)
77 key = append(key, ysweetSubStateVec)
78 return key
79}
80
81func ysweetKeyUpdate(oid uint32, clock uint32) []byte {
82 key := []byte{ysweetV1Prefix, ysweetKeyspaceDoc}
83 key = append(key, uint32ToBytesBE(oid)...)
84 key = append(key, ysweetSubUpdate)
85 key = append(key, uint32ToBytesBE(clock)...)
86 key = append(key, ysweetTerminator)
87 return key
88}
89
90func ysweetKeyUpdateRangeStart(oid uint32) []byte {
91 return ysweetKeyUpdate(oid, 0)
92}
93
94func ysweetKeyUpdateRangeEnd(oid uint32) []byte {
95 key := []byte{ysweetV1Prefix, ysweetKeyspaceDoc}
96 key = append(key, uint32ToBytesBE(oid)...)
97 key = append(key, ysweetSubUpdate)
98 key = append(key, 0xFF, 0xFF, 0xFF, 0xFF)
99 return key
100}
101
102func uint32ToBytesBE(v uint32) []byte {
103 b := make([]byte, 4)
104 binary.BigEndian.PutUint32(b, v)
105 return b
106}
107
108func bytesBEToUint32(b []byte) uint32 {
109 return binary.BigEndian.Uint32(b)
110}
111
112func extractDocNameFromOIDKey(key []byte) string {
113 if len(key) >= 3 && key[0] == ysweetV1Prefix && key[1] == ysweetKeyspaceOID {
114 return string(key[2 : len(key)-1])
115 }
116 return ""
117}
118
119func parseYsweetData(r io.Reader) (map[string][]byte, error) {
120 data, err := io.ReadAll(r)
121 if err != nil {
122 return nil, err
123 }
124
125 return bincodeDeserialize(data), nil
126}
127
128func min(a, b int) int {
129 if a < b {
130 return a
131 }
132 return b
133}
134
135// readBincodeFixint reads a bincode fixint u64 from data at the given offset.
136// Bincode fixint encoding: always 8 bytes, little-endian
137func readBincodeFixint(data []byte, offset int) (uint64, int, error) {
138 if offset+8 > len(data) {
139 return 0, offset, io.EOF
140 }
141
142 val := binary.LittleEndian.Uint64(data[offset : offset+8])
143 return val, offset + 8, nil
144}
145
146// readVarint is kept for backward compatibility but now uses fixint encoding
147var readVarint = readBincodeFixint
148
149// bincodeDeserialize deserializes a bincode-encoded map[string][]byte.
150// Uses bincode fixint encoding (8-byte little-endian u64 for all length fields).
151func bincodeDeserialize(data []byte) map[string][]byte {
152 m := make(map[string][]byte)
153 if len(data) == 0 {
154 return m
155 }
156
157 offset := 0
158
159 // Read map length (u64 LE)
160 mapLen, nextOffset, err := readBincodeFixint(data, offset)
161 if err != nil {
162 return m
163 }
164 offset = nextOffset
165
166 // Now read exactly mapLen entries
167 for i := uint64(0); i < mapLen && offset < len(data); i++ {
168 keyLen, nextOffset, err := readBincodeFixint(data, offset)
169 if err != nil {
170 break
171 }
172 offset = nextOffset
173
174 // Sanity check - key should be reasonable
175 if keyLen > 1000 {
176 break
177 }
178
179 if offset+int(keyLen) > len(data) {
180 break
181 }
182 key := string(data[offset : offset+int(keyLen)])
183 offset += int(keyLen)
184
185 valLen, nextOffset, err := readBincodeFixint(data, offset)
186 if err != nil {
187 break
188 }
189 offset = nextOffset
190
191 // Sanity check - val should be reasonable
192 if valLen > 10000 {
193 break
194 }
195
196 if offset+int(valLen) > len(data) {
197 break
198 }
199 value := data[offset : offset+int(valLen)]
200 offset += int(valLen)
201
202 m[key] = value
203 }
204
205 return m
206}
207
208// NewDocFromYSweet reconstructs a Y document from y-sweet persisted data.
209// The provided io.Reader should contain the binary data stored in a y-sweet
210// data.ysweet file (bincode-serialized BTreeMap).
211//
212// The returned Doc is fully reconstructed and ready for use.
213// Callers must call Destroy on the returned Doc when done to free resources.
214func NewDocFromYSweet(r io.Reader) (*Doc, error) {
215 m, err := parseYsweetData(r)
216 if err != nil {
217 return nil, err
218 }
219
220 var oid uint32
221
222 for _, v := range m {
223 // OID entries have 4-byte big-endian uint32 values
224 if len(v) == 4 {
225 oid = bytesBEToUint32(v)
226 break
227 }
228 }
229
230 if oid == 0 {
231 return nil, ErrNoOID
232 }
233
234 docKey := string(ysweetKeyDoc(oid))
235 docState, ok := m[docKey]
236 if !ok {
237 return nil, ErrNoDocument
238 }
239
240 updateEntries := ysweetMap(m).RangeIterate(ysweetKeyUpdateRangeStart(oid), ysweetKeyUpdateRangeEnd(oid))
241
242 doc, err := NewDoc()
243 if err != nil {
244 return nil, err
245 }
246
247 err = doc.WithWriteTransaction(func(txn *Transaction) error {
248 update := UpdateFromBytes(docState)
249 if err := txn.ApplyUpdate(update); err != nil {
250 return err
251 }
252
253 for _, entry := range updateEntries {
254 update = UpdateFromBytes(entry.value)
255 if err := txn.ApplyUpdate(update); err != nil {
256 return err
257 }
258 }
259
260 return nil
261 })
262 if err != nil {
263 doc.Destroy()
264 return nil, err
265 }
266
267 return doc, nil
268}
269
270// NewDocFromYSweetFile opens a y-sweet data file and reconstructs a Y document.
271// The path should point to a data.ysweet file persisted by a y-sweet server.
272//
273// The returned Doc is fully reconstructed and ready for use.
274// Callers must call Destroy on the returned Doc when done to free resources.
275func NewDocFromYSweetFile(path string) (*Doc, error) {
276 f, err := os.Open(path)
277 if err != nil {
278 return nil, err
279 }
280 defer f.Close()
281
282 return NewDocFromYSweet(f)
283}
