Reformulate data structures
John Smith

John Smith commited on 2023-177 12:35:37
Showing 14 changed files, with 164 additions and 77 deletions.


To make the code make more sense and be more extensible for potential
future card formats, this replaces the list-of-lists-of-prompts
data model with a list-of-cards model, where each card has an associated
list of prompts. This involved many changes to the code internals
and some changes to the Recfile deck format, but there should be no
significant changes for the end user.
... ...
@@ -1,6 +1,6 @@
1 1
 --[[
2 2
     returns a not-necessarily-meaningful number
3
-    iff `s` has mismathched curly-brackets
3
+    iff `s` has mismatched curly-brackets
4 4
 ]]
5 5
 local function mismatched(s)
6 6
     return ("}" .. s):find("}[^{]+}") or (s .. "{"):find("{[^}]+{")
... ...
@@ -10,24 +10,24 @@ local function fn(config, deck)
10 10
     if isatty() then
11 11
         io.write("Reading prompts; press Ctrl-D to stop\n")
12 12
     end
13
-    local rt = io.read("a")
13
+    local readtext = io.read("a")
14 14
     local cardctr = 0
15 15
     local prctr = 0
16
-    for pt in rt:gmatch("(..-)\n\n") do
17
-        local nc = {}
16
+    for notetext in readtext:gmatch("(..-)\n\n") do
17
+        local newcard = Card:new({ note = notetext, ct = "bmc", created = os.time() })
18 18
         -- these Prompts will just be converted to Rec, so we don't need to exclude answers from prompt-text
19
-        for _re in pt:gmatch("%{[^%}]+%}") do
20
-            table.insert(nc, Prompt:new(pt, "", os.time(), 30, 0, 0, os.time(), config:get("StartingEase") or 2.5))
19
+        for _re in pt:gmatch("{[^}]+}") do
20
+            table.insert(newcard.pr, Prompt:new({ card = newcard, ease = config:get("StartingEase") }))
21 21
             prctr = prctr + 1
22 22
         end
23
-        if mismatched(pt) then
24
-            io.write(string.format("Prompt %d:1 (\"%s\") contains mismatched braces\n", #deck + 1, pt))
23
+        if mismatched(notetext) then
24
+            io.write(string.format("Card %d (\"%s\") contains mismatched braces\n", #deck + 1, notetext))
25 25
         end
26
-        if #nc >= 1 then
27
-            table.insert(deck, nc)
26
+        if #newcard.pr >= 1 then
27
+            table.insert(deck, newcard)
28 28
             cardctr = cardctr + 1
29 29
         else
30
-            io.write(string.format("Prompt \"%s\" contains no answers (check braces)\n", pt))
30
+            io.write(string.format("Card \"%s\" contains no answers (check braces)\n", notetext))
31 31
         end
32 32
     end
33 33
     io.write(string.format(
... ...
@@ -0,0 +1,77 @@
1
+--[[
2
+    a card consists of a base note, a card type, a creation timestamp,
3
+    and a list of prompts
4
+
5
+    timestamps are seconds from the Unix epoch
6
+]]
7
+
8
+local Card = {}
9
+Card.__index = Card
10
+
11
+function Card:new(ctable)
12
+    local o = { note = "", ct = "bmc", created = os.time(), pr = {} }
13
+    setmetatable(o, self)
14
+    for k, v in pairs(ctable) do
15
+        if o[k] then o[k] = v end
16
+    end
17
+    return o
18
+end
19
+
20
+function Card:__tostring()
21
+    return string.format("\"%s\" (%s), %d (%d)", self.note, self.ct, #self.pr, self.created)
22
+end
23
+
24
+--[[
25
+    extracts `Card`s from a record-based deck
26
+    returns a table of `Card`s
27
+]]
28
+function Card:fromrecs(recs, conf)
29
+    local ret = {}
30
+    for i, rec in ipairs(recs) do
31
+        local bn
32
+        if rec.PromptText then bn = rec.PromptText[1] end
33
+        if rec.BaseNote then bn = rec.BaseNote[1] end
34
+        if bn then
35
+            local cc = Card:new({ note = bn, ct = (rec.CardType or {})[1], created = (rec.Created or {})[1] })
36
+            if cc.ct == "bmc" then
37
+                local m, n = cc.note:find("{[^}]+}")
38
+                local ctr = 1
39
+                while m do
40
+                    table.insert(cc.pr, Prompt:new({
41
+                        card = cc,
42
+                        ind = m,
43
+                        time = tonumber((rec.LastReview or {})[ctr]),
44
+                        delay = tonumber((rec.LastDelay or {})[ctr]),
45
+                        succ = tonumber((rec.Successes or {})[ctr]),
46
+                        fail = tonumber((rec.Failures or {})[ctr]),
47
+                        ease = tonumber((rec.Ease or {})[ctr]),
48
+                    }))
49
+                    m, n = cc.note:find("{[^}]+}", n)
50
+                    ctr = ctr + 1
51
+                end
52
+                table.insert(ret, cc)
53
+            else
54
+                print("error: unsupported card type " .. cc.ct)
55
+            end
56
+        end
57
+    end
58
+    return ret
59
+end
60
+
61
+--[[
62
+    serialises a `Card` to a record
63
+    returns a table mapping labels to value-lists
64
+]]
65
+function Card:torec()
66
+    local ret = { BaseNote = { self.note }, CardType = { self.ct }, Created = { self.created }, LastReview = {}, LastDelay = {}, Successes = {}, Failures = {}, Ease = {} }
67
+    for i, pr in ipairs(self.pr) do
68
+        ret.LastReview[i] = pr.time
69
+        ret.LastDelay[i] = pr.delay
70
+        ret.Successes[i] = pr.succ
71
+        ret.Failures[i] = pr.fail
72
+        ret.Ease[i] = pr.ease
73
+    end
74
+    return ret
75
+end
76
+
77
+return Card
... ...
@@ -1,9 +1,9 @@
1 1
 local function fn(config, deck, _, ip)
2
-    local sp = deck[ip[1]][ip[2]]
3
-    local rt = sp:spt()
2
+    local sc = deck[ip[1]]
3
+    local ot = sc.note
4 4
     local fname = "memoire_cc.tmp"
5 5
     local fh = io.open(fname, "w")
6
-    fh:write(rt)
6
+    fh:write(ot)
7 7
     fh:close()
8 8
     os.execute(string.format(
9 9
         "%s %q",
... ...
@@ -14,10 +14,8 @@ local function fn(config, deck, _, ip)
14 14
     local nt = fh:read("a"):gsub("%s+$", ""):gsub("^%s+", "")
15 15
     fh:close()
16 16
     os.remove(fname)
17
-    if nt ~= sp:spt() and nt ~= "" then
18
-        local cardrec = Prompt:torec(deck[ip[1]])
19
-        cardrec.PromptText = { nt }
20
-        deck[ip[1]] = Prompt:fromrecs({ cardrec }, config)[1]
17
+    if nt ~= ot and nt ~= "" then
18
+        deck[ip[1]].note = nt
21 19
         return deck
22 20
     end
23 21
 end
... ...
@@ -1,7 +1,7 @@
1 1
 local function fn(config, deck, deckfname)
2 2
     local mcl = {}
3 3
     for i, card in ipairs(deck) do
4
-        for path in card[1]:spt():gmatch(IMG_PATTERN) do
4
+        for path in card.note:gmatch(IMG_PATTERN) do
5 5
             local fh = io.open((deckfname:match(".*/") or "") .. path, "r")
6 6
             if not fh then
7 7
                 table.insert(mcl, i)
... ...
@@ -15,7 +15,7 @@ local function fn(config, deck, deckfname)
15 15
         io.write(string.format("No cards with bad image references\n"))
16 16
     end
17 17
     for _, i in ipairs(mcl) do
18
-        io.write(string.format("%d %s\n", i, deck[i][1]:spt()))
18
+        io.write(string.format("%d %s\n", i, deck[i].note))
19 19
     end
20 20
 end
21 21
 
... ...
@@ -8,18 +8,18 @@ end
8 8
 local function fn(config, deck)
9 9
     local leechqueue = {}
10 10
     for i, cg in ipairs(deck) do
11
-        for j, pr in ipairs(cg) do
11
+        for j, pr in ipairs(cg.pr) do
12 12
             if badness(pr) > 0.3 and pr.delay >= 0 then
13 13
                 table.insert(leechqueue, { i, j })
14 14
             end
15 15
         end
16 16
     end
17 17
     table.sort(leechqueue, function(a, b)
18
-        return badness(deck[a[1]][a[2]]) > badness(deck[b[1]][b[2]])
18
+        return badness(deck[a[1]].pr[a[2]]) > badness(deck[b[1]].pr[b[2]])
19 19
     end)
20 20
     io.write(string.format("Detected %d leeches:\n", #leechqueue))
21 21
     for _, ij in ipairs(leechqueue) do
22
-        io.write(string.format("%d:%d %s\n", ij[1], ij[2], deck[ij[1]][ij[2]]))
22
+        io.write(string.format("%d:%d %s\n", ij[1], ij[2], deck[ij[1]].pr[ij[2]]))
23 23
     end
24 24
 end
25 25
 
... ...
@@ -2,31 +2,32 @@
2 2
     deck loading and saving functions (working with Recfiles)
3 3
 ]]
4 4
 
5
-local Prompt = dofile(SRC_DIR .. "prompt.lua")
5
+local Card = dofile(SRC_DIR .. "card.lua")
6 6
 local recrw = dofile(SRC_DIR .. "recrw.lua")
7 7
 
8 8
 --[[
9
-    load a deck (a table of tables of `Prompt`s) from a Recfile
9
+    load a deck (a table of `Card`s) from a Recfile
10 10
 ]]
11 11
 local function loaddeck(fname, conf)
12 12
     local fh = io.open(fname)
13
-    if not fh then return nil end
13
+    if not fh then print("no fh") return nil end
14 14
     local rt = fh:read "a"
15
-    if not rt then return nil end
15
+    if not rt then print("no read") return nil end
16 16
     fh:close()
17 17
     local pr = recrw.read(rt)
18
-    if not pr then return nil end
19
-    local ep = Prompt:fromrecs(pr, conf)
18
+    if not pr then print("no recs") return nil end
19
+    local ep = Card:fromrecs(pr, conf)
20
+    if not ep then print("no cards") return nil end
20 21
     return ep
21 22
 end
22 23
 
23 24
 --[[
24
-    save a deck (a table of tables of `Prompt`s) to a Recfile
25
+    save a deck (a table of `Card`s) to a Recfile
25 26
 ]]
26 27
 local function savedeck(fname, deck)
27 28
     local rrf = {}
28 29
     for i, cg in ipairs(deck) do
29
-        rrf[i] = Prompt:torec(cg)
30
+        rrf[i] = cg:torec()
30 31
     end
31 32
     local wot = recrw.write(rrf)
32 33
     local fh = io.open(fname, "w")
... ...
@@ -5,6 +5,7 @@
5 5
 
6 6
 SRC_DIR = arg[0]:match("^.*/") or ""
7 7
 
8
+Card = dofile(SRC_DIR .. "card.lua")
8 9
 Prompt = dofile(SRC_DIR .. "prompt.lua")
9 10
 review = dofile(SRC_DIR .. "review.lua")
10 11
 loader = dofile(SRC_DIR .. "loader.lua")
... ...
@@ -78,7 +79,7 @@ end
78 79
     question-text, question-lang, answer-text, answer-lang
79 80
 ]]
80 81
 defaultprspeak = function(pr)
81
-    return {(pr.text:gsub("%{([^{}]+)%}", "%1"):gsub("%%A", "WHAT"))}, {false}, {"ANSWER: " .. pr.ans}, {false}
82
+    return {(pr:text():gsub("%{([^{}]+)%}", "%1"):gsub("%%A", "WHAT"))}, {false}, {"ANSWER: " .. pr:ans()}, {false}
82 83
 end
83 84
 prspeak = defaultprspeak
84 85
 do
... ...
@@ -1,8 +1,7 @@
1 1
 --[[
2
-    a prompt consists of prompt text (with an answer-insertion point),
3
-    a correct answer, the review timestamp, the review delay,
4
-    counts of successful and failed reviews,
5
-    a creation timestamp, and an ease measurement
2
+    a prompt consists of a parent card, an index into the note,
3
+    the review timestamp, the review delay,
4
+    counts of successful and failed reviews, and an ease measurement
6 5
 
7 6
     timestamps are seconds from the Unix epoch
8 7
     review delay is in seconds
... ...
@@ -11,41 +10,52 @@
11 10
 local Prompt = {}
12 11
 Prompt.__index = Prompt
13 12
 
14
---[[
15
-    the answer-insertion point is indicated with a "%A" in the prompt text
16
-]]
17
-function Prompt:new(pt, ca, rts, rd, sc, fc, ct, ez)
18
-    local o = {}
13
+function Prompt:new(prtable)
14
+    local o = { card = {}, ind = 0, time = os.time(), delay = 30, succ = 0, fail = 0, ease = 2.5 }
19 15
     setmetatable(o, self)
20
-    o.text = pt
21
-    o.ans = ca
22
-    o.time = tonumber(rts)
23
-    o.delay = tonumber(rd)
24
-    o.succ = tonumber(sc)
25
-    o.fail = tonumber(fc)
26
-    o.created = tonumber(ct)
27
-    o.ease = tonumber(ez)
16
+    for k, v in pairs(prtable) do
17
+        if o[k] then o[k] = v end
18
+    end
28 19
     return o
29 20
 end
30 21
 
31 22
 function Prompt:__tostring()
32 23
     return string.format(
33 24
         "\"%s\" (\"%s\") %d -> %d+%d*%f %d/%d",
34
-        self.text, self.ans, self.created or 0,
25
+        self:text(), self:ans(), self.card.created or 0,
35 26
         self.time, math.floor(self.delay), self.ease,
36 27
         self.succ, self.fail
37 28
     )
38 29
 end
39 30
 
40
-function Prompt:retention()
41
-    return self.succ / (self.succ + self.fail)
42
-end
43
-
44 31
 --[[
45
-    derive the `PromptText` used to create this `Prompt`
32
+    the answer-insertion point is indicated with a "%A" in the prompt text
46 33
 ]]
47
-function Prompt:spt()
48
-    return self.text:gsub("%%A", "{" .. self.ans:gsub("%%", "%%%%") .. "}")
34
+function Prompt:text()
35
+    if self.ind == 0 then
36
+        return "error: unpopulated ind"
37
+    end
38
+    if self.card.ct == "bmc" then
39
+        local _, eoa = self.card.note:find("{[^}]+}", self.ind)
40
+        return self.card.note:sub(1, self.ind - 1) .. "%A" .. self.card.note:sub(eoa + 1)
41
+    else
42
+        return "error: unrecognised card type " .. self.card.ct
43
+    end
44
+end
45
+
46
+function Prompt:ans()
47
+    if self.ind == 0 then
48
+        return "error: unpopulated ind"
49
+    end
50
+    if self.card.ct == "bmc" then
51
+        return self.card.note:match("{([^}]+)}", self.ind)
52
+    else
53
+        return "error: unrecognised card type " .. self.card.ct
54
+    end
55
+end
56
+
57
+function Prompt:retention()
58
+    return self.succ / (self.succ + self.fail)
49 59
 end
50 60
 
51 61
 --[[
... ...
@@ -1,7 +1,7 @@
1 1
 local function fn(config, deck, deckfname, qs)
2 2
     local mcl = {}
3 3
     for i, card in ipairs(deck) do
4
-        if card[1]:spt():find(qs) then
4
+        if card.note:find(qs) then
5 5
             table.insert(mcl, i)
6 6
         end
7 7
     end
... ...
@@ -11,7 +11,7 @@ local function fn(config, deck, deckfname, qs)
11 11
         io.write(string.format("No matches for %q\n", qs))
12 12
     end
13 13
     for _, i in ipairs(mcl) do
14
-        io.write(string.format("%d %s\n", i, deck[i][1]:spt()))
14
+        io.write(string.format("%d %s\n", i, deck[i].note))
15 15
     end
16 16
 end
17 17
 
... ...
@@ -49,7 +49,7 @@ local function importance(prompt, conf)
49 49
     pb = conf["PriorityBias"] or {}
50 50
     s = 0
51 51
     for i, t in ipairs(pt) do
52
-        if prompt:spt():find(t) then
52
+        if prompt.card.note:find(t) then
53 53
             s = s + pb[i]
54 54
         end
55 55
     end
... ...
@@ -84,7 +84,7 @@ local function toreviewlist(deck, conf, filter)
84 84
     local ctrt = {}
85 85
     local ff = filter or alwaystrue
86 86
     for i, cg in ipairs(deck) do
87
-        for j, pr in ipairs(cg) do
87
+        for j, pr in ipairs(cg.pr) do
88 88
             local sc = should(pr, conf)
89 89
             if sc and ff(pr) then
90 90
                 table.insert(ctrt, { i, j, sc or 0 })
... ...
@@ -124,16 +124,16 @@ local function fn(config, deck, deckfname)
124 124
         if ind > climn then
125 125
             break
126 126
         end
127
-        local pr = deck[ip[1]][ip[2]]
127
+        local pr = deck[ip[1]].pr[ip[2]]
128 128
         -- remove braces for display
129
-        local pst = pr.text:gsub("%{([^{}]+)%}", "%1")
129
+        local pst = pr:text():gsub("%{([^{}]+)%}", "%1")
130 130
         local st = os.time()
131 131
         imgify(pst:gsub("%%A", "___"), deckfname:match(".*/") or "", ind, climn, config, true)
132 132
         io.write("\n")
133 133
         io.read("l")
134 134
         clear()
135 135
         imgify(
136
-            pst:gsub("%%A", "\x1b[1m" .. (pr.ans:gsub("%%", "%%%%")) .. "\x1b[0m"),
136
+            pst:gsub("%%A", "\x1b[1m" .. (pr:ans():gsub("%%", "%%%%")) .. "\x1b[0m"),
137 137
             deckfname:match(".*/") or "",
138 138
             ind,
139 139
             climn,
... ...
@@ -155,7 +155,7 @@ local function fn(config, deck, deckfname)
155 155
             if sc == 0 then
156 156
                 lrc = lrc + 1
157 157
             end
158
-            review.update(deck[ip[1]][ip[2]], st, sc, config)
158
+            review.update(deck[ip[1]].pr[ip[2]], st, sc, config)
159 159
         end
160 160
         clear()
161 161
         if resp:sub(#resp):lower() == "q" then
... ...
@@ -180,7 +180,7 @@ local function fn(config, deck, deckfname)
180 180
     if #editqueue > 0 then
181 181
         io.write(string.format("The following cards are marked for correction:\n"))
182 182
         for _, ip in ipairs(editqueue) do
183
-            io.write(string.format("%d:%d %s\n", ip[1], ip[2], deck[ip[1]][ip[2]]))
183
+            io.write(string.format("%d %s\n", ip[1], deck[ip[1]]))
184 184
         end
185 185
     end
186 186
     if climn > 0 then
... ...
@@ -34,9 +34,9 @@ local function fn(config, deck, deckfname, qs)
34 34
     local expdeck = {}
35 35
     for _, card in ipairs(deck) do
36 36
         local rtc
37
-        for _, pr in ipairs(card) do
38
-            if not qs or (pr.text:find(qs) or pr.ans:find(qs)) then
37
+        if not qs or card.note:find(qs) then
39 38
             rtc = true
39
+            for _, pr in ipairs(card.pr) do
40 40
                 tsc = tsc + pr.succ
41 41
                 tfc = tfc + pr.fail
42 42
                 if pr.delay < 0 then
... ...
@@ -78,11 +78,11 @@ local function fn(config, deck, deckfname, qs)
78 78
     io.write("\nDelay-last review ratio\n" .. histogram(expdeck, 0, 2, 0.1, function(pr) return (os.time() - pr.time) / pr.delay end, 0.1))
79 79
     io.write("\nRetention (%)\n" .. histogram(expdeck, 30, 100, 10, function(pr) return 100 * pr:retention() end, 0.02))
80 80
     io.write("\nReview counts\n" .. histogram(expdeck, 0, 20, 2, function(pr) return pr.succ + pr.fail end, 0.03))
81
-    io.write("\nLog2-age (seconds)\n" .. histogram(expdeck, 18, 28, 0.5, function(pr) return math.log(os.time() - (pr.created or 2 * os.time()), 2) end, 0.05))
82
-    io.write("\nPrompt-count\n" .. histogram(deck, 1, 10, 1, function(card) return #card end, 0.03))
81
+    io.write("\nLog2-age (seconds)\n" .. histogram(deck, 18, 28, 0.5, function(card) return math.log(os.time() - (card.created or 2 * os.time()), 2) end, 0.05))
82
+    io.write("\nPrompt-count\n" .. histogram(deck, 1, 10, 1, function(card) return #card.pr end, 0.03))
83 83
     local cpcs = {}
84 84
     for _, card in ipairs(deck) do
85
-        local cp = card[1].text:match("([^:]+):") or ""
85
+        local cp = card.note:match("([^:]+):") or ""
86 86
         cpcs[cp] = (cpcs[cp] or 0) + 1
87 87
     end
88 88
     local scl = {}
... ...
@@ -1,5 +1,5 @@
1 1
 local function fn(config, deck, _, ip)
2
-    local sp = deck[ip[1]][ip[2]]
2
+    local sp = deck[ip[1]].pr[ip[2]]
3 3
     local dir
4 4
     if sp.delay < 0 then
5 5
         dir = "Restoring"
... ...
@@ -25,7 +25,7 @@ local function fn(config, deck, deckfname)
25 25
         return nil
26 26
     end
27 27
     -- collect list of index-pairs for prompts to be reviewed this session
28
-    local ctrt = review.list(deck, config, function(pr) return not pr:spt():find("IMG%[") end)
28
+    local ctrt = review.list(deck, config, function(pr) return not pr.card.note:find("IMG%[") end)
29 29
     speak(string.format(
30 30
         "%d prompts to review; how many in this session? ", #ctrt
31 31
     ), config)
... ...
@@ -44,9 +44,9 @@ local function fn(config, deck, deckfname)
44 44
         if ind > climn then
45 45
             break
46 46
         end
47
-        local pr = deck[ip[1]][ip[2]]
47
+        local pr = deck[ip[1]].pr[ip[2]]
48 48
         -- remove braces for display
49
-        local pst = pr.text:gsub("%{([^{}]+)%}", "%1")
49
+        local pst = pr:text():gsub("%{([^{}]+)%}", "%1")
50 50
         local st = os.time()
51 51
         io.write(pst .. "\n" .. ind .. "/" .. climn .. "\n")
52 52
         local qt, ql, at, al = prspeak(pr)
... ...
@@ -65,7 +65,7 @@ local function fn(config, deck, deckfname)
65 65
             if sc == 0 then
66 66
                 lrc = lrc + 1
67 67
             end
68
-            review.update(deck[ip[1]][ip[2]], st, sc, config)
68
+            review.update(deck[ip[1]].pr[ip[2]], st, sc, config)
69 69
         end
70 70
         clear()
71 71
         if resp:sub(#resp):lower() == "q" then
... ...
@@ -90,7 +90,7 @@ local function fn(config, deck, deckfname)
90 90
     if #editqueue > 0 then
91 91
         io.write(string.format("The following cards are marked for correction:\n"))
92 92
         for _, ip in ipairs(editqueue) do
93
-            io.write(string.format("%d:%d %s\n", ip[1], ip[2], deck[ip[1]][ip[2]]))
93
+            io.write(string.format("%d %s\n", ip[1], deck[ip[1]]))
94 94
         end
95 95
     end
96 96
     if climn > 0 then
97 97