@@ -0,0 +1,355 @@
1+From deeb435829d73524df851f6f4c2d4be552c99230 Mon Sep 17 00:00:00 2001
2+From: Dmitry Vedenko <dmitry@crsib.me>
3+Date: Fri, 1 Oct 2021 16:21:22 +0300
4+Subject: [PATCH] Use a different approach to estimate the disk space usage
5+
6+New a approach is a bit less precise, but removes the requirement for the "private" SQLite3 table and allows Audacity to be built against system SQLite3.
7+---
8+ cmake-proxies/sqlite/CMakeLists.txt | 5 -
9+ src/DBConnection.h | 4 +-
10+ src/ProjectFileIO.cpp | 269 +++++-----------------------
11+ 3 files changed, 44 insertions(+), 234 deletions(-)
12+
13+diff --git a/cmake-proxies/sqlite/CMakeLists.txt b/cmake-proxies/sqlite/CMakeLists.txt
14+index 63d70637c..d7b9b95ef 100644
15+--- a/cmake-proxies/sqlite/CMakeLists.txt
16++++ b/cmake-proxies/sqlite/CMakeLists.txt
17+@@ -19,11 +19,6 @@ list( APPEND INCLUDES
18+
19+ list( APPEND DEFINES
20+ PRIVATE
21+- #
22+- # We need the dbpage table for space calculations.
23+- #
24+- SQLITE_ENABLE_DBPAGE_VTAB=1
25+-
26+ # Can't be set after a WAL mode database is initialized, so change
27+ # the default here to ensure all project files get the same page
28+ # size.
29+diff --git a/src/DBConnection.h b/src/DBConnection.h
30+index 16a7fc9d4..07d3af95e 100644
31+--- a/src/DBConnection.h
32++++ b/src/DBConnection.h
33+@@ -75,8 +75,8 @@ public:
34+ LoadSampleBlock,
35+ InsertSampleBlock,
36+ DeleteSampleBlock,
37+- GetRootPage,
38+- GetDBPage
39++ GetSampleBlockSize,
40++ GetAllSampleBlocksSize
41+ };
42+ sqlite3_stmt *Prepare(enum StatementID id, const char *sql);
43+
44+diff --git a/src/ProjectFileIO.cpp b/src/ProjectFileIO.cpp
45+index 3b3e2e1fd..c9bc45af4 100644
46+--- a/src/ProjectFileIO.cpp
47++++ b/src/ProjectFileIO.cpp
48+@@ -35,6 +35,7 @@ Paul Licameli split from AudacityProject.cpp
49+ #include "widgets/ProgressDialog.h"
50+ #include "wxFileNameWrapper.h"
51+ #include "xml/XMLFileReader.h"
52++#include "MemoryX.h"`
53+
54+ #undef NO_SHM
55+ #if !defined(__WXMSW__)
56+@@ -2357,255 +2358,69 @@ int64_t ProjectFileIO::GetTotalUsage()
57+ }
58+
59+ //
60+-// Returns the amount of disk space used by the specified sample blockid or all
61+-// of the sample blocks if the blockid is 0. It does this by using the raw SQLite
62+-// pages available from the "sqlite_dbpage" virtual table to traverse the SQLite
63+-// table b-tree described here: https://www.sqlite.org/fileformat.html
64++// Returns the estimation of disk space used by the specified sample blockid or all
65++// of the sample blocks if the blockid is 0. This does not include small overhead
66++// of the internal SQLite structures, only the size used by the data
67+ //
68+ int64_t ProjectFileIO::GetDiskUsage(DBConnection &conn, SampleBlockID blockid /* = 0 */)
69+ {
70+- // Information we need to track our travels through the b-tree
71+- typedef struct
72+- {
73+- int64_t pgno;
74+- int currentCell;
75+- int numCells;
76+- unsigned char data[65536];
77+- } page;
78+- std::vector<page> stack;
79+-
80+- int64_t total = 0;
81+- int64_t found = 0;
82+- int64_t right = 0;
83+- int rc;
84++ sqlite3_stmt* stmt = nullptr;
85+
86+- // Get the rootpage for the sampleblocks table.
87+- sqlite3_stmt *stmt =
88+- conn.Prepare(DBConnection::GetRootPage,
89+- "SELECT rootpage FROM sqlite_master WHERE tbl_name = 'sampleblocks';");
90+- if (stmt == nullptr || sqlite3_step(stmt) != SQLITE_ROW)
91++ if (blockid == 0)
92+ {
93+- return 0;
94+- }
95+-
96+- // And store it in our first stack frame
97+- stack.push_back({sqlite3_column_int64(stmt, 0)});
98++ static const char* statement =
99++R"(SELECT
100++ sum(length(blockid) + length(sampleformat) +
101++ length(summin) + length(summax) + length(sumrms) +
102++ length(summary256) + length(summary64k) +
103++ length(samples))
104++FROM sampleblocks;)";
105+
106+- // All done with the statement
107+- sqlite3_clear_bindings(stmt);
108+- sqlite3_reset(stmt);
109+-
110+- // Prepare/retrieve statement to read raw database page
111+- stmt = conn.Prepare(DBConnection::GetDBPage,
112+- "SELECT data FROM sqlite_dbpage WHERE pgno = ?1;");
113+- if (stmt == nullptr)
114+- {
115+- return 0;
116++ stmt = conn.Prepare(DBConnection::GetAllSampleBlocksSize, statement);
117+ }
118+-
119+- // Traverse the b-tree until we've visited all of the leaf pages or until
120+- // we find the one corresponding to the passed in sample blockid. Because we
121+- // use an integer primary key for the sampleblocks table, the traversal will
122+- // be in ascending blockid sequence.
123+- do
124++ else
125+ {
126+- // Acces the top stack frame
127+- page &pg = stack.back();
128++ static const char* statement =
129++R"(SELECT
130++ length(blockid) + length(sampleformat) +
131++ length(summin) + length(summax) + length(sumrms) +
132++ length(summary256) + length(summary64k) +
133++ length(samples)
134++FROM sampleblocks WHERE blockid = ?1;)";
135+
136+- // Read the page from the sqlite_dbpage table if it hasn't yet been loaded
137+- if (pg.numCells == 0)
138+- {
139+- // Bind the page number
140+- sqlite3_bind_int64(stmt, 1, pg.pgno);
141++ stmt = conn.Prepare(DBConnection::GetSampleBlockSize, statement);
142++ }
143+
144+- // And retrieve the page
145+- if (sqlite3_step(stmt) != SQLITE_ROW)
146++ auto cleanup = finally(
147++ [stmt]() {
148++ // Clear statement bindings and rewind statement
149++ if (stmt != nullptr)
150+ {
151+- // REVIEW: Likely harmless failure - says size is zero on
152+- // this error.
153+- // LLL: Yea, but not much else we can do.
154+- return 0;
155++ sqlite3_clear_bindings(stmt);
156++ sqlite3_reset(stmt);
157+ }
158++ });
159+
160+- // Copy the page content to the stack frame
161+- memcpy(&pg.data,
162+- sqlite3_column_blob(stmt, 0),
163+- sqlite3_column_bytes(stmt, 0));
164+-
165+- // And retrieve the total number of cells within it
166+- pg.numCells = get2(&pg.data[3]);
167+-
168+- // Reset statement for next usage
169+- sqlite3_clear_bindings(stmt);
170+- sqlite3_reset(stmt);
171+- }
172+-
173+- //wxLogDebug("%*.*spgno %lld currentCell %d numCells %d", (stack.size() - 1) * 2, (stack.size() - 1) * 2, "", pg.pgno, pg.currentCell, pg.numCells);
174+-
175+- // Process an interior table b-tree page
176+- if (pg.data[0] == 0x05)
177+- {
178+- // Process the next cell if we haven't examined all of them yet
179+- if (pg.currentCell < pg.numCells)
180+- {
181+- // Remember the right-most leaf page number.
182+- right = get4(&pg.data[8]);
183+-
184+- // Iterate over the cells.
185+- //
186+- // If we're not looking for a specific blockid, then we always push the
187+- // target page onto the stack and leave the loop after a single iteration.
188+- //
189+- // Otherwise, we match the blockid against the highest integer key contained
190+- // within the cell and if the blockid falls within the cell, we stack the
191+- // page and stop the iteration.
192+- //
193+- // In theory, we could do a binary search for a specific blockid here, but
194+- // because our sample blocks are always large, we will get very few cells
195+- // per page...usually 6 or less.
196+- //
197+- // In both cases, the stacked page can be either an internal or leaf page.
198+- bool stacked = false;
199+- while (pg.currentCell < pg.numCells)
200+- {
201+- // Get the offset to this cell using the offset in the cell pointer
202+- // array.
203+- //
204+- // The cell pointer array starts immediately after the page header
205+- // at offset 12 and the retrieved offset is from the beginning of
206+- // the page.
207+- int celloff = get2(&pg.data[12 + (pg.currentCell * 2)]);
208+-
209+- // Bump to the next cell for the next iteration.
210+- pg.currentCell++;
211+-
212+- // Get the page number this cell describes
213+- int pagenum = get4(&pg.data[celloff]);
214+-
215+- // And the highest integer key, which starts at offset 4 within the cell.
216+- int64_t intkey = 0;
217+- get_varint(&pg.data[celloff + 4], &intkey);
218+-
219+- //wxLogDebug("%*.*sinternal - right %lld celloff %d pagenum %d intkey %lld", (stack.size() - 1) * 2, (stack.size() - 1) * 2, " ", right, celloff, pagenum, intkey);
220+-
221+- // Stack the described page if we're not looking for a specific blockid
222+- // or if this page contains the given blockid.
223+- if (!blockid || blockid <= intkey)
224+- {
225+- stack.push_back({pagenum, 0, 0});
226+- stacked = true;
227+- break;
228+- }
229+- }
230+-
231+- // If we pushed a new page onto the stack, we need to jump back up
232+- // to read the page
233+- if (stacked)
234+- {
235+- continue;
236+- }
237+- }
238++ if (blockid != 0)
239++ {
240++ int rc = sqlite3_bind_int64(stmt, 1, blockid);
241+
242+- // We've exhausted all the cells with this page, so we stack the right-most
243+- // leaf page. Ensure we only process it once.
244+- if (right)
245+- {
246+- stack.push_back({right, 0, 0});
247+- right = 0;
248+- continue;
249+- }
250+- }
251+- // Process a leaf table b-tree page
252+- else if (pg.data[0] == 0x0d)
253++ if (rc != SQLITE_OK)
254+ {
255+- // Iterate over the cells
256+- //
257+- // If we're not looking for a specific blockid, then just accumulate the
258+- // payload sizes. We will be reading every leaf page in the sampleblocks
259+- // table.
260+- //
261+- // Otherwise we break out when we find the matching blockid. In this case,
262+- // we only ever look at 1 leaf page.
263+- bool stop = false;
264+- for (int i = 0; i < pg.numCells; i++)
265+- {
266+- // Get the offset to this cell using the offset in the cell pointer
267+- // array.
268+- //
269+- // The cell pointer array starts immediately after the page header
270+- // at offset 8 and the retrieved offset is from the beginning of
271+- // the page.
272+- int celloff = get2(&pg.data[8 + (i * 2)]);
273+-
274+- // Get the total payload size in bytes of the described row.
275+- int64_t payload = 0;
276+- int digits = get_varint(&pg.data[celloff], &payload);
277+-
278+- // Get the integer key for this row.
279+- int64_t intkey = 0;
280+- get_varint(&pg.data[celloff + digits], &intkey);
281+-
282+- //wxLogDebug("%*.*sleaf - celloff %4d intkey %lld payload %lld", (stack.size() - 1) * 2, (stack.size() - 1) * 2, " ", celloff, intkey, payload);
283+-
284+- // Add this payload size to the total if we're not looking for a specific
285+- // blockid
286+- if (!blockid)
287+- {
288+- total += payload;
289+- }
290+- // Otherwise, return the payload size for a matching row
291+- else if (blockid == intkey)
292+- {
293+- return payload;
294+- }
295+- }
296++ conn.ThrowException(false);
297+ }
298++ }
299+
300+- // Done with the current branch, so pop back up to the previous one (if any)
301+- stack.pop_back();
302+- } while (!stack.empty());
303+-
304+- // Return the total used for all sample blocks
305+- return total;
306+-}
307+-
308+-// Retrieves a 2-byte big-endian integer from the page data
309+-unsigned int ProjectFileIO::get2(const unsigned char *ptr)
310+-{
311+- return (ptr[0] << 8) | ptr[1];
312+-}
313+-
314+-// Retrieves a 4-byte big-endian integer from the page data
315+-unsigned int ProjectFileIO::get4(const unsigned char *ptr)
316+-{
317+- return ((unsigned int) ptr[0] << 24) |
318+- ((unsigned int) ptr[1] << 16) |
319+- ((unsigned int) ptr[2] << 8) |
320+- ((unsigned int) ptr[3]);
321+-}
322+-
323+-// Retrieves a variable length integer from the page data. Returns the
324+-// number of digits used to encode the integer and the stores the
325+-// value at the given location.
326+-int ProjectFileIO::get_varint(const unsigned char *ptr, int64_t *out)
327+-{
328+- int64_t val = 0;
329+- int i;
330++ int rc = sqlite3_step(stmt);
331+
332+- for (i = 0; i < 8; ++i)
333++ if (rc != SQLITE_ROW)
334+ {
335+- val = (val << 7) + (ptr[i] & 0x7f);
336+- if ((ptr[i] & 0x80) == 0)
337+- {
338+- *out = val;
339+- return i + 1;
340+- }
341++ conn.ThrowException(false);
342+ }
343+
344+- val = (val << 8) + (ptr[i] & 0xff);
345+- *out = val;
346++ const int64_t size = sqlite3_column_int64(stmt, 0);
347+
348+- return 9;
349++ return size;
350+ }
351+
352+ InvisibleTemporaryProject::InvisibleTemporaryProject()
353+--
354+2.33.1
355+