@@ -1,355 +0,0 @@
1−From deeb435829d73524df851f6f4c2d4be552c99230 Mon Sep 17 00:00:00 2001
2−From: Dmitry Vedenko <dmitry@crsib.me>
3−Date: Fri, 1 Oct 2021 16:21:22 +0300
4−Subject: [PATCH] Use a different approach to estimate the disk space usage
5−
6−New a approach is a bit less precise, but removes the requirement for the "private" SQLite3 table and allows Audacity to be built against system SQLite3.
7−---
8− cmake-proxies/sqlite/CMakeLists.txt | 5 -
9− src/DBConnection.h | 4 +-
10− src/ProjectFileIO.cpp | 269 +++++-----------------------
11− 3 files changed, 44 insertions(+), 234 deletions(-)
12−
13−diff --git a/cmake-proxies/sqlite/CMakeLists.txt b/cmake-proxies/sqlite/CMakeLists.txt
14−index 63d70637c..d7b9b95ef 100644
15−--- a/cmake-proxies/sqlite/CMakeLists.txt
16−+++ b/cmake-proxies/sqlite/CMakeLists.txt
17−@@ -19,11 +19,6 @@ list( APPEND INCLUDES
18−
19− list( APPEND DEFINES
20− PRIVATE
21−- #
22−- # We need the dbpage table for space calculations.
23−- #
24−- SQLITE_ENABLE_DBPAGE_VTAB=1
25−-
26− # Can't be set after a WAL mode database is initialized, so change
27− # the default here to ensure all project files get the same page
28− # size.
29−diff --git a/src/DBConnection.h b/src/DBConnection.h
30−index 16a7fc9d4..07d3af95e 100644
31−--- a/src/DBConnection.h
32−+++ b/src/DBConnection.h
33−@@ -75,8 +75,8 @@ public:
34− LoadSampleBlock,
35− InsertSampleBlock,
36− DeleteSampleBlock,
37−- GetRootPage,
38−- GetDBPage
39−+ GetSampleBlockSize,
40−+ GetAllSampleBlocksSize
41− };
42− sqlite3_stmt *Prepare(enum StatementID id, const char *sql);
43−
44−diff --git a/src/ProjectFileIO.cpp b/src/ProjectFileIO.cpp
45−index 3b3e2e1fd..c9bc45af4 100644
46−--- a/src/ProjectFileIO.cpp
47−+++ b/src/ProjectFileIO.cpp
48−@@ -35,6 +35,7 @@ Paul Licameli split from AudacityProject.cpp
49− #include "widgets/ProgressDialog.h"
50− #include "wxFileNameWrapper.h"
51− #include "xml/XMLFileReader.h"
52−+#include "MemoryX.h"`
53−
54− #undef NO_SHM
55− #if !defined(__WXMSW__)
56−@@ -2357,255 +2358,69 @@ int64_t ProjectFileIO::GetTotalUsage()
57− }
58−
59− //
60−-// Returns the amount of disk space used by the specified sample blockid or all
61−-// of the sample blocks if the blockid is 0. It does this by using the raw SQLite
62−-// pages available from the "sqlite_dbpage" virtual table to traverse the SQLite
63−-// table b-tree described here: https://www.sqlite.org/fileformat.html
64−+// Returns the estimation of disk space used by the specified sample blockid or all
65−+// of the sample blocks if the blockid is 0. This does not include small overhead
66−+// of the internal SQLite structures, only the size used by the data
67− //
68− int64_t ProjectFileIO::GetDiskUsage(DBConnection &conn, SampleBlockID blockid /* = 0 */)
69− {
70−- // Information we need to track our travels through the b-tree
71−- typedef struct
72−- {
73−- int64_t pgno;
74−- int currentCell;
75−- int numCells;
76−- unsigned char data[65536];
77−- } page;
78−- std::vector<page> stack;
79−-
80−- int64_t total = 0;
81−- int64_t found = 0;
82−- int64_t right = 0;
83−- int rc;
84−+ sqlite3_stmt* stmt = nullptr;
85−
86−- // Get the rootpage for the sampleblocks table.
87−- sqlite3_stmt *stmt =
88−- conn.Prepare(DBConnection::GetRootPage,
89−- "SELECT rootpage FROM sqlite_master WHERE tbl_name = 'sampleblocks';");
90−- if (stmt == nullptr || sqlite3_step(stmt) != SQLITE_ROW)
91−+ if (blockid == 0)
92− {
93−- return 0;
94−- }
95−-
96−- // And store it in our first stack frame
97−- stack.push_back({sqlite3_column_int64(stmt, 0)});
98−+ static const char* statement =
99−+R"(SELECT
100−+ sum(length(blockid) + length(sampleformat) +
101−+ length(summin) + length(summax) + length(sumrms) +
102−+ length(summary256) + length(summary64k) +
103−+ length(samples))
104−+FROM sampleblocks;)";
105−
106−- // All done with the statement
107−- sqlite3_clear_bindings(stmt);
108−- sqlite3_reset(stmt);
109−-
110−- // Prepare/retrieve statement to read raw database page
111−- stmt = conn.Prepare(DBConnection::GetDBPage,
112−- "SELECT data FROM sqlite_dbpage WHERE pgno = ?1;");
113−- if (stmt == nullptr)
114−- {
115−- return 0;
116−+ stmt = conn.Prepare(DBConnection::GetAllSampleBlocksSize, statement);
117− }
118−-
119−- // Traverse the b-tree until we've visited all of the leaf pages or until
120−- // we find the one corresponding to the passed in sample blockid. Because we
121−- // use an integer primary key for the sampleblocks table, the traversal will
122−- // be in ascending blockid sequence.
123−- do
124−+ else
125− {
126−- // Acces the top stack frame
127−- page &pg = stack.back();
128−+ static const char* statement =
129−+R"(SELECT
130−+ length(blockid) + length(sampleformat) +
131−+ length(summin) + length(summax) + length(sumrms) +
132−+ length(summary256) + length(summary64k) +
133−+ length(samples)
134−+FROM sampleblocks WHERE blockid = ?1;)";
135−
136−- // Read the page from the sqlite_dbpage table if it hasn't yet been loaded
137−- if (pg.numCells == 0)
138−- {
139−- // Bind the page number
140−- sqlite3_bind_int64(stmt, 1, pg.pgno);
141−+ stmt = conn.Prepare(DBConnection::GetSampleBlockSize, statement);
142−+ }
143−
144−- // And retrieve the page
145−- if (sqlite3_step(stmt) != SQLITE_ROW)
146−+ auto cleanup = finally(
147−+ [stmt]() {
148−+ // Clear statement bindings and rewind statement
149−+ if (stmt != nullptr)
150− {
151−- // REVIEW: Likely harmless failure - says size is zero on
152−- // this error.
153−- // LLL: Yea, but not much else we can do.
154−- return 0;
155−+ sqlite3_clear_bindings(stmt);
156−+ sqlite3_reset(stmt);
157− }
158−+ });
159−
160−- // Copy the page content to the stack frame
161−- memcpy(&pg.data,
162−- sqlite3_column_blob(stmt, 0),
163−- sqlite3_column_bytes(stmt, 0));
164−-
165−- // And retrieve the total number of cells within it
166−- pg.numCells = get2(&pg.data[3]);
167−-
168−- // Reset statement for next usage
169−- sqlite3_clear_bindings(stmt);
170−- sqlite3_reset(stmt);
171−- }
172−-
173−- //wxLogDebug("%*.*spgno %lld currentCell %d numCells %d", (stack.size() - 1) * 2, (stack.size() - 1) * 2, "", pg.pgno, pg.currentCell, pg.numCells);
174−-
175−- // Process an interior table b-tree page
176−- if (pg.data[0] == 0x05)
177−- {
178−- // Process the next cell if we haven't examined all of them yet
179−- if (pg.currentCell < pg.numCells)
180−- {
181−- // Remember the right-most leaf page number.
182−- right = get4(&pg.data[8]);
183−-
184−- // Iterate over the cells.
185−- //
186−- // If we're not looking for a specific blockid, then we always push the
187−- // target page onto the stack and leave the loop after a single iteration.
188−- //
189−- // Otherwise, we match the blockid against the highest integer key contained
190−- // within the cell and if the blockid falls within the cell, we stack the
191−- // page and stop the iteration.
192−- //
193−- // In theory, we could do a binary search for a specific blockid here, but
194−- // because our sample blocks are always large, we will get very few cells
195−- // per page...usually 6 or less.
196−- //
197−- // In both cases, the stacked page can be either an internal or leaf page.
198−- bool stacked = false;
199−- while (pg.currentCell < pg.numCells)
200−- {
201−- // Get the offset to this cell using the offset in the cell pointer
202−- // array.
203−- //
204−- // The cell pointer array starts immediately after the page header
205−- // at offset 12 and the retrieved offset is from the beginning of
206−- // the page.
207−- int celloff = get2(&pg.data[12 + (pg.currentCell * 2)]);
208−-
209−- // Bump to the next cell for the next iteration.
210−- pg.currentCell++;
211−-
212−- // Get the page number this cell describes
213−- int pagenum = get4(&pg.data[celloff]);
214−-
215−- // And the highest integer key, which starts at offset 4 within the cell.
216−- int64_t intkey = 0;
217−- get_varint(&pg.data[celloff + 4], &intkey);
218−-
219−- //wxLogDebug("%*.*sinternal - right %lld celloff %d pagenum %d intkey %lld", (stack.size() - 1) * 2, (stack.size() - 1) * 2, " ", right, celloff, pagenum, intkey);
220−-
221−- // Stack the described page if we're not looking for a specific blockid
222−- // or if this page contains the given blockid.
223−- if (!blockid || blockid <= intkey)
224−- {
225−- stack.push_back({pagenum, 0, 0});
226−- stacked = true;
227−- break;
228−- }
229−- }
230−-
231−- // If we pushed a new page onto the stack, we need to jump back up
232−- // to read the page
233−- if (stacked)
234−- {
235−- continue;
236−- }
237−- }
238−+ if (blockid != 0)
239−+ {
240−+ int rc = sqlite3_bind_int64(stmt, 1, blockid);
241−
242−- // We've exhausted all the cells with this page, so we stack the right-most
243−- // leaf page. Ensure we only process it once.
244−- if (right)
245−- {
246−- stack.push_back({right, 0, 0});
247−- right = 0;
248−- continue;
249−- }
250−- }
251−- // Process a leaf table b-tree page
252−- else if (pg.data[0] == 0x0d)
253−+ if (rc != SQLITE_OK)
254− {
255−- // Iterate over the cells
256−- //
257−- // If we're not looking for a specific blockid, then just accumulate the
258−- // payload sizes. We will be reading every leaf page in the sampleblocks
259−- // table.
260−- //
261−- // Otherwise we break out when we find the matching blockid. In this case,
262−- // we only ever look at 1 leaf page.
263−- bool stop = false;
264−- for (int i = 0; i < pg.numCells; i++)
265−- {
266−- // Get the offset to this cell using the offset in the cell pointer
267−- // array.
268−- //
269−- // The cell pointer array starts immediately after the page header
270−- // at offset 8 and the retrieved offset is from the beginning of
271−- // the page.
272−- int celloff = get2(&pg.data[8 + (i * 2)]);
273−-
274−- // Get the total payload size in bytes of the described row.
275−- int64_t payload = 0;
276−- int digits = get_varint(&pg.data[celloff], &payload);
277−-
278−- // Get the integer key for this row.
279−- int64_t intkey = 0;
280−- get_varint(&pg.data[celloff + digits], &intkey);
281−-
282−- //wxLogDebug("%*.*sleaf - celloff %4d intkey %lld payload %lld", (stack.size() - 1) * 2, (stack.size() - 1) * 2, " ", celloff, intkey, payload);
283−-
284−- // Add this payload size to the total if we're not looking for a specific
285−- // blockid
286−- if (!blockid)
287−- {
288−- total += payload;
289−- }
290−- // Otherwise, return the payload size for a matching row
291−- else if (blockid == intkey)
292−- {
293−- return payload;
294−- }
295−- }
296−+ conn.ThrowException(false);
297− }
298−+ }
299−
300−- // Done with the current branch, so pop back up to the previous one (if any)
301−- stack.pop_back();
302−- } while (!stack.empty());
303−-
304−- // Return the total used for all sample blocks
305−- return total;
306−-}
307−-
308−-// Retrieves a 2-byte big-endian integer from the page data
309−-unsigned int ProjectFileIO::get2(const unsigned char *ptr)
310−-{
311−- return (ptr[0] << 8) | ptr[1];
312−-}
313−-
314−-// Retrieves a 4-byte big-endian integer from the page data
315−-unsigned int ProjectFileIO::get4(const unsigned char *ptr)
316−-{
317−- return ((unsigned int) ptr[0] << 24) |
318−- ((unsigned int) ptr[1] << 16) |
319−- ((unsigned int) ptr[2] << 8) |
320−- ((unsigned int) ptr[3]);
321−-}
322−-
323−-// Retrieves a variable length integer from the page data. Returns the
324−-// number of digits used to encode the integer and the stores the
325−-// value at the given location.
326−-int ProjectFileIO::get_varint(const unsigned char *ptr, int64_t *out)
327−-{
328−- int64_t val = 0;
329−- int i;
330−+ int rc = sqlite3_step(stmt);
331−
332−- for (i = 0; i < 8; ++i)
333−+ if (rc != SQLITE_ROW)
334− {
335−- val = (val << 7) + (ptr[i] & 0x7f);
336−- if ((ptr[i] & 0x80) == 0)
337−- {
338−- *out = val;
339−- return i + 1;
340−- }
341−+ conn.ThrowException(false);
342− }
343−
344−- val = (val << 8) + (ptr[i] & 0xff);
345−- *out = val;
346−+ const int64_t size = sqlite3_column_int64(stmt, 0);
347−
348−- return 9;
349−+ return size;
350− }
351−
352− InvisibleTemporaryProject::InvisibleTemporaryProject()
353−--
354−2.33.1
355−