From b68a8730678c07d597eefd4d3a8b796412416a82 Mon Sep 17 00:00:00 2001 From: asynts Date: Tue, 1 Sep 2020 11:43:32 +0200 Subject: [PATCH] AK: Move memory streams into their own header. --- AK/MemoryStream.h | 299 ++++++++++++++++++ AK/Stream.h | 289 +---------------- .../{TestStream.cpp => TestMemoryStream.cpp} | 4 +- Libraries/LibCompress/Deflate.cpp | 1 + Libraries/LibCompress/Gzip.cpp | 1 + Libraries/LibDebug/DebugInfo.cpp | 2 +- Libraries/LibDebug/Dwarf/AbbreviationsMap.cpp | 2 +- Libraries/LibDebug/Dwarf/DIE.cpp | 2 +- Libraries/LibDebug/Dwarf/DwarfInfo.cpp | 4 +- Libraries/LibDebug/Dwarf/Expression.cpp | 2 +- Libraries/LibDebug/Dwarf/LineProgram.h | 5 +- Shell/AST.cpp | 1 + 12 files changed, 314 insertions(+), 298 deletions(-) create mode 100644 AK/MemoryStream.h rename AK/Tests/{TestStream.cpp => TestMemoryStream.cpp} (98%) diff --git a/AK/MemoryStream.h b/AK/MemoryStream.h new file mode 100644 index 00000000000..96df59f3c97 --- /dev/null +++ b/AK/MemoryStream.h @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2020, the SerenityOS developers. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include +#include +#include +#include + +namespace AK { + +class InputMemoryStream final : public InputStream { +public: + InputMemoryStream(ReadonlyBytes bytes) + : m_bytes(bytes) + { + } + + bool eof() const override { return m_offset >= m_bytes.size(); } + + size_t read(Bytes bytes) override + { + const auto count = min(bytes.size(), remaining()); + __builtin_memcpy(bytes.data(), m_bytes.data() + m_offset, count); + m_offset += count; + return count; + } + + bool read_or_error(Bytes bytes) override + { + if (remaining() < bytes.size()) { + set_recoverable_error(); + return false; + } + + __builtin_memcpy(bytes.data(), m_bytes.data() + m_offset, bytes.size()); + m_offset += bytes.size(); + return true; + } + + bool discard_or_error(size_t count) override + { + if (remaining() < count) { + set_recoverable_error(); + return false; + } + + m_offset += count; + return true; + } + + void seek(size_t offset) + { + ASSERT(offset < m_bytes.size()); + m_offset = offset; + } + + u8 peek_or_error() const + { + if (remaining() == 0) { + set_recoverable_error(); + return 0; + } + + return m_bytes[m_offset]; + } + + // LEB128 is a variable-length encoding for integers + bool read_LEB128_unsigned(size_t& result) + { + const auto backup = m_offset; + + result = 0; + size_t num_bytes = 0; + while (true) { + // Note. The implementation in AK::BufferStream::read_LEB128_unsigned read one + // past the end, this is fixed here. + if (eof()) { + m_offset = backup; + set_recoverable_error(); + return false; + } + + const u8 byte = m_bytes[m_offset]; + result = (result) | (static_cast(byte & ~(1 << 7)) << (num_bytes * 7)); + ++m_offset; + if (!(byte & (1 << 7))) + break; + ++num_bytes; + } + + return true; + } + + // LEB128 is a variable-length encoding for integers + bool read_LEB128_signed(ssize_t& result) + { + const auto backup = m_offset; + + result = 0; + size_t num_bytes = 0; + u8 byte = 0; + + do { + // Note. The implementation in AK::BufferStream::read_LEB128_unsigned read one + // past the end, this is fixed here. + if (eof()) { + m_offset = backup; + set_recoverable_error(); + return false; + } + + byte = m_bytes[m_offset]; + result = (result) | (static_cast(byte & ~(1 << 7)) << (num_bytes * 7)); + ++m_offset; + ++num_bytes; + } while (byte & (1 << 7)); + + if (num_bytes * 7 < sizeof(size_t) * 4 && (byte & 0x40)) { + // sign extend + result |= ((size_t)(-1) << (num_bytes * 7)); + } + + return true; + } + + ReadonlyBytes bytes() const { return m_bytes; } + size_t offset() const { return m_offset; } + size_t remaining() const { return m_bytes.size() - m_offset; } + +private: + ReadonlyBytes m_bytes; + size_t m_offset { 0 }; +}; + +// All data written to this stream can be read from it. Reading and writing is done +// using different offsets, meaning that it is not necessary to seek to the start +// before reading; this behaviour differs from BufferStream. +class DuplexMemoryStream final : public DuplexStream { +public: + static constexpr size_t chunk_size = 4 * 1024; + + bool eof() const override { return m_write_offset == m_read_offset; } + + bool discard_or_error(size_t count) override + { + if (m_write_offset - m_read_offset < count) { + set_recoverable_error(); + return false; + } + + m_read_offset += count; + try_discard_chunks(); + return true; + } + + Optional offset_of(ReadonlyBytes value) const + { + if (value.size() > remaining()) + return {}; + + // First, find which chunk we're in. + auto chunk_index = (m_read_offset - m_base_offset) / chunk_size; + auto last_written_chunk_index = (m_write_offset - m_base_offset) / chunk_size; + auto first_chunk_index = chunk_index; + auto last_written_chunk_offset = m_write_offset % chunk_size; + auto first_chunk_offset = m_read_offset % chunk_size; + size_t last_chunk_offset = 0; + auto found_value = false; + + for (; chunk_index <= last_written_chunk_index; ++chunk_index) { + auto chunk_bytes = m_chunks[chunk_index].bytes(); + size_t chunk_offset = 0; + if (chunk_index == last_written_chunk_index) { + chunk_bytes = chunk_bytes.slice(0, last_written_chunk_offset); + } + if (chunk_index == first_chunk_index) { + chunk_bytes = chunk_bytes.slice(first_chunk_offset); + chunk_offset = first_chunk_offset; + } + + // See if 'value' is in this chunk, + auto position = AK::memmem(chunk_bytes.data(), chunk_bytes.size(), value.data(), value.size()); + if (!position) + continue; // Not in this chunk either :( + + // We found it! + found_value = true; + last_chunk_offset = (const u8*)position - chunk_bytes.data() + chunk_offset; + break; + } + + if (found_value) { + if (first_chunk_index == chunk_index) + return last_chunk_offset - first_chunk_offset; + + return (chunk_index - first_chunk_index) * chunk_size + last_chunk_offset - first_chunk_offset; + } + + // No dice. + return {}; + } + + size_t read(Bytes bytes) override + { + size_t nread = 0; + while (bytes.size() - nread > 0 && m_write_offset - m_read_offset - nread > 0) { + const auto chunk_index = (m_read_offset - m_base_offset) / chunk_size; + const auto chunk_bytes = m_chunks[chunk_index].bytes().slice(m_read_offset % chunk_size).trim(m_write_offset - m_read_offset - nread); + nread += chunk_bytes.copy_trimmed_to(bytes.slice(nread)); + } + + m_read_offset += nread; + + try_discard_chunks(); + + return nread; + } + + bool read_or_error(Bytes bytes) override + { + if (m_write_offset - m_read_offset < bytes.size()) { + set_recoverable_error(); + return false; + } + + read(bytes); + return true; + } + + size_t write(ReadonlyBytes bytes) override + { + size_t nwritten = 0; + while (bytes.size() - nwritten > 0) { + if ((m_write_offset + nwritten) % chunk_size == 0) + m_chunks.append(ByteBuffer::create_uninitialized(chunk_size)); + + nwritten += bytes.copy_trimmed_to(m_chunks.last().bytes().slice(m_write_offset % chunk_size)); + } + + m_write_offset += nwritten; + return nwritten; + } + + bool write_or_error(ReadonlyBytes bytes) override + { + write(bytes); + return true; + } + + size_t roffset() const { return m_read_offset; } + size_t woffset() const { return m_write_offset; } + + size_t remaining() const { return m_write_offset - m_read_offset; } + +private: + void try_discard_chunks() + { + while (m_read_offset - m_base_offset >= chunk_size) { + m_chunks.take_first(); + m_base_offset += chunk_size; + } + } + + Vector m_chunks; + size_t m_write_offset { 0 }; + size_t m_read_offset { 0 }; + size_t m_base_offset { 0 }; +}; + +} + +using AK::DuplexMemoryStream; +using AK::InputMemoryStream; +using AK::InputStream; diff --git a/AK/Stream.h b/AK/Stream.h index 12adc83b09f..c8fd053822f 100644 --- a/AK/Stream.h +++ b/AK/Stream.h @@ -26,15 +26,12 @@ #pragma once -#include #include #include #include -#include #include #include #include -#include namespace AK::Detail { @@ -116,6 +113,7 @@ OutputStream& operator<<(OutputStream& stream, LittleEndian value) { return stream << ReadonlyBytes { &value.m_value, sizeof(value.m_value) }; } + template InputStream& operator>>(InputStream& stream, BigEndian& value) { @@ -136,22 +134,13 @@ InputStream& operator>>(InputStream& stream, Optional& value) return stream; } -#if defined(__cpp_concepts) && !defined(__COVERITY__) -template -#else template::value, int>::Type = 0> -#endif InputStream& operator>>(InputStream& stream, Integral& value) { stream.read_or_error({ &value, sizeof(value) }); return stream; } - -#if defined(__cpp_concepts) && !defined(__COVERITY__) -template -#else template::value, int>::Type = 0> -#endif OutputStream& operator<<(OutputStream& stream, Integral value) { stream.write_or_error({ &value, sizeof(value) }); @@ -160,24 +149,13 @@ OutputStream& operator<<(OutputStream& stream, Integral value) #ifndef KERNEL -// FIXME: clang-format adds spaces before the #if for some reason. -// clang-format off -#if defined(__cpp_concepts) && !defined(__COVERITY__) -template -#else template::value, int>::Type = 0> -#endif InputStream& operator>>(InputStream& stream, FloatingPoint& value) { stream.read_or_error({ &value, sizeof(value) }); return stream; } - -#if defined(__cpp_concepts) && !defined(__COVERITY__) -template -#else template::value, int>::Type = 0> -#endif OutputStream& operator<<(OutputStream& stream, FloatingPoint value) { stream.write_or_error({ &value, sizeof(value) }); @@ -185,281 +163,16 @@ OutputStream& operator<<(OutputStream& stream, FloatingPoint value) } #endif -// clang-format on inline InputStream& operator>>(InputStream& stream, bool& value) { stream.read_or_error({ &value, sizeof(value) }); return stream; } - inline OutputStream& operator<<(OutputStream& stream, bool value) { stream.write_or_error({ &value, sizeof(value) }); return stream; } -class InputMemoryStream final : public InputStream { -public: - InputMemoryStream(ReadonlyBytes bytes) - : m_bytes(bytes) - { - } - - bool eof() const override { return m_offset >= m_bytes.size(); } - - size_t read(Bytes bytes) override - { - const auto count = min(bytes.size(), remaining()); - __builtin_memcpy(bytes.data(), m_bytes.data() + m_offset, count); - m_offset += count; - return count; - } - - bool read_or_error(Bytes bytes) override - { - if (remaining() < bytes.size()) { - set_recoverable_error(); - return false; - } - - __builtin_memcpy(bytes.data(), m_bytes.data() + m_offset, bytes.size()); - m_offset += bytes.size(); - return true; - } - - bool discard_or_error(size_t count) override - { - if (remaining() < count) { - set_recoverable_error(); - return false; - } - - m_offset += count; - return true; - } - - void seek(size_t offset) - { - ASSERT(offset < m_bytes.size()); - m_offset = offset; - } - - u8 peek_or_error() const - { - if (remaining() == 0) { - set_recoverable_error(); - return 0; - } - - return m_bytes[m_offset]; - } - - // LEB128 is a variable-length encoding for integers - bool read_LEB128_unsigned(size_t& result) - { - const auto backup = m_offset; - - result = 0; - size_t num_bytes = 0; - while (true) { - // Note. The implementation in AK::BufferStream::read_LEB128_unsigned read one - // past the end, this is fixed here. - if (eof()) { - m_offset = backup; - set_recoverable_error(); - return false; - } - - const u8 byte = m_bytes[m_offset]; - result = (result) | (static_cast(byte & ~(1 << 7)) << (num_bytes * 7)); - ++m_offset; - if (!(byte & (1 << 7))) - break; - ++num_bytes; - } - - return true; - } - - // LEB128 is a variable-length encoding for integers - bool read_LEB128_signed(ssize_t& result) - { - const auto backup = m_offset; - - result = 0; - size_t num_bytes = 0; - u8 byte = 0; - - do { - // Note. The implementation in AK::BufferStream::read_LEB128_unsigned read one - // past the end, this is fixed here. - if (eof()) { - m_offset = backup; - set_recoverable_error(); - return false; - } - - byte = m_bytes[m_offset]; - result = (result) | (static_cast(byte & ~(1 << 7)) << (num_bytes * 7)); - ++m_offset; - ++num_bytes; - } while (byte & (1 << 7)); - - if (num_bytes * 7 < sizeof(size_t) * 4 && (byte & 0x40)) { - // sign extend - result |= ((size_t)(-1) << (num_bytes * 7)); - } - - return true; - } - - ReadonlyBytes bytes() const { return m_bytes; } - size_t offset() const { return m_offset; } - size_t remaining() const { return m_bytes.size() - m_offset; } - -private: - ReadonlyBytes m_bytes; - size_t m_offset { 0 }; -}; - -// All data written to this stream can be read from it. Reading and writing is done -// using different offsets, meaning that it is not necessary to seek to the start -// before reading; this behaviour differs from BufferStream. -class DuplexMemoryStream final : public DuplexStream { -public: - static constexpr size_t chunk_size = 4 * 1024; - - bool eof() const override { return m_write_offset == m_read_offset; } - - bool discard_or_error(size_t count) override - { - if (m_write_offset - m_read_offset < count) { - set_recoverable_error(); - return false; - } - - m_read_offset += count; - try_discard_chunks(); - return true; - } - - Optional offset_of(ReadonlyBytes value) const - { - if (value.size() > remaining()) - return {}; - - // First, find which chunk we're in. - auto chunk_index = (m_read_offset - m_base_offset) / chunk_size; - auto last_written_chunk_index = (m_write_offset - m_base_offset) / chunk_size; - auto first_chunk_index = chunk_index; - auto last_written_chunk_offset = m_write_offset % chunk_size; - auto first_chunk_offset = m_read_offset % chunk_size; - size_t last_chunk_offset = 0; - auto found_value = false; - - for (; chunk_index <= last_written_chunk_index; ++chunk_index) { - auto chunk_bytes = m_chunks[chunk_index].bytes(); - size_t chunk_offset = 0; - if (chunk_index == last_written_chunk_index) { - chunk_bytes = chunk_bytes.slice(0, last_written_chunk_offset); - } - if (chunk_index == first_chunk_index) { - chunk_bytes = chunk_bytes.slice(first_chunk_offset); - chunk_offset = first_chunk_offset; - } - - // See if 'value' is in this chunk, - auto position = AK::memmem(chunk_bytes.data(), chunk_bytes.size(), value.data(), value.size()); - if (!position) - continue; // Not in this chunk either :( - - // We found it! - found_value = true; - last_chunk_offset = (const u8*)position - chunk_bytes.data() + chunk_offset; - break; - } - - if (found_value) { - if (first_chunk_index == chunk_index) - return last_chunk_offset - first_chunk_offset; - - return (chunk_index - first_chunk_index) * chunk_size + last_chunk_offset - first_chunk_offset; - } - - // No dice. - return {}; - } - - size_t read(Bytes bytes) override - { - size_t nread = 0; - while (bytes.size() - nread > 0 && m_write_offset - m_read_offset - nread > 0) { - const auto chunk_index = (m_read_offset - m_base_offset) / chunk_size; - const auto chunk_bytes = m_chunks[chunk_index].bytes().slice(m_read_offset % chunk_size).trim(m_write_offset - m_read_offset - nread); - nread += chunk_bytes.copy_trimmed_to(bytes.slice(nread)); - } - - m_read_offset += nread; - - try_discard_chunks(); - - return nread; - } - - bool read_or_error(Bytes bytes) override - { - if (m_write_offset - m_read_offset < bytes.size()) { - set_recoverable_error(); - return false; - } - - read(bytes); - return true; - } - - size_t write(ReadonlyBytes bytes) override - { - size_t nwritten = 0; - while (bytes.size() - nwritten > 0) { - if ((m_write_offset + nwritten) % chunk_size == 0) - m_chunks.append(ByteBuffer::create_uninitialized(chunk_size)); - - nwritten += bytes.copy_trimmed_to(m_chunks.last().bytes().slice(m_write_offset % chunk_size)); - } - - m_write_offset += nwritten; - return nwritten; - } - - bool write_or_error(ReadonlyBytes bytes) override - { - write(bytes); - return true; - } - - size_t roffset() const { return m_read_offset; } - size_t woffset() const { return m_write_offset; } - - size_t remaining() const { return m_write_offset - m_read_offset; } - -private: - void try_discard_chunks() - { - while (m_read_offset - m_base_offset >= chunk_size) { - m_chunks.take_first(); - m_base_offset += chunk_size; - } - } - - Vector m_chunks; - size_t m_write_offset { 0 }; - size_t m_read_offset { 0 }; - size_t m_base_offset { 0 }; -}; - } - -using AK::DuplexMemoryStream; -using AK::InputMemoryStream; -using AK::InputStream; diff --git a/AK/Tests/TestStream.cpp b/AK/Tests/TestMemoryStream.cpp similarity index 98% rename from AK/Tests/TestStream.cpp rename to AK/Tests/TestMemoryStream.cpp index d10d45bc5c3..611b24ef604 100644 --- a/AK/Tests/TestStream.cpp +++ b/AK/Tests/TestMemoryStream.cpp @@ -27,7 +27,7 @@ #include #include -#include +#include static bool compare(ReadonlyBytes lhs, ReadonlyBytes rhs) { @@ -160,4 +160,4 @@ TEST_CASE(read_endian_values) EXPECT_EQ(value2, 0x04050607u); } -TEST_MAIN(Stream) +TEST_MAIN(MemoryStream) diff --git a/Libraries/LibCompress/Deflate.cpp b/Libraries/LibCompress/Deflate.cpp index 81ddb6cb938..0ff47adb99e 100644 --- a/Libraries/LibCompress/Deflate.cpp +++ b/Libraries/LibCompress/Deflate.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include diff --git a/Libraries/LibCompress/Gzip.cpp b/Libraries/LibCompress/Gzip.cpp index 56d2a5f9b7e..6f3b40bfba7 100644 --- a/Libraries/LibCompress/Gzip.cpp +++ b/Libraries/LibCompress/Gzip.cpp @@ -26,6 +26,7 @@ #include +#include #include namespace Compress { diff --git a/Libraries/LibDebug/DebugInfo.cpp b/Libraries/LibDebug/DebugInfo.cpp index 0e29e7ca180..98fc80907d0 100644 --- a/Libraries/LibDebug/DebugInfo.cpp +++ b/Libraries/LibDebug/DebugInfo.cpp @@ -25,8 +25,8 @@ */ #include "DebugInfo.h" +#include #include -#include #include #include #include diff --git a/Libraries/LibDebug/Dwarf/AbbreviationsMap.cpp b/Libraries/LibDebug/Dwarf/AbbreviationsMap.cpp index 22c52f19d70..e442681212f 100644 --- a/Libraries/LibDebug/Dwarf/AbbreviationsMap.cpp +++ b/Libraries/LibDebug/Dwarf/AbbreviationsMap.cpp @@ -27,7 +27,7 @@ #include "AbbreviationsMap.h" #include "DwarfInfo.h" -#include +#include namespace Debug::Dwarf { diff --git a/Libraries/LibDebug/Dwarf/DIE.cpp b/Libraries/LibDebug/Dwarf/DIE.cpp index be3f669d558..98dcadf8f79 100644 --- a/Libraries/LibDebug/Dwarf/DIE.cpp +++ b/Libraries/LibDebug/Dwarf/DIE.cpp @@ -28,7 +28,7 @@ #include "CompilationUnit.h" #include "DwarfInfo.h" #include -#include +#include namespace Debug::Dwarf { diff --git a/Libraries/LibDebug/Dwarf/DwarfInfo.cpp b/Libraries/LibDebug/Dwarf/DwarfInfo.cpp index 32ee6d19449..b17001e3590 100644 --- a/Libraries/LibDebug/Dwarf/DwarfInfo.cpp +++ b/Libraries/LibDebug/Dwarf/DwarfInfo.cpp @@ -26,7 +26,7 @@ #include "DwarfInfo.h" -#include +#include namespace Debug::Dwarf { @@ -53,7 +53,7 @@ void DwarfInfo::populate_compilation_units() if (m_debug_info_data.is_null()) return; - InputMemoryStream stream{ m_debug_info_data }; + InputMemoryStream stream { m_debug_info_data }; while (!stream.eof()) { auto unit_offset = stream.offset(); CompilationUnitHeader compilation_unit_header {}; diff --git a/Libraries/LibDebug/Dwarf/Expression.cpp b/Libraries/LibDebug/Dwarf/Expression.cpp index 57c4ac28574..b364beb6de1 100644 --- a/Libraries/LibDebug/Dwarf/Expression.cpp +++ b/Libraries/LibDebug/Dwarf/Expression.cpp @@ -25,7 +25,7 @@ */ #include "Expression.h" -#include +#include #include diff --git a/Libraries/LibDebug/Dwarf/LineProgram.h b/Libraries/LibDebug/Dwarf/LineProgram.h index 7bb2fa3d784..3272125ece3 100644 --- a/Libraries/LibDebug/Dwarf/LineProgram.h +++ b/Libraries/LibDebug/Dwarf/LineProgram.h @@ -26,7 +26,7 @@ #pragma once -#include +#include #include #include @@ -57,7 +57,8 @@ private: void handle_standard_opcode(u8 opcode); void handle_sepcial_opcode(u8 opcode); - struct [[gnu::packed]] UnitHeader32 { + struct [[gnu::packed]] UnitHeader32 + { u32 length; u16 version; u32 header_length; diff --git a/Shell/AST.cpp b/Shell/AST.cpp index fa7cae54841..b71b751d7d9 100644 --- a/Shell/AST.cpp +++ b/Shell/AST.cpp @@ -26,6 +26,7 @@ #include "AST.h" #include "Shell.h" +#include #include #include #include