Add EnterDirectory hook

This hook runs on `change-directory` and is also emitted just before
KakBegin after kakrc has been sourced.

.cirrus.yml

@@ -1,6 +1,6 @@
 freebsd_task:
     freebsd_instance:
-        image_family: freebsd-13-2
+        image_family: freebsd-13-3
     matrix:
         - name: freebsd_clang
           env:
@@ -27,6 +27,7 @@ linux_task:
         - name: linux_gcc
           container:
               image: gcc:10
+              memory: 8G
           env:
               CXX: g++
     test_script: make CXX=$CXX -j$(nproc) test

doc/pages/changelog.asciidoc

@@ -3,6 +3,10 @@
 This changelog contains major and/or breaking changes to Kakoune between
 released versions.
 
+== Development version
+
+* Expose env vars that are mentionned in the arguments passed to shell expansions
+
 == Kakoune 2024.05.18
 
 * Fixed tests on Alpine Linux and *BSD

doc/pages/expansions.asciidoc

@@ -146,14 +146,16 @@ and unquote them, then execute the resulting `set` command, which sets
 the shell's argument variables to the items from `$kak_selections`. The
 `while` loop with `shift` iterates through the arguments one by one.
 
-Only variables actually mentioned in the body of the shell expansion will
-be exported into the shell's environment. For example:
+Only variables actually mentioned in the body of the shell expansion or
+in passed arguments will be exported into the shell's environment.
+
+For example:
 
 ----
 echo %sh{ env | grep ^kak_ }
 ----
 
-... will find none of Kakoune's special environment variables, but:
+... will not find any of Kakoune's special environment variables, but:
 
 ----
 echo %sh{ env | grep ^kak_ # kak_session }
@@ -162,6 +164,15 @@ echo %sh{ env | grep ^kak_ # kak_session }
 ... will find the `$kak_session` variable because it was mentioned by name
 in a comment, even though it wasn't directly used.
 
+----
+define-command -params .. eval-shell %{ echo %sh{ eval "$@" } }
+eval-shell 'echo $kak_session'
+----
+
+... will also find the `$kak_session` variable because it was mentioned by name
+in the command arguments, which are automatically made available to shell blocks
+as "$@"
+
 TIP: These environment variables are also available in other contexts where
 Kakoune uses a shell command, such as the `|`, `!` or `$` normal mode commands
 (See <<keys#,`:doc keys`>>).
@@ -215,8 +226,7 @@ The following expansions are supported (with required context _in italics_):
     number of lines in the current buffer
 
 *%val{buflist}*::
-    quoted list of the names of currently-open buffers (as seen in
-    `%val{bufname}`)
+    list of the names of currently-open buffers (as seen in %val{bufname})
 
 *%val{bufname}*::
     _in buffer, window scope_ +

doc/pages/hooks.asciidoc

@@ -147,6 +147,11 @@ name. Hooks with no description will always use an empty string.
 *SessionRenamed* `<old name>:<new name>`::
     executed when a session is renamed using the `rename-session` command
 
+*EnterDirectory* `path`::
+    executed on startup and when the current working directory is changed
+    using the `change-directory` command. The hook param is an absolute path
+    to the new working directory.
+
 *RuntimeError* `error message`::
     an error was encountered while executing a user command
 

rc/filetype/perf.kak

@@ -0,0 +1,18 @@
+provide-module perf-report %{
+    add-highlighter shared/perf-report group
+    add-highlighter shared/perf-report/above_threshold regex '\b([5-9]|\d{2})\.\d+%' 0:red
+    add-highlighter shared/perf-report/below_threshold regex '\b[0-4]\.\d+%' 0:green
+
+
+    define-command -override perf-report-focus %{
+        execute-keys 'xs...\d+\.\d+%<ret><a-:><a-semicolon>vtv<lt><semicolon>'
+    }
+}
+
+hook -group perf-report-highlight global WinSetOption filetype=perf-report %{
+    require-module perf-report
+    add-highlighter window/perf-report ref perf-report
+    hook -once -always window WinSetOption filetype=.* %{ remove-highlighter window/perf-report }
+
+    map window normal <ret> ': perf-report-focus<ret>'
+}

rc/filetype/python.kak

@@ -167,14 +167,35 @@ define-command -hidden python-insert-on-new-line %{ evaluate-commands -itersel -
     execute-keys <semicolon>
     try %{
         evaluate-commands -draft -save-regs '/"' %{
-            # copy the commenting prefix
-            execute-keys -save-regs '' k x1s^\h*(#+\h*)<ret> y
+            # Ensure previous line is a comment
+            execute-keys -draft kxs^\h*#+\h*<ret>
+
+            # now handle the coment continuation logic
             try %{
-                # if the previous comment isn't empty, create a new one
-                execute-keys x<a-K>^\h*#+\h*$<ret> jxs^\h*<ret>P
+                # try and match a regular block comment, copying the prefix
+                execute-keys -draft -save-regs '' k x 1s^(\h*#+\h*)\S.*$ <ret> y
+                execute-keys -draft P
             } catch %{
-                # if there is no text in the previous comment, remove it completely
-                execute-keys d
+                try %{
+                    # try and match a regular block comment followed by a single
+                    # empty comment line
+                    execute-keys -draft -save-regs '' kKx 1s^(\h*#+\h*)\S+\n\h*#+\h*$ <ret> y
+                    execute-keys -draft P
+                } catch %{
+                    try %{
+                        # try and match a pair of empty comment lines, and delete
+                        # them if we match
+                        execute-keys -draft kKx <a-k> ^\h*#+\h*\n\h*#+\h*$ <ret> <a-d>
+                    } catch %{
+                        # finally, we need a special case for a new line inserted
+                        # into a file that consists of a single empty comment - in
+                        # that case we can't expect to copy the trailing whitespace,
+                        # so we add our own
+                        execute-keys -draft -save-regs '' k x1s^(\h*#+)\h*$<ret> y
+                        execute-keys -draft P
+                        execute-keys -draft i<space>
+                    }
+                }
             }
         }
 

rc/tools/fifo.kak

@@ -0,0 +1,29 @@
+provide-module fifo %{
+
+define-command -params .. -docstring %{
+    fifo [-name <buffer-name>] [-scroll] [--] <command>...: run command in a fifo buffer
+} fifo %{ evaluate-commands %sh{
+    name='*fifo*'
+    while true; do
+        case "$1" in
+            "-scroll") scroll="-scroll"; shift ;;
+            "-name") name="$2"; shift 2 ;;
+            "--") shift; break ;;
+            *) break ;;
+        esac
+    done
+    output=$(mktemp -d "${TMPDIR:-/tmp}"/kak-fifo.XXXXXXXX)/fifo
+    mkfifo ${output}
+    ( eval "$@" > ${output} 2>&1 & ) > /dev/null 2>&1 < /dev/null
+
+    printf %s\\n "
+            edit! -fifo ${output} ${scroll} ${name}
+            hook -always -once buffer BufCloseFifo .* %{ nop %sh{ rm -r $(dirname ${output}) } }
+        "
+    }}
+
+complete-command fifo shell
+
+}
+
+hook -once global KakBegin .* %{ require-module fifo }

rc/tools/grep.kak

@@ -3,38 +3,37 @@ declare-option -docstring "shell command run to search for subtext in a file/dir
 
 provide-module grep %{
 
+require-module fifo
 require-module jump
 
 define-command -params .. -docstring %{
     grep [<arguments>]: grep utility wrapper
     All optional arguments are forwarded to the grep utility
     Passing no argument will perform a literal-string grep for the current selection
-} grep %{ evaluate-commands %sh{
-    if [ $# -eq 0 ]; then
-        case "$kak_opt_grepcmd" in
-        ag\ * | git\ grep\ * | grep\ * | rg\ * | ripgrep\ * | ugrep\ * | ug\ *)
-            set -- -F "${kak_selection}"
-            ;;
-        ack\ *)
-            set -- -Q "${kak_selection}"
-            ;;
-        *)
-            set -- "${kak_selection}"
-            ;;
-        esac
-    fi
-
-     output=$(mktemp -d "${TMPDIR:-/tmp}"/kak-grep.XXXXXXXX)/fifo
-     mkfifo ${output}
-     ( { trap - INT QUIT; ${kak_opt_grepcmd} "$@" 2>&1 | tr -d '\r'; } > ${output} 2>&1 & ) > /dev/null 2>&1 < /dev/null
-
-     printf %s\\n "evaluate-commands -try-client '$kak_opt_toolsclient' %{
-               edit! -fifo ${output} *grep*
-               set-option buffer filetype grep
-               set-option buffer jump_current_line 0
-               hook -always -once buffer BufCloseFifo .* %{ nop %sh{ rm -r $(dirname ${output}) } }
-           }"
-}}
+} grep %{
+    evaluate-commands -try-client %opt{toolsclient} %{
+        fifo -name *grep* %{
+            shift 2
+            trap - INT QUIT
+            if [ $# -eq 0 ]; then
+                case "$kak_opt_grepcmd" in
+                ag\ * | git\ grep\ * | grep\ * | rg\ * | ripgrep\ * | ugrep\ * | ug\ *)
+                    set -- -F "$kak_selection"
+                    ;;
+                ack\ *)
+                    set -- -Q "$kak_selection"
+                    ;;
+                *)
+                    set -- "$kak_selection"
+                    ;;
+                esac
+            fi
+            $kak_opt_grepcmd "$@" 2>&1 | tr -d '\r'
+        } 'exit;' %arg{@} # pass arguments for "$@" above, exit to avoid evaluating them
+        set-option buffer filetype grep
+        set-option buffer jump_current_line 0
+    }
+}
 complete-command grep file 
 
 hook -group grep-highlight global WinSetOption filetype=grep %{

rc/tools/make.kak

@@ -5,24 +5,23 @@ declare-option -docstring "pattern that describes lines containing information a
 
 provide-module make %{
 
+require-module fifo
 require-module jump
 
-define-command -params .. \
-    -docstring %{
-        make [<arguments>]: make utility wrapper
-        All the optional arguments are forwarded to the make utility
-     } make %{ evaluate-commands %sh{
-     output=$(mktemp -d "${TMPDIR:-/tmp}"/kak-make.XXXXXXXX)/fifo
-     mkfifo ${output}
-     ( { trap - INT QUIT; eval "${kak_opt_makecmd}" "$@"; } > ${output} 2>&1 & ) > /dev/null 2>&1 < /dev/null
-
-     printf %s\\n "evaluate-commands -try-client '$kak_opt_toolsclient' %{
-               edit! -fifo ${output} -scroll *make*
-               set-option buffer filetype make
-               set-option buffer jump_current_line 0
-               hook -always -once buffer BufCloseFifo .* %{ nop %sh{ rm -r $(dirname ${output}) } }
-           }"
-}}
+define-command -params .. -docstring %{
+    make [<arguments>]: make utility wrapper
+    All the optional arguments are forwarded to the make utility
+} make %{
+    evaluate-commands -try-client %opt{toolsclient} %{
+        fifo -scroll -name *make* %{
+            shift 2
+            trap - INT QUIT
+            $kak_opt_makecmd "$@"
+        } 'exit;' %arg{@} # pass arguments for "$@" above, exit to avoid evaluating them
+        set-option buffer filetype make
+        set-option buffer jump_current_line 0
+    }
+}
 
 add-highlighter shared/make group
 add-highlighter shared/make/ regex "^([^:\n]+):(\d+):(?:(\d+):)?\h+(?:((?:fatal )?error)|(warning)|(note)|(required from(?: here)?))?.*?$" 1:cyan 2:green 3:green 4:red 5:yellow 6:blue 7:yellow

src/commands.cc

@@ -2600,13 +2600,15 @@ const CommandDesc change_directory_cmd = {
                                         cursor_pos, FilenameFlags::OnlyDirectories),
                       Completions::Flags::Menu };
         }),
-    [](const ParametersParser& parser, Context&, const ShellContext&)
+    [](const ParametersParser& parser, Context& ctx, const ShellContext&)
     {
         StringView target = parser.positional_count() == 1 ? StringView{parser[0]} : "~";
-        if (chdir(parse_filename(target).c_str()) != 0)
+        auto path = real_path(parse_filename(target));
+        if (chdir(path.c_str()) != 0)
             throw runtime_error(format("unable to change to directory: '{}'", target));
         for (auto& buffer : BufferManager::instance())
             buffer->update_display_name();
+        ctx.hooks().run_hook(Hook::EnterDirectory, path, ctx);
     }
 };
 

src/hook_manager.hh

@@ -50,6 +50,7 @@ enum class Hook
     NextKeyIdle,
     NormalKey,
     ModeChange,
+    EnterDirectory,
     RawKey,
     RegisterModified,
     WinClose,
@@ -97,6 +98,7 @@ constexpr auto enum_desc(Meta::Type<Hook>)
         {Hook::NextKeyIdle, "NextKeyIdle"},
         {Hook::NormalKey, "NormalKey"},
         {Hook::ModeChange, "ModeChange"},
+        {Hook::EnterDirectory, "EnterDirectory"},
         {Hook::RawKey, "RawKey"},
         {Hook::RegisterModified, "RegisterModified"},
         {Hook::WinClose, "WinClose"},

src/main.cc

@@ -45,6 +45,9 @@ struct {
     unsigned int version;
     StringView notes;
 } constexpr version_notes[] = { {
+        0,
+        "» kak_* appearing in shell arguments will be added to the environment\n"
+    }, {
         20240518,
         "» Fix tests failing on some platforms\n"
     }, {
@@ -839,6 +842,7 @@ int run_server(StringView session, StringView server_init,
 
     {
         Context empty_context{Context::EmptyContextFlag{}};
+        global_scope.hooks().run_hook(Hook::EnterDirectory, real_path("."), empty_context);
         global_scope.hooks().run_hook(Hook::KakBegin, session, empty_context);
     }
 

src/shell_manager.cc

@@ -141,30 +141,35 @@ Shell spawn_shell(const char* shell, StringView cmdline,
 }
 
 template<typename GetValue>
-Vector<String> generate_env(StringView cmdline, const Context& context, GetValue&& get_value)
+Vector<String> generate_env(StringView cmdline, ConstArrayView<String> params, const Context& context, GetValue&& get_value)
 {
     static const Regex re(R"(\bkak_(quoted_)?(\w+)\b)");
 
     Vector<String> env;
-    for (auto&& match : RegexIterator{cmdline.begin(), cmdline.end(), re})
-    {
-        StringView name{match[2].first, match[2].second};
-        StringView shell_name{match[0].first, match[0].second};
-
-        auto match_name = [&](const String& s) {
-            return s.substr(0_byte, shell_name.length()) == shell_name and
-                   s.substr(shell_name.length(), 1_byte) == "=";
-        };
-        if (any_of(env, match_name))
-            continue;
-
-        try
+    auto add_matches = [&](StringView s) {
+        for (auto&& match : RegexIterator{s.begin(), s.end(), re})
         {
-            StringView quoted{match[1].first, match[1].second};
-            Quoting quoting = match[1].matched ? Quoting::Shell : Quoting::Raw;
-            env.push_back(format("kak_{}{}={}", quoted, name, get_value(name, quoting)));
-        } catch (runtime_error&) {}
-    }
+            StringView name{match[2].first, match[2].second};
+            StringView shell_name{match[0].first, match[0].second};
+
+            auto match_name = [&](const String& s) {
+                return s.substr(0_byte, shell_name.length()) == shell_name and
+                       s.substr(shell_name.length(), 1_byte) == "=";
+            };
+            if (any_of(env, match_name))
+                continue;
+
+            try
+            {
+                StringView quoted{match[1].first, match[1].second};
+                Quoting quoting = match[1].matched ? Quoting::Shell : Quoting::Raw;
+                env.push_back(format("kak_{}{}={}", quoted, name, get_value(name, quoting)));
+            } catch (runtime_error&) {}
+        }
+    };
+    add_matches(cmdline);
+    for (auto&& param : params)
+        add_matches(param);
 
     return env;
 }
@@ -265,13 +270,13 @@ std::pair<String, int> ShellManager::eval(
     const DebugFlags debug_flags = context.options()["debug"].get<DebugFlags>();
     const bool profile = debug_flags & DebugFlags::Profile;
     if (debug_flags & DebugFlags::Shell)
-        write_to_debug_buffer(format("shell:\n{}\n----\n", cmdline));
+        write_to_debug_buffer(format("shell:\n{}\n----\nargs: {}\n----\n", cmdline, join(shell_context.params | transform(shell_quote), ' ')));
 
     auto start_time = profile ? Clock::now() : Clock::time_point{};
 
     Optional<CommandFifos> command_fifos;
 
-    auto kak_env = generate_env(cmdline, context, [&](StringView name, Quoting quoting) {
+    auto kak_env = generate_env(cmdline, shell_context.params, context, [&](StringView name, Quoting quoting) {
         if (name == "command_fifo" or name == "response_fifo")
         {
             if (not command_fifos)
@@ -377,7 +382,7 @@ std::pair<String, int> ShellManager::eval(
 Shell ShellManager::spawn(StringView cmdline, const Context& context,
                           bool open_stdin, const ShellContext& shell_context)
 {
-    auto kak_env = generate_env(cmdline, context, [&](StringView name, Quoting quoting) {
+    auto kak_env = generate_env(cmdline, shell_context.params, context, [&](StringView name, Quoting quoting) {
         if (auto it = shell_context.env_vars.find(name); it != shell_context.env_vars.end())
             return it->value;
         return join(get_val(name, context) | transform(quoter(quoting)), ' ', false);

src/string.cc

@@ -1,22 +1,27 @@
 #include "string.hh"
 
-#include <cstdio>
 #include <cstring>
+#include "assert.hh"
+#include "unit_tests.hh"
 
 namespace Kakoune
 {
+namespace 
+{
+// Avoid including all of <algorithm> just for this.
+constexpr auto max(auto lhs, auto rhs) { return lhs > rhs ? lhs : rhs;}
+constexpr auto min(auto lhs, auto rhs) { return lhs < rhs ? lhs : rhs;}
+}
 
 String::Data::Data(const char* data, size_t size, size_t capacity)
 {
     if (capacity > Short::capacity)
     {
-        if (capacity & 1)
-            ++capacity;
-
-        kak_assert(capacity < Long::max_capacity);
+        kak_assert(capacity <= Long::max_capacity);
         u.l.ptr = Alloc{}.allocate(capacity+1);
         u.l.size = size;
-        u.l.capacity = capacity;
+        u.l.capacity = (capacity & Long::max_capacity);
+        u.l.mode = Long::active_mask;
 
         if (data != nullptr)
             memcpy(u.l.ptr, data, size);
@@ -60,26 +65,28 @@ String::Data& String::Data::operator=(Data&& other) noexcept
 template<bool copy>
 void String::Data::reserve(size_t new_capacity)
 {
-    if (capacity() != 0 and new_capacity <= capacity())
+    auto const current_capacity = capacity();
+    if (current_capacity != 0 and new_capacity <= current_capacity)
         return;
 
-    if (is_long())
-        new_capacity = std::max(u.l.capacity * 2, new_capacity);
+    if (!is_long() and new_capacity <= Short::capacity) 
+        return;
 
-    if (new_capacity & 1)
-        ++new_capacity;
+    kak_assert(new_capacity <= Long::max_capacity);
+    new_capacity = max(new_capacity, // Do not upgrade new_capacity to be over limit.
+                       min(current_capacity * 2, Long::max_capacity));
 
-    kak_assert(new_capacity < Long::max_capacity);
     char* new_ptr = Alloc{}.allocate(new_capacity+1);
     if (copy)
     {
         memcpy(new_ptr, data(), size()+1);
-        u.l.size = size();
     }
     release();
 
+    u.l.size = size();
     u.l.ptr = new_ptr;
-    u.l.capacity = new_capacity;
+    u.l.capacity = (new_capacity & Long::max_capacity);
+    u.l.mode = Long::active_mask;
 }
 
 template void String::Data::reserve<true>(size_t);
@@ -89,6 +96,7 @@ void String::Data::force_size(size_t new_size)
 {
     reserve<false>(new_size);
     set_size(new_size);
+    data()[new_size] = 0;
 }
 
 void String::Data::append(const char* str, size_t len)
@@ -131,17 +139,47 @@ void String::Data::set_size(size_t size)
     if (is_long())
         u.l.size = size;
     else
-        u.s.size = (size << 1) | 1;
+        u.s.remaining_size = Short::capacity - size;
 }
 
 void String::Data::set_short(const char* data, size_t size)
 {
-    u.s.size = (size << 1) | 1;
+    kak_assert(size <= Short::capacity);
+    u.s.remaining_size = Short::capacity - size;
     if (data != nullptr)
         memcpy(u.s.string, data, size);
     u.s.string[size] = 0;
 }
 
+UnitTest test_data{[]{
+  using Data = String::Data;
+    { // Basic data usage.
+        Data data;
+        kak_assert(data.size() == 0);
+        kak_assert(not data.is_long());
+        kak_assert(data.capacity() == 23);
+
+        // Should be SSO-ed.
+        data.append("test", 4);
+        kak_assert(data.size() == 4);
+        kak_assert(data.capacity() == 23);
+        kak_assert(not data.is_long());
+        kak_assert(data.data() == StringView("test"));
+    }
+    {
+        char large_buf[2048];
+        memset(large_buf, 'x', 2048);
+        Data data(large_buf, 2048);
+        kak_assert(data.size() == 2048);
+        kak_assert(data.capacity() >= 2048);
+
+        data.clear();
+        kak_assert(data.size() == 0);
+        kak_assert(not data.is_long());
+        kak_assert(data.capacity() == 23);
+    }
+}};
+
 const String String::ms_empty;
 
 }

src/string.hh

@@ -1,12 +1,13 @@
 #ifndef string_hh_INCLUDED
 #define string_hh_INCLUDED
 
+#include <climits>
+#include <cstddef>
 #include "memory.hh"
 #include "hash.hh"
 #include "units.hh"
 #include "utf8.hh"
 
-#include <climits>
 
 namespace Kakoune
 {
@@ -156,17 +157,22 @@ public:
 
     // String data storage using small string optimization.
     //
-    // the LSB of the last byte is used to flag if we are using the small buffer
-    // or an allocated one. On big endian systems that means the allocated
-    // capacity must be pair, on little endian systems that means the allocated
-    // capacity cannot use its most significant byte, so we effectively limit
-    // capacity to 2^24 on 32bit arch, and 2^60 on 64.
+    // The MSB of the last byte is used to flag if we are using the allocated buffer
+    // (1) or in-situ storage, the small one (0). That means the allocated capacity
+    // cannot use its most significant byte, so we effectively limit capacity to
+    // 2^24 on 32bit arch, and 2^56 on 64bit.
+    //
+    // There is also a special NoCopy mode in which the data referred to is un-owned.
+    // It is indicated by being in Long mode with capacity == 0.
     struct Data
     {
         using Alloc = Allocator<char, MemoryDomain::String>;
 
         Data() { set_empty(); }
-        Data(NoCopy, const char* data, size_t size) : u{Long{const_cast<char*>(data), size, 0}} {}
+        Data(NoCopy, const char* data, size_t size) : u{Long{const_cast<char*>(data), 
+                                                             size,
+                                                             /*capacity=*/0, 
+                                                             /*mode=*/Long::active_mask}} {}
 
         Data(const char* data, size_t size, size_t capacity);
         Data(const char* data, size_t size) : Data(data, size, size) {}
@@ -177,8 +183,8 @@ public:
         Data& operator=(const Data& other);
         Data& operator=(Data&& other) noexcept;
 
-        bool is_long() const { return (u.s.size & 1) == 0; }
-        size_t size() const { return is_long() ? u.l.size : (u.s.size >> 1); }
+        bool is_long() const { return (u.l.mode& Long::active_mask) > 0; }
+        size_t size() const { return is_long() ? u.l.size : (Short::capacity - u.s.remaining_size); }
         size_t capacity() const { return is_long() ? u.l.capacity : Short::capacity; }
 
         const char* data() const { return is_long() ? u.l.ptr : u.s.string; }
@@ -194,20 +200,27 @@ public:
     private:
         struct Long
         {
-            static constexpr size_t max_capacity =
-                (size_t)1 << 8 * (sizeof(size_t) - 1);
+            static constexpr size_t capacity_bits = CHAR_BIT * (sizeof(size_t) - 1);
+            static constexpr size_t max_capacity = ((size_t)1 << capacity_bits) - 1;
 
             char* ptr;
             size_t size;
-            size_t capacity;
+            size_t capacity : capacity_bits;
+            unsigned char mode;
+            static constexpr unsigned char active_mask = 0b1000'0000;
         };
+        static_assert(sizeof(Long) == sizeof(char*) * 3);
 
         struct Short
         {
-            static constexpr size_t capacity = sizeof(Long) - 2;
-            char string[capacity+1];
-            unsigned char size;
+            static constexpr size_t capacity = sizeof(Long) - 1;
+            char string[capacity];
+            // When string is full remaining_size will be 0 and be the null terminator.
+            // When string is empty remaining size will be 23 (0b00010111)
+            // and not collide with Long::active_mask.
+            unsigned char remaining_size;
         };
+        static_assert(offsetof(Long, mode) == offsetof(Short, remaining_size));
 
         union
         {
@@ -217,11 +230,11 @@ public:
 
         void release()
         {
-            if (is_long() and u.l.capacity != 0)
+            if (is_long() and (u.l.capacity != 0))
                 Alloc{}.deallocate(u.l.ptr, u.l.capacity+1);
         }
 
-        void set_empty() { u.s.size = 1; u.s.string[0] = 0; }
+        void set_empty() { u.s.remaining_size = Short::capacity; u.s.string[0] = '\0'; }
         void set_short(const char* data, size_t size);
     };
 

test/indent/python/empty-start-of-file-indented/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/empty-start-of-file-indented/in

@@ -0,0 +1 @@
+    #

test/indent/python/empty-start-of-file-indented/out

@@ -0,0 +1,2 @@
+    #
+    # 

test/indent/python/empty-start-of-file-indented/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/empty-start-of-file/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/empty-start-of-file/in

@@ -0,0 +1 @@
+#

test/indent/python/empty-start-of-file/out

@@ -0,0 +1,2 @@
+#
+# 

test/indent/python/empty-start-of-file/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/exit-block-indented/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/exit-block-indented/in

@@ -0,0 +1,4 @@
+    # A new line after a pair of empty comment lines should exit the
+    # block comment.
+    #
+    #

test/indent/python/exit-block-indented/out

@@ -0,0 +1,3 @@
+    # A new line after a pair of empty comment lines should exit the
+    # block comment.
+    

test/indent/python/exit-block-indented/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/exit-block/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/exit-block/in

@@ -0,0 +1,4 @@
+# A new line after a pair of empty comment lines should exit the
+# block comment.
+#
+#

test/indent/python/exit-block/out

@@ -0,0 +1,3 @@
+# A new line after a pair of empty comment lines should exit the
+# block comment.
+

test/indent/python/exit-block/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/not-a-comment-indented/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/not-a-comment-indented/in

@@ -0,0 +1,4 @@
+    # If we're not in a comment at all, make sure we do the right thing.
+    # (one empty line and a 4-space prefixed line follows)
+
+    

test/indent/python/not-a-comment-indented/out

@@ -0,0 +1,5 @@
+    # If we're not in a comment at all, make sure we do the right thing.
+    # (one empty line and a 4-space prefixed line follows)
+
+
+    

test/indent/python/not-a-comment-indented/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/not-a-comment/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/not-a-comment/in

@@ -0,0 +1,4 @@
+# If we're not in a comment at all, make sure we do the right thing.
+# (two empty lines follow)
+
+

test/indent/python/not-a-comment/out

@@ -0,0 +1,5 @@
+# If we're not in a comment at all, make sure we do the right thing.
+# (two empty lines follow)
+
+
+

test/indent/python/not-a-comment/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/paragraph-break-indented/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/paragraph-break-indented/in

@@ -0,0 +1,3 @@
+    # A new line after a single empty comment line should leave the comment in
+    # place as a possible paragraph separator.
+    #

test/indent/python/paragraph-break-indented/out

@@ -0,0 +1,4 @@
+    # A new line after a single empty comment line should leave the comment in
+    # place as a possible paragraph separator.
+    #
+    # 

test/indent/python/paragraph-break-indented/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python

test/indent/python/paragraph-break/cmd

@@ -0,0 +1 @@
+gjA<ret>

test/indent/python/paragraph-break/in

@@ -0,0 +1,4 @@
+# A new line after a single empty comment line should leave the empty comment
+# in place as a possible paragraph separator, starting a new comment with the
+# prefix copied.
+#

test/indent/python/paragraph-break/out

@@ -0,0 +1,5 @@
+# A new line after a single empty comment line should leave the empty comment
+# in place as a possible paragraph separator, starting a new comment with the
+# prefix copied.
+#
+# 

test/indent/python/paragraph-break/rc

@@ -0,0 +1,3 @@
+source "%val{runtime}/colors/default.kak"
+source "%val{runtime}/rc/filetype/python.kak"
+set buffer filetype python