diff --git a/game/src/ltn/partition.rs b/game/src/ltn/partition.rs
index adb1af9398..406c37e824 100644
--- a/game/src/ltn/partition.rs
+++ b/game/src/ltn/partition.rs
@@ -62,9 +62,6 @@ impl Partitioning {
             // If we got more than one result back, merging partially failed. Oh well?
             merged.extend(Perimeter::merge_all(perimeters, false));
         }
-
-        let mut colors = Perimeter::calculate_coloring(&merged, COLORS.len())
-            .unwrap_or_else(|| (0..merged.len()).collect());
         timer.stop("partition");
 
         timer.start_iter("blockify", merged.len());
@@ -76,22 +73,38 @@ impl Partitioning {
                     blocks.push(block);
                 }
                 Err(err) => {
-                    warn!("Failed to make a block from a perimeter: {}", err);
-                    // We assigned a color, so don't let the indices get out of sync!
-                    colors.remove(blocks.len());
+                    warn!("Failed to make a block from a merged perimeter: {}", err);
                 }
             }
         }
 
         let mut neighborhoods = BTreeMap::new();
-        for (block, color_idx) in blocks.into_iter().zip(colors.into_iter()) {
-            let color = COLORS[color_idx % COLORS.len()];
-            neighborhoods.insert(NeighborhoodID(neighborhoods.len()), (block, color));
+        for block in blocks {
+            neighborhoods.insert(NeighborhoodID(neighborhoods.len()), (block, Color::RED));
         }
-        Partitioning {
+        let mut p = Partitioning {
             neighborhoods,
             single_blocks,
+        };
+        p.recalculate_coloring();
+        p
+    }
+
+    /// True if the coloring changed
+    pub fn recalculate_coloring(&mut self) -> bool {
+        let perims: Vec<Perimeter> = self
+            .neighborhoods
+            .values()
+            .map(|pair| pair.0.perimeter.clone())
+            .collect();
+        let colors = Perimeter::calculate_coloring(&perims, COLORS.len())
+            .unwrap_or_else(|| (0..perims.len()).collect());
+        let orig_coloring: Vec<Color> = self.neighborhoods.values().map(|pair| pair.1).collect();
+        for (pair, color_idx) in self.neighborhoods.values_mut().zip(colors.into_iter()) {
+            pair.1 = COLORS[color_idx % COLORS.len()];
         }
+        let new_coloring: Vec<Color> = self.neighborhoods.values().map(|pair| pair.1).collect();
+        orig_coloring != new_coloring
     }
 
     pub fn neighborhood_containing(&self, find_block: &Block) -> Option<NeighborhoodID> {
diff --git a/game/src/ltn/select_boundary.rs b/game/src/ltn/select_boundary.rs
index f48064704b..c1e92bd3f0 100644
--- a/game/src/ltn/select_boundary.rs
+++ b/game/src/ltn/select_boundary.rs
@@ -144,8 +144,11 @@ impl SelectBoundary {
         match self.try_block_changed(app, id) {
             Ok(()) => {
                 let old_frontier = std::mem::take(&mut self.frontier);
-                let new_perimeter = &app.session.partitioning.neighborhoods[&self.id].0.perimeter;
-                self.frontier = calculate_frontier(new_perimeter, &self.blocks);
+                let new_perimeter = app.session.partitioning.neighborhoods[&self.id]
+                    .0
+                    .perimeter
+                    .clone();
+                self.frontier = calculate_frontier(&new_perimeter, &self.blocks);
 
                 // Redraw all of the blocks that changed
                 let mut changed_blocks: Vec<BlockID> = old_frontier
@@ -154,13 +157,21 @@ impl SelectBoundary {
                     .collect();
                 // And always the current block
                 changed_blocks.push(id);
+
+                if app.session.partitioning.recalculate_coloring() {
+                    // The coloring of neighborhoods changed; this could possibly have impact far
+                    // away. Just redraw all blocks.
+                    changed_blocks.clear();
+                    changed_blocks.extend(self.blocks.keys().cloned());
+                }
+
                 for changed in changed_blocks {
                     self.world.delete_before_replacement(changed);
                     self.add_block(ctx, app, changed);
                 }
 
                 // TODO Pass in the Block
-                self.redraw_outline(ctx, app, new_perimeter.clone());
+                self.redraw_outline(ctx, app, new_perimeter);
                 self.panel = make_panel(ctx, app);
             }
             Err(err) => {
@@ -232,7 +243,6 @@ impl SelectBoundary {
             } else {
                 let old_neighborhood_block = self.make_merged_block(app, old_blocks)?;
                 // Great! Do the transfer.
-                // TODO May need to recalculate colors!
                 app.session
                     .partitioning
                     .neighborhoods