fix: allow node to heal itself after not_prepared_yet

Unclear what causes this error, but when we encounter it we'll
clear out the entropy sync record so that it can be regnerated
next time

apps/arweave/e2e/ar_repack_mine_tests.erl

@@ -59,8 +59,6 @@ test_repack_mine({FromPackingType, ToPackingType}) ->
 		mining_addr = AddrB
 	}),
 
-	Overlap = ar_storage_module:get_overlap(ToPacking),
-
 	ar_e2e:assert_syncs_range(RepackerNode, 0, 4*?PARTITION_SIZE),
 	ar_e2e:assert_partition_size(RepackerNode, 0, ToPacking),
 	ar_e2e:assert_partition_size(RepackerNode, 1, ToPacking),

apps/arweave/e2e/ar_sync_pack_mine_tests.erl

@@ -68,7 +68,7 @@ unpacked_sync_pack_mine_test_() ->
 				]
 		end}.
 
-%% Note: we should limit the number of tests run per setup_source_node to ~4, if it gets
+%% Note: we should limit the number of tests run per setup_source_node to 5, if it gets
 %% too long then the source node may hit a difficulty adjustment, which can impact the
 %% results.
 unpacked_edge_case_test_() ->
@@ -252,6 +252,16 @@ test_entropy_first_sync_pack_mine({{Blocks, Chunks, SourcePackingType}, SinkPack
 	ar_e2e:assert_empty_partition(SinkNode, 1, unpacked),
 	ar_e2e:assert_empty_partition(SinkNode, 1, unpacked_padded),
 
+	%% Delete two chunks of entropy from storage to test that the node will heal itself.
+	%% 1. Delete the chunk from disk as well as all sync records.
+	%% 2. Delete the chunk only from disk, but keep it in the sync records.
+	DeleteOffset1 = RangeStart + ?DATA_CHUNK_SIZE,
+	ar_test_node:remote_call(SinkNode, ar_chunk_storage, delete,
+		[DeleteOffset1, StoreID]),
+	DeleteOffset2 = DeleteOffset1 + ?DATA_CHUNK_SIZE,
+	ar_test_node:remote_call(SinkNode, ar_chunk_storage, delete_chunk,
+		[DeleteOffset2, StoreID]),
+
 	%% 2. Run node with sync jobs so that it syncs and packs data
 	ar_test_node:restart_with_config(SinkNode, Config2#config{
 		sync_jobs = 100

apps/arweave/src/ar_chunk_storage.erl

@@ -15,6 +15,9 @@
 
 -export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]).
 
+%% Used in tests.
+-export([delete_chunk/2]).
+
 -include("../include/ar.hrl").
 -include("../include/ar_sup.hrl").
 -include("../include/ar_config.hrl").

apps/arweave/src/ar_data_sync.erl

@@ -3170,6 +3170,14 @@ log_failed_to_store_chunk(already_stored,
 			{data_path_hash, ar_util:encode(DataPathHash)},
 			{data_root, ar_util:encode(DataRoot)},
 			{store_id, StoreID}]);
+log_failed_to_store_chunk(not_prepared_yet,
+		AbsoluteOffset, Offset, DataRoot, DataPathHash, StoreID) ->
+	?LOG_WARNING([{event, chunk_not_prepared_yet},
+			{absolute_end_offset, AbsoluteOffset},
+			{relative_offset, Offset},
+			{data_path_hash, ar_util:encode(DataPathHash)},
+			{data_root, ar_util:encode(DataRoot)},
+			{store_id, StoreID}]);
 log_failed_to_store_chunk(Reason, AbsoluteOffset, Offset, DataRoot, DataPathHash, StoreID) ->
 	?LOG_ERROR([{event, failed_to_store_chunk},
 			{reason, io_lib:format("~p", [Reason])},

apps/arweave/src/ar_entropy_storage.erl

@@ -227,7 +227,8 @@ record_chunk(
 		{error, _} = Error2 ->
 			Error2;
 		not_found ->
-			{error, not_prepared_yet2};
+			delete_record(PaddedEndOffset, StoreID),
+			{error, not_prepared_yet};
 		missing_entropy ->
 			Packing = {replica_2_9, RewardAddr},
 			?LOG_WARNING([{event, missing_entropy}, {padded_end_offset, PaddedEndOffset},