Deduped Implementation

examples/clients/mycelium.vsh

@@ -1,7 +1,7 @@
 #!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
 
 import freeflowuniverse.herolib.clients.mycelium
-import freeflowuniverse.herolib.installers.net.mycelium as mycelium_installer
+import freeflowuniverse.herolib.installers.net.mycelium_installer
 import freeflowuniverse.herolib.osal
 import time
 import os

examples/data/deduped_mycelium_master.vsh

@@ -0,0 +1,72 @@
+#!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
+
+import freeflowuniverse.herolib.clients.mycelium
+import freeflowuniverse.herolib.installers.net.mycelium_installer
+import freeflowuniverse.herolib.data.ourdb
+import freeflowuniverse.herolib.osal
+import time
+import os
+import encoding.base64
+import json
+
+// NOTE: Before running this script, ensure that the mycelium binary is installed and in the PATH
+
+const master_port = 9000
+const slave_public_key = '46a9f9cee1ce98ef7478f3dea759589bbf6da9156533e63fed9f233640ac072c'
+const slave_address = '59c:28ee:8597:6c20:3b2f:a9ee:2e18:9d4f'
+
+// Struct to hold data for syncing
+struct SyncData {
+	id    u32
+	data  string
+	topic string = 'db_sync'
+}
+
+mycelium.delete()!
+
+// Initialize mycelium clients
+mut master := mycelium.get()!
+master.server_url = 'http://localhost:${master_port}'
+master.name = 'master_node'
+
+// Get public keys for communication
+master_inspect := mycelium.inspect(key_file_path: '/tmp/mycelium_server1/priv_key.bin')!
+
+println('Server 1 (Master Node) public key: ${master_inspect.public_key}')
+
+// Initialize ourdb instances
+mut db := ourdb.new(
+	record_nr_max:   16777216 - 1
+	record_size_max: 1024
+	path:            '/tmp/ourdb1'
+	reset:           true
+)!
+
+defer {
+	db.destroy() or { panic('failed to destroy db1: ${err}') }
+}
+
+// Store in master db
+println('\nStoring data in master node DB...')
+data := 'Test data for sync - ' + time.now().str()
+id := db.set(data: data.bytes())!
+println('Successfully stored data in master node DB with ID: ${id}')
+
+// Create sync data
+sync_data := SyncData{
+	id:   id
+	data: data
+}
+
+// Convert to JSON
+json_data := json.encode(sync_data)
+
+// Send sync message to slave
+println('\nSending sync message to slave...')
+msg := master.send_msg(
+	public_key: '46a9f9cee1ce98ef7478f3dea759589bbf6da9156533e63fed9f233640ac072c'
+	payload:    json_data
+	topic:      'db_sync'
+)!
+
+println('Sync message sent with ID: ${msg.id} to slave with public key: 46a9f9cee1ce98ef7478f3dea759589bbf6da9156533e63fed9f233640ac072c')

examples/data/deduped_mycelium_slave.vsh

@@ -0,0 +1,78 @@
+#!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
+
+import freeflowuniverse.herolib.clients.mycelium
+import freeflowuniverse.herolib.installers.net.mycelium_installer
+import freeflowuniverse.herolib.data.ourdb
+import freeflowuniverse.herolib.osal
+import time
+import os
+import encoding.base64
+import json
+
+// NOTE: Before running this script, ensure that the mycelium binary is installed and in the PATH
+
+const slave_port = 9001
+const master_public_key = '89c2eeb24bcdfaaac78c0023a166d88f760c097c1a57748770e432ba10757179'
+const master_address = '458:90d4:a3ef:b285:6d32:a22d:9e73:697f'
+
+// Struct to hold data for syncing
+struct SyncData {
+	id    u32
+	data  string
+	topic string = 'db_sync'
+}
+
+mycelium.delete()!
+
+// Initialize mycelium clients
+mut slave := mycelium.get()!
+slave.server_url = 'http://localhost:${slave_port}'
+slave.name = 'slave_node'
+
+// Get public keys for communication
+slave_inspect := mycelium.inspect(key_file_path: '/tmp/mycelium_server1/priv_key.bin')!
+
+println('Server 2 (slave Node) public key: ${slave_inspect.public_key}')
+
+// Initialize ourdb instances
+mut db := ourdb.new(
+	record_nr_max:   16777216 - 1
+	record_size_max: 1024
+	path:            '/tmp/ourdb1'
+	reset:           true
+)!
+
+defer {
+	db.destroy() or { panic('failed to destroy db1: ${err}') }
+}
+
+// Receive messages
+// Parameters: wait_for_message, peek_only, topic_filter
+received := slave.receive_msg(wait: true, peek: false, topic: 'db_sync')!
+println('Received message from: ${received.src_pk}')
+println('Message payload: ${base64.decode_str(received.payload)}')
+
+// // Store in slave db
+// println('\nStoring data in slave node DB...')
+// data := 'Test data for sync - ' + time.now().str()
+// id := db.set(data: data.bytes())!
+// println('Successfully stored data in slave node DB with ID: ${id}')
+
+// // Create sync data
+// sync_data := SyncData{
+// 	id:   id
+// 	data: data
+// }
+
+// // Convert to JSON
+// json_data := json.encode(sync_data)
+
+// // Send sync message to slave
+// println('\nSending sync message to slave...')
+// msg := slave.send_msg(
+// 	public_key: '46a9f9cee1ce98ef7478f3dea759589bbf6da9156533e63fed9f233640ac072c'
+// 	payload:    json_data
+// 	topic:      'db_sync'
+// )!
+
+// println('Sync message sent with ID: ${msg.id} to slave with public key: 46a9f9cee1ce98ef7478f3dea759589bbf6da9156533e63fed9f233640ac072c')

examples/vfs/vfs_db/.gitignore

@@ -0,0 +1 @@
+example_db

examples/vfs/vfs_db/dedupestor_vfs.vsh

@@ -0,0 +1,74 @@
+#!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
+
+import os
+import rand
+import freeflowuniverse.herolib.vfs.vfs_db
+import freeflowuniverse.herolib.data.dedupestor
+import freeflowuniverse.herolib.data.ourdb
+
+pub struct VFSDedupeDB {
+	dedupestor.DedupeStore
+}
+
+pub fn (mut db VFSDedupeDB) set(args ourdb.OurDBSetArgs) !u32 {
+	return db.store(args.data, 
+		dedupestor.Reference{owner: u16(1), id: args.id or {panic('VFS Must provide id')}}
+	)!
+}
+
+pub fn (mut db VFSDedupeDB) delete(id u32) ! {
+	db.DedupeStore.delete(id, dedupestor.Reference{owner: u16(1), id: id})!
+}
+
+example_data_dir := os.join_path(os.dir(@FILE), 'example_db')
+os.mkdir_all(example_data_dir)!
+
+// Create separate databases for data and metadata
+mut db_data := VFSDedupeDB{
+	DedupeStore: dedupestor.new(
+		path: os.join_path(example_data_dir, 'data')
+	)!
+}
+
+mut db_metadata := ourdb.new(
+	path: os.join_path(example_data_dir, 'metadata')
+	incremental_mode: false
+)!
+
+// Create VFS with separate databases for data and metadata
+mut vfs := vfs_db.new_with_separate_dbs(
+	mut db_data, 
+	mut db_metadata, 
+	data_dir: os.join_path(example_data_dir, 'data'),
+	metadata_dir: os.join_path(example_data_dir, 'metadata')
+)!
+
+println('\n---------BEGIN EXAMPLE')
+println('---------WRITING FILES')
+vfs.file_create('some_file.txt')!
+vfs.file_create('another_file.txt')!
+
+vfs.file_write('some_file.txt', 'gibberish'.bytes())!
+vfs.file_write('another_file.txt', 'abcdefg'.bytes())!
+
+println('\n---------READING FILES')
+println(vfs.file_read('some_file.txt')!.bytestr())
+println(vfs.file_read('another_file.txt')!.bytestr())
+
+println("\n---------WRITING DUPLICATE FILE (DB SIZE: ${os.file_size(os.join_path(example_data_dir, 'data/0.db'))})")
+vfs.file_create('duplicate.txt')!
+vfs.file_write('duplicate.txt', 'gibberish'.bytes())!
+
+println("\n---------WROTE DUPLICATE FILE (DB SIZE: ${os.file_size(os.join_path(example_data_dir, 'data/0.db'))})")
+println('---------READING FILES')
+println(vfs.file_read('some_file.txt')!.bytestr())
+println(vfs.file_read('another_file.txt')!.bytestr())
+println(vfs.file_read('duplicate.txt')!.bytestr())
+
+println("\n---------DELETING DUPLICATE FILE (DB SIZE: ${os.file_size(os.join_path(example_data_dir, 'data/0.db'))})")
+vfs.file_delete('duplicate.txt')!
+println("---------READING FILES (DB SIZE: ${os.file_size(os.join_path(example_data_dir, 'data/0.db'))})")
+println(vfs.file_read('some_file.txt')!.bytestr())
+println(vfs.file_read('another_file.txt')!.bytestr())
+// FAILS SUCCESSFULLY
+// println(vfs.file_read('duplicate.txt')!.bytestr())

examples/vfs/vfs_db/ourdb_vfs.vsh

@@ -0,0 +1,73 @@
+#!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
+
+import os
+import rand
+import freeflowuniverse.herolib.vfs.vfs_db
+import freeflowuniverse.herolib.data.ourdb
+
+example_data_dir := os.join_path(os.temp_dir(), 'ourdb_example_data_${rand.string(3)}')
+os.mkdir_all(example_data_dir)!
+
+// Create separate directories for data and metadata
+data_dir := os.join_path(example_data_dir, 'data')
+metadata_dir := os.join_path(example_data_dir, 'metadata')
+os.mkdir_all(data_dir)!
+os.mkdir_all(metadata_dir)!
+
+// Create separate databases for data and metadata
+mut db_data := ourdb.new(
+	path: data_dir
+	incremental_mode: false
+)!
+
+mut db_metadata := ourdb.new(
+	path: metadata_dir
+	incremental_mode: false
+)!
+
+// Create VFS with separate databases for data and metadata
+mut vfs := vfs_db.new_with_separate_dbs(
+	mut db_data, 
+	mut db_metadata, 
+	data_dir: data_dir,
+	metadata_dir: metadata_dir
+)!
+
+// Create a root directory if it doesn't exist
+if !vfs.exists('/') {
+    vfs.dir_create('/')!
+}
+
+// Create some files and directories
+vfs.dir_create('/test_dir')!
+vfs.file_create('/test_file.txt')!
+vfs.file_write('/test_file.txt', 'Hello, world!'.bytes())!
+
+// Create a file in the directory
+vfs.file_create('/test_dir/nested_file.txt')!
+vfs.file_write('/test_dir/nested_file.txt', 'This is a nested file.'.bytes())!
+
+// Read the files
+println('File content: ${vfs.file_read('/test_file.txt')!.bytestr()}')
+println('Nested file content: ${vfs.file_read('/test_dir/nested_file.txt')!.bytestr()}')
+
+// List directory contents
+println('Root directory contents:')
+root_entries := vfs.dir_list('/')!
+for entry in root_entries {
+    println('- ${entry.get_metadata().name} (${entry.get_metadata().file_type})')
+}
+
+println('Test directory contents:')
+test_dir_entries := vfs.dir_list('/test_dir')!
+for entry in test_dir_entries {
+    println('- ${entry.get_metadata().name} (${entry.get_metadata().file_type})')
+}
+
+// Create a duplicate file with the same content
+vfs.file_create('/duplicate_file.txt')!
+vfs.file_write('/duplicate_file.txt', 'Hello, world!'.bytes())!
+
+// Demonstrate that data and metadata are stored separately
+println('Data DB Size: ${os.file_size(os.join_path(data_dir, '0.ourdb'))} bytes')
+println('Metadata DB Size: ${os.file_size(os.join_path(metadata_dir, '0.ourdb'))} bytes')