From 9e35025a2f1d3b5ac418f20935eeacc5041b00a4 Mon Sep 17 00:00:00 2001 From: Rob Widmer Date: Sat, 18 Nov 2023 22:49:21 -0500 Subject: [PATCH 1/5] Sync more SQLite adapter code from AR to fix tests --- lib/arjdbc/sqlite3/adapter.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/arjdbc/sqlite3/adapter.rb b/lib/arjdbc/sqlite3/adapter.rb index f298d02eb..09486949a 100644 --- a/lib/arjdbc/sqlite3/adapter.rb +++ b/lib/arjdbc/sqlite3/adapter.rb @@ -230,14 +230,6 @@ def disable_referential_integrity # :nodoc: end end - def all_foreign_keys_valid? # :nodoc: - # Rails 7 - check_all_foreign_keys_valid! - true - rescue ActiveRecord::StatementInvalid - false - end - def check_all_foreign_keys_valid! # :nodoc: # Rails 7.1 sql = "PRAGMA foreign_key_check" @@ -269,7 +261,8 @@ def remove_index(table_name, column_name = nil, **options) # :nodoc: # # Example: # rename_table('octopuses', 'octopi') - def rename_table(table_name, new_name) + def rename_table(table_name, new_name, **options) + validate_table_length!(new_name) unless options[:_uses_legacy_table_name] schema_cache.clear_data_source_cache!(table_name.to_s) schema_cache.clear_data_source_cache!(new_name.to_s) internal_exec_query "ALTER TABLE #{quote_table_name(table_name)} RENAME TO #{quote_table_name(new_name)}" @@ -312,6 +305,8 @@ def change_column_default(table_name, column_name, default_or_changes) #:nodoc: end def change_column_null(table_name, column_name, null, default = nil) #:nodoc: + validate_change_column_null_argument!(null) + unless null || default.nil? internal_exec_query("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote(default)} WHERE #{quote_column_name(column_name)} IS NULL") end @@ -720,6 +715,16 @@ class SQLite3Adapter < AbstractAdapter include ArJdbc::Abstract::StatementCache include ArJdbc::Abstract::TransactionSupport + ## + # :singleton-method: + # Configure the SQLite3Adapter to be used in a strict strings mode. + # This will disable double-quoted string literals, because otherwise typos can silently go unnoticed. + # For example, it is possible to create an index for a non existing column. + # If you wish to enable this mode you can add the following line to your application.rb file: + # + # config.active_record.sqlite3_adapter_strict_strings_by_default = true + class_attribute :strict_strings_by_default, default: false # Does not actually do anything right now + def self.represent_boolean_as_integer=(value) # :nodoc: if value == false raise "`.represent_boolean_as_integer=` is now always true, so make sure your application can work with it and remove this settings." From 88928740fb12f2b8e2967eead4565dc3f5122c6a Mon Sep 17 00:00:00 2001 From: Rob Widmer Date: Sat, 18 Nov 2023 22:49:51 -0500 Subject: [PATCH 2/5] No longer configure connection on init so connection is deferred --- lib/arjdbc/sqlite3/adapter.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/arjdbc/sqlite3/adapter.rb b/lib/arjdbc/sqlite3/adapter.rb index 09486949a..df87b918d 100644 --- a/lib/arjdbc/sqlite3/adapter.rb +++ b/lib/arjdbc/sqlite3/adapter.rb @@ -93,7 +93,6 @@ def dealloc(stmt) def initialize(config) @memory_database = config[:database] == ":memory:" super - configure_connection end def self.database_exists?(config) From dd8820756fa28ba5d2048d834efe9c678f77b910 Mon Sep 17 00:00:00 2001 From: Rob Widmer Date: Mon, 15 Jan 2024 22:32:12 -0500 Subject: [PATCH 3/5] Fix some sqlite failures due to exception mapping --- lib/arjdbc/sqlite3/adapter.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/arjdbc/sqlite3/adapter.rb b/lib/arjdbc/sqlite3/adapter.rb index df87b918d..27b6ecdff 100644 --- a/lib/arjdbc/sqlite3/adapter.rb +++ b/lib/arjdbc/sqlite3/adapter.rb @@ -621,6 +621,11 @@ def translate_exception(exception, message:, sql:, binds:) elsif exception.message.match?(/called on a closed database/i) # DIFFERENCE: FQN ::ActiveRecord::ConnectionNotEstablished.new(exception, connection_pool: @pool) + elsif exception.message.match?(/sql error/i) + ::ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds, connection_pool: @pool) + elsif exception.message.match?(/write a readonly database/i) + message = message.sub('org.sqlite.SQLiteException', 'SQLite3::ReadOnlyException') + ::ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds, connection_pool: @pool) else super end From ad5a5a927ba5fe81bf985b5c363656c174a0f162 Mon Sep 17 00:00:00 2001 From: Rob Widmer Date: Mon, 15 Jan 2024 22:35:07 -0500 Subject: [PATCH 4/5] Match sqlite initialization to default AR settings --- lib/arjdbc/abstract/database_statements.rb | 16 +++++++--- lib/arjdbc/sqlite3/adapter.rb | 37 ++++++++++++++++++++-- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/lib/arjdbc/abstract/database_statements.rb b/lib/arjdbc/abstract/database_statements.rb index 27a8d8ed7..75d76b9b1 100644 --- a/lib/arjdbc/abstract/database_statements.rb +++ b/lib/arjdbc/abstract/database_statements.rb @@ -87,11 +87,7 @@ def execute(sql, name = nil, async: false, allow_retry: false, materialize_trans mark_transaction_written_if_write(sql) - log(sql, name, async: async) do - with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn| - conn.execute(sql) - end - end + raw_execute(sql, name, async: async, allow_retry: allow_retry, materialize_transactions: materialize_transactions) end # overridden to support legacy binds @@ -108,6 +104,16 @@ def convert_legacy_binds_to_attributes(binds) end end + def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false) + log(sql, name, async: async) do + with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn| + # We sometimes return an ActiveRecord::Result and sometimes a raw, so force to raw + result = conn.execute(sql) + result.is_a?(ActiveRecord::Result) ? result.to_a : result + end + end + end + end end end diff --git a/lib/arjdbc/sqlite3/adapter.rb b/lib/arjdbc/sqlite3/adapter.rb index 27b6ecdff..c34a39759 100644 --- a/lib/arjdbc/sqlite3/adapter.rb +++ b/lib/arjdbc/sqlite3/adapter.rb @@ -693,10 +693,41 @@ def connect end def configure_connection - # FIXME: missing from adapter -# @connection.busy_timeout(self.class.type_cast_config_to_integer(@config[:timeout])) if @config[:timeout] + if @config[:timeout] && @config[:retries] + raise ArgumentError, "Cannot specify both timeout and retries arguments" + elsif @config[:timeout] + # FIXME: missing from adapter + # @raw_connection.busy_timeout(self.class.type_cast_config_to_integer(@config[:timeout])) + elsif @config[:retries] + retries = self.class.type_cast_config_to_integer(@config[:retries]) + raw_connection.busy_handler do |count| + count <= retries + end + end - execute("PRAGMA foreign_keys = ON", "SCHEMA") + # Enforce foreign key constraints + # https://www.sqlite.org/pragma.html#pragma_foreign_keys + # https://www.sqlite.org/foreignkeys.html + raw_execute("PRAGMA foreign_keys = ON", "SCHEMA") + unless @memory_database + # Journal mode WAL allows for greater concurrency (many readers + one writer) + # https://www.sqlite.org/pragma.html#pragma_journal_mode + raw_execute("PRAGMA journal_mode = WAL", "SCHEMA") + # Set more relaxed level of database durability + # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE" + # https://www.sqlite.org/pragma.html#pragma_synchronous + raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA") + # Set the global memory map so all processes can share some data + # https://www.sqlite.org/pragma.html#pragma_mmap_size + # https://www.sqlite.org/mmap.html + raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA") + end + # Impose a limit on the WAL file to prevent unlimited growth + # https://www.sqlite.org/pragma.html#pragma_journal_size_limit + raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA") + # Set the local connection cache to 2000 pages + # https://www.sqlite.org/pragma.html#pragma_cache_size + raw_execute("PRAGMA cache_size = 2000", "SCHEMA") end end From 78175ecd6010a86cf3133fa306b41dc650bafb10 Mon Sep 17 00:00:00 2001 From: Rob Widmer Date: Mon, 15 Jan 2024 22:35:40 -0500 Subject: [PATCH 5/5] Support sqlite columns that have string manipulation as a default --- lib/arjdbc/sqlite3/adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/arjdbc/sqlite3/adapter.rb b/lib/arjdbc/sqlite3/adapter.rb index c34a39759..ee128a3fd 100644 --- a/lib/arjdbc/sqlite3/adapter.rb +++ b/lib/arjdbc/sqlite3/adapter.rb @@ -469,7 +469,7 @@ def extract_default_function(default_value, default) end def has_default_function?(default_value, default) - !default_value && %r{\w+\(.*\)|CURRENT_TIME|CURRENT_DATE|CURRENT_TIMESTAMP}.match?(default) + !default_value && %r{\w+\(.*\)|CURRENT_TIME|CURRENT_DATE|CURRENT_TIMESTAMP|\|\|}.match?(default) end # See: https://www.sqlite.org/lang_altertable.html