From 5a3d9f05f86fe472ae95d78412f7bb0760aa11d3 Mon Sep 17 00:00:00 2001 From: "Daniel J. Summers" Date: Thu, 4 Jan 2024 17:33:41 -0500 Subject: [PATCH] Rework migration; add PG 2.1 changes --- src/MyWebLog.Data/PostgresData.fs | 79 ++++++++++++++++++------------ src/MyWebLog.Data/RethinkDbData.fs | 14 +++--- src/MyWebLog.Data/SQLiteData.fs | 25 ++++------ src/MyWebLog.Data/Utils.fs | 35 ++++++++++--- 4 files changed, 94 insertions(+), 59 deletions(-) diff --git a/src/MyWebLog.Data/PostgresData.fs b/src/MyWebLog.Data/PostgresData.fs index 6bfce4e..43d9bef 100644 --- a/src/MyWebLog.Data/PostgresData.fs +++ b/src/MyWebLog.Data/PostgresData.fs @@ -77,8 +77,8 @@ type PostgresData(log: ILogger, ser: JsonSerializer) = Table.Post "status" [ nameof Post.Empty.WebLogId; nameof Post.Empty.Status; nameof Post.Empty.UpdatedOn ] - $"CREATE INDEX post_category_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))" - $"CREATE INDEX post_tag_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))" + $"CREATE INDEX idx_post_category ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))" + $"CREATE INDEX idx_post_tag ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))" if needsTable Table.PostRevision then $"CREATE TABLE {Table.PostRevision} ( post_id TEXT NOT NULL, @@ -104,13 +104,13 @@ type PostgresData(log: ILogger, ser: JsonSerializer) = path TEXT NOT NULL, updated_on TIMESTAMPTZ NOT NULL, data BYTEA NOT NULL)" - $"CREATE INDEX upload_web_log_idx ON {Table.Upload} (web_log_id)" - $"CREATE INDEX upload_path_idx ON {Table.Upload} (web_log_id, path)" + $"CREATE INDEX idx_upload_web_log ON {Table.Upload} (web_log_id)" + $"CREATE INDEX idx_upload_path ON {Table.Upload} (web_log_id, path)" // Database version table if needsTable Table.DbVersion then $"CREATE TABLE {Table.DbVersion} (id TEXT NOT NULL PRIMARY KEY)" - $"INSERT INTO {Table.DbVersion} VALUES ('{Utils.currentDbVersion}')" + $"INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')" } Configuration.dataSource () @@ -134,35 +134,54 @@ type PostgresData(log: ILogger, ser: JsonSerializer) = /// Migrate from v2-rc2 to v2 (manual migration required) let migrateV2Rc2ToV2 () = backgroundTask { - Utils.logMigrationStep log "v2-rc2 to v2" "Requires user action" - let! webLogs = - Configuration.dataSource () - |> Sql.fromDataSource - |> Sql.query $"SELECT url_base, slug FROM {Table.WebLog}" - |> Sql.executeAsync (fun row -> row.string "url_base", row.string "slug") - - [ "** MANUAL DATABASE UPGRADE REQUIRED **"; "" - "The data structure for PostgreSQL changed significantly between v2-rc2 and v2." - "To migrate your data:" - " - Use a v2-rc2 executable to back up each web log" - " - Drop all tables from the database" - " - Use this executable to restore each backup"; "" - "Commands to back up all web logs:" - yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup {url} v2-rc2.{slug}.json") ] - |> String.concat "\n" - |> log.LogWarning - - log.LogCritical "myWebLog will now exit" - exit 1 + Custom.list + $"SELECT url_base, slug FROM {Table.WebLog}" [] (fun row -> row.string "url_base", row.string "slug") + Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs } /// Migrate from v2 to v2.1 let migrateV2ToV2point1 () = backgroundTask { - Utils.logMigrationStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs" + let migration = "v2 to v2.1" + Utils.Migration.logStep log migration "Adding empty redirect rule set to all weblogs" do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data + '{{ "RedirectRules": [] }}'::json""" [] - Utils.logMigrationStep log "v2 to v2.1" "Setting database to version 2.1" + let tables = + [ Table.Category; Table.Page; Table.Post; Table.PostComment; Table.TagMap; Table.Theme; Table.WebLog + Table.WebLogUser ] + + Utils.Migration.logStep log migration "Adding unique indexes on ID fields" + do! Custom.nonQuery (tables |> List.map Query.Definition.ensureKey |> String.concat "; ") [] + + Utils.Migration.logStep log migration "Dropping old ID columns" + do! Custom.nonQuery (tables |> List.map (sprintf "ALTER TABLE %s DROP COLUMN id") |> String.concat "; ") [] + + Utils.Migration.logStep log migration "Adjusting indexes" + let toDrop = [ "page_web_log_idx"; "post_web_log_idx" ] + do! Custom.nonQuery (toDrop |> List.map (sprintf "DROP INDEX %s") |> String.concat "; ") [] + + let toRename = + [ "idx_category", "idx_category_document" + "idx_tag_map", "idx_tag_map_document" + "idx_web_log", "idx_web_log_document" + "idx_web_log_user", "idx_web_log_user_document" + "page_author_idx", "idx_page_author" + "page_permalink_idx", "idx_page_permalink" + "post_author_idx", "idx_post_author" + "post_status_idx", "idx_post_status" + "post_permalink_idx", "idx_post_permalink" + "post_category_idx", "idx_post_category" + "post_tag_idx", "idx_post_tag" + "post_comment_post_idx", "idx_post_comment_post" + "upload_web_log_idx", "idx_upload_web_log" + "upload_path_idx", "idx_upload_path" ] + do! Custom.nonQuery + (toRename + |> List.map (fun (oldName, newName) -> $"ALTER INDEX {oldName} RENAME TO {newName}") + |> String.concat "; ") + [] + + Utils.Migration.logStep log migration "Setting database to version 2.1" do! setDbVersion "v2.1" } @@ -178,9 +197,9 @@ type PostgresData(log: ILogger, ser: JsonSerializer) = do! migrateV2ToV2point1 () v <- "v2.1" - if v <> "v2.1" then - log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}" - do! setDbVersion Utils.currentDbVersion + if v <> Utils.Migration.currentDbVersion then + log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" + do! setDbVersion Utils.Migration.currentDbVersion } interface IData with diff --git a/src/MyWebLog.Data/RethinkDbData.fs b/src/MyWebLog.Data/RethinkDbData.fs index 59bf934..4ab4294 100644 --- a/src/MyWebLog.Data/RethinkDbData.fs +++ b/src/MyWebLog.Data/RethinkDbData.fs @@ -207,7 +207,7 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger obj ] write; withRetryOnce; ignoreResult conn } - Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1" + Utils.Migration.logStep log "v2 to v2.1" "Setting database version to v2.1" do! setDbVersion "v2.1" } @@ -250,9 +250,9 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger "v2.1" then - log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}" - do! setDbVersion Utils.currentDbVersion + if v <> Utils.Migration.currentDbVersion then + log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" + do! setDbVersion Utils.Migration.currentDbVersion } /// The connection for this instance diff --git a/src/MyWebLog.Data/SQLiteData.fs b/src/MyWebLog.Data/SQLiteData.fs index 80b1ac4..33fb56b 100644 --- a/src/MyWebLog.Data/SQLiteData.fs +++ b/src/MyWebLog.Data/SQLiteData.fs @@ -18,7 +18,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger, ser: JsonSeria Configuration.useSerializer (Utils.createDocumentSerializer ser) - let! tables = conn.customList "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0) + let! tables = conn.customList "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0) let needsTable table = not (List.contains table tables) @@ -107,7 +107,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger, ser: JsonSeria // Database version table if needsTable Table.DbVersion then $"CREATE TABLE {Table.DbVersion} (id TEXT PRIMARY KEY); - INSERT INTO {Table.DbVersion} VALUES ('v2.1')" + INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')" } |> Seq.map (fun sql -> log.LogInformation $"""Creating {(sql.Replace("IF NOT EXISTS ", "").Split ' ')[2]} table...""" @@ -123,7 +123,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger, ser: JsonSeria /// Implement the changes between v2-rc1 and v2-rc2 let migrateV2Rc1ToV2Rc2 () = backgroundTask { - let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2" + let logStep = Utils.Migration.logStep log "v2-rc1 to v2-rc2" // Move meta items, podcast settings, and episode details to JSON-encoded text fields use cmd = conn.CreateCommand() logStep "Adding new columns" @@ -418,20 +418,15 @@ type SQLiteData(conn: SqliteConnection, log: ILogger, ser: JsonSeria /// Migrate from v2-rc2 to v2 let migrateV2Rc2ToV2 () = backgroundTask { - Utils.logMigrationStep log "v2-rc2 to v2" "Setting database version; no migration required" + Utils.Migration.logStep log "v2-rc2 to v2" "Setting database version; no migration required" do! setDbVersion "v2" } /// Migrate from v2 to v2.1 let migrateV2ToV2point1 () = backgroundTask { - // FIXME: This will be a backup/restore scenario, as we're changing to documents for most tables - Utils.logMigrationStep log "v2 to v2.1" "Adding redirect rules to web_log table" - use cmd = conn.CreateCommand() - cmd.CommandText <- "ALTER TABLE web_log ADD COLUMN redirect_rules TEXT NOT NULL DEFAULT '[]'" - do! write cmd - - Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1" - do! setDbVersion "v2.1" + let! webLogs = + Custom.list $"SELECT url_base, slug FROM {Table.WebLog}" [] (fun rdr -> rdr.GetString(0), rdr.GetString(1)) + Utils.Migration.backupAndRestoreRequired log "v2" "v2.1" webLogs } /// Migrate data among versions (up only) @@ -450,9 +445,9 @@ type SQLiteData(conn: SqliteConnection, log: ILogger, ser: JsonSeria do! migrateV2ToV2point1 () v <- "v2.1" - if v <> "v2.1" then - log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}" - do! setDbVersion Utils.currentDbVersion + if v <> Utils.Migration.currentDbVersion then + log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" + do! setDbVersion Utils.Migration.currentDbVersion } /// The connection for this instance diff --git a/src/MyWebLog.Data/Utils.fs b/src/MyWebLog.Data/Utils.fs index 228cb97..eb87162 100644 --- a/src/MyWebLog.Data/Utils.fs +++ b/src/MyWebLog.Data/Utils.fs @@ -5,9 +5,6 @@ module internal MyWebLog.Data.Utils open MyWebLog open MyWebLog.ViewModels -/// The current database version -let currentDbVersion = "v2.1" - /// Create a category hierarchy from the given list of categories let rec orderByHierarchy (cats: Category list) parentId slugBase parentNames = seq { for cat in cats |> List.filter (fun c -> c.ParentId = parentId) do @@ -59,9 +56,33 @@ let createDocumentSerializer ser = member _.Deserialize<'T>(it: string) : 'T = deserialize ser it } +/// Data migration utilities +module Migration = + + open Microsoft.Extensions.Logging -open Microsoft.Extensions.Logging + /// The current database version + let currentDbVersion = "v2.1" -/// Log a migration step -let logMigrationStep<'T> (log: ILogger<'T>) migration message = - log.LogInformation $"Migrating %s{migration}: %s{message}" + /// Log a migration step + let logStep<'T> (log: ILogger<'T>) migration message = + log.LogInformation $"Migrating %s{migration}: %s{message}" + + /// Notify the user that a backup/restore + let backupAndRestoreRequired log oldVersion newVersion webLogs = + logStep log $"%s{oldVersion} to %s{newVersion}" "Requires Using Action" + + [ "** MANUAL DATABASE UPGRADE REQUIRED **"; "" + $"The data structure changed between {oldVersion} and {newVersion}." + "To migrate your data:" + $" - Use a {oldVersion} executable to back up each web log" + " - Drop all tables from the database" + " - Use this executable to restore each backup"; "" + "Commands to back up all web logs:" + yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup %s{url} {oldVersion}.%s{slug}.json") ] + |> String.concat "\n" + |> log.LogWarning + + log.LogCritical "myWebLog will now exit" + exit 1 |> ignore + \ No newline at end of file