Fix PostgreSQL v2.1 migration #44

Merged
danieljsummers merged 2 commits from issue-43 into main 2024-03-29 02:25:10 +00:00
7 changed files with 91 additions and 41 deletions

View File

@ -129,8 +129,10 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
} }
/// Set a specific database version /// Set a specific database version
let setDbVersion version = let setDbVersion version = backgroundTask {
Custom.nonQuery $"DELETE FROM db_version; INSERT INTO db_version VALUES ('%s{version}')" [] do! Custom.nonQuery $"DELETE FROM db_version; INSERT INTO db_version VALUES ('%s{version}')" []
return version
}
/// Migrate from v2-rc2 to v2 (manual migration required) /// Migrate from v2-rc2 to v2 (manual migration required)
let migrateV2Rc2ToV2 () = backgroundTask { let migrateV2Rc2ToV2 () = backgroundTask {
@ -140,11 +142,11 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs
} }
/// Migrate from v2 to v2.1 /// Migrate from v2 to v2.1.1
let migrateV2ToV2point1 () = backgroundTask { let migrateV2ToV2point1point1 () = backgroundTask {
let migration = "v2 to v2.1" let migration = "v2 to v2.1.1"
Utils.Migration.logStep log migration "Adding empty redirect rule set to all weblogs" Utils.Migration.logStep log migration "Adding empty redirect rule set to all weblogs"
do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data + '{{ "RedirectRules": [] }}'::json""" [] do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data || '{{ "RedirectRules": [] }}'::jsonb""" []
let tables = let tables =
[ Table.Category; Table.Page; Table.Post; Table.PostComment; Table.TagMap; Table.Theme; Table.WebLog [ Table.Category; Table.Page; Table.Post; Table.PostComment; Table.TagMap; Table.Theme; Table.WebLog
@ -153,36 +155,60 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
Utils.Migration.logStep log migration "Adding unique indexes on ID fields" Utils.Migration.logStep log migration "Adding unique indexes on ID fields"
do! Custom.nonQuery (tables |> List.map Query.Definition.ensureKey |> String.concat "; ") [] do! Custom.nonQuery (tables |> List.map Query.Definition.ensureKey |> String.concat "; ") []
Utils.Migration.logStep log migration "Dropping old ID columns" Utils.Migration.logStep log migration "Removing constraints"
do! Custom.nonQuery (tables |> List.map (sprintf "ALTER TABLE %s DROP COLUMN id") |> String.concat "; ") [] let fkToDrop =
[ "page_revision", "page_revision_page_id_fkey"
Utils.Migration.logStep log migration "Adjusting indexes" "post_revision", "post_revision_post_id_fkey"
let toDrop = [ "page_web_log_idx"; "post_web_log_idx" ] "theme_asset", "theme_asset_theme_id_fkey"
do! Custom.nonQuery (toDrop |> List.map (sprintf "DROP INDEX %s") |> String.concat "; ") [] "upload", "upload_web_log_id_fkey"
"category", "category_pkey"
let toRename = "page", "page_pkey"
[ "idx_category", "idx_category_document" "post", "post_pkey"
"idx_tag_map", "idx_tag_map_document" "post_comment", "post_comment_pkey"
"idx_web_log", "idx_web_log_document" "tag_map", "tag_map_pkey"
"idx_web_log_user", "idx_web_log_user_document" "theme", "theme_pkey"
"page_author_idx", "idx_page_author" "web_log", "web_log_pkey"
"page_permalink_idx", "idx_page_permalink" "web_log_user", "web_log_user_pkey" ]
"post_author_idx", "idx_post_author"
"post_status_idx", "idx_post_status"
"post_permalink_idx", "idx_post_permalink"
"post_category_idx", "idx_post_category"
"post_tag_idx", "idx_post_tag"
"post_comment_post_idx", "idx_post_comment_post"
"upload_web_log_idx", "idx_upload_web_log"
"upload_path_idx", "idx_upload_path" ]
do! Custom.nonQuery do! Custom.nonQuery
(toRename (fkToDrop
|> List.map (fun (oldName, newName) -> $"ALTER INDEX {oldName} RENAME TO {newName}") |> List.map (fun (tbl, fk) -> $"ALTER TABLE {tbl} DROP CONSTRAINT {fk}")
|> String.concat "; ") |> String.concat "; ")
[] []
Utils.Migration.logStep log migration "Setting database to version 2.1" Utils.Migration.logStep log migration "Dropping old indexes"
do! setDbVersion "v2.1" let toDrop =
[ "idx_category"; "page_author_idx"; "page_permalink_idx"; "page_web_log_idx"; "post_author_idx"
"post_category_idx"; "post_permalink_idx"; "post_status_idx"; "post_tag_idx"; "post_web_log_idx"
"post_comment_post_idx"; "idx_tag_map"; "idx_web_log"; "idx_web_log_user" ]
do! Custom.nonQuery (toDrop |> List.map (sprintf "DROP INDEX %s") |> String.concat "; ") []
Utils.Migration.logStep log migration "Dropping old ID columns"
do! Custom.nonQuery (tables |> List.map (sprintf "ALTER TABLE %s DROP COLUMN id") |> String.concat "; ") []
Utils.Migration.logStep log migration "Adding new indexes"
let newIdx =
[ yield! tables |> List.map Query.Definition.ensureKey
Query.Definition.ensureDocumentIndex Table.Category Optimized
Query.Definition.ensureDocumentIndex Table.TagMap Optimized
Query.Definition.ensureDocumentIndex Table.WebLog Optimized
Query.Definition.ensureDocumentIndex Table.WebLogUser Optimized
Query.Definition.ensureIndexOn Table.Page "author" [ nameof Page.Empty.AuthorId ]
Query.Definition.ensureIndexOn
Table.Page "permalink" [ nameof Page.Empty.WebLogId; nameof Page.Empty.Permalink ]
Query.Definition.ensureIndexOn Table.Post "author" [ nameof Post.Empty.AuthorId ]
Query.Definition.ensureIndexOn
Table.Post "permalink" [ nameof Post.Empty.WebLogId; nameof Post.Empty.Permalink ]
Query.Definition.ensureIndexOn
Table.Post
"status"
[ nameof Post.Empty.WebLogId; nameof Post.Empty.Status; nameof Post.Empty.UpdatedOn ]
$"CREATE INDEX idx_post_category ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))"
$"CREATE INDEX idx_post_tag ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))"
Query.Definition.ensureIndexOn Table.PostComment "post" [ nameof Comment.Empty.PostId ] ]
do! Custom.nonQuery (newIdx |> String.concat "; ") []
Utils.Migration.logStep log migration "Setting database to version 2.1.1"
return! setDbVersion "v2.1.1"
} }
/// Do required data migration between versions /// Do required data migration between versions
@ -190,16 +216,20 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
let mutable v = defaultArg version "" let mutable v = defaultArg version ""
if v = "v2-rc2" then if v = "v2-rc2" then
do! migrateV2Rc2ToV2 () let! webLogs =
v <- "v2" Custom.list
$"SELECT url_base, slug FROM {Table.WebLog}" []
(fun row -> row.string "url_base", row.string "slug")
Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs
if v = "v2" then if v = "v2" then
do! migrateV2ToV2point1 () let! ver = migrateV2ToV2point1point1 ()
v <- "v2.1" v <- ver
if v <> Utils.Migration.currentDbVersion then if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion let! _ = setDbVersion Utils.Migration.currentDbVersion
()
} }
interface IData with interface IData with

View File

@ -238,6 +238,12 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
do! setDbVersion "v2.1" do! setDbVersion "v2.1"
} }
/// Migrate from v2.1 to v2.1.1
let migrateV2ToV2point1point1 () = backgroundTask {
Utils.Migration.logStep log "v2.1 to v2.1.1" "Setting database version; no migration required"
do! setDbVersion "v2.1.1"
}
/// Migrate data between versions /// Migrate data between versions
let migrate version = backgroundTask { let migrate version = backgroundTask {
let mutable v = defaultArg version "" let mutable v = defaultArg version ""
@ -254,6 +260,10 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
do! migrateV2ToV2point1 () do! migrateV2ToV2point1 ()
v <- "v2.1" v <- "v2.1"
if v = "v2.1" then
do! migrateV2ToV2point1point1 ()
v <- "v2.1.1"
if v <> Utils.Migration.currentDbVersion then if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion do! setDbVersion Utils.Migration.currentDbVersion

View File

@ -430,6 +430,12 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
Utils.Migration.backupAndRestoreRequired log "v2" "v2.1" webLogs Utils.Migration.backupAndRestoreRequired log "v2" "v2.1" webLogs
} }
/// Migrate from v2.1 to v2.1.1
let migrateV2ToV2point1point1 () = backgroundTask {
Utils.Migration.logStep log "v2.1 to v2.1.1" "Setting database version; no migration required"
do! setDbVersion "v2.1.1"
}
/// Migrate data among versions (up only) /// Migrate data among versions (up only)
let migrate version = backgroundTask { let migrate version = backgroundTask {
let mutable v = defaultArg version "" let mutable v = defaultArg version ""
@ -446,6 +452,10 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
do! migrateV2ToV2point1 () do! migrateV2ToV2point1 ()
v <- "v2.1" v <- "v2.1"
if v = "v2.1" then
do! migrateV2ToV2point1point1 ()
v <- "v2.1.1"
if v <> Utils.Migration.currentDbVersion then if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}" log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion do! setDbVersion Utils.Migration.currentDbVersion

View File

@ -54,7 +54,7 @@ module Migration =
open Microsoft.Extensions.Logging open Microsoft.Extensions.Logging
/// The current database version /// The current database version
let currentDbVersion = "v2.1" let currentDbVersion = "v2.1.1"
/// Log a migration step /// Log a migration step
let logStep<'T> (log: ILogger<'T>) migration message = let logStep<'T> (log: ILogger<'T>) migration message =

View File

@ -1,5 +1,5 @@
{ {
"Generator": "myWebLog 2.1", "Generator": "myWebLog 2.1.1",
"Logging": { "Logging": {
"LogLevel": { "LogLevel": {
"MyWebLog.Handlers": "Information" "MyWebLog.Handlers": "Information"

View File

@ -1,2 +1,2 @@
myWebLog Admin myWebLog Admin
2.1.0 2.1.1

View File

@ -1,2 +1,2 @@
myWebLog Default Theme myWebLog Default Theme
2.1.0 2.1.1