bit-badger/myWebLog
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 11 Activity

Rework migration; add PG 2.1 changes

Browse Source
This commit is contained in:
Daniel J. Summers 2024-01-04 17:33:41 -05:00
parent b69b6c37a4
commit 5a3d9f05f8
4 changed files with 94 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
src/MyWebLog.Data/PostgresData.fs
View File

@ -77,8 +77,8 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
Table.Post
"status"
[ nameof Post.Empty.WebLogId; nameof Post.Empty.Status; nameof Post.Empty.UpdatedOn ]
$"CREATE INDEX post_category_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))"
$"CREATE INDEX post_tag_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))"
$"CREATE INDEX idx_post_category ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))"
$"CREATE INDEX idx_post_tag ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))"
if needsTable Table.PostRevision then
$"CREATE TABLE {Table.PostRevision} (
post_id TEXT NOT NULL,
@ -104,13 +104,13 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
path TEXT NOT NULL,
updated_on TIMESTAMPTZ NOT NULL,
data BYTEA NOT NULL)"
$"CREATE INDEX upload_web_log_idx ON {Table.Upload} (web_log_id)"
$"CREATE INDEX upload_path_idx ON {Table.Upload} (web_log_id, path)"
$"CREATE INDEX idx_upload_web_log ON {Table.Upload} (web_log_id)"
$"CREATE INDEX idx_upload_path ON {Table.Upload} (web_log_id, path)"
// Database version table
if needsTable Table.DbVersion then
$"CREATE TABLE {Table.DbVersion} (id TEXT NOT NULL PRIMARY KEY)"
$"INSERT INTO {Table.DbVersion} VALUES ('{Utils.currentDbVersion}')"
$"INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')"
}
Configuration.dataSource ()
@ -134,35 +134,54 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
/// Migrate from v2-rc2 to v2 (manual migration required)
let migrateV2Rc2ToV2 () = backgroundTask {
Utils.logMigrationStep log "v2-rc2 to v2" "Requires user action"
let! webLogs =
Configuration.dataSource ()
|> Sql.fromDataSource
|> Sql.query $"SELECT url_base, slug FROM {Table.WebLog}"
|> Sql.executeAsync (fun row -> row.string "url_base", row.string "slug")
[ "** MANUAL DATABASE UPGRADE REQUIRED **"; ""
"The data structure for PostgreSQL changed significantly between v2-rc2 and v2."
"To migrate your data:"
" - Use a v2-rc2 executable to back up each web log"
" - Drop all tables from the database"
" - Use this executable to restore each backup"; ""
"Commands to back up all web logs:"
yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup {url} v2-rc2.{slug}.json") ]
|> String.concat "\n"
|> log.LogWarning
log.LogCritical "myWebLog will now exit"
exit 1
Custom.list
$"SELECT url_base, slug FROM {Table.WebLog}" [] (fun row -> row.string "url_base", row.string "slug")
Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs
}
/// Migrate from v2 to v2.1
let migrateV2ToV2point1 () = backgroundTask {
Utils.logMigrationStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
let migration = "v2 to v2.1"
Utils.Migration.logStep log migration "Adding empty redirect rule set to all weblogs"
do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data + '{{ "RedirectRules": [] }}'::json""" []
Utils.logMigrationStep log "v2 to v2.1" "Setting database to version 2.1"
let tables =
[ Table.Category; Table.Page; Table.Post; Table.PostComment; Table.TagMap; Table.Theme; Table.WebLog
Table.WebLogUser ]
Utils.Migration.logStep log migration "Adding unique indexes on ID fields"
do! Custom.nonQuery (tables |> List.map Query.Definition.ensureKey |> String.concat "; ") []
Utils.Migration.logStep log migration "Dropping old ID columns"
do! Custom.nonQuery (tables |> List.map (sprintf "ALTER TABLE %s DROP COLUMN id") |> String.concat "; ") []
Utils.Migration.logStep log migration "Adjusting indexes"
let toDrop = [ "page_web_log_idx"; "post_web_log_idx" ]
do! Custom.nonQuery (toDrop |> List.map (sprintf "DROP INDEX %s") |> String.concat "; ") []
let toRename =
[ "idx_category", "idx_category_document"
"idx_tag_map", "idx_tag_map_document"
"idx_web_log", "idx_web_log_document"
"idx_web_log_user", "idx_web_log_user_document"
"page_author_idx", "idx_page_author"
"page_permalink_idx", "idx_page_permalink"
"post_author_idx", "idx_post_author"
"post_status_idx", "idx_post_status"
"post_permalink_idx", "idx_post_permalink"
"post_category_idx", "idx_post_category"
"post_tag_idx", "idx_post_tag"
"post_comment_post_idx", "idx_post_comment_post"
"upload_web_log_idx", "idx_upload_web_log"
"upload_path_idx", "idx_upload_path" ]
do! Custom.nonQuery
(toRename
|> List.map (fun (oldName, newName) -> $"ALTER INDEX {oldName} RENAME TO {newName}")
|> String.concat "; ")
[]
Utils.Migration.logStep log migration "Setting database to version 2.1"
do! setDbVersion "v2.1"
}
@ -178,9 +197,9 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
do! migrateV2ToV2point1 ()
v <- "v2.1"
if v <> "v2.1" then
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
do! setDbVersion Utils.currentDbVersion
if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion
}
interface IData with

14
src/MyWebLog.Data/RethinkDbData.fs
View File

@ -207,7 +207,7 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
/// Migrate from v2-rc1 to v2-rc2
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2"
let logStep = Utils.Migration.logStep log "v2-rc1 to v2-rc2"
logStep "**IMPORTANT**"
logStep "See release notes about required backup/restoration for RethinkDB."
logStep "If there is an error immediately below this message, this is why."
@ -217,20 +217,20 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
/// Migrate from v2-rc2 to v2
let migrateV2Rc2ToV2 () = backgroundTask {
Utils.logMigrationStep log "v2-rc2 to v2" "Setting database version; no migration required"
Utils.Migration.logStep log "v2-rc2 to v2" "Setting database version; no migration required"
do! setDbVersion "v2"
}
/// Migrate from v2 to v2.1
let migrateV2ToV2point1 () = backgroundTask {
Utils.logMigrationStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
Utils.Migration.logStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
do! rethink {
withTable Table.WebLog
update [ nameof WebLog.Empty.RedirectRules, [] :> obj ]
write; withRetryOnce; ignoreResult conn
}
Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1"
Utils.Migration.logStep log "v2 to v2.1" "Setting database version to v2.1"
do! setDbVersion "v2.1"
}
@ -250,9 +250,9 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
do! migrateV2ToV2point1 ()
v <- "v2.1"
if v <> "v2.1" then
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
do! setDbVersion Utils.currentDbVersion
if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion
}
/// The connection for this instance

25
src/MyWebLog.Data/SQLiteData.fs
View File

@ -18,7 +18,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
Configuration.useSerializer (Utils.createDocumentSerializer ser)
let! tables = conn.customList<string> "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0)
let! tables = conn.customList "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0)
let needsTable table =
not (List.contains table tables)
@ -107,7 +107,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
// Database version table
if needsTable Table.DbVersion then
$"CREATE TABLE {Table.DbVersion} (id TEXT PRIMARY KEY);
INSERT INTO {Table.DbVersion} VALUES ('v2.1')"
INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')"
}
|> Seq.map (fun sql ->
log.LogInformation $"""Creating {(sql.Replace("IF NOT EXISTS ", "").Split ' ')[2]} table..."""
@ -123,7 +123,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
/// Implement the changes between v2-rc1 and v2-rc2
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2"
let logStep = Utils.Migration.logStep log "v2-rc1 to v2-rc2"
// Move meta items, podcast settings, and episode details to JSON-encoded text fields
use cmd = conn.CreateCommand()
logStep "Adding new columns"
@ -418,20 +418,15 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
/// Migrate from v2-rc2 to v2
let migrateV2Rc2ToV2 () = backgroundTask {
Utils.logMigrationStep log "v2-rc2 to v2" "Setting database version; no migration required"
Utils.Migration.logStep log "v2-rc2 to v2" "Setting database version; no migration required"
do! setDbVersion "v2"
}
/// Migrate from v2 to v2.1
let migrateV2ToV2point1 () = backgroundTask {
// FIXME: This will be a backup/restore scenario, as we're changing to documents for most tables
Utils.logMigrationStep log "v2 to v2.1" "Adding redirect rules to web_log table"
use cmd = conn.CreateCommand()
cmd.CommandText <- "ALTER TABLE web_log ADD COLUMN redirect_rules TEXT NOT NULL DEFAULT '[]'"
do! write cmd
Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1"
do! setDbVersion "v2.1"
let! webLogs =
Custom.list $"SELECT url_base, slug FROM {Table.WebLog}" [] (fun rdr -> rdr.GetString(0), rdr.GetString(1))
Utils.Migration.backupAndRestoreRequired log "v2" "v2.1" webLogs
}
/// Migrate data among versions (up only)
@ -450,9 +445,9 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
do! migrateV2ToV2point1 ()
v <- "v2.1"
if v <> "v2.1" then
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
do! setDbVersion Utils.currentDbVersion
if v <> Utils.Migration.currentDbVersion then
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
do! setDbVersion Utils.Migration.currentDbVersion
}
/// The connection for this instance

35
src/MyWebLog.Data/Utils.fs
View File

@ -5,9 +5,6 @@ module internal MyWebLog.Data.Utils
open MyWebLog
open MyWebLog.ViewModels
/// The current database version
let currentDbVersion = "v2.1"
/// Create a category hierarchy from the given list of categories
let rec orderByHierarchy (cats: Category list) parentId slugBase parentNames = seq {
for cat in cats |> List.filter (fun c -> c.ParentId = parentId) do
@ -59,9 +56,33 @@ let createDocumentSerializer ser =
member _.Deserialize<'T>(it: string) : 'T = deserialize ser it
}
/// Data migration utilities
module Migration =
open Microsoft.Extensions.Logging
open Microsoft.Extensions.Logging
/// The current database version
let currentDbVersion = "v2.1"
/// Log a migration step
let logMigrationStep<'T> (log: ILogger<'T>) migration message =
log.LogInformation $"Migrating %s{migration}: %s{message}"
/// Log a migration step
let logStep<'T> (log: ILogger<'T>) migration message =
log.LogInformation $"Migrating %s{migration}: %s{message}"
/// Notify the user that a backup/restore
let backupAndRestoreRequired log oldVersion newVersion webLogs =
logStep log $"%s{oldVersion} to %s{newVersion}" "Requires Using Action"
[ "** MANUAL DATABASE UPGRADE REQUIRED **"; ""
$"The data structure changed between {oldVersion} and {newVersion}."
"To migrate your data:"
$" - Use a {oldVersion} executable to back up each web log"
" - Drop all tables from the database"
" - Use this executable to restore each backup"; ""
"Commands to back up all web logs:"
yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup %s{url} {oldVersion}.%s{slug}.json") ]
|> String.concat "\n"
|> log.LogWarning
log.LogCritical "myWebLog will now exit"
exit 1 |> ignore