Version 2.1 #41
src/MyWebLog.Data
@ -77,8 +77,8 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
|
|||||||
Table.Post
|
Table.Post
|
||||||
"status"
|
"status"
|
||||||
[ nameof Post.Empty.WebLogId; nameof Post.Empty.Status; nameof Post.Empty.UpdatedOn ]
|
[ nameof Post.Empty.WebLogId; nameof Post.Empty.Status; nameof Post.Empty.UpdatedOn ]
|
||||||
$"CREATE INDEX post_category_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))"
|
$"CREATE INDEX idx_post_category ON {Table.Post} USING GIN ((data['{nameof Post.Empty.CategoryIds}']))"
|
||||||
$"CREATE INDEX post_tag_idx ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))"
|
$"CREATE INDEX idx_post_tag ON {Table.Post} USING GIN ((data['{nameof Post.Empty.Tags}']))"
|
||||||
if needsTable Table.PostRevision then
|
if needsTable Table.PostRevision then
|
||||||
$"CREATE TABLE {Table.PostRevision} (
|
$"CREATE TABLE {Table.PostRevision} (
|
||||||
post_id TEXT NOT NULL,
|
post_id TEXT NOT NULL,
|
||||||
@ -104,13 +104,13 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
|
|||||||
path TEXT NOT NULL,
|
path TEXT NOT NULL,
|
||||||
updated_on TIMESTAMPTZ NOT NULL,
|
updated_on TIMESTAMPTZ NOT NULL,
|
||||||
data BYTEA NOT NULL)"
|
data BYTEA NOT NULL)"
|
||||||
$"CREATE INDEX upload_web_log_idx ON {Table.Upload} (web_log_id)"
|
$"CREATE INDEX idx_upload_web_log ON {Table.Upload} (web_log_id)"
|
||||||
$"CREATE INDEX upload_path_idx ON {Table.Upload} (web_log_id, path)"
|
$"CREATE INDEX idx_upload_path ON {Table.Upload} (web_log_id, path)"
|
||||||
|
|
||||||
// Database version table
|
// Database version table
|
||||||
if needsTable Table.DbVersion then
|
if needsTable Table.DbVersion then
|
||||||
$"CREATE TABLE {Table.DbVersion} (id TEXT NOT NULL PRIMARY KEY)"
|
$"CREATE TABLE {Table.DbVersion} (id TEXT NOT NULL PRIMARY KEY)"
|
||||||
$"INSERT INTO {Table.DbVersion} VALUES ('{Utils.currentDbVersion}')"
|
$"INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')"
|
||||||
}
|
}
|
||||||
|
|
||||||
Configuration.dataSource ()
|
Configuration.dataSource ()
|
||||||
@ -134,35 +134,54 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
|
|||||||
|
|
||||||
/// Migrate from v2-rc2 to v2 (manual migration required)
|
/// Migrate from v2-rc2 to v2 (manual migration required)
|
||||||
let migrateV2Rc2ToV2 () = backgroundTask {
|
let migrateV2Rc2ToV2 () = backgroundTask {
|
||||||
Utils.logMigrationStep log "v2-rc2 to v2" "Requires user action"
|
|
||||||
|
|
||||||
let! webLogs =
|
let! webLogs =
|
||||||
Configuration.dataSource ()
|
Custom.list
|
||||||
|> Sql.fromDataSource
|
$"SELECT url_base, slug FROM {Table.WebLog}" [] (fun row -> row.string "url_base", row.string "slug")
|
||||||
|> Sql.query $"SELECT url_base, slug FROM {Table.WebLog}"
|
Utils.Migration.backupAndRestoreRequired log "v2-rc2" "v2" webLogs
|
||||||
|> Sql.executeAsync (fun row -> row.string "url_base", row.string "slug")
|
|
||||||
|
|
||||||
[ "** MANUAL DATABASE UPGRADE REQUIRED **"; ""
|
|
||||||
"The data structure for PostgreSQL changed significantly between v2-rc2 and v2."
|
|
||||||
"To migrate your data:"
|
|
||||||
" - Use a v2-rc2 executable to back up each web log"
|
|
||||||
" - Drop all tables from the database"
|
|
||||||
" - Use this executable to restore each backup"; ""
|
|
||||||
"Commands to back up all web logs:"
|
|
||||||
yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup {url} v2-rc2.{slug}.json") ]
|
|
||||||
|> String.concat "\n"
|
|
||||||
|> log.LogWarning
|
|
||||||
|
|
||||||
log.LogCritical "myWebLog will now exit"
|
|
||||||
exit 1
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Migrate from v2 to v2.1
|
/// Migrate from v2 to v2.1
|
||||||
let migrateV2ToV2point1 () = backgroundTask {
|
let migrateV2ToV2point1 () = backgroundTask {
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
|
let migration = "v2 to v2.1"
|
||||||
|
Utils.Migration.logStep log migration "Adding empty redirect rule set to all weblogs"
|
||||||
do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data + '{{ "RedirectRules": [] }}'::json""" []
|
do! Custom.nonQuery $"""UPDATE {Table.WebLog} SET data = data + '{{ "RedirectRules": [] }}'::json""" []
|
||||||
|
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Setting database to version 2.1"
|
let tables =
|
||||||
|
[ Table.Category; Table.Page; Table.Post; Table.PostComment; Table.TagMap; Table.Theme; Table.WebLog
|
||||||
|
Table.WebLogUser ]
|
||||||
|
|
||||||
|
Utils.Migration.logStep log migration "Adding unique indexes on ID fields"
|
||||||
|
do! Custom.nonQuery (tables |> List.map Query.Definition.ensureKey |> String.concat "; ") []
|
||||||
|
|
||||||
|
Utils.Migration.logStep log migration "Dropping old ID columns"
|
||||||
|
do! Custom.nonQuery (tables |> List.map (sprintf "ALTER TABLE %s DROP COLUMN id") |> String.concat "; ") []
|
||||||
|
|
||||||
|
Utils.Migration.logStep log migration "Adjusting indexes"
|
||||||
|
let toDrop = [ "page_web_log_idx"; "post_web_log_idx" ]
|
||||||
|
do! Custom.nonQuery (toDrop |> List.map (sprintf "DROP INDEX %s") |> String.concat "; ") []
|
||||||
|
|
||||||
|
let toRename =
|
||||||
|
[ "idx_category", "idx_category_document"
|
||||||
|
"idx_tag_map", "idx_tag_map_document"
|
||||||
|
"idx_web_log", "idx_web_log_document"
|
||||||
|
"idx_web_log_user", "idx_web_log_user_document"
|
||||||
|
"page_author_idx", "idx_page_author"
|
||||||
|
"page_permalink_idx", "idx_page_permalink"
|
||||||
|
"post_author_idx", "idx_post_author"
|
||||||
|
"post_status_idx", "idx_post_status"
|
||||||
|
"post_permalink_idx", "idx_post_permalink"
|
||||||
|
"post_category_idx", "idx_post_category"
|
||||||
|
"post_tag_idx", "idx_post_tag"
|
||||||
|
"post_comment_post_idx", "idx_post_comment_post"
|
||||||
|
"upload_web_log_idx", "idx_upload_web_log"
|
||||||
|
"upload_path_idx", "idx_upload_path" ]
|
||||||
|
do! Custom.nonQuery
|
||||||
|
(toRename
|
||||||
|
|> List.map (fun (oldName, newName) -> $"ALTER INDEX {oldName} RENAME TO {newName}")
|
||||||
|
|> String.concat "; ")
|
||||||
|
[]
|
||||||
|
|
||||||
|
Utils.Migration.logStep log migration "Setting database to version 2.1"
|
||||||
do! setDbVersion "v2.1"
|
do! setDbVersion "v2.1"
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -178,9 +197,9 @@ type PostgresData(log: ILogger<PostgresData>, ser: JsonSerializer) =
|
|||||||
do! migrateV2ToV2point1 ()
|
do! migrateV2ToV2point1 ()
|
||||||
v <- "v2.1"
|
v <- "v2.1"
|
||||||
|
|
||||||
if v <> "v2.1" then
|
if v <> Utils.Migration.currentDbVersion then
|
||||||
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
|
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
|
||||||
do! setDbVersion Utils.currentDbVersion
|
do! setDbVersion Utils.Migration.currentDbVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
interface IData with
|
interface IData with
|
||||||
|
@ -207,7 +207,7 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
|
|||||||
|
|
||||||
/// Migrate from v2-rc1 to v2-rc2
|
/// Migrate from v2-rc1 to v2-rc2
|
||||||
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
|
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
|
||||||
let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2"
|
let logStep = Utils.Migration.logStep log "v2-rc1 to v2-rc2"
|
||||||
logStep "**IMPORTANT**"
|
logStep "**IMPORTANT**"
|
||||||
logStep "See release notes about required backup/restoration for RethinkDB."
|
logStep "See release notes about required backup/restoration for RethinkDB."
|
||||||
logStep "If there is an error immediately below this message, this is why."
|
logStep "If there is an error immediately below this message, this is why."
|
||||||
@ -217,20 +217,20 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
|
|||||||
|
|
||||||
/// Migrate from v2-rc2 to v2
|
/// Migrate from v2-rc2 to v2
|
||||||
let migrateV2Rc2ToV2 () = backgroundTask {
|
let migrateV2Rc2ToV2 () = backgroundTask {
|
||||||
Utils.logMigrationStep log "v2-rc2 to v2" "Setting database version; no migration required"
|
Utils.Migration.logStep log "v2-rc2 to v2" "Setting database version; no migration required"
|
||||||
do! setDbVersion "v2"
|
do! setDbVersion "v2"
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Migrate from v2 to v2.1
|
/// Migrate from v2 to v2.1
|
||||||
let migrateV2ToV2point1 () = backgroundTask {
|
let migrateV2ToV2point1 () = backgroundTask {
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
|
Utils.Migration.logStep log "v2 to v2.1" "Adding empty redirect rule set to all weblogs"
|
||||||
do! rethink {
|
do! rethink {
|
||||||
withTable Table.WebLog
|
withTable Table.WebLog
|
||||||
update [ nameof WebLog.Empty.RedirectRules, [] :> obj ]
|
update [ nameof WebLog.Empty.RedirectRules, [] :> obj ]
|
||||||
write; withRetryOnce; ignoreResult conn
|
write; withRetryOnce; ignoreResult conn
|
||||||
}
|
}
|
||||||
|
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1"
|
Utils.Migration.logStep log "v2 to v2.1" "Setting database version to v2.1"
|
||||||
do! setDbVersion "v2.1"
|
do! setDbVersion "v2.1"
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -250,9 +250,9 @@ type RethinkDbData(conn: Net.IConnection, config: DataConfig, log: ILogger<Rethi
|
|||||||
do! migrateV2ToV2point1 ()
|
do! migrateV2ToV2point1 ()
|
||||||
v <- "v2.1"
|
v <- "v2.1"
|
||||||
|
|
||||||
if v <> "v2.1" then
|
if v <> Utils.Migration.currentDbVersion then
|
||||||
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
|
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
|
||||||
do! setDbVersion Utils.currentDbVersion
|
do! setDbVersion Utils.Migration.currentDbVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The connection for this instance
|
/// The connection for this instance
|
||||||
|
@ -18,7 +18,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
|
|||||||
|
|
||||||
Configuration.useSerializer (Utils.createDocumentSerializer ser)
|
Configuration.useSerializer (Utils.createDocumentSerializer ser)
|
||||||
|
|
||||||
let! tables = conn.customList<string> "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0)
|
let! tables = conn.customList "SELECT name FROM sqlite_master WHERE type = 'table'" [] _.GetString(0)
|
||||||
|
|
||||||
let needsTable table =
|
let needsTable table =
|
||||||
not (List.contains table tables)
|
not (List.contains table tables)
|
||||||
@ -107,7 +107,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
|
|||||||
// Database version table
|
// Database version table
|
||||||
if needsTable Table.DbVersion then
|
if needsTable Table.DbVersion then
|
||||||
$"CREATE TABLE {Table.DbVersion} (id TEXT PRIMARY KEY);
|
$"CREATE TABLE {Table.DbVersion} (id TEXT PRIMARY KEY);
|
||||||
INSERT INTO {Table.DbVersion} VALUES ('v2.1')"
|
INSERT INTO {Table.DbVersion} VALUES ('{Utils.Migration.currentDbVersion}')"
|
||||||
}
|
}
|
||||||
|> Seq.map (fun sql ->
|
|> Seq.map (fun sql ->
|
||||||
log.LogInformation $"""Creating {(sql.Replace("IF NOT EXISTS ", "").Split ' ')[2]} table..."""
|
log.LogInformation $"""Creating {(sql.Replace("IF NOT EXISTS ", "").Split ' ')[2]} table..."""
|
||||||
@ -123,7 +123,7 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
|
|||||||
|
|
||||||
/// Implement the changes between v2-rc1 and v2-rc2
|
/// Implement the changes between v2-rc1 and v2-rc2
|
||||||
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
|
let migrateV2Rc1ToV2Rc2 () = backgroundTask {
|
||||||
let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2"
|
let logStep = Utils.Migration.logStep log "v2-rc1 to v2-rc2"
|
||||||
// Move meta items, podcast settings, and episode details to JSON-encoded text fields
|
// Move meta items, podcast settings, and episode details to JSON-encoded text fields
|
||||||
use cmd = conn.CreateCommand()
|
use cmd = conn.CreateCommand()
|
||||||
logStep "Adding new columns"
|
logStep "Adding new columns"
|
||||||
@ -418,20 +418,15 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
|
|||||||
|
|
||||||
/// Migrate from v2-rc2 to v2
|
/// Migrate from v2-rc2 to v2
|
||||||
let migrateV2Rc2ToV2 () = backgroundTask {
|
let migrateV2Rc2ToV2 () = backgroundTask {
|
||||||
Utils.logMigrationStep log "v2-rc2 to v2" "Setting database version; no migration required"
|
Utils.Migration.logStep log "v2-rc2 to v2" "Setting database version; no migration required"
|
||||||
do! setDbVersion "v2"
|
do! setDbVersion "v2"
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Migrate from v2 to v2.1
|
/// Migrate from v2 to v2.1
|
||||||
let migrateV2ToV2point1 () = backgroundTask {
|
let migrateV2ToV2point1 () = backgroundTask {
|
||||||
// FIXME: This will be a backup/restore scenario, as we're changing to documents for most tables
|
let! webLogs =
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Adding redirect rules to web_log table"
|
Custom.list $"SELECT url_base, slug FROM {Table.WebLog}" [] (fun rdr -> rdr.GetString(0), rdr.GetString(1))
|
||||||
use cmd = conn.CreateCommand()
|
Utils.Migration.backupAndRestoreRequired log "v2" "v2.1" webLogs
|
||||||
cmd.CommandText <- "ALTER TABLE web_log ADD COLUMN redirect_rules TEXT NOT NULL DEFAULT '[]'"
|
|
||||||
do! write cmd
|
|
||||||
|
|
||||||
Utils.logMigrationStep log "v2 to v2.1" "Setting database version to v2.1"
|
|
||||||
do! setDbVersion "v2.1"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Migrate data among versions (up only)
|
/// Migrate data among versions (up only)
|
||||||
@ -450,9 +445,9 @@ type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSeria
|
|||||||
do! migrateV2ToV2point1 ()
|
do! migrateV2ToV2point1 ()
|
||||||
v <- "v2.1"
|
v <- "v2.1"
|
||||||
|
|
||||||
if v <> "v2.1" then
|
if v <> Utils.Migration.currentDbVersion then
|
||||||
log.LogWarning $"Unknown database version; assuming {Utils.currentDbVersion}"
|
log.LogWarning $"Unknown database version; assuming {Utils.Migration.currentDbVersion}"
|
||||||
do! setDbVersion Utils.currentDbVersion
|
do! setDbVersion Utils.Migration.currentDbVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The connection for this instance
|
/// The connection for this instance
|
||||||
|
@ -5,9 +5,6 @@ module internal MyWebLog.Data.Utils
|
|||||||
open MyWebLog
|
open MyWebLog
|
||||||
open MyWebLog.ViewModels
|
open MyWebLog.ViewModels
|
||||||
|
|
||||||
/// The current database version
|
|
||||||
let currentDbVersion = "v2.1"
|
|
||||||
|
|
||||||
/// Create a category hierarchy from the given list of categories
|
/// Create a category hierarchy from the given list of categories
|
||||||
let rec orderByHierarchy (cats: Category list) parentId slugBase parentNames = seq {
|
let rec orderByHierarchy (cats: Category list) parentId slugBase parentNames = seq {
|
||||||
for cat in cats |> List.filter (fun c -> c.ParentId = parentId) do
|
for cat in cats |> List.filter (fun c -> c.ParentId = parentId) do
|
||||||
@ -59,9 +56,33 @@ let createDocumentSerializer ser =
|
|||||||
member _.Deserialize<'T>(it: string) : 'T = deserialize ser it
|
member _.Deserialize<'T>(it: string) : 'T = deserialize ser it
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Data migration utilities
|
||||||
|
module Migration =
|
||||||
|
|
||||||
open Microsoft.Extensions.Logging
|
open Microsoft.Extensions.Logging
|
||||||
|
|
||||||
|
/// The current database version
|
||||||
|
let currentDbVersion = "v2.1"
|
||||||
|
|
||||||
|
/// Log a migration step
|
||||||
|
let logStep<'T> (log: ILogger<'T>) migration message =
|
||||||
|
log.LogInformation $"Migrating %s{migration}: %s{message}"
|
||||||
|
|
||||||
|
/// Notify the user that a backup/restore
|
||||||
|
let backupAndRestoreRequired log oldVersion newVersion webLogs =
|
||||||
|
logStep log $"%s{oldVersion} to %s{newVersion}" "Requires Using Action"
|
||||||
|
|
||||||
|
[ "** MANUAL DATABASE UPGRADE REQUIRED **"; ""
|
||||||
|
$"The data structure changed between {oldVersion} and {newVersion}."
|
||||||
|
"To migrate your data:"
|
||||||
|
$" - Use a {oldVersion} executable to back up each web log"
|
||||||
|
" - Drop all tables from the database"
|
||||||
|
" - Use this executable to restore each backup"; ""
|
||||||
|
"Commands to back up all web logs:"
|
||||||
|
yield! webLogs |> List.map (fun (url, slug) -> $"./myWebLog backup %s{url} {oldVersion}.%s{slug}.json") ]
|
||||||
|
|> String.concat "\n"
|
||||||
|
|> log.LogWarning
|
||||||
|
|
||||||
|
log.LogCritical "myWebLog will now exit"
|
||||||
|
exit 1 |> ignore
|
||||||
|
|
||||||
/// Log a migration step
|
|
||||||
let logMigrationStep<'T> (log: ILogger<'T>) migration message =
|
|
||||||
log.LogInformation $"Migrating %s{migration}: %s{message}"
|
|
||||||
|
Loading…
Reference in New Issue
Block a user