From e0e2d97ac22333fd319432ba22d90cf5f79bafb8 Mon Sep 17 00:00:00 2001 From: "Daniel J. Summers" Date: Sun, 21 Aug 2022 00:31:34 -0400 Subject: [PATCH] Finish SQLite v2-rc2 migration --- src/MyWebLog.Data/SQLite/Helpers.fs | 18 +- src/MyWebLog.Data/SQLite/SQLitePageData.fs | 2 +- src/MyWebLog.Data/SQLiteData.fs | 184 ++++++++++++++++++++- src/MyWebLog.Domain/SupportTypes.fs | 7 +- src/MyWebLog/Maintenance.fs | 4 +- 5 files changed, 198 insertions(+), 17 deletions(-) diff --git a/src/MyWebLog.Data/SQLite/Helpers.fs b/src/MyWebLog.Data/SQLite/Helpers.fs index c68f926..1513987 100644 --- a/src/MyWebLog.Data/SQLite/Helpers.fs +++ b/src/MyWebLog.Data/SQLite/Helpers.fs @@ -32,21 +32,24 @@ let write (cmd : SqliteCommand) = backgroundTask { () } +/// Add a possibly-missing parameter, substituting null for None +let maybe<'T> (it : 'T option) : obj = match it with Some x -> x :> obj | None -> DBNull.Value + /// Create a value for a Duration let durationParam = DurationPattern.Roundtrip.Format /// Create a value for an Instant let instantParam = - InstantPattern.ExtendedIso.Format + InstantPattern.General.Format /// Create an optional value for a Duration let maybeDuration = - Option.map durationParam + Option.map durationParam >> maybe /// Create an optional value for an Instant let maybeInstant = - Option.map instantParam + Option.map instantParam >> maybe /// Create the SQL and parameters for an IN clause let inClause<'T> colNameAndPrefix paramName (valueFunc: 'T -> string) (items : 'T list) = @@ -260,9 +263,9 @@ module Map = dataStream.ToArray () else [||] - { Id = getString "id" rdr |> UploadId - WebLogId = getString "web_log_id" rdr |> WebLogId - Path = getString "path" rdr |> Permalink + { Id = getString "id" rdr |> UploadId + WebLogId = getString "web_log_id" rdr |> WebLogId + Path = getString "path" rdr |> Permalink UpdatedOn = getInstant "updated_on" rdr Data = data } @@ -307,9 +310,6 @@ module Map = LastSeenOn = tryInstant "last_seen_on" rdr } -/// Add a possibly-missing parameter, substituting null for None -let maybe<'T> (it : 'T option) : obj = match it with Some x -> x :> obj | None -> DBNull.Value - /// Add a web log ID parameter let addWebLogId (cmd : SqliteCommand) webLogId = cmd.Parameters.AddWithValue ("@webLogId", WebLogId.toString webLogId) |> ignore diff --git a/src/MyWebLog.Data/SQLite/SQLitePageData.fs b/src/MyWebLog.Data/SQLite/SQLitePageData.fs index 1854cb5..5562bcc 100644 --- a/src/MyWebLog.Data/SQLite/SQLitePageData.fs +++ b/src/MyWebLog.Data/SQLite/SQLitePageData.fs @@ -115,7 +115,7 @@ type SQLitePageData (conn : SqliteConnection, ser : JsonSerializer) = page_text, meta_items ) VALUES ( @id, @webLogId, @authorId, @title, @permalink, @publishedOn, @updatedOn, @isInPageList, @template, - @text, @meta_items + @text, @metaItems )" addPageParameters cmd page do! write cmd diff --git a/src/MyWebLog.Data/SQLiteData.fs b/src/MyWebLog.Data/SQLiteData.fs index 00c4808..7b732c6 100644 --- a/src/MyWebLog.Data/SQLiteData.fs +++ b/src/MyWebLog.Data/SQLiteData.fs @@ -24,7 +24,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger, ser : JsonS return tableList } let needsTable table = - List.contains table tables + not (List.contains table tables) seq { // Theme tables if needsTable "theme" then @@ -230,7 +230,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger, ser : JsonS /// Log a migration step let logMigrationStep migration message = - log.LogInformation $"[%s{migration}] %s{message}" + log.LogInformation $"Migrating %s{migration}: %s{message}" /// Implement the changes between v2-rc1 and v2-rc2 let migrateV2Rc1ToV2Rc2 () = backgroundTask { @@ -335,6 +335,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger, ser : JsonS EpisodeDescription = Map.tryString "episode_description" epRdr } } |> List.ofSeq + epRdr.Close () episodes |> List.iter (fun (postId, episode) -> cmd.CommandText <- "UPDATE post SET episode = @episode WHERE id = @id" @@ -343,12 +344,189 @@ type SQLiteData (conn : SqliteConnection, log : ILogger, ser : JsonS let _ = cmd.ExecuteNonQuery () cmd.Parameters.Clear ()) + logStep "Migrating dates/times" + let inst (dt : System.DateTime) = + System.DateTime (dt.Ticks, System.DateTimeKind.Utc) + |> (Instant.FromDateTimeUtc >> Noda.toSecondsPrecision) + // page.updated_on, page.published_on + cmd.CommandText <- "SELECT id, updated_on, published_on FROM page" + use! pageRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while pageRdr.Read () do + Map.getString "id" pageRdr, + inst (Map.getDateTime "updated_on" pageRdr), + inst (Map.getDateTime "published_on" pageRdr) + } |> List.ofSeq + pageRdr.Close () + cmd.CommandText <- "UPDATE page SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id" + [ cmd.Parameters.Add ("@id", SqliteType.Text) + cmd.Parameters.Add ("@updatedOn", SqliteType.Text) + cmd.Parameters.Add ("@publishedOn", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (pageId, updatedOn, publishedOn) -> + cmd.Parameters["@id" ].Value <- pageId + cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn + cmd.Parameters["@publishedOn"].Value <- instantParam publishedOn + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // page_revision.as_of + cmd.CommandText <- "SELECT * FROM page_revision" + use! pageRevRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while pageRevRdr.Read () do + let asOf = Map.getDateTime "as_of" pageRevRdr + Map.getString "page_id" pageRevRdr, asOf, inst asOf, Map.getString "revision_text" pageRevRdr + } |> List.ofSeq + pageRevRdr.Close () + cmd.CommandText <- + "DELETE FROM page_revision WHERE page_id = @pageId AND as_of = @oldAsOf; + INSERT INTO page_revision (page_id, as_of, revision_text) VALUES (@pageId, @asOf, @text)" + [ cmd.Parameters.Add ("@pageId", SqliteType.Text) + cmd.Parameters.Add ("@oldAsOf", SqliteType.Text) + cmd.Parameters.Add ("@asOf", SqliteType.Text) + cmd.Parameters.Add ("@text", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (pageId, oldAsOf, asOf, text) -> + cmd.Parameters["@pageId" ].Value <- pageId + cmd.Parameters["@oldAsOf"].Value <- oldAsOf + cmd.Parameters["@asOf" ].Value <- instantParam asOf + cmd.Parameters["@text" ].Value <- text + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // post.updated_on, post.published_on (opt) + cmd.CommandText <- "SELECT id, updated_on, published_on FROM post" + use! postRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while postRdr.Read () do + Map.getString "id" postRdr, + inst (Map.getDateTime "updated_on" postRdr), + (Map.tryDateTime "published_on" postRdr |> Option.map inst) + } |> List.ofSeq + postRdr.Close () + cmd.CommandText <- "UPDATE post SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id" + [ cmd.Parameters.Add ("@id", SqliteType.Text) + cmd.Parameters.Add ("@updatedOn", SqliteType.Text) + cmd.Parameters.Add ("@publishedOn", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (postId, updatedOn, publishedOn) -> + cmd.Parameters["@id" ].Value <- postId + cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn + cmd.Parameters["@publishedOn"].Value <- maybeInstant publishedOn + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // post_revision.as_of + cmd.CommandText <- "SELECT * FROM post_revision" + use! postRevRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while postRevRdr.Read () do + let asOf = Map.getDateTime "as_of" postRevRdr + Map.getString "post_id" postRevRdr, asOf, inst asOf, Map.getString "revision_text" postRevRdr + } |> List.ofSeq + postRevRdr.Close () + cmd.CommandText <- + "DELETE FROM post_revision WHERE post_id = @postId AND as_of = @oldAsOf; + INSERT INTO post_revision (post_id, as_of, revision_text) VALUES (@postId, @asOf, @text)" + [ cmd.Parameters.Add ("@postId", SqliteType.Text) + cmd.Parameters.Add ("@oldAsOf", SqliteType.Text) + cmd.Parameters.Add ("@asOf", SqliteType.Text) + cmd.Parameters.Add ("@text", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (postId, oldAsOf, asOf, text) -> + cmd.Parameters["@postId" ].Value <- postId + cmd.Parameters["@oldAsOf"].Value <- oldAsOf + cmd.Parameters["@asOf" ].Value <- instantParam asOf + cmd.Parameters["@text" ].Value <- text + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // theme_asset.updated_on + cmd.CommandText <- "SELECT theme_id, path, updated_on FROM theme_asset" + use! assetRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while assetRdr.Read () do + Map.getString "theme_id" assetRdr, Map.getString "path" assetRdr, + inst (Map.getDateTime "updated_on" assetRdr) + } |> List.ofSeq + assetRdr.Close () + cmd.CommandText <- "UPDATE theme_asset SET updated_on = @updatedOn WHERE theme_id = @themeId AND path = @path" + [ cmd.Parameters.Add ("@updatedOn", SqliteType.Text) + cmd.Parameters.Add ("@themeId", SqliteType.Text) + cmd.Parameters.Add ("@path", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (themeId, path, updatedOn) -> + cmd.Parameters["@themeId" ].Value <- themeId + cmd.Parameters["@path" ].Value <- path + cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // upload.updated_on + cmd.CommandText <- "SELECT id, updated_on FROM upload" + use! upRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while upRdr.Read () do + Map.getString "id" upRdr, inst (Map.getDateTime "updated_on" upRdr) + } |> List.ofSeq + upRdr.Close () + cmd.CommandText <- "UPDATE upload SET updated_on = @updatedOn WHERE id = @id" + [ cmd.Parameters.Add ("@updatedOn", SqliteType.Text) + cmd.Parameters.Add ("@id", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (upId, updatedOn) -> + cmd.Parameters["@id" ].Value <- upId + cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + // web_log_user.created_on, web_log_user.last_seen_on (opt) + cmd.CommandText <- "SELECT id, created_on, last_seen_on FROM web_log_user" + use! userRdr = cmd.ExecuteReaderAsync () + let toUpdate = + seq { + while userRdr.Read () do + Map.getString "id" userRdr, + inst (Map.getDateTime "created_on" userRdr), + (Map.tryDateTime "last_seen_on" userRdr |> Option.map inst) + } |> List.ofSeq + userRdr.Close () + cmd.CommandText <- "UPDATE web_log_user SET created_on = @createdOn, last_seen_on = @lastSeenOn WHERE id = @id" + [ cmd.Parameters.Add ("@id", SqliteType.Text) + cmd.Parameters.Add ("@createdOn", SqliteType.Text) + cmd.Parameters.Add ("@lastSeenOn", SqliteType.Text) + ] |> ignore + toUpdate + |> List.iter (fun (userId, createdOn, lastSeenOn) -> + cmd.Parameters["@id" ].Value <- userId + cmd.Parameters["@createdOn" ].Value <- instantParam createdOn + cmd.Parameters["@lastSeenOn"].Value <- maybeInstant lastSeenOn + let _ = cmd.ExecuteNonQuery () + ()) + cmd.Parameters.Clear () + + conn.Close () + conn.Open () + logStep "Dropping old tables" cmd.CommandText <- "DROP TABLE post_episode; DROP TABLE post_meta; DROP TABLE page_meta; - DROP TABLE web_log_podcast" + DROP TABLE web_log_feed_podcast" do! write cmd logStep "Setting database version" diff --git a/src/MyWebLog.Domain/SupportTypes.fs b/src/MyWebLog.Domain/SupportTypes.fs index 3785293..49bb09c 100644 --- a/src/MyWebLog.Domain/SupportTypes.fs +++ b/src/MyWebLog.Domain/SupportTypes.fs @@ -22,8 +22,13 @@ module Noda = /// The Unix epoch let epoch = Instant.FromUnixTimeSeconds 0L + /// Truncate an instant to remove fractional seconds + let toSecondsPrecision (value : Instant) = + Instant.FromUnixTimeSeconds (value.ToUnixTimeSeconds ()) + /// The current Instant, with fractional seconds truncated - let now () = Instant.FromUnixTimeSeconds (clock.GetCurrentInstant().ToUnixTimeSeconds ()) + let now () = + toSecondsPrecision (clock.GetCurrentInstant ()) /// A user's access level diff --git a/src/MyWebLog/Maintenance.fs b/src/MyWebLog/Maintenance.fs index 4d4fbe9..9fb32d8 100644 --- a/src/MyWebLog/Maintenance.fs +++ b/src/MyWebLog/Maintenance.fs @@ -397,9 +397,7 @@ module Backup = if not (List.isEmpty restore.Categories) then do! data.Category.Restore restore.Categories printfn "- Restoring pages..." - if not (List.isEmpty restore.Pages) then - printfn "here" - do! data.Page.Restore restore.Pages + if not (List.isEmpty restore.Pages) then do! data.Page.Restore restore.Pages printfn "- Restoring posts..." if not (List.isEmpty restore.Posts) then do! data.Post.Restore restore.Posts