bit-badger/myWebLog
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects Releases 11 Activity

Version 2.1 #41

Merged
danieljsummers merged 123 commits from version-2.1 into main 2024-03-27 00:13:28 +00:00
Conversation 0 Commits 123 Files Changed 116 +195 -194
3 changed files with 195 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 58b83b8d28 - Show all commits

33
src/MyWebLog.Data/SQLite/Helpers.fs
View File

@ -66,26 +66,26 @@ open MyWebLog.Data
open NodaTime.Text open NodaTime.Text
/// Run a command that returns a count /// Run a command that returns a count
let count (cmd : SqliteCommand) = backgroundTask { let count (cmd: SqliteCommand) = backgroundTask {
let! it = cmd.ExecuteScalarAsync () let! it = cmd.ExecuteScalarAsync()
return int (it :?> int64) return int (it :?> int64)
} }
/// Create a list of items from the given data reader /// Create a list of items from the given data reader
let toList<'T> (it : SqliteDataReader -> 'T) (rdr : SqliteDataReader) = let toList<'T> (it: SqliteDataReader -> 'T) (rdr: SqliteDataReader) =
seq { while rdr.Read () do it rdr } seq { while rdr.Read () do it rdr }
|> List.ofSeq |> List.ofSeq
/// Verify that the web log ID matches before returning an item /// Verify that the web log ID matches before returning an item
let verifyWebLog<'T> webLogId (prop : 'T -> WebLogId) (it : SqliteDataReader -> 'T) (rdr : SqliteDataReader) = let verifyWebLog<'T> webLogId (prop : 'T -> WebLogId) (it : SqliteDataReader -> 'T) (rdr : SqliteDataReader) =
if rdr.Read () then if rdr.Read() then
let item = it rdr let item = it rdr
if prop item = webLogId then Some item else None if prop item = webLogId then Some item else None
else None else None
/// Execute a command that returns no data /// Execute a command that returns no data
let write (cmd : SqliteCommand) = backgroundTask { let write (cmd: SqliteCommand) = backgroundTask {
let! _ = cmd.ExecuteNonQueryAsync () let! _ = cmd.ExecuteNonQueryAsync()
() ()
} }
@ -367,6 +367,25 @@ module Map =
LastSeenOn = tryInstant "last_seen_on" rdr LastSeenOn = tryInstant "last_seen_on" rdr
} }
/// Map from a document to a domain type, specifying the field name for the document
let fromData<'T> ser rdr fieldName : 'T =
Utils.deserialize<'T> ser (getString fieldName rdr)
/// Map from a document to a domain type
let fromDoc<'T> ser rdr : 'T =
fromData<'T> ser rdr "data"
/// Queries to assist with document manipulation
module Query =
/// Fragment to add an ID condition to a WHERE clause
let whereById =
"data ->> 'Id' = @id"
/// Fragment to add a web log ID condition to a WHERE clause
let whereWebLogId =
"data ->> 'WebLogId' = @webLogId"
/// Add a web log ID parameter /// Add a web log ID parameter
let addWebLogId (cmd: SqliteCommand) (webLogId: WebLogId) = let addWebLogId (cmd: SqliteCommand) (webLogId: WebLogId) =
cmd.Parameters.AddWithValue ("@webLogId", string webLogId) |> ignore cmd.Parameters.AddWithValue("@webLogId", string webLogId) |> ignore

35
src/MyWebLog.Data/SQLite/SQLiteCategoryData.fs
View File

@ -4,9 +4,10 @@ open System.Threading.Tasks
open Microsoft.Data.Sqlite open Microsoft.Data.Sqlite
open MyWebLog open MyWebLog
open MyWebLog.Data open MyWebLog.Data
open Newtonsoft.Json
/// SQLite myWebLog category data implementation /// SQLite myWebLog category data implementation
type SQLiteCategoryData(conn: SqliteConnection) = type SQLiteCategoryData(conn: SqliteConnection, ser: JsonSerializer) =
/// Add parameters for category INSERT or UPDATE statements /// Add parameters for category INSERT or UPDATE statements
let addCategoryParameters (cmd: SqliteCommand) (cat: Category) = let addCategoryParameters (cmd: SqliteCommand) (cat: Category) =
@ -34,8 +35,8 @@ type SQLiteCategoryData(conn: SqliteConnection) =
/// Count all categories for the given web log /// Count all categories for the given web log
let countAll webLogId = backgroundTask { let countAll webLogId = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
cmd.CommandText <- "SELECT COUNT(id) FROM category WHERE web_log_id = @webLogId" cmd.CommandText <- $"SELECT COUNT(*) FROM {Table.Category} WHERE {whereWebLogId}"
addWebLogId cmd webLogId addWebLogId cmd webLogId
return! count cmd return! count cmd
} }
@ -44,25 +45,27 @@ type SQLiteCategoryData(conn: SqliteConnection) =
let countTopLevel webLogId = backgroundTask { let countTopLevel webLogId = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand ()
cmd.CommandText <- cmd.CommandText <-
"SELECT COUNT(id) FROM category WHERE web_log_id = @webLogId AND parent_id IS NULL" $"SELECT COUNT(*) FROM {Table.Category}
WHERE {whereWebLogId} AND data ->> '{nameof Category.Empty.ParentId}' IS NULL"
addWebLogId cmd webLogId addWebLogId cmd webLogId
return! count cmd return! count cmd
} }
// TODO: need to get SQLite in clause format for JSON documents
/// Retrieve all categories for the given web log in a DotLiquid-friendly format /// Retrieve all categories for the given web log in a DotLiquid-friendly format
let findAllForView webLogId = backgroundTask { let findAllForView webLogId = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
cmd.CommandText <- "SELECT * FROM category WHERE web_log_id = @webLogId" cmd.CommandText <- $"SELECT data FROM {Table.Category} WHERE {whereWebLogId}"
addWebLogId cmd webLogId addWebLogId cmd webLogId
use! rdr = cmd.ExecuteReaderAsync () use! rdr = cmd.ExecuteReaderAsync()
let cats = let cats =
seq { seq {
while rdr.Read () do while rdr.Read() do
Map.toCategory rdr Map.fromDoc<Category> ser rdr
} }
|> Seq.sortBy (fun cat -> cat.Name.ToLowerInvariant ()) |> Seq.sortBy _.Name.ToLowerInvariant()
|> List.ofSeq |> List.ofSeq
do! rdr.CloseAsync () do! rdr.CloseAsync()
let ordered = Utils.orderByHierarchy cats None None [] let ordered = Utils.orderByHierarchy cats None None []
let! counts = let! counts =
ordered ordered
@ -71,7 +74,7 @@ type SQLiteCategoryData(conn: SqliteConnection) =
let catSql, catParams = let catSql, catParams =
ordered ordered
|> Seq.filter (fun cat -> cat.ParentNames |> Array.contains it.Name) |> Seq.filter (fun cat -> cat.ParentNames |> Array.contains it.Name)
|> Seq.map (fun cat -> cat.Id) |> Seq.map _.Id
|> Seq.append (Seq.singleton it.Id) |> Seq.append (Seq.singleton it.Id)
|> List.ofSeq |> List.ofSeq
|> inClause "AND pc.category_id" "catId" id |> inClause "AND pc.category_id" "catId" id
@ -103,12 +106,12 @@ type SQLiteCategoryData(conn: SqliteConnection) =
/// Find a category by its ID for the given web log /// Find a category by its ID for the given web log
let findById (catId: CategoryId) webLogId = backgroundTask { let findById (catId: CategoryId) webLogId = backgroundTask {
use cmd = conn.CreateCommand() use cmd = conn.CreateCommand()
cmd.CommandText <- "SELECT * FROM category WHERE id = @id" cmd.CommandText <- $"SELECT * FROM {Table.Category} WHERE {Query.whereById}"
cmd.Parameters.AddWithValue ("@id", string catId) |> ignore cmd.Parameters.AddWithValue("@id", string catId) |> ignore
use! rdr = cmd.ExecuteReaderAsync() use! rdr = cmd.ExecuteReaderAsync()
return verifyWebLog<Category> webLogId (_.WebLogId) Map.toCategory rdr return verifyWebLog<Category> webLogId (_.WebLogId) (Map.fromDoc ser) rdr
} }
// TODO: stopped here
/// Find all categories for the given web log /// Find all categories for the given web log
let findByWebLog (webLogId: WebLogId) = backgroundTask { let findByWebLog (webLogId: WebLogId) = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand ()

319
src/MyWebLog.Data/SQLiteData.fs
View File

@ -8,57 +8,56 @@ open Newtonsoft.Json
open NodaTime open NodaTime
/// SQLite myWebLog data implementation /// SQLite myWebLog data implementation
type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonSerializer) = type SQLiteData(conn: SqliteConnection, log: ILogger<SQLiteData>, ser: JsonSerializer) =
let ensureTables () = backgroundTask { let ensureTables () = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
let! tables = backgroundTask { let! tables = backgroundTask {
cmd.CommandText <- "SELECT name FROM sqlite_master WHERE type = 'table'" cmd.CommandText <- "SELECT name FROM sqlite_master WHERE type = 'table'"
let! rdr = cmd.ExecuteReaderAsync () let! rdr = cmd.ExecuteReaderAsync()
let mutable tableList = [] let mutable tableList = []
while rdr.Read() do while! rdr.ReadAsync() do
tableList <- Map.getString "name" rdr :: tableList tableList <- Map.getString "name" rdr :: tableList
do! rdr.CloseAsync () do! rdr.CloseAsync()
return tableList return tableList
} }
let needsTable table = let needsTable table =
not (List.contains table tables) not (List.contains table tables)
let jsonTable table =
$"CREATE TABLE {table} (data TEXT NOT NULL);
CREATE UNIQUE INDEX idx_{table}_key ON {table} (data ->> 'Id')"
seq { seq {
// Theme tables // Theme tables
if needsTable Table.Theme then if needsTable Table.Theme then jsonTable Table.Theme
$"CREATE TABLE {Table.Theme} (data TEXT NOT NULL); if needsTable Table.ThemeAsset then
CREATE UNIQUE INDEX idx_{Table.Theme}_key ON {Table.Theme} (data ->> 'Id')"; $"CREATE TABLE {Table.ThemeAsset} (
if needsTable "theme_asset" then theme_id TEXT NOT NULL,
"CREATE TABLE theme_asset (
theme_id TEXT NOT NULL REFERENCES theme (id),
path TEXT NOT NULL, path TEXT NOT NULL,
updated_on TEXT NOT NULL, updated_on TEXT NOT NULL,
data BLOB NOT NULL, data BLOB NOT NULL,
PRIMARY KEY (theme_id, path))" PRIMARY KEY (theme_id, path))"
// Web log table // Web log table
if needsTable Table.WebLog then if needsTable Table.WebLog then jsonTable Table.WebLog
$"CREATE TABLE {Table.WebLog} (data TEXT NOT NULL);
CREATE UNIQUE INDEX idx_{Table.WebLog}_key ON {Table.WebLog} (data ->> 'Id')"
// Category table // Category table
if needsTable Table.Category then if needsTable Table.Category then
$"CREATE TABLE {Table.Category} (data TEXT NOT NULL); $"{jsonTable Table.Category};
CREATE UNIQUE INDEX idx_{Table.Category}_key ON {Table.Category} (data -> 'Id');
CREATE INDEX idx_{Table.Category}_web_log ON {Table.Category} (data ->> 'WebLogId')" CREATE INDEX idx_{Table.Category}_web_log ON {Table.Category} (data ->> 'WebLogId')"
// Web log user table // Web log user table
if needsTable Table.WebLogUser then if needsTable Table.WebLogUser then
$"CREATE TABLE web_log_user (data TEXT NOT NULL); $"{jsonTable Table.WebLogUser};
CREATE UNIQUE INDEX idx_{Table.WebLogUser}_key ON {Table.WebLogUser} (data ->> 'Id');
CREATE INDEX idx_{Table.WebLogUser}_email ON {Table.WebLogUser} (data ->> 'WebLogId', data ->> 'Email')" CREATE INDEX idx_{Table.WebLogUser}_email ON {Table.WebLogUser} (data ->> 'WebLogId', data ->> 'Email')"
// Page tables // Page tables
if needsTable Table.Page then if needsTable Table.Page then
$"CREATE TABLE {Table.Page} (data TEXT NOT NULL); $"{jsonTable Table.Page};
CREATE UNIQUE INDEX idx_{Table.Page}_key ON {Table.Page} (data ->> 'Id');
CREATE INDEX idx_{Table.Page}_author ON {Table.Page} (data ->> 'AuthorId'); CREATE INDEX idx_{Table.Page}_author ON {Table.Page} (data ->> 'AuthorId');
CREATE INDEX idx_{Table.Page}_permalink ON {Table.Page} (data ->> 'WebLogId', data ->> 'Permalink')" CREATE INDEX idx_{Table.Page}_permalink ON {Table.Page} (data ->> 'WebLogId', data ->> 'Permalink')"
if needsTable Table.PageRevision then if needsTable Table.PageRevision then
@ -70,8 +69,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
// Post tables // Post tables
if needsTable Table.Post then if needsTable Table.Post then
$"CREATE TABLE {Table.Post} (data TEXT NOT NULL); $"{jsonTable Table.Post};
CREATE UNIQUE INDEX idx_{Table.Post}_key ON {Table.Post} (data ->> 'Id');
CREATE INDEX idx_{Table.Post}_author ON {Table.Post} (data ->> 'AuthorId'); CREATE INDEX idx_{Table.Post}_author ON {Table.Post} (data ->> 'AuthorId');
CREATE INDEX idx_{Table.Post}_status ON {Table.Post} (data ->> 'WebLogId', data ->> 'Status', data ->> 'UpdatedOn'); CREATE INDEX idx_{Table.Post}_status ON {Table.Post} (data ->> 'WebLogId', data ->> 'Status', data ->> 'UpdatedOn');
CREATE INDEX idx_{Table.Post}_permalink ON {Table.Post} (data ->> 'WebLogId', data ->> 'Permalink')" CREATE INDEX idx_{Table.Post}_permalink ON {Table.Post} (data ->> 'WebLogId', data ->> 'Permalink')"
@ -83,22 +81,12 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
revision_text TEXT NOT NULL, revision_text TEXT NOT NULL,
PRIMARY KEY (post_id, as_of))" PRIMARY KEY (post_id, as_of))"
if needsTable Table.PostComment then if needsTable Table.PostComment then
$"CREATE TABLE {Table.PostComment} ( $"{jsonTable Table.PostComment};
id TEXT PRIMARY KEY, CREATE INDEX idx_{Table.PostComment}_post ON {Table.PostComment} (data ->> 'PostId')"
post_id TEXT NOT NULL,
in_reply_to_id TEXT,
name TEXT NOT NULL,
email TEXT NOT NULL,
url TEXT,
status TEXT NOT NULL,
posted_on TEXT NOT NULL,
comment_text TEXT NOT NULL);
CREATE INDEX idx_{Table.PostComment}_post ON {Table.PostComment} (post_id)"
// Tag map table // Tag map table
if needsTable Table.TagMap then if needsTable Table.TagMap then
$"CREATE TABLE {Table.TagMap} (data TEXT NOT NULL); $"{jsonTable Table.TagMap};
CREATE UNIQUE INDEX idx_{Table.TagMap}_key ON {Table.TagMap} (data ->> 'Id');
CREATE INDEX idx_{Table.TagMap}_tag ON {Table.TagMap} (data ->> 'WebLogId', data ->> 'UrlValue')"; CREATE INDEX idx_{Table.TagMap}_tag ON {Table.TagMap} (data ->> 'WebLogId', data ->> 'UrlValue')";
// Uploaded file table // Uploaded file table
@ -126,7 +114,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
/// Set the database version to the specified version /// Set the database version to the specified version
let setDbVersion version = backgroundTask { let setDbVersion version = backgroundTask {
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
cmd.CommandText <- $"DELETE FROM {Table.DbVersion}; INSERT INTO {Table.DbVersion} VALUES ('%s{version}')" cmd.CommandText <- $"DELETE FROM {Table.DbVersion}; INSERT INTO {Table.DbVersion} VALUES ('%s{version}')"
do! write cmd do! write cmd
} }
@ -135,7 +123,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
let migrateV2Rc1ToV2Rc2 () = backgroundTask { let migrateV2Rc1ToV2Rc2 () = backgroundTask {
let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2" let logStep = Utils.logMigrationStep log "v2-rc1 to v2-rc2"
// Move meta items, podcast settings, and episode details to JSON-encoded text fields // Move meta items, podcast settings, and episode details to JSON-encoded text fields
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
logStep "Adding new columns" logStep "Adding new columns"
cmd.CommandText <- cmd.CommandText <-
"ALTER TABLE web_log_feed ADD COLUMN podcast TEXT; "ALTER TABLE web_log_feed ADD COLUMN podcast TEXT;
@ -146,10 +134,10 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
logStep "Migrating meta items" logStep "Migrating meta items"
let migrateMeta entity = backgroundTask { let migrateMeta entity = backgroundTask {
cmd.CommandText <- $"SELECT * FROM %s{entity}_meta" cmd.CommandText <- $"SELECT * FROM %s{entity}_meta"
use! metaRdr = cmd.ExecuteReaderAsync () use! metaRdr = cmd.ExecuteReaderAsync()
let allMetas = let allMetas =
seq { seq {
while metaRdr.Read () do while metaRdr.Read() do
Map.getString $"{entity}_id" metaRdr, Map.getString $"{entity}_id" metaRdr,
{ Name = Map.getString "name" metaRdr; Value = Map.getString "value" metaRdr } { Name = Map.getString "name" metaRdr; Value = Map.getString "value" metaRdr }
} |> List.ofSeq } |> List.ofSeq
@ -165,120 +153,117 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
"UPDATE post "UPDATE post
SET meta_items = @metaItems SET meta_items = @metaItems
WHERE id = @postId" WHERE id = @postId"
[ cmd.Parameters.AddWithValue ("@metaItems", Utils.serialize ser items) [ cmd.Parameters.AddWithValue("@metaItems", Utils.serialize ser items)
cmd.Parameters.AddWithValue ("@id", entityId) ] |> ignore cmd.Parameters.AddWithValue("@id", entityId) ] |> ignore
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
cmd.Parameters.Clear ()) cmd.Parameters.Clear())
} }
do! migrateMeta "page" do! migrateMeta "page"
do! migrateMeta "post" do! migrateMeta "post"
logStep "Migrating podcasts and episodes" logStep "Migrating podcasts and episodes"
cmd.CommandText <- "SELECT * FROM web_log_feed_podcast" cmd.CommandText <- "SELECT * FROM web_log_feed_podcast"
use! podcastRdr = cmd.ExecuteReaderAsync () use! podcastRdr = cmd.ExecuteReaderAsync()
let podcasts = let podcasts =
seq { seq {
while podcastRdr.Read () do while podcastRdr.Read() do
CustomFeedId (Map.getString "feed_id" podcastRdr), CustomFeedId (Map.getString "feed_id" podcastRdr),
{ Title = Map.getString "title" podcastRdr { Title = Map.getString "title" podcastRdr
Subtitle = Map.tryString "subtitle" podcastRdr Subtitle = Map.tryString "subtitle" podcastRdr
ItemsInFeed = Map.getInt "items_in_feed" podcastRdr ItemsInFeed = Map.getInt "items_in_feed" podcastRdr
Summary = Map.getString "summary" podcastRdr Summary = Map.getString "summary" podcastRdr
DisplayedAuthor = Map.getString "displayed_author" podcastRdr DisplayedAuthor = Map.getString "displayed_author" podcastRdr
Email = Map.getString "email" podcastRdr Email = Map.getString "email" podcastRdr
ImageUrl = Map.getString "image_url" podcastRdr |> Permalink ImageUrl = Map.getString "image_url" podcastRdr |> Permalink
AppleCategory = Map.getString "apple_category" podcastRdr AppleCategory = Map.getString "apple_category" podcastRdr
AppleSubcategory = Map.tryString "apple_subcategory" podcastRdr AppleSubcategory = Map.tryString "apple_subcategory" podcastRdr
Explicit = Map.getString "explicit" podcastRdr |> ExplicitRating.Parse Explicit = Map.getString "explicit" podcastRdr |> ExplicitRating.Parse
DefaultMediaType = Map.tryString "default_media_type" podcastRdr DefaultMediaType = Map.tryString "default_media_type" podcastRdr
MediaBaseUrl = Map.tryString "media_base_url" podcastRdr MediaBaseUrl = Map.tryString "media_base_url" podcastRdr
PodcastGuid = Map.tryGuid "podcast_guid" podcastRdr PodcastGuid = Map.tryGuid "podcast_guid" podcastRdr
FundingUrl = Map.tryString "funding_url" podcastRdr FundingUrl = Map.tryString "funding_url" podcastRdr
FundingText = Map.tryString "funding_text" podcastRdr FundingText = Map.tryString "funding_text" podcastRdr
Medium = Map.tryString "medium" podcastRdr Medium = Map.tryString "medium" podcastRdr
|> Option.map PodcastMedium.Parse |> Option.map PodcastMedium.Parse }
}
} |> List.ofSeq } |> List.ofSeq
podcastRdr.Close () podcastRdr.Close()
podcasts podcasts
|> List.iter (fun (feedId, podcast) -> |> List.iter (fun (feedId, podcast) ->
cmd.CommandText <- "UPDATE web_log_feed SET podcast = @podcast WHERE id = @id" cmd.CommandText <- "UPDATE web_log_feed SET podcast = @podcast WHERE id = @id"
[ cmd.Parameters.AddWithValue ("@podcast", Utils.serialize ser podcast) [ cmd.Parameters.AddWithValue("@podcast", Utils.serialize ser podcast)
cmd.Parameters.AddWithValue ("@id", string feedId) ] |> ignore cmd.Parameters.AddWithValue("@id", string feedId) ] |> ignore
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
cmd.Parameters.Clear ()) cmd.Parameters.Clear())
cmd.CommandText <- "SELECT * FROM post_episode" cmd.CommandText <- "SELECT * FROM post_episode"
use! epRdr = cmd.ExecuteReaderAsync () use! epRdr = cmd.ExecuteReaderAsync()
let episodes = let episodes =
seq { seq {
while epRdr.Read () do while epRdr.Read() do
PostId (Map.getString "post_id" epRdr), PostId (Map.getString "post_id" epRdr),
{ Media = Map.getString "media" epRdr { Media = Map.getString "media" epRdr
Length = Map.getLong "length" epRdr Length = Map.getLong "length" epRdr
Duration = Map.tryTimeSpan "duration" epRdr Duration = Map.tryTimeSpan "duration" epRdr
|> Option.map Duration.FromTimeSpan |> Option.map Duration.FromTimeSpan
MediaType = Map.tryString "media_type" epRdr MediaType = Map.tryString "media_type" epRdr
ImageUrl = Map.tryString "image_url" epRdr ImageUrl = Map.tryString "image_url" epRdr
Subtitle = Map.tryString "subtitle" epRdr Subtitle = Map.tryString "subtitle" epRdr
Explicit = Map.tryString "explicit" epRdr Explicit = Map.tryString "explicit" epRdr
|> Option.map ExplicitRating.Parse |> Option.map ExplicitRating.Parse
Chapters = Map.tryString "chapters" epRdr Chapters = Map.tryString "chapters" epRdr
|> Option.map (Utils.deserialize<Chapter list> ser) |> Option.map (Utils.deserialize<Chapter list> ser)
ChapterFile = Map.tryString "chapter_file" epRdr ChapterFile = Map.tryString "chapter_file" epRdr
ChapterType = Map.tryString "chapter_type" epRdr ChapterType = Map.tryString "chapter_type" epRdr
TranscriptUrl = Map.tryString "transcript_url" epRdr TranscriptUrl = Map.tryString "transcript_url" epRdr
TranscriptType = Map.tryString "transcript_type" epRdr TranscriptType = Map.tryString "transcript_type" epRdr
TranscriptLang = Map.tryString "transcript_lang" epRdr TranscriptLang = Map.tryString "transcript_lang" epRdr
TranscriptCaptions = Map.tryBoolean "transcript_captions" epRdr TranscriptCaptions = Map.tryBoolean "transcript_captions" epRdr
SeasonNumber = Map.tryInt "season_number" epRdr SeasonNumber = Map.tryInt "season_number" epRdr
SeasonDescription = Map.tryString "season_description" epRdr SeasonDescription = Map.tryString "season_description" epRdr
EpisodeNumber = Map.tryString "episode_number" epRdr EpisodeNumber = Map.tryString "episode_number" epRdr
|> Option.map System.Double.Parse |> Option.map System.Double.Parse
EpisodeDescription = Map.tryString "episode_description" epRdr EpisodeDescription = Map.tryString "episode_description" epRdr }
}
} |> List.ofSeq } |> List.ofSeq
epRdr.Close () epRdr.Close()
episodes episodes
|> List.iter (fun (postId, episode) -> |> List.iter (fun (postId, episode) ->
cmd.CommandText <- "UPDATE post SET episode = @episode WHERE id = @id" cmd.CommandText <- "UPDATE post SET episode = @episode WHERE id = @id"
[ cmd.Parameters.AddWithValue ("@episode", Utils.serialize ser episode) [ cmd.Parameters.AddWithValue("@episode", Utils.serialize ser episode)
cmd.Parameters.AddWithValue ("@id", string postId) ] |> ignore cmd.Parameters.AddWithValue("@id", string postId) ] |> ignore
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
cmd.Parameters.Clear ()) cmd.Parameters.Clear())
logStep "Migrating dates/times" logStep "Migrating dates/times"
let inst (dt : System.DateTime) = let inst (dt: System.DateTime) =
System.DateTime (dt.Ticks, System.DateTimeKind.Utc) System.DateTime(dt.Ticks, System.DateTimeKind.Utc)
|> (Instant.FromDateTimeUtc >> Noda.toSecondsPrecision) |> (Instant.FromDateTimeUtc >> Noda.toSecondsPrecision)
// page.updated_on, page.published_on // page.updated_on, page.published_on
cmd.CommandText <- "SELECT id, updated_on, published_on FROM page" cmd.CommandText <- "SELECT id, updated_on, published_on FROM page"
use! pageRdr = cmd.ExecuteReaderAsync () use! pageRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while pageRdr.Read () do while pageRdr.Read() do
Map.getString "id" pageRdr, Map.getString "id" pageRdr,
inst (Map.getDateTime "updated_on" pageRdr), inst (Map.getDateTime "updated_on" pageRdr),
inst (Map.getDateTime "published_on" pageRdr) inst (Map.getDateTime "published_on" pageRdr)
} |> List.ofSeq } |> List.ofSeq
pageRdr.Close () pageRdr.Close()
cmd.CommandText <- "UPDATE page SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id" cmd.CommandText <- "UPDATE page SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id"
[ cmd.Parameters.Add ("@id", SqliteType.Text) [ cmd.Parameters.Add("@id", SqliteType.Text)
cmd.Parameters.Add ("@updatedOn", SqliteType.Text) cmd.Parameters.Add("@updatedOn", SqliteType.Text)
cmd.Parameters.Add ("@publishedOn", SqliteType.Text) cmd.Parameters.Add("@publishedOn", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (pageId, updatedOn, publishedOn) -> |> List.iter (fun (pageId, updatedOn, publishedOn) ->
cmd.Parameters["@id" ].Value <- pageId cmd.Parameters["@id" ].Value <- pageId
cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn
cmd.Parameters["@publishedOn"].Value <- instantParam publishedOn cmd.Parameters["@publishedOn"].Value <- instantParam publishedOn
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// page_revision.as_of // page_revision.as_of
cmd.CommandText <- "SELECT * FROM page_revision" cmd.CommandText <- "SELECT * FROM page_revision"
use! pageRevRdr = cmd.ExecuteReaderAsync () use! pageRevRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while pageRevRdr.Read () do while pageRevRdr.Read() do
let asOf = Map.getDateTime "as_of" pageRevRdr let asOf = Map.getDateTime "as_of" pageRevRdr
Map.getString "page_id" pageRevRdr, asOf, inst asOf, Map.getString "revision_text" pageRevRdr Map.getString "page_id" pageRevRdr, asOf, inst asOf, Map.getString "revision_text" pageRevRdr
} |> List.ofSeq } |> List.ofSeq
@ -286,141 +271,135 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
cmd.CommandText <- cmd.CommandText <-
"DELETE FROM page_revision WHERE page_id = @pageId AND as_of = @oldAsOf; "DELETE FROM page_revision WHERE page_id = @pageId AND as_of = @oldAsOf;
INSERT INTO page_revision (page_id, as_of, revision_text) VALUES (@pageId, @asOf, @text)" INSERT INTO page_revision (page_id, as_of, revision_text) VALUES (@pageId, @asOf, @text)"
[ cmd.Parameters.Add ("@pageId", SqliteType.Text) [ cmd.Parameters.Add("@pageId", SqliteType.Text)
cmd.Parameters.Add ("@oldAsOf", SqliteType.Text) cmd.Parameters.Add("@oldAsOf", SqliteType.Text)
cmd.Parameters.Add ("@asOf", SqliteType.Text) cmd.Parameters.Add("@asOf", SqliteType.Text)
cmd.Parameters.Add ("@text", SqliteType.Text) cmd.Parameters.Add("@text", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (pageId, oldAsOf, asOf, text) -> |> List.iter (fun (pageId, oldAsOf, asOf, text) ->
cmd.Parameters["@pageId" ].Value <- pageId cmd.Parameters["@pageId" ].Value <- pageId
cmd.Parameters["@oldAsOf"].Value <- oldAsOf cmd.Parameters["@oldAsOf"].Value <- oldAsOf
cmd.Parameters["@asOf" ].Value <- instantParam asOf cmd.Parameters["@asOf" ].Value <- instantParam asOf
cmd.Parameters["@text" ].Value <- text cmd.Parameters["@text" ].Value <- text
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// post.updated_on, post.published_on (opt) // post.updated_on, post.published_on (opt)
cmd.CommandText <- "SELECT id, updated_on, published_on FROM post" cmd.CommandText <- "SELECT id, updated_on, published_on FROM post"
use! postRdr = cmd.ExecuteReaderAsync () use! postRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while postRdr.Read () do while postRdr.Read() do
Map.getString "id" postRdr, Map.getString "id" postRdr,
inst (Map.getDateTime "updated_on" postRdr), inst (Map.getDateTime "updated_on" postRdr),
(Map.tryDateTime "published_on" postRdr |> Option.map inst) (Map.tryDateTime "published_on" postRdr |> Option.map inst)
} |> List.ofSeq } |> List.ofSeq
postRdr.Close () postRdr.Close()
cmd.CommandText <- "UPDATE post SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id" cmd.CommandText <- "UPDATE post SET updated_on = @updatedOn, published_on = @publishedOn WHERE id = @id"
[ cmd.Parameters.Add ("@id", SqliteType.Text) [ cmd.Parameters.Add("@id", SqliteType.Text)
cmd.Parameters.Add ("@updatedOn", SqliteType.Text) cmd.Parameters.Add("@updatedOn", SqliteType.Text)
cmd.Parameters.Add ("@publishedOn", SqliteType.Text) cmd.Parameters.Add("@publishedOn", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (postId, updatedOn, publishedOn) -> |> List.iter (fun (postId, updatedOn, publishedOn) ->
cmd.Parameters["@id" ].Value <- postId cmd.Parameters["@id" ].Value <- postId
cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn cmd.Parameters["@updatedOn" ].Value <- instantParam updatedOn
cmd.Parameters["@publishedOn"].Value <- maybeInstant publishedOn cmd.Parameters["@publishedOn"].Value <- maybeInstant publishedOn
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// post_revision.as_of // post_revision.as_of
cmd.CommandText <- "SELECT * FROM post_revision" cmd.CommandText <- "SELECT * FROM post_revision"
use! postRevRdr = cmd.ExecuteReaderAsync () use! postRevRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while postRevRdr.Read () do while postRevRdr.Read() do
let asOf = Map.getDateTime "as_of" postRevRdr let asOf = Map.getDateTime "as_of" postRevRdr
Map.getString "post_id" postRevRdr, asOf, inst asOf, Map.getString "revision_text" postRevRdr Map.getString "post_id" postRevRdr, asOf, inst asOf, Map.getString "revision_text" postRevRdr
} |> List.ofSeq } |> List.ofSeq
postRevRdr.Close () postRevRdr.Close()
cmd.CommandText <- cmd.CommandText <-
"DELETE FROM post_revision WHERE post_id = @postId AND as_of = @oldAsOf; "DELETE FROM post_revision WHERE post_id = @postId AND as_of = @oldAsOf;
INSERT INTO post_revision (post_id, as_of, revision_text) VALUES (@postId, @asOf, @text)" INSERT INTO post_revision (post_id, as_of, revision_text) VALUES (@postId, @asOf, @text)"
[ cmd.Parameters.Add ("@postId", SqliteType.Text) [ cmd.Parameters.Add("@postId", SqliteType.Text)
cmd.Parameters.Add ("@oldAsOf", SqliteType.Text) cmd.Parameters.Add("@oldAsOf", SqliteType.Text)
cmd.Parameters.Add ("@asOf", SqliteType.Text) cmd.Parameters.Add("@asOf", SqliteType.Text)
cmd.Parameters.Add ("@text", SqliteType.Text) cmd.Parameters.Add("@text", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (postId, oldAsOf, asOf, text) -> |> List.iter (fun (postId, oldAsOf, asOf, text) ->
cmd.Parameters["@postId" ].Value <- postId cmd.Parameters["@postId" ].Value <- postId
cmd.Parameters["@oldAsOf"].Value <- oldAsOf cmd.Parameters["@oldAsOf"].Value <- oldAsOf
cmd.Parameters["@asOf" ].Value <- instantParam asOf cmd.Parameters["@asOf" ].Value <- instantParam asOf
cmd.Parameters["@text" ].Value <- text cmd.Parameters["@text" ].Value <- text
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// theme_asset.updated_on // theme_asset.updated_on
cmd.CommandText <- "SELECT theme_id, path, updated_on FROM theme_asset" cmd.CommandText <- "SELECT theme_id, path, updated_on FROM theme_asset"
use! assetRdr = cmd.ExecuteReaderAsync () use! assetRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while assetRdr.Read () do while assetRdr.Read() do
Map.getString "theme_id" assetRdr, Map.getString "path" assetRdr, Map.getString "theme_id" assetRdr, Map.getString "path" assetRdr,
inst (Map.getDateTime "updated_on" assetRdr) inst (Map.getDateTime "updated_on" assetRdr)
} |> List.ofSeq } |> List.ofSeq
assetRdr.Close () assetRdr.Close ()
cmd.CommandText <- "UPDATE theme_asset SET updated_on = @updatedOn WHERE theme_id = @themeId AND path = @path" cmd.CommandText <- "UPDATE theme_asset SET updated_on = @updatedOn WHERE theme_id = @themeId AND path = @path"
[ cmd.Parameters.Add ("@updatedOn", SqliteType.Text) [ cmd.Parameters.Add("@updatedOn", SqliteType.Text)
cmd.Parameters.Add ("@themeId", SqliteType.Text) cmd.Parameters.Add("@themeId", SqliteType.Text)
cmd.Parameters.Add ("@path", SqliteType.Text) cmd.Parameters.Add("@path", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (themeId, path, updatedOn) -> |> List.iter (fun (themeId, path, updatedOn) ->
cmd.Parameters["@themeId" ].Value <- themeId cmd.Parameters["@themeId" ].Value <- themeId
cmd.Parameters["@path" ].Value <- path cmd.Parameters["@path" ].Value <- path
cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// upload.updated_on // upload.updated_on
cmd.CommandText <- "SELECT id, updated_on FROM upload" cmd.CommandText <- "SELECT id, updated_on FROM upload"
use! upRdr = cmd.ExecuteReaderAsync () use! upRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while upRdr.Read () do while upRdr.Read() do
Map.getString "id" upRdr, inst (Map.getDateTime "updated_on" upRdr) Map.getString "id" upRdr, inst (Map.getDateTime "updated_on" upRdr)
} |> List.ofSeq } |> List.ofSeq
upRdr.Close () upRdr.Close ()
cmd.CommandText <- "UPDATE upload SET updated_on = @updatedOn WHERE id = @id" cmd.CommandText <- "UPDATE upload SET updated_on = @updatedOn WHERE id = @id"
[ cmd.Parameters.Add ("@updatedOn", SqliteType.Text) [ cmd.Parameters.Add("@updatedOn", SqliteType.Text)
cmd.Parameters.Add ("@id", SqliteType.Text) cmd.Parameters.Add("@id", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (upId, updatedOn) -> |> List.iter (fun (upId, updatedOn) ->
cmd.Parameters["@id" ].Value <- upId cmd.Parameters["@id" ].Value <- upId
cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn cmd.Parameters["@updatedOn"].Value <- instantParam updatedOn
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
// web_log_user.created_on, web_log_user.last_seen_on (opt) // web_log_user.created_on, web_log_user.last_seen_on (opt)
cmd.CommandText <- "SELECT id, created_on, last_seen_on FROM web_log_user" cmd.CommandText <- "SELECT id, created_on, last_seen_on FROM web_log_user"
use! userRdr = cmd.ExecuteReaderAsync () use! userRdr = cmd.ExecuteReaderAsync()
let toUpdate = let toUpdate =
seq { seq {
while userRdr.Read () do while userRdr.Read() do
Map.getString "id" userRdr, Map.getString "id" userRdr,
inst (Map.getDateTime "created_on" userRdr), inst (Map.getDateTime "created_on" userRdr),
(Map.tryDateTime "last_seen_on" userRdr |> Option.map inst) (Map.tryDateTime "last_seen_on" userRdr |> Option.map inst)
} |> List.ofSeq } |> List.ofSeq
userRdr.Close () userRdr.Close()
cmd.CommandText <- "UPDATE web_log_user SET created_on = @createdOn, last_seen_on = @lastSeenOn WHERE id = @id" cmd.CommandText <- "UPDATE web_log_user SET created_on = @createdOn, last_seen_on = @lastSeenOn WHERE id = @id"
[ cmd.Parameters.Add ("@id", SqliteType.Text) [ cmd.Parameters.Add("@id", SqliteType.Text)
cmd.Parameters.Add ("@createdOn", SqliteType.Text) cmd.Parameters.Add("@createdOn", SqliteType.Text)
cmd.Parameters.Add ("@lastSeenOn", SqliteType.Text) cmd.Parameters.Add("@lastSeenOn", SqliteType.Text) ] |> ignore
] |> ignore
toUpdate toUpdate
|> List.iter (fun (userId, createdOn, lastSeenOn) -> |> List.iter (fun (userId, createdOn, lastSeenOn) ->
cmd.Parameters["@id" ].Value <- userId cmd.Parameters["@id" ].Value <- userId
cmd.Parameters["@createdOn" ].Value <- instantParam createdOn cmd.Parameters["@createdOn" ].Value <- instantParam createdOn
cmd.Parameters["@lastSeenOn"].Value <- maybeInstant lastSeenOn cmd.Parameters["@lastSeenOn"].Value <- maybeInstant lastSeenOn
let _ = cmd.ExecuteNonQuery () let _ = cmd.ExecuteNonQuery()
()) ())
cmd.Parameters.Clear () cmd.Parameters.Clear()
conn.Close () conn.Close()
conn.Open () conn.Open()
logStep "Dropping old tables and columns" logStep "Dropping old tables and columns"
cmd.CommandText <- cmd.CommandText <-
@ -444,7 +423,7 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
/// Migrate from v2 to v2.1 /// Migrate from v2 to v2.1
let migrateV2ToV2point1 () = backgroundTask { let migrateV2ToV2point1 () = backgroundTask {
Utils.logMigrationStep log "v2 to v2.1" "Adding redirect rules to web_log table" Utils.logMigrationStep log "v2 to v2.1" "Adding redirect rules to web_log table"
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
cmd.CommandText <- "ALTER TABLE web_log ADD COLUMN redirect_rules TEXT NOT NULL DEFAULT '[]'" cmd.CommandText <- "ALTER TABLE web_log ADD COLUMN redirect_rules TEXT NOT NULL DEFAULT '[]'"
do! write cmd do! write cmd
@ -477,17 +456,17 @@ type SQLiteData (conn : SqliteConnection, log : ILogger<SQLiteData>, ser : JsonS
member _.Conn = conn member _.Conn = conn
/// Make a SQLite connection ready to execute commends /// Make a SQLite connection ready to execute commends
static member setUpConnection (conn : SqliteConnection) = backgroundTask { static member setUpConnection (conn: SqliteConnection) = backgroundTask {
do! conn.OpenAsync () do! conn.OpenAsync()
use cmd = conn.CreateCommand () use cmd = conn.CreateCommand()
cmd.CommandText <- "PRAGMA foreign_keys = TRUE" cmd.CommandText <- "PRAGMA foreign_keys = TRUE"
let! _ = cmd.ExecuteNonQueryAsync () let! _ = cmd.ExecuteNonQueryAsync()
() ()
} }
interface IData with interface IData with
member _.Category = SQLiteCategoryData conn member _.Category = SQLiteCategoryData (conn, ser)
member _.Page = SQLitePageData (conn, ser) member _.Page = SQLitePageData (conn, ser)
member _.Post = SQLitePostData (conn, ser) member _.Post = SQLitePostData (conn, ser)
member _.TagMap = SQLiteTagMapData conn member _.TagMap = SQLiteTagMapData conn