From 21957330feeb5a9f40795e4f67eb27387a7e3150 Mon Sep 17 00:00:00 2001 From: "Daniel J. Summers" Date: Sat, 27 Aug 2022 22:34:51 -0400 Subject: [PATCH] Migration to pg docs complete --- .gitignore | 1 + src/JobsJobsJobs/Domain/SharedTypes.fs | 24 +- src/JobsJobsJobs/Domain/Types.fs | 221 +++++++++--------- src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs | 208 +++++++++-------- .../JobsJobsJobs.Data.fsproj | 3 +- src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs | 5 + .../JobsJobsJobs.V3Migration/Program.fs | 156 +++++++++++-- src/JobsJobsJobs/Server/Auth.fs | 2 +- src/JobsJobsJobs/Server/Handlers.fs | 118 +++++----- 9 files changed, 442 insertions(+), 296 deletions(-) diff --git a/.gitignore b/.gitignore index 2d0cfe8..16bf782 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ src/**/obj src/**/appsettings.*.json src/.vs src/.idea +src/JobsJobsJobs/JobsJobsJobs.V3Migration/appsettings.json diff --git a/src/JobsJobsJobs/Domain/SharedTypes.fs b/src/JobsJobsJobs/Domain/SharedTypes.fs index 649f7bf..a2c714e 100644 --- a/src/JobsJobsJobs/Domain/SharedTypes.fs +++ b/src/JobsJobsJobs/Domain/SharedTypes.fs @@ -203,20 +203,20 @@ module ProfileForm = /// Create an instance of this form from the given profile let fromProfile (profile : Profile) = - { isSeekingEmployment = profile.seekingEmployment - isPublic = profile.isPublic + { isSeekingEmployment = profile.IsSeekingEmployment + isPublic = profile.IsPubliclySearchable realName = "" - continentId = string profile.continentId - region = profile.region - remoteWork = profile.remoteWork - fullTime = profile.fullTime - biography = MarkdownString.toString profile.biography - experience = profile.experience |> Option.map MarkdownString.toString - skills = profile.skills + continentId = string profile.ContinentId + region = profile.Region + remoteWork = profile.IsRemote + fullTime = profile.IsFullTime + biography = MarkdownString.toString profile.Biography + experience = profile.Experience |> Option.map MarkdownString.toString + skills = profile.Skills |> List.map (fun s -> - { id = string s.id - description = s.description - notes = s.notes + { id = string s.Id + description = s.Description + notes = s.Notes }) } diff --git a/src/JobsJobsJobs/Domain/Types.fs b/src/JobsJobsJobs/Domain/Types.fs index eab5d3c..52cab1c 100644 --- a/src/JobsJobsJobs/Domain/Types.fs +++ b/src/JobsJobsJobs/Domain/Types.fs @@ -9,139 +9,140 @@ open System [] type Citizen = { /// The ID of the user - id : CitizenId + Id : CitizenId /// When the user joined Jobs, Jobs, Jobs - joinedOn : Instant + JoinedOn : Instant /// When the user last logged in - lastSeenOn : Instant + LastSeenOn : Instant /// The user's e-mail address - email : string + Email : string /// The user's first name - firstName : string + FirstName : string /// The user's last name - lastName : string + LastName : string /// The hash of the user's password - passwordHash : string + PasswordHash : string /// The name displayed for this user throughout the site - displayName : string option + DisplayName : string option /// The other contacts for this user - otherContacts : OtherContact list + OtherContacts : OtherContact list /// Whether this is a legacy citizen - isLegacy : bool + IsLegacy : bool } /// Support functions for citizens module Citizen = /// An empty citizen - let empty = - { id = CitizenId Guid.Empty - joinedOn = Instant.MinValue - lastSeenOn = Instant.MinValue - email = "" - firstName = "" - lastName = "" - passwordHash = "" - displayName = None - otherContacts = [] - isLegacy = false - } + let empty = { + Id = CitizenId Guid.Empty + JoinedOn = Instant.MinValue + LastSeenOn = Instant.MinValue + Email = "" + FirstName = "" + LastName = "" + PasswordHash = "" + DisplayName = None + OtherContacts = [] + IsLegacy = false + } + /// Get the name of the citizen (either their preferred display name or first/last names) let name x = - match x.displayName with Some it -> it | None -> $"{x.firstName} {x.lastName}" + match x.DisplayName with Some it -> it | None -> $"{x.FirstName} {x.LastName}" /// A continent [] type Continent = { /// The ID of the continent - id : ContinentId + Id : ContinentId /// The name of the continent - name : string + Name : string } /// Support functions for continents module Continent = /// An empty continent - let empty = - { id = ContinentId Guid.Empty - name = "" - } + let empty ={ + Id = ContinentId Guid.Empty + Name = "" + } /// A job listing [] type Listing = { /// The ID of the job listing - id : ListingId + Id : ListingId /// The ID of the citizen who posted the job listing - citizenId : CitizenId + CitizenId : CitizenId /// When this job listing was created - createdOn : Instant + CreatedOn : Instant /// The short title of the job listing - title : string + Title : string /// The ID of the continent on which the job is located - continentId : ContinentId + ContinentId : ContinentId /// The region in which the job is located - region : string + Region : string /// Whether this listing is for remote work - remoteWork : bool + IsRemote : bool /// Whether this listing has expired - isExpired : bool + IsExpired : bool /// When this listing was last updated - updatedOn : Instant + UpdatedOn : Instant /// The details of this job - text : MarkdownString + Text : MarkdownString /// When this job needs to be filled - neededBy : LocalDate option + NeededBy : LocalDate option /// Was this job filled as part of its appearance on Jobs, Jobs, Jobs? - wasFilledHere : bool option + WasFilledHere : bool option /// Whether this is a legacy listing - isLegacy : bool + IsLegacy : bool } /// Support functions for job listings module Listing = /// An empty job listing - let empty = - { id = ListingId Guid.Empty - citizenId = CitizenId Guid.Empty - createdOn = Instant.MinValue - title = "" - continentId = ContinentId Guid.Empty - region = "" - remoteWork = false - isExpired = false - updatedOn = Instant.MinValue - text = Text "" - neededBy = None - wasFilledHere = None - isLegacy = false - } + let empty = { + Id = ListingId Guid.Empty + CitizenId = CitizenId Guid.Empty + CreatedOn = Instant.MinValue + Title = "" + ContinentId = ContinentId Guid.Empty + Region = "" + IsRemote = false + IsExpired = false + UpdatedOn = Instant.MinValue + Text = Text "" + NeededBy = None + WasFilledHere = None + IsLegacy = false + } /// Security settings for a user @@ -169,26 +170,26 @@ type SecurityInfo = module SecurityInfo = /// An empty set of security info - let empty = - { Id = CitizenId Guid.Empty - FailedLogOnAttempts = 0 - AccountLocked = false - Token = None - TokenUsage = None - TokenExpires = None - } + let empty = { + Id = CitizenId Guid.Empty + FailedLogOnAttempts = 0 + AccountLocked = false + Token = None + TokenUsage = None + TokenExpires = None + } /// A skill the job seeker possesses type Skill = { /// The ID of the skill - id : SkillId + Id : SkillId /// A description of the skill - description : string + Description : string /// Notes regarding this skill (level, duration, etc.) - notes : string option + Notes : string option } @@ -196,97 +197,97 @@ type Skill = [] type Profile = { /// The ID of the citizen to whom this profile belongs - id : CitizenId + Id : CitizenId /// Whether this citizen is actively seeking employment - seekingEmployment : bool + IsSeekingEmployment : bool /// Whether this citizen allows their profile to be a part of the publicly-viewable, anonymous data - isPublic : bool + IsPubliclySearchable : bool /// Whether this citizen allows their profile to be viewed via a public link - isPublicLinkable : bool + IsPubliclyLinkable : bool /// The ID of the continent on which the citizen resides - continentId : ContinentId + ContinentId : ContinentId /// The region in which the citizen resides - region : string + Region : string /// Whether the citizen is looking for remote work - remoteWork : bool + IsRemote : bool /// Whether the citizen is looking for full-time work - fullTime : bool + IsFullTime : bool /// The citizen's professional biography - biography : MarkdownString + Biography : MarkdownString /// When the citizen last updated their profile - lastUpdatedOn : Instant + LastUpdatedOn : Instant /// The citizen's experience (topical / chronological) - experience : MarkdownString option + Experience : MarkdownString option /// Skills this citizen possesses - skills : Skill list + Skills : Skill list /// Whether this is a legacy profile - isLegacy : bool + IsLegacy : bool } /// Support functions for Profiles module Profile = // An empty profile - let empty = - { id = CitizenId Guid.Empty - seekingEmployment = false - isPublic = false - isPublicLinkable = false - continentId = ContinentId Guid.Empty - region = "" - remoteWork = false - fullTime = false - biography = Text "" - lastUpdatedOn = Instant.MinValue - experience = None - skills = [] - isLegacy = false - } + let empty = { + Id = CitizenId Guid.Empty + IsSeekingEmployment = false + IsPubliclySearchable = false + IsPubliclyLinkable = false + ContinentId = ContinentId Guid.Empty + Region = "" + IsRemote = false + IsFullTime = false + Biography = Text "" + LastUpdatedOn = Instant.MinValue + Experience = None + Skills = [] + IsLegacy = false + } /// A record of success finding employment [] type Success = { /// The ID of the success report - id : SuccessId + Id : SuccessId /// The ID of the citizen who wrote this success report - citizenId : CitizenId + CitizenId : CitizenId /// When this success report was recorded - recordedOn : Instant + RecordedOn : Instant /// Whether the success was due, at least in part, to Jobs, Jobs, Jobs - fromHere : bool + IsFromHere : bool /// The source of this success (listing or profile) - source : string + Source : string /// The success story - story : MarkdownString option + Story : MarkdownString option } /// Support functions for success stories module Success = /// An empty success story - let empty = - { id = SuccessId Guid.Empty - citizenId = CitizenId Guid.Empty - recordedOn = Instant.MinValue - fromHere = false - source = "" - story = None - } + let empty = { + Id = SuccessId Guid.Empty + CitizenId = CitizenId Guid.Empty + RecordedOn = Instant.MinValue + IsFromHere = false + Source = "" + Story = None + } diff --git a/src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs b/src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs index b8b645e..083d1c6 100644 --- a/src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs +++ b/src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs @@ -5,27 +5,27 @@ module Table = /// Citizens [] - let Citizen = "citizen" + let Citizen = "jjj.citizen" /// Continents [] - let Continent = "continent" + let Continent = "jjj.continent" /// Job Listings [] - let Listing = "listing" + let Listing = "jjj.listing" /// Employment Profiles [] - let Profile = "profile" + let Profile = "jjj.profile" /// User Security Information [] - let SecurityInfo = "security_info" + let SecurityInfo = "jjj.security_info" /// Success Stories [] - let Success = "success" + let Success = "jjj.success" open Npgsql.FSharp @@ -46,15 +46,25 @@ module DataConnection = /// Create tables let private createTables () = backgroundTask { - let sql = - [ Table.Citizen; Table.Continent; Table.Listing; Table.Profile; Table.SecurityInfo; Table.Success ] - |> List.map (fun table -> - $"CREATE TABLE IF NOT EXISTS jjj.{table} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)") - |> String.concat "; " + let sql = [ + $"CREATE SCHEMA IF NOT EXISTS jjj" + $"CREATE TABLE IF NOT EXISTS {Table.Citizen} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)" + $"CREATE TABLE IF NOT EXISTS {Table.Continent} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)" + $"CREATE TABLE IF NOT EXISTS {Table.Listing} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)" + $"CREATE TABLE IF NOT EXISTS {Table.Profile} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL, + CONSTRAINT fk_profile_citizen FOREIGN KEY (id) REFERENCES {Table.Citizen} (id) ON DELETE CASCADE)" + $"CREATE TABLE IF NOT EXISTS {Table.SecurityInfo} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL, + CONSTRAINT fk_security_info_citizen FOREIGN KEY (id) REFERENCES {Table.Citizen} (id) ON DELETE CASCADE)" + $"CREATE TABLE IF NOT EXISTS {Table.Success} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)" + $"CREATE INDEX IF NOT EXISTS idx_citizen_email ON {Table.Citizen} USING GIN ((data -> 'email'))" + $"CREATE INDEX IF NOT EXISTS idx_listing_citizen ON {Table.Listing} USING GIN ((data -> 'citizenId'))" + $"CREATE INDEX IF NOT EXISTS idx_listing_continent ON {Table.Listing} USING GIN ((data -> 'continentId'))" + $"CREATE INDEX IF NOT EXISTS idx_profile_continent ON {Table.Profile} USING GIN ((data -> 'continentId'))" + $"CREATE INDEX IF NOT EXISTS idx_success_citizen ON {Table.Success} USING GIN ((data -> 'citizenId'))" + ] let! _ = connection () - |> Sql.executeTransactionAsync [ sql, [ [] ] ] - // TODO: prudent indexes + |> Sql.executeTransactionAsync (sql |> List.map (fun sql -> sql, [ [] ])) () } @@ -84,22 +94,26 @@ module private Helpers = /// Get a document let getDocument<'T> table docId sqlProps : Task<'T option> = backgroundTask { let! doc = - Sql.query $"SELECT * FROM jjj.%s{table} where id = @id" sqlProps + Sql.query $"SELECT * FROM %s{table} where id = @id" sqlProps |> Sql.parameters [ "@id", Sql.string docId ] |> Sql.executeAsync toDocument return List.tryHead doc } + /// Serialize a document to JSON + let mkDoc<'T> (doc : 'T) = + JsonSerializer.Serialize<'T> (doc, Json.options) + /// Save a document - let saveDocument<'T> table docId (doc : 'T) sqlProps = backgroundTask { + let saveDocument table docId sqlProps doc = backgroundTask { let! _ = Sql.query - $"INSERT INTO jjj.%s{table} (id, data) VALUES (@id, @data) - ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data" + $"INSERT INTO %s{table} (id, data) VALUES (@id, @data) + ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data" sqlProps |> Sql.parameters - [ "@id", Sql.string docId - "@data", Sql.jsonb (JsonSerializer.Serialize (doc, Json.options)) ] + [ "@id", Sql.string docId + "@data", Sql.jsonb doc ] |> Sql.executeNonQueryAsync () } @@ -128,59 +142,60 @@ module Citizens = let deleteById citizenId = backgroundTask { let! _ = connection () - |> Sql.executeTransactionAsync [ - "DELETE FROM jjj.success WHERE data->>'citizenId' = @id; - DELETE FROM jjj.listing WHERE data->>'citizenId' = @id; - DELETE FROM jjj.profile WHERE id = @id; - DELETE FROM jjj.security_info WHERE id = @id; - DELETE FROM jjj.citizen WHERE id = @id", - [ [ "@id", Sql.string (CitizenId.toString citizenId) ] ] - ] + |> Sql.query $" + DELETE FROM {Table.Success} WHERE data ->> 'citizenId' = @id; + DELETE FROM {Table.Listing} WHERE data ->> 'citizenId' = @id; + DELETE FROM {Table.Citizen} WHERE id = @id" + |> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ] + |> Sql.executeNonQueryAsync () } /// Find a citizen by their ID let findById citizenId = backgroundTask { match! connection () |> getDocument Table.Citizen (CitizenId.toString citizenId) with - | Some c when not c.isLegacy -> return Some c + | Some c when not c.IsLegacy -> return Some c | Some _ | None -> return None } /// Save a citizen let save (citizen : Citizen) = - connection () |> saveDocument Table.Citizen (CitizenId.toString citizen.id) citizen + connection () |> saveDocument Table.Citizen (CitizenId.toString citizen.Id) <| mkDoc citizen /// Attempt a user log on let tryLogOn email (pwCheck : string -> bool) now = backgroundTask { let connProps = connection () let! tryCitizen = connProps - |> Sql.query $"SELECT * FROM jjj.{Table.Citizen} WHERE data->>email = @email AND data->>isValue <> 'true'" + |> Sql.query $" + SELECT * + FROM {Table.Citizen} + WHERE data ->> 'email' = @email + AND data ->> 'isLegacy' = 'false'" |> Sql.parameters [ "@email", Sql.string email ] |> Sql.executeAsync toDocument match List.tryHead tryCitizen with | Some citizen -> - let citizenId = CitizenId.toString citizen.id + let citizenId = CitizenId.toString citizen.Id let! tryInfo = getDocument Table.SecurityInfo citizenId connProps let! info = backgroundTask { match tryInfo with | Some it -> return it | None -> - let it = { SecurityInfo.empty with Id = citizen.id } - do! saveDocument Table.SecurityInfo citizenId it connProps + let it = { SecurityInfo.empty with Id = citizen.Id } + do! saveDocument Table.SecurityInfo citizenId connProps (mkDoc it) return it } if info.AccountLocked then return Error "Log on unsuccessful (Account Locked)" - elif pwCheck citizen.passwordHash then - do! saveDocument Table.SecurityInfo citizenId { info with FailedLogOnAttempts = 0 } connProps - do! saveDocument Table.Citizen citizenId { citizen with lastSeenOn = now } connProps - return Ok { citizen with lastSeenOn = now } + elif pwCheck citizen.PasswordHash then + do! saveDocument Table.SecurityInfo citizenId connProps (mkDoc { info with FailedLogOnAttempts = 0 }) + do! saveDocument Table.Citizen citizenId connProps (mkDoc { citizen with LastSeenOn = now }) + return Ok { citizen with LastSeenOn = now } else let locked = info.FailedLogOnAttempts >= 4 - do! saveDocument Table.SecurityInfo citizenId - { info with FailedLogOnAttempts = info.FailedLogOnAttempts + 1; AccountLocked = locked } - connProps + do! mkDoc { info with FailedLogOnAttempts = info.FailedLogOnAttempts + 1; AccountLocked = locked } + |> saveDocument Table.SecurityInfo citizenId connProps return Error $"""Log on unsuccessful{if locked then " - Account is now locked" else ""}""" | None -> return Error "Log on unsuccessful" } @@ -193,7 +208,7 @@ module Continents = /// Retrieve all continents let all () = connection () - |> Sql.query $"SELECT * FROM jjj.{Table.Continent}" + |> Sql.query $"SELECT * FROM {Table.Continent}" |> Sql.executeAsync toDocument /// Retrieve a continent by its ID @@ -210,8 +225,8 @@ module Listings = /// The SQL to select a listing view let viewSql = $"SELECT l.*, c.data AS cont_data - FROM jjj.{Table.Listing} l - INNER JOIN jjj.{Table.Continent} c ON c.id = l.data->>'continentId'" + FROM {Table.Listing} l + INNER JOIN {Table.Continent} c ON c.id = l.data ->> 'continentId'" /// Map a result for a listing view let private toListingForView row = @@ -220,14 +235,14 @@ module Listings = /// Find all job listings posted by the given citizen let findByCitizen citizenId = connection () - |> Sql.query $"{viewSql} WHERE l.data->>'citizenId' = @citizenId AND l.data->>'isLegacy' <> 'true'" + |> Sql.query $"{viewSql} WHERE l.data ->> 'citizenId' = @citizenId AND l.data ->> 'isLegacy' = 'false'" |> Sql.parameters [ "@citizenId", Sql.string (CitizenId.toString citizenId) ] |> Sql.executeAsync toListingForView /// Find a listing by its ID let findById listingId = backgroundTask { match! connection () |> getDocument Table.Listing (ListingId.toString listingId) with - | Some listing when not listing.isLegacy -> return Some listing + | Some listing when not listing.IsLegacy -> return Some listing | Some _ | None -> return None } @@ -236,7 +251,7 @@ module Listings = let findByIdForView listingId = backgroundTask { let! tryListing = connection () - |> Sql.query $"{viewSql} WHERE id = @id AND l.data->>'isLegacy' <> 'true'" + |> Sql.query $"{viewSql} WHERE id = @id AND l.data ->> 'isLegacy' = 'false'" |> Sql.parameters [ "@id", Sql.string (ListingId.toString listingId) ] |> Sql.executeAsync toListingForView return List.tryHead tryListing @@ -244,27 +259,27 @@ module Listings = /// Save a listing let save (listing : Listing) = - connection () |> saveDocument Table.Listing (ListingId.toString listing.id) listing + connection () |> saveDocument Table.Listing (ListingId.toString listing.Id) <| mkDoc listing /// Search job listings let search (search : ListingSearch) = let searches = [ match search.continentId with - | Some contId -> "l.data->>'continentId' = @continentId", [ "@continentId", Sql.string contId ] + | Some contId -> "l.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ] | None -> () match search.region with - | Some region -> "l.data->>'region' ILIKE @region", [ "@region", like region ] + | Some region -> "l.data ->> 'region' ILIKE @region", [ "@region", like region ] | None -> () if search.remoteWork <> "" then - "l.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] + "l.data ->> 'isRemote' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] match search.text with - | Some text -> "l.data->>'text' ILIKE @text", [ "@text", like text ] + | Some text -> "l.data ->> 'text' ILIKE @text", [ "@text", like text ] | None -> () ] connection () |> Sql.query $" {viewSql} - WHERE l.data->>'isExpired' = 'false' AND l.data->>'isLegacy' = 'false' + WHERE l.data ->> 'isExpired' = 'false' AND l.data ->> 'isLegacy' = 'false' {searchSql searches}" |> Sql.parameters (searches |> List.collect snd) |> Sql.executeAsync toListingForView @@ -277,14 +292,14 @@ module Profiles = /// Count the current profiles let count () = connection () - |> Sql.query $"SELECT COUNT(id) AS the_count FROM jjj.{Table.Profile} WHERE data->>'isLegacy' <> 'true'" + |> Sql.query $"SELECT COUNT(id) AS the_count FROM {Table.Profile} WHERE data ->> 'isLegacy' = 'false'" |> Sql.executeRowAsync (fun row -> row.int64 "the_count") /// Delete a profile by its ID let deleteById citizenId = backgroundTask { let! _ = connection () - |> Sql.query $"DELETE FROM jjj.{Table.Profile} WHERE id = @id" + |> Sql.query $"DELETE FROM {Table.Profile} WHERE id = @id" |> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ] |> Sql.executeNonQueryAsync () @@ -293,7 +308,7 @@ module Profiles = /// Find a profile by citizen ID let findById citizenId = backgroundTask { match! connection () |> getDocument Table.Profile (CitizenId.toString citizenId) with - | Some profile when not profile.isLegacy -> return Some profile + | Some profile when not profile.IsLegacy -> return Some profile | Some _ | None -> return None } @@ -304,11 +319,11 @@ module Profiles = connection () |> Sql.query $" SELECT p.*, c.data AS cit_data, o.data AS cont_data - FROM jjj.{Table.Profile} p - INNER JOIN jjj.{Table.Citizen} c ON c.id = p.id - INNER JOIN jjj.{Table.Continent} o ON o.id = p.data->>'continentId' - WHERE p.id = @id - AND p.data->>'isLegacy' = 'false'" + FROM {Table.Profile} p + INNER JOIN {Table.Citizen} c ON c.id = p.id + INNER JOIN {Table.Continent} o ON o.id = p.data ->> 'continentId' + WHERE p.id = @id + AND p.data ->> 'isLegacy' = 'false'" |> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ] |> Sql.executeAsync (fun row -> { profile = toDocument row @@ -320,42 +335,43 @@ module Profiles = /// Save a profile let save (profile : Profile) = - connection () |> saveDocument Table.Profile (CitizenId.toString profile.id) profile + connection () |> saveDocument Table.Profile (CitizenId.toString profile.Id) <| mkDoc profile /// Search profiles (logged-on users) let search (search : ProfileSearch) = backgroundTask { let searches = [ match search.continentId with - | Some contId -> "p.data ->>'continentId' = @continentId", [ "@continentId", Sql.string contId ] + | Some contId -> "p.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ] | None -> () if search.remoteWork <> "" then - "p.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] + "p.data ->> 'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] match search.skill with - | Some skl -> "p.data->'skills'->>'description' ILIKE @description", [ "@description", like skl ] + | Some skl -> "p.data -> 'skills' ->> 'description' ILIKE @description", [ "@description", like skl ] | None -> () match search.bioExperience with | Some text -> - "(p.data->>'biography' ILIKE @text OR p.data->>'experience' ILIKE @text)", [ "@text", Sql.string text ] + "(p.data ->> 'biography' ILIKE @text OR p.data ->> 'experience' ILIKE @text)", + [ "@text", Sql.string text ] | None -> () ] let! results = connection () |> Sql.query $" SELECT p.*, c.data AS cit_data - FROM jjj.{Table.Profile} p - INNER JOIN jjj.{Table.Citizen} c ON c.id = p.id - WHERE p.data->>'isLegacy' = 'false' + FROM {Table.Profile} p + INNER JOIN {Table.Citizen} c ON c.id = p.id + WHERE p.data ->> 'isLegacy' = 'false' {searchSql searches}" |> Sql.parameters (searches |> List.collect snd) |> Sql.executeAsync (fun row -> let profile = toDocument row let citizen = toDocumentFrom "cit_data" row - { citizenId = profile.id + { citizenId = profile.Id displayName = Citizen.name citizen - seekingEmployment = profile.seekingEmployment - remoteWork = profile.remoteWork - fullTime = profile.fullTime - lastUpdatedOn = profile.lastUpdatedOn + seekingEmployment = profile.IsSeekingEmployment + remoteWork = profile.IsRemote + fullTime = profile.IsFullTime + lastUpdatedOn = profile.LastUpdatedOn }) return results |> List.sortBy (fun psr -> psr.displayName.ToLowerInvariant ()) } @@ -364,36 +380,36 @@ module Profiles = let publicSearch (search : PublicSearch) = let searches = [ match search.continentId with - | Some contId -> "p.data->>'continentId' = @continentId", [ "@continentId", Sql.string contId ] + | Some contId -> "p.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ] | None -> () match search.region with - | Some region -> "p.data->>'region' ILIKE @region", [ "@region", like region ] + | Some region -> "p.data ->> 'region' ILIKE @region", [ "@region", like region ] | None -> () if search.remoteWork <> "" then - "p.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] + "p.data ->> 'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ] match search.skill with | Some skl -> - "p.data->'skills'->>'description' ILIKE @description", [ "@description", like skl ] + "p.data -> 'skills' ->> 'description' ILIKE @description", [ "@description", like skl ] | None -> () ] connection () |> Sql.query $" SELECT p.*, c.data AS cont_data - FROM jjj.{Table.Profile} p - INNER JOIN jjj.{Table.Continent} c ON c.id = p.data->>'continentId' - WHERE p.data->>'isPublic' = 'true' - AND p.data->>'isLegacy' = 'false' + FROM {Table.Profile} p + INNER JOIN {Table.Continent} c ON c.id = p.data ->> 'continentId' + WHERE p.data ->> 'isPublic' = 'true' + AND p.data ->> 'isLegacy' = 'false' {searchSql searches}" |> Sql.executeAsync (fun row -> let profile = toDocument row let continent = toDocumentFrom "cont_data" row - { continent = continent.name - region = profile.region - remoteWork = profile.remoteWork - skills = profile.skills + { continent = continent.Name + region = profile.Region + remoteWork = profile.IsRemote + skills = profile.Skills |> List.map (fun s -> - let notes = match s.notes with Some n -> $" ({n})" | None -> "" - $"{s.description}{notes}") + let notes = match s.Notes with Some n -> $" ({n})" | None -> "" + $"{s.Description}{notes}") }) /// Success story data access functions @@ -405,18 +421,18 @@ module Successes = connection () |> Sql.query $" SELECT s.*, c.data AS cit_data - FROM jjj.{Table.Success} s - INNER JOIN jjj.{Table.Citizen} c ON c.id = s.data->>'citizenId' - ORDER BY s.data->>'recordedOn' DESC" + FROM {Table.Success} s + INNER JOIN {Table.Citizen} c ON c.id = s.data ->> 'citizenId' + ORDER BY s.data ->> 'recordedOn' DESC" |> Sql.executeAsync (fun row -> let success = toDocument row let citizen = toDocumentFrom "cit_data" row - { id = success.id - citizenId = success.citizenId + { id = success.Id + citizenId = success.CitizenId citizenName = Citizen.name citizen - recordedOn = success.recordedOn - fromHere = success.fromHere - hasStory = Option.isSome success.story + recordedOn = success.RecordedOn + fromHere = success.IsFromHere + hasStory = Option.isSome success.Story }) /// Find a success story by its ID @@ -425,5 +441,5 @@ module Successes = /// Save a success story let save (success : Success) = - connection () |> saveDocument Table.Success (SuccessId.toString success.id) success + connection () |> saveDocument Table.Success (SuccessId.toString success.Id) <| mkDoc success \ No newline at end of file diff --git a/src/JobsJobsJobs/JobsJobsJobs.Data/JobsJobsJobs.Data.fsproj b/src/JobsJobsJobs/JobsJobsJobs.Data/JobsJobsJobs.Data.fsproj index 4cfa081..153d84c 100644 --- a/src/JobsJobsJobs/JobsJobsJobs.Data/JobsJobsJobs.Data.fsproj +++ b/src/JobsJobsJobs/JobsJobsJobs.Data/JobsJobsJobs.Data.fsproj @@ -16,9 +16,8 @@ - - + diff --git a/src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs b/src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs index 5c96745..a7c58a9 100644 --- a/src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs +++ b/src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs @@ -12,6 +12,9 @@ type WrappedJsonConverter<'T> (wrap : string -> 'T, unwrap : 'T -> string) = override _.Write(writer, value, _) = writer.WriteStringValue (unwrap value) +open NodaTime +open NodaTime.Serialization.SystemTextJson + /// JsonSerializer options that use the custom converters let options = let opts = JsonSerializerOptions () @@ -24,4 +27,6 @@ let options = JsonFSharpConverter () ] |> List.iter opts.Converters.Add + let _ = opts.ConfigureForNodaTime DateTimeZoneProviders.Tzdb + opts.PropertyNamingPolicy <- JsonNamingPolicy.CamelCase opts diff --git a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs b/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs index d96da5a..3c531f0 100644 --- a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs +++ b/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs @@ -1,4 +1,5 @@  +open System.Text.Json open Microsoft.Extensions.Configuration /// Data access for v2 Jobs, Jobs, Jobs @@ -44,8 +45,8 @@ let r = RethinkDb.Driver.RethinkDB.R open JobsJobsJobs.Data open JobsJobsJobs.Domain open Newtonsoft.Json.Linq -open NodaTime open NodaTime.Text +open Npgsql.FSharp open RethinkDb.Driver.FSharp.Functions /// Retrieve an instant from a JObject field @@ -62,32 +63,155 @@ task { // Establish database connections let cfg = ConfigurationBuilder().AddJsonFile("appsettings.json").Build () use rethinkConn = Rethink.Startup.createConnection (cfg.GetConnectionString "RethinkDB") - match! DataConnection.setUp cfg with - | Ok _ -> () - | Error msg -> failwith msg + do! DataConnection.setUp cfg + let pgConn = DataConnection.connection () - // Migrate citizens - let! oldCitizens = - fromTable Rethink.Table.Citizen + let getOld table = + fromTable table |> runResult |> withRetryOnce |> withConn rethinkConn + + // Migrate citizens + let! oldCitizens = getOld Rethink.Table.Citizen let newCitizens = oldCitizens |> List.map (fun c -> let user = c["mastodonUser"].Value () { Citizen.empty with - id = CitizenId.ofString (c["id"].Value ()) - joinedOn = getInstant c "joinedOn" - lastSeenOn = getInstant c "lastSeenOn" - email = $"""{user}@{c["instance"].Value ()}""" - firstName = user - lastName = user - isLegacy = true + Id = CitizenId.ofString (c["id"].Value ()) + JoinedOn = getInstant c "joinedOn" + LastSeenOn = getInstant c "lastSeenOn" + Email = $"""{user}@{c["instance"].Value ()}""" + FirstName = user + LastName = user + IsLegacy = true }) for citizen in newCitizens do do! Citizens.save citizen - printfn $"** Migrated {List.length newCitizens} citizen(s)" - () + let! _ = + pgConn + |> Sql.executeTransactionAsync [ + $"INSERT INTO jjj.{Table.SecurityInfo} VALUES (@id, @data)", + newCitizens |> List.map (fun c -> + let info = { SecurityInfo.empty with Id = c.Id; AccountLocked = true } + [ "@id", Sql.string (CitizenId.toString c.Id) + "@data", Sql.jsonb (JsonSerializer.Serialize (info, Json.options)) + ]) + ] + printfn $"** Migrated {List.length newCitizens} citizens" + + // Migrate continents + let! oldContinents = getOld Rethink.Table.Continent + let newContinents = + oldContinents + |> List.map (fun c -> + { Continent.empty with + Id = ContinentId.ofString (c["id"].Value ()) + Name = c["name"].Value () + }) + let! _ = + pgConn + |> Sql.executeTransactionAsync [ + "INSERT INTO jjj.continent VALUES (@id, @data)", + newContinents |> List.map (fun c -> [ + "@id", Sql.string (ContinentId.toString c.Id) + "@data", Sql.jsonb (JsonSerializer.Serialize (c, Json.options)) + ]) + ] + printfn $"** Migrated {List.length newContinents} continents" + + // Migrate profiles + let! oldProfiles = getOld Rethink.Table.Profile + let newProfiles = + oldProfiles + |> List.map (fun p -> + let experience = p["experience"].Value () + { Profile.empty with + Id = CitizenId.ofString (p["id"].Value ()) + IsSeekingEmployment = p["seekingEmployment"].Value () + IsPubliclySearchable = p["isPublic"].Value () + ContinentId = ContinentId.ofString (p["continentId"].Value ()) + Region = p["region"].Value () + IsRemote = p["remoteWork"].Value () + IsFullTime = p["fullTime"].Value () + Biography = Text (p["biography"].Value ()) + LastUpdatedOn = getInstant p "lastUpdatedOn" + Experience = if isNull experience then None else Some (Text experience) + Skills = p["skills"].Children() + |> Seq.map (fun s -> + let notes = s["notes"].Value () + { Skill.Id = SkillId.ofString (s["id"].Value ()) + Description = s["description"].Value () + Notes = if isNull notes then None else Some notes + }) + |> List.ofSeq + IsLegacy = true + }) + for profile in newProfiles do + do! Profiles.save profile + printfn $"** Migrated {List.length newProfiles} profiles" + + // Migrate listings + let! oldListings = getOld Rethink.Table.Listing + let newListings = + oldListings + |> List.map (fun l -> + let neededBy = l["neededBy"].Value () + let wasFilledHere = l["wasFilledHere"].Value () + { Listing.empty with + Id = ListingId.ofString (l["id"].Value ()) + CitizenId = CitizenId.ofString (l["citizenId"].Value ()) + CreatedOn = getInstant l "createdOn" + Title = l["title"].Value () + ContinentId = ContinentId.ofString (l["continentId"].Value ()) + Region = l["region"].Value () + IsRemote = l["remoteWork"].Value () + IsExpired = l["isExpired"].Value () + UpdatedOn = getInstant l "updatedOn" + Text = Text (l["text"].Value ()) + NeededBy = if isNull neededBy then None else + match LocalDatePattern.Iso.Parse neededBy with + | it when it.Success -> Some it.Value + | it -> + eprintfn $"Error parsing date - {it.Exception.Message}" + None + WasFilledHere = if isNull wasFilledHere then None else Some (bool.Parse wasFilledHere) + IsLegacy = true + }) + for listing in newListings do + do! Listings.save listing + printfn $"** Migrated {List.length newListings} listings" + + // Migrate success stories + let! oldSuccesses = getOld Rethink.Table.Success + let newSuccesses = + oldSuccesses + |> List.map (fun s -> + let story = s["story"].Value () + { Success.empty with + Id = SuccessId.ofString (s["id"].Value ()) + CitizenId = CitizenId.ofString (s["citizenId"].Value ()) + RecordedOn = getInstant s "recordedOn" + Source = s["source"].Value () + Story = if isNull story then None else Some (Text story) + }) + for success in newSuccesses do + do! Successes.save success + printfn $"** Migrated {List.length newSuccesses} successes" + + // Delete any citizens who have no profile, no listing, and no success story recorded + let! deleted = + pgConn + |> Sql.query $" + DELETE FROM jjj.{Table.Citizen} + WHERE id NOT IN (SELECT id FROM jjj.{Table.Profile}) + AND id NOT IN (SELECT DISTINCT data->>'citizenId' FROM jjj.{Table.Listing}) + AND id NOT IN (SELECT DISTINCT data->>'citizenId' FROM jjj.{Table.Success})" + |> Sql.executeNonQueryAsync + printfn $"** Deleted {deleted} citizens who had no profile, listings, or success stories" + + printfn "" + printfn "Migration complete" } |> Async.AwaitTask |> Async.RunSynchronously diff --git a/src/JobsJobsJobs/Server/Auth.fs b/src/JobsJobsJobs/Server/Auth.fs index 8ea26e3..6d49875 100644 --- a/src/JobsJobsJobs/Server/Auth.fs +++ b/src/JobsJobsJobs/Server/Auth.fs @@ -91,7 +91,7 @@ let createJwt (citizen : Citizen) (cfg : AuthOptions) = tokenHandler.CreateToken ( SecurityTokenDescriptor ( Subject = ClaimsIdentity [| - Claim (ClaimTypes.NameIdentifier, CitizenId.toString citizen.id) + Claim (ClaimTypes.NameIdentifier, CitizenId.toString citizen.Id) Claim (ClaimTypes.Name, Citizen.name citizen) |], Expires = DateTime.UtcNow.AddHours 2., diff --git a/src/JobsJobsJobs/Server/Handlers.fs b/src/JobsJobsJobs/Server/Handlers.fs index ecd51ab..5918924 100644 --- a/src/JobsJobsJobs/Server/Handlers.fs +++ b/src/JobsJobsJobs/Server/Handlers.fs @@ -109,7 +109,7 @@ module Citizen = return! json { jwt = Auth.createJwt citizen (authConfig ctx) - citizenId = CitizenId.toString citizen.id + citizenId = CitizenId.toString citizen.Id name = Citizen.name citizen } next ctx | Error msg -> @@ -238,19 +238,19 @@ module Listing = let! form = ctx.BindJsonAsync () let now = now ctx do! Listings.save { - id = ListingId.create () - citizenId = currentCitizenId ctx - createdOn = now - title = form.title - continentId = ContinentId.ofString form.continentId - region = form.region - remoteWork = form.remoteWork - isExpired = false - updatedOn = now - text = Text form.text - neededBy = (form.neededBy |> Option.map parseDate) - wasFilledHere = None - isLegacy = false + Id = ListingId.create () + CitizenId = currentCitizenId ctx + CreatedOn = now + Title = form.title + ContinentId = ContinentId.ofString form.continentId + Region = form.region + IsRemote = form.remoteWork + IsExpired = false + UpdatedOn = now + Text = Text form.text + NeededBy = (form.neededBy |> Option.map parseDate) + WasFilledHere = None + IsLegacy = false } return! ok next ctx } @@ -258,18 +258,18 @@ module Listing = // PUT: /api/listing/[id] let update listingId : HttpHandler = authorize >=> fun next ctx -> task { match! Listings.findById (ListingId listingId) with - | Some listing when listing.citizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx + | Some listing when listing.CitizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx | Some listing -> let! form = ctx.BindJsonAsync () do! Listings.save { listing with - title = form.title - continentId = ContinentId.ofString form.continentId - region = form.region - remoteWork = form.remoteWork - text = Text form.text - neededBy = form.neededBy |> Option.map parseDate - updatedOn = now ctx + Title = form.title + ContinentId = ContinentId.ofString form.continentId + Region = form.region + IsRemote = form.remoteWork + Text = Text form.text + NeededBy = form.neededBy |> Option.map parseDate + UpdatedOn = now ctx } return! ok next ctx | None -> return! Error.notFound next ctx @@ -279,24 +279,24 @@ module Listing = let expire listingId : HttpHandler = authorize >=> fun next ctx -> task { let now = now ctx match! Listings.findById (ListingId listingId) with - | Some listing when listing.citizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx + | Some listing when listing.CitizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx | Some listing -> let! form = ctx.BindJsonAsync () do! Listings.save { listing with - isExpired = true - wasFilledHere = Some form.fromHere - updatedOn = now + IsExpired = true + WasFilledHere = Some form.fromHere + UpdatedOn = now } match form.successStory with | Some storyText -> do! Successes.save - { id = SuccessId.create() - citizenId = currentCitizenId ctx - recordedOn = now - fromHere = form.fromHere - source = "listing" - story = (Text >> Some) storyText + { Id = SuccessId.create() + CitizenId = currentCitizenId ctx + RecordedOn = now + IsFromHere = form.fromHere + Source = "listing" + Story = (Text >> Some) storyText } | None -> () return! ok next ctx @@ -351,26 +351,26 @@ module Profile = let! profile = task { match! Profiles.findById citizenId with | Some p -> return p - | None -> return { Profile.empty with id = citizenId } + | None -> return { Profile.empty with Id = citizenId } } do! Profiles.save { profile with - seekingEmployment = form.isSeekingEmployment - isPublic = form.isPublic - continentId = ContinentId.ofString form.continentId - region = form.region - remoteWork = form.remoteWork - fullTime = form.fullTime - biography = Text form.biography - lastUpdatedOn = now ctx - experience = noneIfBlank form.experience |> Option.map Text - skills = form.skills - |> List.map (fun s -> - { id = if s.id.StartsWith "new" then SkillId.create () - else SkillId.ofString s.id - description = s.description - notes = noneIfBlank s.notes - }) + IsSeekingEmployment = form.isSeekingEmployment + IsPubliclySearchable = form.isPublic + ContinentId = ContinentId.ofString form.continentId + Region = form.region + IsRemote = form.remoteWork + IsFullTime = form.fullTime + Biography = Text form.biography + LastUpdatedOn = now ctx + Experience = noneIfBlank form.experience |> Option.map Text + Skills = form.skills + |> List.map (fun s -> + { Id = if s.id.StartsWith "new" then SkillId.create () + else SkillId.ofString s.id + Description = s.description + Notes = noneIfBlank s.notes + }) } return! ok next ctx } @@ -379,7 +379,7 @@ module Profile = let employmentFound : HttpHandler = authorize >=> fun next ctx -> task { match! Profiles.findById (currentCitizenId ctx) with | Some profile -> - do! Profiles.save { profile with seekingEmployment = false } + do! Profiles.save { profile with IsSeekingEmployment = false } return! ok next ctx | None -> return! Error.notFound next ctx } @@ -429,19 +429,19 @@ module Success = let! success = task { match form.id with | "new" -> - return Some { id = SuccessId.create () - citizenId = citizenId - recordedOn = now ctx - fromHere = form.fromHere - source = "profile" - story = noneIfEmpty form.story |> Option.map Text + return Some { Id = SuccessId.create () + CitizenId = citizenId + RecordedOn = now ctx + IsFromHere = form.fromHere + Source = "profile" + Story = noneIfEmpty form.story |> Option.map Text } | successId -> match! Successes.findById (SuccessId.ofString successId) with - | Some story when story.citizenId = citizenId -> + | Some story when story.CitizenId = citizenId -> return Some { story with - fromHere = form.fromHere - story = noneIfEmpty form.story |> Option.map Text + IsFromHere = form.fromHere + Story = noneIfEmpty form.story |> Option.map Text } | Some _ | None -> return None }