bit-badger/jobs-jobs-jobs
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 19 Activity

Migration to pg docs complete

Browse Source
This commit is contained in:
Daniel J. Summers 2022-08-27 22:34:51 -04:00
parent 1a91f10da2
commit 21957330fe
9 changed files with 442 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ src/**/obj
src/**/appsettings.*.json
src/.vs
src/.idea
src/JobsJobsJobs/JobsJobsJobs.V3Migration/appsettings.json

24
src/JobsJobsJobs/Domain/SharedTypes.fs
View File

@ -203,20 +203,20 @@ module ProfileForm =
/// Create an instance of this form from the given profile
let fromProfile (profile : Profile) =
{ isSeekingEmployment = profile.seekingEmployment
isPublic = profile.isPublic
{ isSeekingEmployment = profile.IsSeekingEmployment
isPublic = profile.IsPubliclySearchable
realName = ""
continentId = string profile.continentId
region = profile.region
remoteWork = profile.remoteWork
fullTime = profile.fullTime
biography = MarkdownString.toString profile.biography
experience = profile.experience |> Option.map MarkdownString.toString
skills = profile.skills
continentId = string profile.ContinentId
region = profile.Region
remoteWork = profile.IsRemote
fullTime = profile.IsFullTime
biography = MarkdownString.toString profile.Biography
experience = profile.Experience |> Option.map MarkdownString.toString
skills = profile.Skills
|> List.map (fun s ->
{ id = string s.id
description = s.description
notes = s.notes
{ id = string s.Id
description = s.Description
notes = s.Notes
})
}

199
src/JobsJobsJobs/Domain/Types.fs
View File

@ -9,74 +9,75 @@ open System
[<CLIMutable; NoComparison; NoEquality>]
type Citizen =
{ /// The ID of the user
id : CitizenId
Id : CitizenId
/// When the user joined Jobs, Jobs, Jobs
joinedOn : Instant
JoinedOn : Instant
/// When the user last logged in
lastSeenOn : Instant
LastSeenOn : Instant
/// The user's e-mail address
email : string
Email : string
/// The user's first name
firstName : string
FirstName : string
/// The user's last name
lastName : string
LastName : string
/// The hash of the user's password
passwordHash : string
PasswordHash : string
/// The name displayed for this user throughout the site
displayName : string option
DisplayName : string option
/// The other contacts for this user
otherContacts : OtherContact list
OtherContacts : OtherContact list
/// Whether this is a legacy citizen
isLegacy : bool
IsLegacy : bool
}
/// Support functions for citizens
module Citizen =
/// An empty citizen
let empty =
{ id = CitizenId Guid.Empty
joinedOn = Instant.MinValue
lastSeenOn = Instant.MinValue
email = ""
firstName = ""
lastName = ""
passwordHash = ""
displayName = None
otherContacts = []
isLegacy = false
let empty = {
Id = CitizenId Guid.Empty
JoinedOn = Instant.MinValue
LastSeenOn = Instant.MinValue
Email = ""
FirstName = ""
LastName = ""
PasswordHash = ""
DisplayName = None
OtherContacts = []
IsLegacy = false
}
/// Get the name of the citizen (either their preferred display name or first/last names)
let name x =
match x.displayName with Some it -> it | None -> $"{x.firstName} {x.lastName}"
match x.DisplayName with Some it -> it | None -> $"{x.FirstName} {x.LastName}"
/// A continent
[<CLIMutable; NoComparison; NoEquality>]
type Continent =
{ /// The ID of the continent
id : ContinentId
Id : ContinentId
/// The name of the continent
name : string
Name : string
}
/// Support functions for continents
module Continent =
/// An empty continent
let empty =
{ id = ContinentId Guid.Empty
name = ""
let empty ={
Id = ContinentId Guid.Empty
Name = ""
}
@ -84,63 +85,63 @@ module Continent =
[<CLIMutable; NoComparison; NoEquality>]
type Listing =
{ /// The ID of the job listing
id : ListingId
Id : ListingId
/// The ID of the citizen who posted the job listing
citizenId : CitizenId
CitizenId : CitizenId
/// When this job listing was created
createdOn : Instant
CreatedOn : Instant
/// The short title of the job listing
title : string
Title : string
/// The ID of the continent on which the job is located
continentId : ContinentId
ContinentId : ContinentId
/// The region in which the job is located
region : string
Region : string
/// Whether this listing is for remote work
remoteWork : bool
IsRemote : bool
/// Whether this listing has expired
isExpired : bool
IsExpired : bool
/// When this listing was last updated
updatedOn : Instant
UpdatedOn : Instant
/// The details of this job
text : MarkdownString
Text : MarkdownString
/// When this job needs to be filled
neededBy : LocalDate option
NeededBy : LocalDate option
/// Was this job filled as part of its appearance on Jobs, Jobs, Jobs?
wasFilledHere : bool option
WasFilledHere : bool option
/// Whether this is a legacy listing
isLegacy : bool
IsLegacy : bool
}
/// Support functions for job listings
module Listing =
/// An empty job listing
let empty =
{ id = ListingId Guid.Empty
citizenId = CitizenId Guid.Empty
createdOn = Instant.MinValue
title = ""
continentId = ContinentId Guid.Empty
region = ""
remoteWork = false
isExpired = false
updatedOn = Instant.MinValue
text = Text ""
neededBy = None
wasFilledHere = None
isLegacy = false
let empty = {
Id = ListingId Guid.Empty
CitizenId = CitizenId Guid.Empty
CreatedOn = Instant.MinValue
Title = ""
ContinentId = ContinentId Guid.Empty
Region = ""
IsRemote = false
IsExpired = false
UpdatedOn = Instant.MinValue
Text = Text ""
NeededBy = None
WasFilledHere = None
IsLegacy = false
}
@ -169,8 +170,8 @@ type SecurityInfo =
module SecurityInfo =
/// An empty set of security info
let empty =
{ Id = CitizenId Guid.Empty
let empty = {
Id = CitizenId Guid.Empty
FailedLogOnAttempts = 0
AccountLocked = false
Token = None
@ -182,13 +183,13 @@ module SecurityInfo =
/// A skill the job seeker possesses
type Skill =
{ /// The ID of the skill
id : SkillId
Id : SkillId
/// A description of the skill
description : string
Description : string
/// Notes regarding this skill (level, duration, etc.)
notes : string option
Notes : string option
}
@ -196,63 +197,63 @@ type Skill =
[<CLIMutable; NoComparison; NoEquality>]
type Profile =
{ /// The ID of the citizen to whom this profile belongs
id : CitizenId
Id : CitizenId
/// Whether this citizen is actively seeking employment
seekingEmployment : bool
IsSeekingEmployment : bool
/// Whether this citizen allows their profile to be a part of the publicly-viewable, anonymous data
isPublic : bool
IsPubliclySearchable : bool
/// Whether this citizen allows their profile to be viewed via a public link
isPublicLinkable : bool
IsPubliclyLinkable : bool
/// The ID of the continent on which the citizen resides
continentId : ContinentId
ContinentId : ContinentId
/// The region in which the citizen resides
region : string
Region : string
/// Whether the citizen is looking for remote work
remoteWork : bool
IsRemote : bool
/// Whether the citizen is looking for full-time work
fullTime : bool
IsFullTime : bool
/// The citizen's professional biography
biography : MarkdownString
Biography : MarkdownString
/// When the citizen last updated their profile
lastUpdatedOn : Instant
LastUpdatedOn : Instant
/// The citizen's experience (topical / chronological)
experience : MarkdownString option
Experience : MarkdownString option
/// Skills this citizen possesses
skills : Skill list
Skills : Skill list
/// Whether this is a legacy profile
isLegacy : bool
IsLegacy : bool
}
/// Support functions for Profiles
module Profile =
// An empty profile
let empty =
{ id = CitizenId Guid.Empty
seekingEmployment = false
isPublic = false
isPublicLinkable = false
continentId = ContinentId Guid.Empty
region = ""
remoteWork = false
fullTime = false
biography = Text ""
lastUpdatedOn = Instant.MinValue
experience = None
skills = []
isLegacy = false
let empty = {
Id = CitizenId Guid.Empty
IsSeekingEmployment = false
IsPubliclySearchable = false
IsPubliclyLinkable = false
ContinentId = ContinentId Guid.Empty
Region = ""
IsRemote = false
IsFullTime = false
Biography = Text ""
LastUpdatedOn = Instant.MinValue
Experience = None
Skills = []
IsLegacy = false
}
@ -260,33 +261,33 @@ module Profile =
[<CLIMutable; NoComparison; NoEquality>]
type Success =
{ /// The ID of the success report
id : SuccessId
Id : SuccessId
/// The ID of the citizen who wrote this success report
citizenId : CitizenId
CitizenId : CitizenId
/// When this success report was recorded
recordedOn : Instant
RecordedOn : Instant
/// Whether the success was due, at least in part, to Jobs, Jobs, Jobs
fromHere : bool
IsFromHere : bool
/// The source of this success (listing or profile)
source : string
Source : string
/// The success story
story : MarkdownString option
Story : MarkdownString option
}
/// Support functions for success stories
module Success =
/// An empty success story
let empty =
{ id = SuccessId Guid.Empty
citizenId = CitizenId Guid.Empty
recordedOn = Instant.MinValue
fromHere = false
source = ""
story = None
let empty = {
Id = SuccessId Guid.Empty
CitizenId = CitizenId Guid.Empty
RecordedOn = Instant.MinValue
IsFromHere = false
Source = ""
Story = None
}

202
src/JobsJobsJobs/JobsJobsJobs.Data/Data.fs
View File

@ -5,27 +5,27 @@ module Table =
/// Citizens
[<Literal>]
let Citizen = "citizen"
let Citizen = "jjj.citizen"
/// Continents
[<Literal>]
let Continent = "continent"
let Continent = "jjj.continent"
/// Job Listings
[<Literal>]
let Listing = "listing"
let Listing = "jjj.listing"
/// Employment Profiles
[<Literal>]
let Profile = "profile"
let Profile = "jjj.profile"
/// User Security Information
[<Literal>]
let SecurityInfo = "security_info"
let SecurityInfo = "jjj.security_info"
/// Success Stories
[<Literal>]
let Success = "success"
let Success = "jjj.success"
open Npgsql.FSharp
@ -46,15 +46,25 @@ module DataConnection =
/// Create tables
let private createTables () = backgroundTask {
let sql =
[ Table.Citizen; Table.Continent; Table.Listing; Table.Profile; Table.SecurityInfo; Table.Success ]
|> List.map (fun table ->
$"CREATE TABLE IF NOT EXISTS jjj.{table} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)")
|> String.concat "; "
let sql = [
$"CREATE SCHEMA IF NOT EXISTS jjj"
$"CREATE TABLE IF NOT EXISTS {Table.Citizen} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)"
$"CREATE TABLE IF NOT EXISTS {Table.Continent} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)"
$"CREATE TABLE IF NOT EXISTS {Table.Listing} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)"
$"CREATE TABLE IF NOT EXISTS {Table.Profile} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL,
CONSTRAINT fk_profile_citizen FOREIGN KEY (id) REFERENCES {Table.Citizen} (id) ON DELETE CASCADE)"
$"CREATE TABLE IF NOT EXISTS {Table.SecurityInfo} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL,
CONSTRAINT fk_security_info_citizen FOREIGN KEY (id) REFERENCES {Table.Citizen} (id) ON DELETE CASCADE)"
$"CREATE TABLE IF NOT EXISTS {Table.Success} (id TEXT NOT NULL PRIMARY KEY, data JSONB NOT NULL)"
$"CREATE INDEX IF NOT EXISTS idx_citizen_email ON {Table.Citizen} USING GIN ((data -> 'email'))"
$"CREATE INDEX IF NOT EXISTS idx_listing_citizen ON {Table.Listing} USING GIN ((data -> 'citizenId'))"
$"CREATE INDEX IF NOT EXISTS idx_listing_continent ON {Table.Listing} USING GIN ((data -> 'continentId'))"
$"CREATE INDEX IF NOT EXISTS idx_profile_continent ON {Table.Profile} USING GIN ((data -> 'continentId'))"
$"CREATE INDEX IF NOT EXISTS idx_success_citizen ON {Table.Success} USING GIN ((data -> 'citizenId'))"
]
let! _ =
connection ()
|> Sql.executeTransactionAsync [ sql, [ [] ] ]
// TODO: prudent indexes
|> Sql.executeTransactionAsync (sql |> List.map (fun sql -> sql, [ [] ]))
()
}
@ -84,22 +94,26 @@ module private Helpers =
/// Get a document
let getDocument<'T> table docId sqlProps : Task<'T option> = backgroundTask {
let! doc =
Sql.query $"SELECT * FROM jjj.%s{table} where id = @id" sqlProps
Sql.query $"SELECT * FROM %s{table} where id = @id" sqlProps
|> Sql.parameters [ "@id", Sql.string docId ]
|> Sql.executeAsync toDocument
return List.tryHead doc
}
/// Serialize a document to JSON
let mkDoc<'T> (doc : 'T) =
JsonSerializer.Serialize<'T> (doc, Json.options)
/// Save a document
let saveDocument<'T> table docId (doc : 'T) sqlProps = backgroundTask {
let saveDocument table docId sqlProps doc = backgroundTask {
let! _ =
Sql.query
$"INSERT INTO jjj.%s{table} (id, data) VALUES (@id, @data)
$"INSERT INTO %s{table} (id, data) VALUES (@id, @data)
ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data"
sqlProps
|> Sql.parameters
[ "@id", Sql.string docId
"@data", Sql.jsonb (JsonSerializer.Serialize (doc, Json.options)) ]
"@data", Sql.jsonb doc ]
|> Sql.executeNonQueryAsync
()
}
@ -128,59 +142,60 @@ module Citizens =
let deleteById citizenId = backgroundTask {
let! _ =
connection ()
|> Sql.executeTransactionAsync [
"DELETE FROM jjj.success WHERE data->>'citizenId' = @id;
DELETE FROM jjj.listing WHERE data->>'citizenId' = @id;
DELETE FROM jjj.profile WHERE id = @id;
DELETE FROM jjj.security_info WHERE id = @id;
DELETE FROM jjj.citizen WHERE id = @id",
[ [ "@id", Sql.string (CitizenId.toString citizenId) ] ]
]
|> Sql.query $"
DELETE FROM {Table.Success} WHERE data ->> 'citizenId' = @id;
DELETE FROM {Table.Listing} WHERE data ->> 'citizenId' = @id;
DELETE FROM {Table.Citizen} WHERE id = @id"
|> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ]
|> Sql.executeNonQueryAsync
()
}
/// Find a citizen by their ID
let findById citizenId = backgroundTask {
match! connection () |> getDocument<Citizen> Table.Citizen (CitizenId.toString citizenId) with
| Some c when not c.isLegacy -> return Some c
| Some c when not c.IsLegacy -> return Some c
| Some _
| None -> return None
}
/// Save a citizen
let save (citizen : Citizen) =
connection () |> saveDocument Table.Citizen (CitizenId.toString citizen.id) citizen
connection () |> saveDocument Table.Citizen (CitizenId.toString citizen.Id) <| mkDoc citizen
/// Attempt a user log on
let tryLogOn email (pwCheck : string -> bool) now = backgroundTask {
let connProps = connection ()
let! tryCitizen =
connProps
|> Sql.query $"SELECT * FROM jjj.{Table.Citizen} WHERE data->>email = @email AND data->>isValue <> 'true'"
|> Sql.query $"
SELECT *
FROM {Table.Citizen}
WHERE data ->> 'email' = @email
AND data ->> 'isLegacy' = 'false'"
|> Sql.parameters [ "@email", Sql.string email ]
|> Sql.executeAsync toDocument<Citizen>
match List.tryHead tryCitizen with
| Some citizen ->
let citizenId = CitizenId.toString citizen.id
let citizenId = CitizenId.toString citizen.Id
let! tryInfo = getDocument<SecurityInfo> Table.SecurityInfo citizenId connProps
let! info = backgroundTask {
match tryInfo with
| Some it -> return it
| None ->
let it = { SecurityInfo.empty with Id = citizen.id }
do! saveDocument Table.SecurityInfo citizenId it connProps
let it = { SecurityInfo.empty with Id = citizen.Id }
do! saveDocument Table.SecurityInfo citizenId connProps (mkDoc it)
return it
}
if info.AccountLocked then return Error "Log on unsuccessful (Account Locked)"
elif pwCheck citizen.passwordHash then
do! saveDocument Table.SecurityInfo citizenId { info with FailedLogOnAttempts = 0 } connProps
do! saveDocument Table.Citizen citizenId { citizen with lastSeenOn = now } connProps
return Ok { citizen with lastSeenOn = now }
elif pwCheck citizen.PasswordHash then
do! saveDocument Table.SecurityInfo citizenId connProps (mkDoc { info with FailedLogOnAttempts = 0 })
do! saveDocument Table.Citizen citizenId connProps (mkDoc { citizen with LastSeenOn = now })
return Ok { citizen with LastSeenOn = now }
else
let locked = info.FailedLogOnAttempts >= 4
do! saveDocument Table.SecurityInfo citizenId
{ info with FailedLogOnAttempts = info.FailedLogOnAttempts + 1; AccountLocked = locked }
connProps
do! mkDoc { info with FailedLogOnAttempts = info.FailedLogOnAttempts + 1; AccountLocked = locked }
|> saveDocument Table.SecurityInfo citizenId connProps
return Error $"""Log on unsuccessful{if locked then " - Account is now locked" else ""}"""
| None -> return Error "Log on unsuccessful"
}
@ -193,7 +208,7 @@ module Continents =
/// Retrieve all continents
let all () =
connection ()
|> Sql.query $"SELECT * FROM jjj.{Table.Continent}"
|> Sql.query $"SELECT * FROM {Table.Continent}"
|> Sql.executeAsync toDocument<Continent>
/// Retrieve a continent by its ID
@ -210,8 +225,8 @@ module Listings =
/// The SQL to select a listing view
let viewSql =
$"SELECT l.*, c.data AS cont_data
FROM jjj.{Table.Listing} l
INNER JOIN jjj.{Table.Continent} c ON c.id = l.data->>'continentId'"
FROM {Table.Listing} l
INNER JOIN {Table.Continent} c ON c.id = l.data ->> 'continentId'"
/// Map a result for a listing view
let private toListingForView row =
@ -220,14 +235,14 @@ module Listings =
/// Find all job listings posted by the given citizen
let findByCitizen citizenId =
connection ()
|> Sql.query $"{viewSql} WHERE l.data->>'citizenId' = @citizenId AND l.data->>'isLegacy' <> 'true'"
|> Sql.query $"{viewSql} WHERE l.data ->> 'citizenId' = @citizenId AND l.data ->> 'isLegacy' = 'false'"
|> Sql.parameters [ "@citizenId", Sql.string (CitizenId.toString citizenId) ]
|> Sql.executeAsync toListingForView
/// Find a listing by its ID
let findById listingId = backgroundTask {
match! connection () |> getDocument<Listing> Table.Listing (ListingId.toString listingId) with
| Some listing when not listing.isLegacy -> return Some listing
| Some listing when not listing.IsLegacy -> return Some listing
| Some _
| None -> return None
}
@ -236,7 +251,7 @@ module Listings =
let findByIdForView listingId = backgroundTask {
let! tryListing =
connection ()
|> Sql.query $"{viewSql} WHERE id = @id AND l.data->>'isLegacy' <> 'true'"
|> Sql.query $"{viewSql} WHERE id = @id AND l.data ->> 'isLegacy' = 'false'"
|> Sql.parameters [ "@id", Sql.string (ListingId.toString listingId) ]
|> Sql.executeAsync toListingForView
return List.tryHead tryListing
@ -244,27 +259,27 @@ module Listings =
/// Save a listing
let save (listing : Listing) =
connection () |> saveDocument Table.Listing (ListingId.toString listing.id) listing
connection () |> saveDocument Table.Listing (ListingId.toString listing.Id) <| mkDoc listing
/// Search job listings
let search (search : ListingSearch) =
let searches = [
match search.continentId with
| Some contId -> "l.data->>'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| Some contId -> "l.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| None -> ()
match search.region with
| Some region -> "l.data->>'region' ILIKE @region", [ "@region", like region ]
| Some region -> "l.data ->> 'region' ILIKE @region", [ "@region", like region ]
| None -> ()
if search.remoteWork <> "" then
"l.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
"l.data ->> 'isRemote' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
match search.text with
| Some text -> "l.data->>'text' ILIKE @text", [ "@text", like text ]
| Some text -> "l.data ->> 'text' ILIKE @text", [ "@text", like text ]
| None -> ()
]
connection ()
|> Sql.query $"
{viewSql}
WHERE l.data->>'isExpired' = 'false' AND l.data->>'isLegacy' = 'false'
WHERE l.data ->> 'isExpired' = 'false' AND l.data ->> 'isLegacy' = 'false'
{searchSql searches}"
|> Sql.parameters (searches |> List.collect snd)
|> Sql.executeAsync toListingForView
@ -277,14 +292,14 @@ module Profiles =
/// Count the current profiles
let count () =
connection ()
|> Sql.query $"SELECT COUNT(id) AS the_count FROM jjj.{Table.Profile} WHERE data->>'isLegacy' <> 'true'"
|> Sql.query $"SELECT COUNT(id) AS the_count FROM {Table.Profile} WHERE data ->> 'isLegacy' = 'false'"
|> Sql.executeRowAsync (fun row -> row.int64 "the_count")
/// Delete a profile by its ID
let deleteById citizenId = backgroundTask {
let! _ =
connection ()
|> Sql.query $"DELETE FROM jjj.{Table.Profile} WHERE id = @id"
|> Sql.query $"DELETE FROM {Table.Profile} WHERE id = @id"
|> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ]
|> Sql.executeNonQueryAsync
()
@ -293,7 +308,7 @@ module Profiles =
/// Find a profile by citizen ID
let findById citizenId = backgroundTask {
match! connection () |> getDocument<Profile> Table.Profile (CitizenId.toString citizenId) with
| Some profile when not profile.isLegacy -> return Some profile
| Some profile when not profile.IsLegacy -> return Some profile
| Some _
| None -> return None
}
@ -304,11 +319,11 @@ module Profiles =
connection ()
|> Sql.query $"
SELECT p.*, c.data AS cit_data, o.data AS cont_data
FROM jjj.{Table.Profile} p
INNER JOIN jjj.{Table.Citizen} c ON c.id = p.id
INNER JOIN jjj.{Table.Continent} o ON o.id = p.data->>'continentId'
FROM {Table.Profile} p
INNER JOIN {Table.Citizen} c ON c.id = p.id
INNER JOIN {Table.Continent} o ON o.id = p.data ->> 'continentId'
WHERE p.id = @id
AND p.data->>'isLegacy' = 'false'"
AND p.data ->> 'isLegacy' = 'false'"
|> Sql.parameters [ "@id", Sql.string (CitizenId.toString citizenId) ]
|> Sql.executeAsync (fun row ->
{ profile = toDocument<Profile> row
@ -320,42 +335,43 @@ module Profiles =
/// Save a profile
let save (profile : Profile) =
connection () |> saveDocument Table.Profile (CitizenId.toString profile.id) profile
connection () |> saveDocument Table.Profile (CitizenId.toString profile.Id) <| mkDoc profile
/// Search profiles (logged-on users)
let search (search : ProfileSearch) = backgroundTask {
let searches = [
match search.continentId with
| Some contId -> "p.data ->>'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| Some contId -> "p.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| None -> ()
if search.remoteWork <> "" then
"p.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
"p.data ->> 'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
match search.skill with
| Some skl -> "p.data->'skills'->>'description' ILIKE @description", [ "@description", like skl ]
| Some skl -> "p.data -> 'skills' ->> 'description' ILIKE @description", [ "@description", like skl ]
| None -> ()
match search.bioExperience with
| Some text ->
"(p.data->>'biography' ILIKE @text OR p.data->>'experience' ILIKE @text)", [ "@text", Sql.string text ]
"(p.data ->> 'biography' ILIKE @text OR p.data ->> 'experience' ILIKE @text)",
[ "@text", Sql.string text ]
| None -> ()
]
let! results =
connection ()
|> Sql.query $"
SELECT p.*, c.data AS cit_data
FROM jjj.{Table.Profile} p
INNER JOIN jjj.{Table.Citizen} c ON c.id = p.id
WHERE p.data->>'isLegacy' = 'false'
FROM {Table.Profile} p
INNER JOIN {Table.Citizen} c ON c.id = p.id
WHERE p.data ->> 'isLegacy' = 'false'
{searchSql searches}"
|> Sql.parameters (searches |> List.collect snd)
|> Sql.executeAsync (fun row ->
let profile = toDocument<Profile> row
let citizen = toDocumentFrom<Citizen> "cit_data" row
{ citizenId = profile.id
{ citizenId = profile.Id
displayName = Citizen.name citizen
seekingEmployment = profile.seekingEmployment
remoteWork = profile.remoteWork
fullTime = profile.fullTime
lastUpdatedOn = profile.lastUpdatedOn
seekingEmployment = profile.IsSeekingEmployment
remoteWork = profile.IsRemote
fullTime = profile.IsFullTime
lastUpdatedOn = profile.LastUpdatedOn
})
return results |> List.sortBy (fun psr -> psr.displayName.ToLowerInvariant ())
}
@ -364,36 +380,36 @@ module Profiles =
let publicSearch (search : PublicSearch) =
let searches = [
match search.continentId with
| Some contId -> "p.data->>'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| Some contId -> "p.data ->> 'continentId' = @continentId", [ "@continentId", Sql.string contId ]
| None -> ()
match search.region with
| Some region -> "p.data->>'region' ILIKE @region", [ "@region", like region ]
| Some region -> "p.data ->> 'region' ILIKE @region", [ "@region", like region ]
| None -> ()
if search.remoteWork <> "" then
"p.data->>'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
"p.data ->> 'remoteWork' = @remote", [ "@remote", jsonBool (search.remoteWork = "yes") ]
match search.skill with
| Some skl ->
"p.data->'skills'->>'description' ILIKE @description", [ "@description", like skl ]
"p.data -> 'skills' ->> 'description' ILIKE @description", [ "@description", like skl ]
| None -> ()
]
connection ()
|> Sql.query $"
SELECT p.*, c.data AS cont_data
FROM jjj.{Table.Profile} p
INNER JOIN jjj.{Table.Continent} c ON c.id = p.data->>'continentId'
WHERE p.data->>'isPublic' = 'true'
AND p.data->>'isLegacy' = 'false'
FROM {Table.Profile} p
INNER JOIN {Table.Continent} c ON c.id = p.data ->> 'continentId'
WHERE p.data ->> 'isPublic' = 'true'
AND p.data ->> 'isLegacy' = 'false'
{searchSql searches}"
|> Sql.executeAsync (fun row ->
let profile = toDocument<Profile> row
let continent = toDocumentFrom<Continent> "cont_data" row
{ continent = continent.name
region = profile.region
remoteWork = profile.remoteWork
skills = profile.skills
{ continent = continent.Name
region = profile.Region
remoteWork = profile.IsRemote
skills = profile.Skills
|> List.map (fun s ->
let notes = match s.notes with Some n -> $" ({n})" | None -> ""
$"{s.description}{notes}")
let notes = match s.Notes with Some n -> $" ({n})" | None -> ""
$"{s.Description}{notes}")
})
/// Success story data access functions
@ -405,18 +421,18 @@ module Successes =
connection ()
|> Sql.query $"
SELECT s.*, c.data AS cit_data
FROM jjj.{Table.Success} s
INNER JOIN jjj.{Table.Citizen} c ON c.id = s.data->>'citizenId'
ORDER BY s.data->>'recordedOn' DESC"
FROM {Table.Success} s
INNER JOIN {Table.Citizen} c ON c.id = s.data ->> 'citizenId'
ORDER BY s.data ->> 'recordedOn' DESC"
|> Sql.executeAsync (fun row ->
let success = toDocument<Success> row
let citizen = toDocumentFrom<Citizen> "cit_data" row
{ id = success.id
citizenId = success.citizenId
{ id = success.Id
citizenId = success.CitizenId
citizenName = Citizen.name citizen
recordedOn = success.recordedOn
fromHere = success.fromHere
hasStory = Option.isSome success.story
recordedOn = success.RecordedOn
fromHere = success.IsFromHere
hasStory = Option.isSome success.Story
})
/// Find a success story by its ID
@ -425,5 +441,5 @@ module Successes =
/// Save a success story
let save (success : Success) =
connection () |> saveDocument Table.Success (SuccessId.toString success.id) success
connection () |> saveDocument Table.Success (SuccessId.toString success.Id) <| mkDoc success

3
src/JobsJobsJobs/JobsJobsJobs.Data/JobsJobsJobs.Data.fsproj
View File

@ -16,9 +16,8 @@
<ItemGroup>
<PackageReference Include="FSharp.SystemTextJson" Version="0.19.13" />
<PackageReference Include="Marten" Version="5.8.0" />
<PackageReference Include="Marten.NodaTime" Version="5.8.0" />
<PackageReference Update="FSharp.Core" Version="6.0.5" />
<PackageReference Include="NodaTime.Serialization.SystemTextJson" Version="1.0.0" />
<PackageReference Include="Npgsql" Version="6.0.6" />
<PackageReference Include="Npgsql.FSharp" Version="5.3.0" />
<PackageReference Include="Npgsql.NodaTime" Version="6.0.6" />

5
src/JobsJobsJobs/JobsJobsJobs.Data/Json.fs
View File

@ -12,6 +12,9 @@ type WrappedJsonConverter<'T> (wrap : string -> 'T, unwrap : 'T -> string) =
override _.Write(writer, value, _) =
writer.WriteStringValue (unwrap value)
open NodaTime
open NodaTime.Serialization.SystemTextJson
/// JsonSerializer options that use the custom converters
let options =
let opts = JsonSerializerOptions ()
@ -24,4 +27,6 @@ let options =
JsonFSharpConverter ()
]
|> List.iter opts.Converters.Add
let _ = opts.ConfigureForNodaTime DateTimeZoneProviders.Tzdb
opts.PropertyNamingPolicy <- JsonNamingPolicy.CamelCase
opts

156
src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs
View File

@ -1,4 +1,5 @@

open System.Text.Json
open Microsoft.Extensions.Configuration
/// Data access for v2 Jobs, Jobs, Jobs
@ -44,8 +45,8 @@ let r = RethinkDb.Driver.RethinkDB.R
open JobsJobsJobs.Data
open JobsJobsJobs.Domain
open Newtonsoft.Json.Linq
open NodaTime
open NodaTime.Text
open Npgsql.FSharp
open RethinkDb.Driver.FSharp.Functions
/// Retrieve an instant from a JObject field
@ -62,32 +63,155 @@ task {
// Establish database connections
let cfg = ConfigurationBuilder().AddJsonFile("appsettings.json").Build ()
use rethinkConn = Rethink.Startup.createConnection (cfg.GetConnectionString "RethinkDB")
match! DataConnection.setUp cfg with
| Ok _ -> ()
| Error msg -> failwith msg
do! DataConnection.setUp cfg
let pgConn = DataConnection.connection ()
// Migrate citizens
let! oldCitizens =
fromTable Rethink.Table.Citizen
let getOld table =
fromTable table
|> runResult<JObject list>
|> withRetryOnce
|> withConn rethinkConn
// Migrate citizens
let! oldCitizens = getOld Rethink.Table.Citizen
let newCitizens =
oldCitizens
|> List.map (fun c ->
let user = c["mastodonUser"].Value<string> ()
{ Citizen.empty with
id = CitizenId.ofString (c["id"].Value<string> ())
joinedOn = getInstant c "joinedOn"
lastSeenOn = getInstant c "lastSeenOn"
email = $"""{user}@{c["instance"].Value<string> ()}"""
firstName = user
lastName = user
isLegacy = true
Id = CitizenId.ofString (c["id"].Value<string> ())
JoinedOn = getInstant c "joinedOn"
LastSeenOn = getInstant c "lastSeenOn"
Email = $"""{user}@{c["instance"].Value<string> ()}"""
FirstName = user
LastName = user
IsLegacy = true
})
for citizen in newCitizens do
do! Citizens.save citizen
printfn $"** Migrated {List.length newCitizens} citizen(s)"
()
let! _ =
pgConn
|> Sql.executeTransactionAsync [
$"INSERT INTO jjj.{Table.SecurityInfo} VALUES (@id, @data)",
newCitizens |> List.map (fun c ->
let info = { SecurityInfo.empty with Id = c.Id; AccountLocked = true }
[ "@id", Sql.string (CitizenId.toString c.Id)
"@data", Sql.jsonb (JsonSerializer.Serialize (info, Json.options))
])
]
printfn $"** Migrated {List.length newCitizens} citizens"
// Migrate continents
let! oldContinents = getOld Rethink.Table.Continent
let newContinents =
oldContinents
|> List.map (fun c ->
{ Continent.empty with
Id = ContinentId.ofString (c["id"].Value<string> ())
Name = c["name"].Value<string> ()
})
let! _ =
pgConn
|> Sql.executeTransactionAsync [
"INSERT INTO jjj.continent VALUES (@id, @data)",
newContinents |> List.map (fun c -> [
"@id", Sql.string (ContinentId.toString c.Id)
"@data", Sql.jsonb (JsonSerializer.Serialize (c, Json.options))
])
]
printfn $"** Migrated {List.length newContinents} continents"
// Migrate profiles
let! oldProfiles = getOld Rethink.Table.Profile
let newProfiles =
oldProfiles
|> List.map (fun p ->
let experience = p["experience"].Value<string> ()
{ Profile.empty with
Id = CitizenId.ofString (p["id"].Value<string> ())
IsSeekingEmployment = p["seekingEmployment"].Value<bool> ()
IsPubliclySearchable = p["isPublic"].Value<bool> ()
ContinentId = ContinentId.ofString (p["continentId"].Value<string> ())
Region = p["region"].Value<string> ()
IsRemote = p["remoteWork"].Value<bool> ()
IsFullTime = p["fullTime"].Value<bool> ()
Biography = Text (p["biography"].Value<string> ())
LastUpdatedOn = getInstant p "lastUpdatedOn"
Experience = if isNull experience then None else Some (Text experience)
Skills = p["skills"].Children()
|> Seq.map (fun s ->
let notes = s["notes"].Value<string> ()
{ Skill.Id = SkillId.ofString (s["id"].Value<string> ())
Description = s["description"].Value<string> ()
Notes = if isNull notes then None else Some notes
})
|> List.ofSeq
IsLegacy = true
})
for profile in newProfiles do
do! Profiles.save profile
printfn $"** Migrated {List.length newProfiles} profiles"
// Migrate listings
let! oldListings = getOld Rethink.Table.Listing
let newListings =
oldListings
|> List.map (fun l ->
let neededBy = l["neededBy"].Value<string> ()
let wasFilledHere = l["wasFilledHere"].Value<string> ()
{ Listing.empty with
Id = ListingId.ofString (l["id"].Value<string> ())
CitizenId = CitizenId.ofString (l["citizenId"].Value<string> ())
CreatedOn = getInstant l "createdOn"
Title = l["title"].Value<string> ()
ContinentId = ContinentId.ofString (l["continentId"].Value<string> ())
Region = l["region"].Value<string> ()
IsRemote = l["remoteWork"].Value<bool> ()
IsExpired = l["isExpired"].Value<bool> ()
UpdatedOn = getInstant l "updatedOn"
Text = Text (l["text"].Value<string> ())
NeededBy = if isNull neededBy then None else
match LocalDatePattern.Iso.Parse neededBy with
| it when it.Success -> Some it.Value
| it ->
eprintfn $"Error parsing date - {it.Exception.Message}"
None
WasFilledHere = if isNull wasFilledHere then None else Some (bool.Parse wasFilledHere)
IsLegacy = true
})
for listing in newListings do
do! Listings.save listing
printfn $"** Migrated {List.length newListings} listings"
// Migrate success stories
let! oldSuccesses = getOld Rethink.Table.Success
let newSuccesses =
oldSuccesses
|> List.map (fun s ->
let story = s["story"].Value<string> ()
{ Success.empty with
Id = SuccessId.ofString (s["id"].Value<string> ())
CitizenId = CitizenId.ofString (s["citizenId"].Value<string> ())
RecordedOn = getInstant s "recordedOn"
Source = s["source"].Value<string> ()
Story = if isNull story then None else Some (Text story)
})
for success in newSuccesses do
do! Successes.save success
printfn $"** Migrated {List.length newSuccesses} successes"
// Delete any citizens who have no profile, no listing, and no success story recorded
let! deleted =
pgConn
|> Sql.query $"
DELETE FROM jjj.{Table.Citizen}
WHERE id NOT IN (SELECT id FROM jjj.{Table.Profile})
AND id NOT IN (SELECT DISTINCT data->>'citizenId' FROM jjj.{Table.Listing})
AND id NOT IN (SELECT DISTINCT data->>'citizenId' FROM jjj.{Table.Success})"
|> Sql.executeNonQueryAsync
printfn $"** Deleted {deleted} citizens who had no profile, listings, or success stories"
printfn ""
printfn "Migration complete"
} |> Async.AwaitTask |> Async.RunSynchronously

2
src/JobsJobsJobs/Server/Auth.fs
View File

@ -91,7 +91,7 @@ let createJwt (citizen : Citizen) (cfg : AuthOptions) =
tokenHandler.CreateToken (
SecurityTokenDescriptor (
Subject = ClaimsIdentity [|
Claim (ClaimTypes.NameIdentifier, CitizenId.toString citizen.id)
Claim (ClaimTypes.NameIdentifier, CitizenId.toString citizen.Id)
Claim (ClaimTypes.Name, Citizen.name citizen)
|],
Expires = DateTime.UtcNow.AddHours 2.,

112
src/JobsJobsJobs/Server/Handlers.fs
View File

@ -109,7 +109,7 @@ module Citizen =
return!
json
{ jwt = Auth.createJwt citizen (authConfig ctx)
citizenId = CitizenId.toString citizen.id
citizenId = CitizenId.toString citizen.Id
name = Citizen.name citizen
} next ctx
| Error msg ->
@ -238,19 +238,19 @@ module Listing =
let! form = ctx.BindJsonAsync<ListingForm> ()
let now = now ctx
do! Listings.save {
id = ListingId.create ()
citizenId = currentCitizenId ctx
createdOn = now
title = form.title
continentId = ContinentId.ofString form.continentId
region = form.region
remoteWork = form.remoteWork
isExpired = false
updatedOn = now
text = Text form.text
neededBy = (form.neededBy |> Option.map parseDate)
wasFilledHere = None
isLegacy = false
Id = ListingId.create ()
CitizenId = currentCitizenId ctx
CreatedOn = now
Title = form.title
ContinentId = ContinentId.ofString form.continentId
Region = form.region
IsRemote = form.remoteWork
IsExpired = false
UpdatedOn = now
Text = Text form.text
NeededBy = (form.neededBy |> Option.map parseDate)
WasFilledHere = None
IsLegacy = false
}
return! ok next ctx
}
@ -258,18 +258,18 @@ module Listing =
// PUT: /api/listing/[id]
let update listingId : HttpHandler = authorize >=> fun next ctx -> task {
match! Listings.findById (ListingId listingId) with
| Some listing when listing.citizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx
| Some listing when listing.CitizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx
| Some listing ->
let! form = ctx.BindJsonAsync<ListingForm> ()
do! Listings.save
{ listing with
title = form.title
continentId = ContinentId.ofString form.continentId
region = form.region
remoteWork = form.remoteWork
text = Text form.text
neededBy = form.neededBy |> Option.map parseDate
updatedOn = now ctx
Title = form.title
ContinentId = ContinentId.ofString form.continentId
Region = form.region
IsRemote = form.remoteWork
Text = Text form.text
NeededBy = form.neededBy |> Option.map parseDate
UpdatedOn = now ctx
}
return! ok next ctx
| None -> return! Error.notFound next ctx
@ -279,24 +279,24 @@ module Listing =
let expire listingId : HttpHandler = authorize >=> fun next ctx -> task {
let now = now ctx
match! Listings.findById (ListingId listingId) with
| Some listing when listing.citizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx
| Some listing when listing.CitizenId <> (currentCitizenId ctx) -> return! Error.notAuthorized next ctx
| Some listing ->
let! form = ctx.BindJsonAsync<ListingExpireForm> ()
do! Listings.save
{ listing with
isExpired = true
wasFilledHere = Some form.fromHere
updatedOn = now
IsExpired = true
WasFilledHere = Some form.fromHere
UpdatedOn = now
}
match form.successStory with
| Some storyText ->
do! Successes.save
{ id = SuccessId.create()
citizenId = currentCitizenId ctx
recordedOn = now
fromHere = form.fromHere
source = "listing"
story = (Text >> Some) storyText
{ Id = SuccessId.create()
CitizenId = currentCitizenId ctx
RecordedOn = now
IsFromHere = form.fromHere
Source = "listing"
Story = (Text >> Some) storyText
}
| None -> ()
return! ok next ctx
@ -351,25 +351,25 @@ module Profile =
let! profile = task {
match! Profiles.findById citizenId with
| Some p -> return p
| None -> return { Profile.empty with id = citizenId }
| None -> return { Profile.empty with Id = citizenId }
}
do! Profiles.save
{ profile with
seekingEmployment = form.isSeekingEmployment
isPublic = form.isPublic
continentId = ContinentId.ofString form.continentId
region = form.region
remoteWork = form.remoteWork
fullTime = form.fullTime
biography = Text form.biography
lastUpdatedOn = now ctx
experience = noneIfBlank form.experience |> Option.map Text
skills = form.skills
IsSeekingEmployment = form.isSeekingEmployment
IsPubliclySearchable = form.isPublic
ContinentId = ContinentId.ofString form.continentId
Region = form.region
IsRemote = form.remoteWork
IsFullTime = form.fullTime
Biography = Text form.biography
LastUpdatedOn = now ctx
Experience = noneIfBlank form.experience |> Option.map Text
Skills = form.skills
|> List.map (fun s ->
{ id = if s.id.StartsWith "new" then SkillId.create ()
{ Id = if s.id.StartsWith "new" then SkillId.create ()
else SkillId.ofString s.id
description = s.description
notes = noneIfBlank s.notes
Description = s.description
Notes = noneIfBlank s.notes
})
}
return! ok next ctx
@ -379,7 +379,7 @@ module Profile =
let employmentFound : HttpHandler = authorize >=> fun next ctx -> task {
match! Profiles.findById (currentCitizenId ctx) with
| Some profile ->
do! Profiles.save { profile with seekingEmployment = false }
do! Profiles.save { profile with IsSeekingEmployment = false }
return! ok next ctx
| None -> return! Error.notFound next ctx
}
@ -429,19 +429,19 @@ module Success =
let! success = task {
match form.id with
| "new" ->
return Some { id = SuccessId.create ()
citizenId = citizenId
recordedOn = now ctx
fromHere = form.fromHere
source = "profile"
story = noneIfEmpty form.story |> Option.map Text
return Some { Id = SuccessId.create ()
CitizenId = citizenId
RecordedOn = now ctx
IsFromHere = form.fromHere
Source = "profile"
Story = noneIfEmpty form.story |> Option.map Text
}
| successId ->
match! Successes.findById (SuccessId.ofString successId) with
| Some story when story.citizenId = citizenId ->
| Some story when story.CitizenId = citizenId ->
return Some { story with
fromHere = form.fromHere
story = noneIfEmpty form.story |> Option.map Text
IsFromHere = form.fromHere
Story = noneIfEmpty form.story |> Option.map Text
}
| Some _ | None -> return None
}