diff --git a/src/JobsJobsJobs.sln b/src/JobsJobsJobs.sln index c5aab85..3ff2d43 100644 --- a/src/JobsJobsJobs.sln +++ b/src/JobsJobsJobs.sln @@ -27,8 +27,6 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Profiles", "JobsJobsJobs\Pr EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "SuccessStories", "JobsJobsJobs\SuccessStories\JobsJobsJobs.SuccessStories.fsproj", "{8DAFA6F6-0415-4507-B31C-7FEBE0D2E9D7}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "JobsJobsJobs.V3Migration", "JobsJobsJobs\JobsJobsJobs.V3Migration\JobsJobsJobs.V3Migration.fsproj", "{DC3E225D-9720-44E8-86AE-DEE71262C9F0}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -39,10 +37,6 @@ Global {8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Debug|Any CPU.Build.0 = Debug|Any CPU {8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.ActiveCfg = Release|Any CPU {8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.Build.0 = Release|Any CPU - {DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Release|Any CPU.Build.0 = Release|Any CPU {D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.Build.0 = Debug|Any CPU {D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -76,7 +70,6 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {8F5A3D1E-562B-4F27-9787-6CB14B35E69E} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF} - {DC3E225D-9720-44E8-86AE-DEE71262C9F0} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF} {D6E4A943-5113-41ED-A547-8D3BE5516DC0} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF} {4C184AB8-DDA7-4545-BC84-A4ACCBE29764} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF} {0B89D606-A094-4E82-8F8A-9D72D6A0E805} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF} diff --git a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/JobsJobsJobs.V3Migration.fsproj b/src/JobsJobsJobs/JobsJobsJobs.V3Migration/JobsJobsJobs.V3Migration.fsproj deleted file mode 100644 index c380f67..0000000 --- a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/JobsJobsJobs.V3Migration.fsproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - Exe - true - false - - - - - - - - - - - - - - - - - diff --git a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs b/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs deleted file mode 100644 index 23dde0d..0000000 --- a/src/JobsJobsJobs/JobsJobsJobs.V3Migration/Program.fs +++ /dev/null @@ -1,217 +0,0 @@ - -open System.Text.Json -open Microsoft.Extensions.Configuration - -/// Data access for v2 Jobs, Jobs, Jobs -module Rethink = - - /// Table names - [] - module Table = - /// The user (citizen of Gitmo Nation) table - let Citizen = "citizen" - /// The continent table - let Continent = "continent" - /// The job listing table - let Listing = "listing" - /// The citizen employment profile table - let Profile = "profile" - /// The success story table - let Success = "success" - /// All tables - let all () = [ Citizen; Continent; Listing; Profile; Success ] - - open RethinkDb.Driver.Net - - /// Functions run at startup - [] - module Startup = - - open NodaTime - open NodaTime.Serialization.JsonNet - open RethinkDb.Driver.FSharp - - /// Create a RethinkDB connection - let createConnection (connStr : string) = - // Add all required JSON converters - Converter.Serializer.ConfigureForNodaTime DateTimeZoneProviders.Tzdb |> ignore - // Connect to the database - let config = DataConfig.FromUri connStr - config.CreateConnection () - -/// Shorthand for the RethinkDB R variable (how every command starts) -let r = RethinkDb.Driver.RethinkDB.R - -open JobsJobsJobs -open JobsJobsJobs.Common.Data -open JobsJobsJobs.Domain -open Newtonsoft.Json.Linq -open NodaTime.Text -open Npgsql.FSharp -open RethinkDb.Driver.FSharp.Functions - -/// Retrieve an instant from a JObject field -let getInstant (doc : JObject) name = - let text = doc[name].Value () - match InstantPattern.General.Parse text with - | it when it.Success -> it.Value - | _ -> - match InstantPattern.ExtendedIso.Parse text with - | it when it.Success -> it.Value - | it -> raise it.Exception - -task { - // Establish database connections - let cfg = ConfigurationBuilder().AddJsonFile("appsettings.Migration.json").Build () - use rethinkConn = Rethink.Startup.createConnection (cfg.GetConnectionString "RethinkDB") - do! setUp cfg - let pgConn = dataSource () - - let getOld table = - fromTable table - |> runResult - |> withRetryOnce - |> withConn rethinkConn - - // Migrate citizens - let! oldCitizens = getOld Rethink.Table.Citizen - let newCitizens = - oldCitizens - |> List.map (fun c -> - let user = c["mastodonUser"].Value () - { Citizen.empty with - Id = CitizenId.ofString (c["id"].Value ()) - JoinedOn = getInstant c "joinedOn" - LastSeenOn = getInstant c "lastSeenOn" - Email = $"""{user}@{c["instance"].Value ()}""" - FirstName = user - LastName = user - IsLegacy = true - }) - for citizen in newCitizens do - do! Citizens.Data.save citizen - let! _ = - pgConn - |> Sql.executeTransactionAsync [ - $"INSERT INTO {Table.SecurityInfo} VALUES (@id, @data)", - newCitizens |> List.map (fun c -> - let info = { SecurityInfo.empty with Id = c.Id; AccountLocked = true } - [ "@id", Sql.string (CitizenId.toString c.Id) - "@data", Sql.jsonb (JsonSerializer.Serialize (info, Json.options)) - ]) - ] - printfn $"** Migrated {List.length newCitizens} citizens" - - // Migrate continents - let! oldContinents = getOld Rethink.Table.Continent - let newContinents = - oldContinents - |> List.map (fun c -> - { Continent.empty with - Id = ContinentId.ofString (c["id"].Value ()) - Name = c["name"].Value () - }) - let! _ = - pgConn - |> Sql.executeTransactionAsync [ - $"INSERT INTO {Table.Continent} VALUES (@id, @data)", - newContinents |> List.map (fun c -> [ - "@id", Sql.string (ContinentId.toString c.Id) - "@data", Sql.jsonb (JsonSerializer.Serialize (c, Json.options)) - ]) - ] - printfn $"** Migrated {List.length newContinents} continents" - - // Migrate profiles - let! oldProfiles = getOld Rethink.Table.Profile - let newProfiles = - oldProfiles - |> List.map (fun p -> - let experience = p["experience"].Value () - { Profile.empty with - Id = CitizenId.ofString (p["id"].Value ()) - ContinentId = ContinentId.ofString (p["continentId"].Value ()) - Region = p["region"].Value () - IsSeekingEmployment = p["seekingEmployment"].Value () - IsRemote = p["remoteWork"].Value () - IsFullTime = p["fullTime"].Value () - Biography = Text (p["biography"].Value ()) - Experience = if isNull experience then None else Some (Text experience) - Skills = p["skills"].Children() - |> Seq.map (fun s -> - let notes = s["notes"].Value () - { Description = s["description"].Value () - Notes = if isNull notes then None else Some notes - }) - |> List.ofSeq - Visibility = if p["isPublic"].Value () then Anonymous else Private - LastUpdatedOn = getInstant p "lastUpdatedOn" - IsLegacy = true - }) - for profile in newProfiles do - do! Profiles.Data.save profile - printfn $"** Migrated {List.length newProfiles} profiles" - - // Migrate listings - let! oldListings = getOld Rethink.Table.Listing - let newListings = - oldListings - |> List.map (fun l -> - let neededBy = l["neededBy"].Value () - let wasFilledHere = l["wasFilledHere"].Value () - { Listing.empty with - Id = ListingId.ofString (l["id"].Value ()) - CitizenId = CitizenId.ofString (l["citizenId"].Value ()) - CreatedOn = getInstant l "createdOn" - Title = l["title"].Value () - ContinentId = ContinentId.ofString (l["continentId"].Value ()) - Region = l["region"].Value () - IsRemote = l["remoteWork"].Value () - IsExpired = l["isExpired"].Value () - UpdatedOn = getInstant l "updatedOn" - Text = Text (l["text"].Value ()) - NeededBy = if isNull neededBy then None else - match LocalDatePattern.Iso.Parse neededBy with - | it when it.Success -> Some it.Value - | it -> - eprintfn $"Error parsing date - {it.Exception.Message}" - None - WasFilledHere = if isNull wasFilledHere then None else Some (bool.Parse wasFilledHere) - IsLegacy = true - }) - for listing in newListings do - do! Listings.Data.save listing - printfn $"** Migrated {List.length newListings} listings" - - // Migrate success stories - let! oldSuccesses = getOld Rethink.Table.Success - let newSuccesses = - oldSuccesses - |> List.map (fun s -> - let story = s["story"].Value () - { Success.empty with - Id = SuccessId.ofString (s["id"].Value ()) - CitizenId = CitizenId.ofString (s["citizenId"].Value ()) - RecordedOn = getInstant s "recordedOn" - Source = s["source"].Value () - Story = if isNull story then None else Some (Text story) - }) - for success in newSuccesses do - do! SuccessStories.Data.save success - printfn $"** Migrated {List.length newSuccesses} successes" - - // Delete any citizens who have no profile, no listing, and no success story recorded - let! deleted = - pgConn - |> Sql.query $" - DELETE FROM {Table.Citizen} - WHERE id NOT IN (SELECT id FROM {Table.Profile}) - AND id NOT IN (SELECT DISTINCT data ->> 'citizenId' FROM {Table.Listing}) - AND id NOT IN (SELECT DISTINCT data ->> 'citizenId' FROM {Table.Success})" - |> Sql.executeNonQueryAsync - printfn $"** Deleted {deleted} citizens who had no profile, listings, or success stories" - - printfn "" - printfn "Migration complete" -} |> Async.AwaitTask |> Async.RunSynchronously -