Remove v2/v3 migration program
This commit is contained in:
parent
ce30af17fe
commit
26fc214f36
|
|
@ -27,8 +27,6 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Profiles", "JobsJobsJobs\Pr
|
||||||
EndProject
|
EndProject
|
||||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "SuccessStories", "JobsJobsJobs\SuccessStories\JobsJobsJobs.SuccessStories.fsproj", "{8DAFA6F6-0415-4507-B31C-7FEBE0D2E9D7}"
|
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "SuccessStories", "JobsJobsJobs\SuccessStories\JobsJobsJobs.SuccessStories.fsproj", "{8DAFA6F6-0415-4507-B31C-7FEBE0D2E9D7}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "JobsJobsJobs.V3Migration", "JobsJobsJobs\JobsJobsJobs.V3Migration\JobsJobsJobs.V3Migration.fsproj", "{DC3E225D-9720-44E8-86AE-DEE71262C9F0}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
|
@ -39,10 +37,6 @@ Global
|
||||||
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.Build.0 = Release|Any CPU
|
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{DC3E225D-9720-44E8-86AE-DEE71262C9F0}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{D6E4A943-5113-41ED-A547-8D3BE5516DC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
|
@ -76,7 +70,6 @@ Global
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(NestedProjects) = preSolution
|
GlobalSection(NestedProjects) = preSolution
|
||||||
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
{8F5A3D1E-562B-4F27-9787-6CB14B35E69E} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
||||||
{DC3E225D-9720-44E8-86AE-DEE71262C9F0} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
|
||||||
{D6E4A943-5113-41ED-A547-8D3BE5516DC0} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
{D6E4A943-5113-41ED-A547-8D3BE5516DC0} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
||||||
{4C184AB8-DDA7-4545-BC84-A4ACCBE29764} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
{4C184AB8-DDA7-4545-BC84-A4ACCBE29764} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
||||||
{0B89D606-A094-4E82-8F8A-9D72D6A0E805} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
{0B89D606-A094-4E82-8F8A-9D72D6A0E805} = {FA833B24-B8F6-4CE6-A044-99257EAC02FF}
|
||||||
|
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<PublishSingleFile>true</PublishSingleFile>
|
|
||||||
<SelfContained>false</SelfContained>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<Compile Include="Program.fs" />
|
|
||||||
<Content Include="appsettings.Migration.json" CopyToOutputDirectory="Always" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="NodaTime.Serialization.JsonNet" Version="3.0.0" />
|
|
||||||
<PackageReference Include="RethinkDb.Driver.FSharp" Version="0.9.0-beta-07" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\Application\JobsJobsJobs.Application.fsproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
|
|
@ -1,217 +0,0 @@
|
||||||
|
|
||||||
open System.Text.Json
|
|
||||||
open Microsoft.Extensions.Configuration
|
|
||||||
|
|
||||||
/// Data access for v2 Jobs, Jobs, Jobs
|
|
||||||
module Rethink =
|
|
||||||
|
|
||||||
/// Table names
|
|
||||||
[<RequireQualifiedAccess>]
|
|
||||||
module Table =
|
|
||||||
/// The user (citizen of Gitmo Nation) table
|
|
||||||
let Citizen = "citizen"
|
|
||||||
/// The continent table
|
|
||||||
let Continent = "continent"
|
|
||||||
/// The job listing table
|
|
||||||
let Listing = "listing"
|
|
||||||
/// The citizen employment profile table
|
|
||||||
let Profile = "profile"
|
|
||||||
/// The success story table
|
|
||||||
let Success = "success"
|
|
||||||
/// All tables
|
|
||||||
let all () = [ Citizen; Continent; Listing; Profile; Success ]
|
|
||||||
|
|
||||||
open RethinkDb.Driver.Net
|
|
||||||
|
|
||||||
/// Functions run at startup
|
|
||||||
[<RequireQualifiedAccess>]
|
|
||||||
module Startup =
|
|
||||||
|
|
||||||
open NodaTime
|
|
||||||
open NodaTime.Serialization.JsonNet
|
|
||||||
open RethinkDb.Driver.FSharp
|
|
||||||
|
|
||||||
/// Create a RethinkDB connection
|
|
||||||
let createConnection (connStr : string) =
|
|
||||||
// Add all required JSON converters
|
|
||||||
Converter.Serializer.ConfigureForNodaTime DateTimeZoneProviders.Tzdb |> ignore
|
|
||||||
// Connect to the database
|
|
||||||
let config = DataConfig.FromUri connStr
|
|
||||||
config.CreateConnection ()
|
|
||||||
|
|
||||||
/// Shorthand for the RethinkDB R variable (how every command starts)
|
|
||||||
let r = RethinkDb.Driver.RethinkDB.R
|
|
||||||
|
|
||||||
open JobsJobsJobs
|
|
||||||
open JobsJobsJobs.Common.Data
|
|
||||||
open JobsJobsJobs.Domain
|
|
||||||
open Newtonsoft.Json.Linq
|
|
||||||
open NodaTime.Text
|
|
||||||
open Npgsql.FSharp
|
|
||||||
open RethinkDb.Driver.FSharp.Functions
|
|
||||||
|
|
||||||
/// Retrieve an instant from a JObject field
|
|
||||||
let getInstant (doc : JObject) name =
|
|
||||||
let text = doc[name].Value<string> ()
|
|
||||||
match InstantPattern.General.Parse text with
|
|
||||||
| it when it.Success -> it.Value
|
|
||||||
| _ ->
|
|
||||||
match InstantPattern.ExtendedIso.Parse text with
|
|
||||||
| it when it.Success -> it.Value
|
|
||||||
| it -> raise it.Exception
|
|
||||||
|
|
||||||
task {
|
|
||||||
// Establish database connections
|
|
||||||
let cfg = ConfigurationBuilder().AddJsonFile("appsettings.Migration.json").Build ()
|
|
||||||
use rethinkConn = Rethink.Startup.createConnection (cfg.GetConnectionString "RethinkDB")
|
|
||||||
do! setUp cfg
|
|
||||||
let pgConn = dataSource ()
|
|
||||||
|
|
||||||
let getOld table =
|
|
||||||
fromTable table
|
|
||||||
|> runResult<JObject list>
|
|
||||||
|> withRetryOnce
|
|
||||||
|> withConn rethinkConn
|
|
||||||
|
|
||||||
// Migrate citizens
|
|
||||||
let! oldCitizens = getOld Rethink.Table.Citizen
|
|
||||||
let newCitizens =
|
|
||||||
oldCitizens
|
|
||||||
|> List.map (fun c ->
|
|
||||||
let user = c["mastodonUser"].Value<string> ()
|
|
||||||
{ Citizen.empty with
|
|
||||||
Id = CitizenId.ofString (c["id"].Value<string> ())
|
|
||||||
JoinedOn = getInstant c "joinedOn"
|
|
||||||
LastSeenOn = getInstant c "lastSeenOn"
|
|
||||||
Email = $"""{user}@{c["instance"].Value<string> ()}"""
|
|
||||||
FirstName = user
|
|
||||||
LastName = user
|
|
||||||
IsLegacy = true
|
|
||||||
})
|
|
||||||
for citizen in newCitizens do
|
|
||||||
do! Citizens.Data.save citizen
|
|
||||||
let! _ =
|
|
||||||
pgConn
|
|
||||||
|> Sql.executeTransactionAsync [
|
|
||||||
$"INSERT INTO {Table.SecurityInfo} VALUES (@id, @data)",
|
|
||||||
newCitizens |> List.map (fun c ->
|
|
||||||
let info = { SecurityInfo.empty with Id = c.Id; AccountLocked = true }
|
|
||||||
[ "@id", Sql.string (CitizenId.toString c.Id)
|
|
||||||
"@data", Sql.jsonb (JsonSerializer.Serialize (info, Json.options))
|
|
||||||
])
|
|
||||||
]
|
|
||||||
printfn $"** Migrated {List.length newCitizens} citizens"
|
|
||||||
|
|
||||||
// Migrate continents
|
|
||||||
let! oldContinents = getOld Rethink.Table.Continent
|
|
||||||
let newContinents =
|
|
||||||
oldContinents
|
|
||||||
|> List.map (fun c ->
|
|
||||||
{ Continent.empty with
|
|
||||||
Id = ContinentId.ofString (c["id"].Value<string> ())
|
|
||||||
Name = c["name"].Value<string> ()
|
|
||||||
})
|
|
||||||
let! _ =
|
|
||||||
pgConn
|
|
||||||
|> Sql.executeTransactionAsync [
|
|
||||||
$"INSERT INTO {Table.Continent} VALUES (@id, @data)",
|
|
||||||
newContinents |> List.map (fun c -> [
|
|
||||||
"@id", Sql.string (ContinentId.toString c.Id)
|
|
||||||
"@data", Sql.jsonb (JsonSerializer.Serialize (c, Json.options))
|
|
||||||
])
|
|
||||||
]
|
|
||||||
printfn $"** Migrated {List.length newContinents} continents"
|
|
||||||
|
|
||||||
// Migrate profiles
|
|
||||||
let! oldProfiles = getOld Rethink.Table.Profile
|
|
||||||
let newProfiles =
|
|
||||||
oldProfiles
|
|
||||||
|> List.map (fun p ->
|
|
||||||
let experience = p["experience"].Value<string> ()
|
|
||||||
{ Profile.empty with
|
|
||||||
Id = CitizenId.ofString (p["id"].Value<string> ())
|
|
||||||
ContinentId = ContinentId.ofString (p["continentId"].Value<string> ())
|
|
||||||
Region = p["region"].Value<string> ()
|
|
||||||
IsSeekingEmployment = p["seekingEmployment"].Value<bool> ()
|
|
||||||
IsRemote = p["remoteWork"].Value<bool> ()
|
|
||||||
IsFullTime = p["fullTime"].Value<bool> ()
|
|
||||||
Biography = Text (p["biography"].Value<string> ())
|
|
||||||
Experience = if isNull experience then None else Some (Text experience)
|
|
||||||
Skills = p["skills"].Children()
|
|
||||||
|> Seq.map (fun s ->
|
|
||||||
let notes = s["notes"].Value<string> ()
|
|
||||||
{ Description = s["description"].Value<string> ()
|
|
||||||
Notes = if isNull notes then None else Some notes
|
|
||||||
})
|
|
||||||
|> List.ofSeq
|
|
||||||
Visibility = if p["isPublic"].Value<bool> () then Anonymous else Private
|
|
||||||
LastUpdatedOn = getInstant p "lastUpdatedOn"
|
|
||||||
IsLegacy = true
|
|
||||||
})
|
|
||||||
for profile in newProfiles do
|
|
||||||
do! Profiles.Data.save profile
|
|
||||||
printfn $"** Migrated {List.length newProfiles} profiles"
|
|
||||||
|
|
||||||
// Migrate listings
|
|
||||||
let! oldListings = getOld Rethink.Table.Listing
|
|
||||||
let newListings =
|
|
||||||
oldListings
|
|
||||||
|> List.map (fun l ->
|
|
||||||
let neededBy = l["neededBy"].Value<string> ()
|
|
||||||
let wasFilledHere = l["wasFilledHere"].Value<string> ()
|
|
||||||
{ Listing.empty with
|
|
||||||
Id = ListingId.ofString (l["id"].Value<string> ())
|
|
||||||
CitizenId = CitizenId.ofString (l["citizenId"].Value<string> ())
|
|
||||||
CreatedOn = getInstant l "createdOn"
|
|
||||||
Title = l["title"].Value<string> ()
|
|
||||||
ContinentId = ContinentId.ofString (l["continentId"].Value<string> ())
|
|
||||||
Region = l["region"].Value<string> ()
|
|
||||||
IsRemote = l["remoteWork"].Value<bool> ()
|
|
||||||
IsExpired = l["isExpired"].Value<bool> ()
|
|
||||||
UpdatedOn = getInstant l "updatedOn"
|
|
||||||
Text = Text (l["text"].Value<string> ())
|
|
||||||
NeededBy = if isNull neededBy then None else
|
|
||||||
match LocalDatePattern.Iso.Parse neededBy with
|
|
||||||
| it when it.Success -> Some it.Value
|
|
||||||
| it ->
|
|
||||||
eprintfn $"Error parsing date - {it.Exception.Message}"
|
|
||||||
None
|
|
||||||
WasFilledHere = if isNull wasFilledHere then None else Some (bool.Parse wasFilledHere)
|
|
||||||
IsLegacy = true
|
|
||||||
})
|
|
||||||
for listing in newListings do
|
|
||||||
do! Listings.Data.save listing
|
|
||||||
printfn $"** Migrated {List.length newListings} listings"
|
|
||||||
|
|
||||||
// Migrate success stories
|
|
||||||
let! oldSuccesses = getOld Rethink.Table.Success
|
|
||||||
let newSuccesses =
|
|
||||||
oldSuccesses
|
|
||||||
|> List.map (fun s ->
|
|
||||||
let story = s["story"].Value<string> ()
|
|
||||||
{ Success.empty with
|
|
||||||
Id = SuccessId.ofString (s["id"].Value<string> ())
|
|
||||||
CitizenId = CitizenId.ofString (s["citizenId"].Value<string> ())
|
|
||||||
RecordedOn = getInstant s "recordedOn"
|
|
||||||
Source = s["source"].Value<string> ()
|
|
||||||
Story = if isNull story then None else Some (Text story)
|
|
||||||
})
|
|
||||||
for success in newSuccesses do
|
|
||||||
do! SuccessStories.Data.save success
|
|
||||||
printfn $"** Migrated {List.length newSuccesses} successes"
|
|
||||||
|
|
||||||
// Delete any citizens who have no profile, no listing, and no success story recorded
|
|
||||||
let! deleted =
|
|
||||||
pgConn
|
|
||||||
|> Sql.query $"
|
|
||||||
DELETE FROM {Table.Citizen}
|
|
||||||
WHERE id NOT IN (SELECT id FROM {Table.Profile})
|
|
||||||
AND id NOT IN (SELECT DISTINCT data ->> 'citizenId' FROM {Table.Listing})
|
|
||||||
AND id NOT IN (SELECT DISTINCT data ->> 'citizenId' FROM {Table.Success})"
|
|
||||||
|> Sql.executeNonQueryAsync
|
|
||||||
printfn $"** Deleted {deleted} citizens who had no profile, listings, or success stories"
|
|
||||||
|
|
||||||
printfn ""
|
|
||||||
printfn "Migration complete"
|
|
||||||
} |> Async.AwaitTask |> Async.RunSynchronously
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user