WPB-22959 migrate team feature data to postgres

changelog.d/0-release-notes/WPB-22959

@@ -0,0 +1,6 @@
+Team features can now be migrated from Cassandra to Postgres. To migrate:
+- Set galley `postgresMigration.teamFeatures` to `migration-to-postgresql`.
+- Enable the background-worker flag `migrateTeamFeatures=true` to run the backfill.
+- Monitor the `wire_team_features_migration_finished` metric to confirm completion.
+- Switch `postgresMigration.teamFeatures` to `postgresql` and restart Galley and background-worker.
+- Once fully cut over, drop the Cassandra `team_features_dyn` table.

changelog.d/5-internal/WPB-22959

@@ -1,3 +1 @@
-- Generalized the migration lock for better reuse
-- Move logic from `TeamFeatureStore` interpreter to `FeatureConfigSubsystem` 
-(#4982, #4983)
+Migration from Cassandra to Postgres of Team Features (#4982, #4983, #4979)

charts/background-worker/templates/configmap.yaml

@@ -91,6 +91,7 @@ data:
 
     migrateConversations: {{ .migrateConversations }}
     migrateConversationCodes: {{ .migrateConversationCodes }}
+    migrateTeamFeatures: {{ .migrateTeamFeatures }}
     migrateConversationsOptions:
 {{toYaml .migrateConversationsOptions | indent 6 }}
 

charts/background-worker/values.yaml

@@ -73,6 +73,10 @@ config:
   # It's important to set `settings.postgresMigration.conversationCodes` to `migration-to-postgresql`
   # before starting the migration.
   migrateConversationCodes: false
+  # This will start the migration of team features.
+  # It's important to set `settings.postgresMigration.teamFeatures` to `migration-to-postgresql`
+  # before starting the migration.
+  migrateTeamFeatures: false
 
   backendNotificationPusher:
     pushBackoffMinWait: 10000 # in microseconds, so 10ms
@@ -92,6 +96,7 @@ config:
   postgresMigration:
     conversation: cassandra
     conversationCodes: cassandra
+    teamFeatures: cassandra
 
 secrets:
   {}

charts/galley/values.yaml

@@ -72,6 +72,7 @@ config:
   postgresMigration:
     conversation: cassandra
     conversationCodes: cassandra
+    teamFeatures: cassandra
   settings:
     httpPoolSize: 128
     maxTeamSize: 10000

docs/src/developer/reference/config-options.md

@@ -1812,11 +1812,13 @@ galley:
     postgresMigration:
       conversation: postgresql
       conversationCodes: postgresql
+      teamFeatures: postgresql
 background-worker:
   config:
     postgresMigration:
       conversation: postgresql
       conversationCodes: postgresql
+      teamFeatures: postgresql
     migrateConversations: false
 ```
 
@@ -1847,13 +1849,16 @@ pattern below applies per store. Use it for `conversation` and
        postgresMigration:
          conversation: migration-to-postgresql
          conversationCodes: migration-to-postgresql
+         teamFeatures: migration-to-postgresql
    background-worker:
      config:
-       postgresMigration:
-         conversation: migration-to-postgresql
-         conversationCodes: migration-to-postgresql
-       migrateConversations: false
-       migrateConversationCodes: false
+      postgresMigration:
+        conversation: migration-to-postgresql
+        conversationCodes: migration-to-postgresql
+        teamFeatures: migration-to-postgresql
+      migrateConversations: false
+      migrateConversationCodes: false
+      migrateTeamFeatures: false
    ```
 
    This change should restart all the galley pods, and new writes will follow
@@ -1866,8 +1871,14 @@ pattern below applies per store. Use it for `conversation` and
      config:
        migrateConversations: true
        migrateConversationCodes: true
+       migrateTeamFeatures: true
    ```
 
+   During migration, Cassandra rows are not deleted. Writes and migration share
+   per-row locks to avoid races, so there is no need to delete early. Deletion is
+   deferred to keep rollback options and to remove Cassandra only after a full
+   cutover to PostgreSQL-only.
+
    Wait for the store-specific migration metrics to reach `1.0`. For
    conversations: `wire_local_convs_migration_finished` and
    `wire_user_remote_convs_migration_finished`. For conversation codes:
@@ -1882,13 +1893,16 @@ pattern below applies per store. Use it for `conversation` and
        postgresMigration:
          conversation: postgresql
          conversationCodes: postgresql
+         teamFeatures: postgresql
    background-worker:
      config:
-       postgresMigration:
-         conversation: postgresql
-         conversationCodes: postgresql
-       migrateConversations: false
-       migrateConversationCodes: false
+      postgresMigration:
+        conversation: postgresql
+        conversationCodes: postgresql
+        teamFeatures: postgresql
+      migrateConversations: false
+      migrateConversationCodes: false
+      migrateTeamFeatures: false
    ```
 
 **How to run migrations independently or in batches**
@@ -1956,6 +1970,8 @@ postgresqlPool:
 postgresMigration:
   # Valid: cassandra | migration-to-postgresql | postgresql
   conversation: postgresql
+  conversationCodes: postgresql
+  teamFeatures: postgresql
 
 # Start the migration worker when true
 migrateConversations: false
@@ -1978,7 +1994,7 @@ Notes
 
 - `postgresql` values follow libpq keywords; password is sourced via `secrets.pgPassword`.
 - RabbitMQ admin fields (`adminHost`, `adminPort`) are templated only when `config.enableFederation` is true.
-- `postgresMigration.conversation` must match `galley.config.postgresMigration.conversation` during migration phases.
-- `migrateConversations: true` triggers the migration job; leave it `false` for new installs and after migration.
+- `postgresMigration.<store>` must match between `galley` and `background-worker` during migration phases.
+- `migrateConversations: true` triggers the conversation migration job; leave it `false` for new installs and after migration.
 - `concurrency`, `jobTimeout`, and `maxAttempts` control parallelism and retry behavior of the consumer.
 - `brig` and `gundeck` endpoints default to in-cluster services; override via `background-worker.config.brig` and `.gundeck` if your service DNS/ports differ.

hack/helm_vars/common.yaml.gotmpl

@@ -16,6 +16,7 @@ dynBackendDomain3: dynamic-backend-3.{{ requiredEnv "NAMESPACE_1" }}.svc.cluster
 {{- $preferredStore := default "cassandra" (env "PREFERRED_STORE") }}
 conversationStore: {{ $preferredStore }}
 conversationCodesStore: {{ $preferredStore }}
+teamFeaturesStore: {{ $preferredStore }}
 
 {{- if (eq (env "UPLOAD_XML_S3_BASE_URL") "") }}
 uploadXml: {}

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -307,6 +307,7 @@ galley:
     postgresMigration:
       conversation: {{ .Values.conversationStore }}
       conversationCodes: {{ .Values.conversationCodesStore }}
+      teamFeatures: {{ .Values.teamFeaturesStore }}
     settings:
       maxConvAndTeamSize: 16
       maxTeamSize: 32
@@ -675,6 +676,7 @@ background-worker:
     postgresMigration:
       conversation: {{ .Values.conversationStore }}
       conversationCodes: {{ .Values.conversationCodesStore }}
+      teamFeatures: {{ .Values.teamFeaturesStore }}
     rabbitmq:
       port: 5671
       adminPort: 15671

integration/integration.cabal

@@ -175,6 +175,7 @@ library
     Test.MessageTimer
     Test.Migration.Conversation
     Test.Migration.ConversationCodes
+    Test.Migration.TeamFeatures
     Test.Migration.Util
     Test.MLS
     Test.MLS.Clients

integration/test/Test/Migration/TeamFeatures.hs

@@ -0,0 +1,178 @@
+module Test.Migration.TeamFeatures where
+
+import qualified API.Galley as Public
+import qualified API.GalleyInternal as Internal
+import Control.Monad.Codensity
+import Control.Monad.Reader
+import SetupHelpers
+import Test.FeatureFlags.Util
+import Test.Migration.Util (waitForMigration)
+import Testlib.Prelude hiding (pairs)
+import Testlib.ResourcePool
+
+testTeamFeaturesMigration :: (HasCallStack) => App ()
+testTeamFeaturesMigration = do
+  resourcePool <- asks (.resourcePool)
+  runCodensity (acquireResources 1 resourcePool) $ \[backend] -> do
+    let preMigration = runCodensity (startDynamicBackend backend (conf "cassandra" False)) . const
+        switchToMigratingInterpreter = runCodensity (startDynamicBackend backend (conf "migration-to-postgresql" False)) . const
+        startMigration = runCodensity (startDynamicBackend backend (conf "migration-to-postgresql" True)) . const
+        stopMigration = runCodensity (startDynamicBackend backend (conf "migration-to-postgresql" False)) . const
+        switchToPostgresInterpreter = runCodensity (startDynamicBackend backend (conf "postgresql" False)) . const
+        domain = backend.berDomain
+
+    (teams0, teams1) <-
+      preMigration $ do
+        teams0 <- replicateM 3 $ createTeam domain 2
+        teams1@(team1 : _) <- replicateM 5 $ createTeam domain 1
+        for_ teams0 $ \(owner, tid, _) -> enableFeatures owner tid unlockableFeatures
+        testSetFeatures team1
+        testGetFeatures team1
+        pure (teams0, teams1)
+
+    team1 : team2 : team3 : team4 : team5 : _ <- pure teams1
+
+    switchToMigratingInterpreter $ do
+      assertModifiedFeatures domain teams0
+      testSetFeatures team2
+      testGetFeatures team1
+      testGetFeatures team2
+
+    startMigration $ do
+      assertModifiedFeatures domain teams0
+      testSetFeatures team3
+      testGetFeatures team1
+      testGetFeatures team2
+      testGetFeatures team3
+      waitForMigration domain counterName
+
+    stopMigration $ do
+      assertModifiedFeatures domain teams0
+      testSetFeatures team4
+      testGetFeatures team1
+      testGetFeatures team2
+      testGetFeatures team3
+      testGetFeatures team4
+
+    switchToPostgresInterpreter $ do
+      assertModifiedFeatures domain teams0
+      testSetFeatures team5
+      testGetFeatures team1
+      testGetFeatures team2
+      testGetFeatures team3
+      testGetFeatures team4
+      testGetFeatures team5
+  where
+    unlockableFeatures :: [String]
+    unlockableFeatures =
+      [ "fileSharing",
+        "conferenceCalling",
+        "selfDeletingMessages",
+        "conversationGuestLinks",
+        "sndFactorPasswordChallenge",
+        "mls",
+        "outlookCalIntegration",
+        "mlsE2EId",
+        "mlsMigration",
+        "enforceFileDownloadLocation",
+        "domainRegistration",
+        "channels",
+        "cells",
+        "consumableNotifications",
+        "chatBubbles",
+        "apps",
+        "simplifiedUserConnectionRequestQRCode",
+        "stealthUsers",
+        "meetings",
+        "meetingsPremium"
+      ]
+
+    assertModifiedFeatures :: String -> [(Value, String, [Value])] -> App ()
+    assertModifiedFeatures domain teams = do
+      expectedModifiedFeatures <-
+        mkExpectedModifiedFeatures defAllFeatures
+          >>= setField "classifiedDomains.config.domains" [domain]
+      for_ teams $ \(owner, tid, _) ->
+        bindResponse (Public.getTeamFeatures owner tid) $ \resp -> do
+          resp.status `shouldMatchInt` 200
+          for_ unlockableFeatures $ \feat -> do
+            resp.json %. feat %. "status" `shouldMatch` "enabled"
+            resp.json %. feat %. "lockStatus" `shouldMatch` "unlocked"
+          resp.json `shouldMatch` expectedModifiedFeatures
+
+    enableFeatures :: Value -> String -> [String] -> App ()
+    enableFeatures owner tid features = do
+      for_ features $ \name -> do
+        Internal.setTeamFeatureLockStatus owner tid name "unlocked"
+        assertSuccess =<< Internal.setTeamFeatureStatus owner tid name "enabled"
+
+    testSetFeatures :: (HasCallStack) => (Value, String, [Value]) -> App ()
+    testSetFeatures (owner, tid, _) = do
+      Internal.setTeamFeatureLockStatus owner tid "channels" "unlocked"
+      Internal.setTeamFeatureLockStatus owner tid "enforceFileDownloadLocation" "unlocked"
+      assertSuccess =<< Internal.setTeamFeatureConfig owner tid "channels" channelsConfig
+      assertSuccess =<< Internal.setTeamFeatureConfig owner tid "enforceFileDownloadLocation" enforceDownloadLocationConfig
+      where
+        channelsConfig :: Value
+        channelsConfig =
+          object
+            [ "status" .= "enabled",
+              "config"
+                .= object
+                  [ "allowed_to_create_channels" .= "team-members",
+                    "allowed_to_open_channels" .= "admins"
+                  ]
+            ]
+
+        enforceDownloadLocationConfig :: Value
+        enforceDownloadLocationConfig =
+          object
+            [ "status" .= "enabled",
+              "config" .= object ["enforcedDownloadLocation" .= "/tmp/migration-test"]
+            ]
+
+    testGetFeatures :: (HasCallStack) => (Value, String, [Value]) -> App ()
+    testGetFeatures (owner, tid, _) = do
+      expectedChannels <- expectedChannelsConfig
+      expectedDownloadLocation <- expectedEnforceDownloadLocationConfig
+      bindResponse (Public.getTeamFeature owner tid "channels") $ \resp -> do
+        resp.status `shouldMatchInt` 200
+        resp.json `shouldMatch` expectedChannels
+      bindResponse (Public.getTeamFeature owner tid "enforceFileDownloadLocation") $ \resp -> do
+        resp.status `shouldMatchInt` 200
+        resp.json `shouldMatch` expectedDownloadLocation
+      where
+        expectedChannelsConfig :: App Value
+        expectedChannelsConfig = do
+          defChannels <- defAllFeatures %. "channels"
+          defChannels
+            & setField "lockStatus" "unlocked"
+              >>= setField "status" "enabled"
+              >>= setField "config.allowed_to_create_channels" "team-members"
+              >>= setField "config.allowed_to_open_channels" "admins"
+
+        expectedEnforceDownloadLocationConfig :: App Value
+        expectedEnforceDownloadLocationConfig = do
+          defFeature <- defAllFeatures %. "enforceFileDownloadLocation"
+          defFeature
+            & setField "lockStatus" "unlocked"
+              >>= setField "status" "enabled"
+              >>= setField "config.enforcedDownloadLocation" "/tmp/migration-test"
+
+    mkExpectedModifiedFeatures :: Value -> App Value
+    mkExpectedModifiedFeatures features =
+      foldl (flip update) (pure features) unlockableFeatures
+      where
+        update feat =
+          setField (feat <> ".status") "enabled"
+            >=> setField (feat <> ".lockStatus") "unlocked"
+
+    conf :: String -> Bool -> ServiceOverrides
+    conf db runMigration =
+      def
+        { galleyCfg = setField "postgresMigration.teamFeatures" db,
+          backgroundWorkerCfg = setField "migrateTeamFeatures" runMigration
+        }
+
+counterName :: String
+counterName = "^wire_team_features_migration_finished"