Summary
We’re in the process of migrating from MySQL to Postgres and I’m having some trouble getting a test instance of Mattermost to start when pointed at the newly migrated database.
Steps to reproduce
We are currently running version 10.5.9 on AWS ECS and a MySQL RDS instance. I’ve used Database Migration Service to both convert the schema to Postgres and migrate (one time at first then continuously later) the current database to a new Postgres Aurora instance. I’m trying to use the Kubernetes operator and the instructions here: Deploy Mattermost on Kubernetes - Mattermost documentation to test this new database.
Expected behavior
Mattermost pod starts and we can access Mattermost with all of our current data now in Postgres.
Observed behavior
The pod will start, crash once with a “connection refused” error from the database then restart and hang indefinitely with no logs. I tried sending a quit signal to the running process in the pod and captured the output. Based on this it appears to be getting stuck at the `initializeConfigurationsTable` function. Checking idle transactions in the database this seems to support this as there as a single transaction stuck running `ROLLBACK`. While testing I set the ready/liveness check’s timeouts to multiple days and this roll back never seems to complete.
Extra Information
We decided to use DMS over the official migration scripts from Mattermost because of the number of plugins we have that aren’t covered by those scripts.
I’m fairly certain at this point its not a database configuration issue given the number of mistakes I made and corrected to get to this point. The Mattermost schema has been created properly and the db user has been given all the correct permissions to it.
At this point I’m not sure what to troubleshoot. Adjusting logging doesn’t produce anymore logs. I’ve thought about copying the config from our running instances but we have so many plugins there I’m worried I’d cause a conflict with something.
Stack Trace From SIGQUIT
SIGQUIT: quit
PC=0x47c6a1 m=0 sigcode=0
goroutine 0 gp=0x5358880 m=0 mp=0x5359ec0 [idle]:
runtime.futex(0x535a000, 0x80, 0x0, 0x0, 0x0, 0x0)
runtime/sys_linux_amd64.s:557 +0x21 fp=0x7ffe0e0ee5f0 sp=0x7ffe0e0ee5e8 pc=0x47c6a1
runtime.futexsleep(0x7ffe0e0ee668?, 0x411f30?, 0x4504e7?)
runtime/os_linux.go:69 +0x30 fp=0x7ffe0e0ee640 sp=0x7ffe0e0ee5f0 pc=0x43c470
runtime.notesleep(0x535a000)
runtime/lock_futex.go:170 +0x87 fp=0x7ffe0e0ee678 sp=0x7ffe0e0ee640 pc=0x4120c7
runtime.mPark(...)
runtime/proc.go:1761
runtime.stopm()
runtime/proc.go:2782 +0x8c fp=0x7ffe0e0ee6a8 sp=0x7ffe0e0ee678 pc=0x4471ac
runtime.findRunnable()
runtime/proc.go:3512 +0xd5f fp=0x7ffe0e0ee820 sp=0x7ffe0e0ee6a8 pc=0x448d1f
runtime.schedule()
runtime/proc.go:3868 +0xb1 fp=0x7ffe0e0ee858 sp=0x7ffe0e0ee820 pc=0x449df1
runtime.park_m(0xc000102c40)
runtime/proc.go:4036 +0x1ec fp=0x7ffe0e0ee8b0 sp=0x7ffe0e0ee858 pc=0x44a3cc
runtime.mcall()
runtime/asm_amd64.s:458 +0x4e fp=0x7ffe0e0ee8c8 sp=0x7ffe0e0ee8b0 pc=0x47884e
goroutine 1 gp=0xc0000061c0 m=nil [select]:
runtime.gopark(0xc0000558a8?, 0x2?, 0x60?, 0x57?, 0xc00005587c?)
runtime/proc.go:402 +0xce fp=0xc000055728 sp=0xc000055708 pc=0x442b0e
runtime.selectgo(0xc0000558a8, 0xc000055878, 0xc00320fe40?, 0x0, 0x52ee540?, 0x1)
runtime/select.go:327 +0x725 fp=0xc000055848 sp=0xc000055728 pc=0x454b85
github.com/mattermost/morph/drivers/postgres.(*Mutex).Lock(0xc001e86180, {0x3882c20, 0x53bbae0})
github.com/mattermost/morph@v1.1.0/drivers/postgres/lock.go:176 +0xa9 fp=0xc0000558d8 sp=0xc000055848 pc=0x1199089
github.com/mattermost/morph.New({0x3882c20, 0x53bbae0}, {0x3886780, 0xc0016ad920}, {0x3872020, 0xc00320f0c0}, {0xc000055a98, 0x3, 0x10?})
github.com/mattermost/morph@v1.1.0/morph.go:136 +0x3ba fp=0xc000055960 sp=0xc0000558d8 pc=0x138f11a
github.com/mattermost/mattermost/server/v8/config.(*DatabaseStore).initializeConfigurationsTable(0xc0013b8f40)
github.com/mattermost/mattermost/server/v8/config/database.go:157 +0x66d fp=0xc000055b48 sp=0xc000055960 pc=0x139b48d
github.com/mattermost/mattermost/server/v8/config.NewDatabaseStore({0xc00004e00a, 0xe1})
github.com/mattermost/mattermost/server/v8/config/database.go:89 +0x19f fp=0xc000055c08 sp=0xc000055b48 pc=0x139ac3f
github.com/mattermost/mattermost/server/v8/config.NewStoreFromDSN({0xc00004e00a, 0xe1}, 0x0, 0x0, 0x1)
github.com/mattermost/mattermost/server/v8/config/store.go:92 +0xcd fp=0xc000055c58 sp=0xc000055c08 pc=0x13a456d
github.com/mattermost/mattermost/server/v8/cmd/mattermost/commands.serverCmdF(0x5325d00, {0x2ee1a4e?, 0x4?, 0x2ee1a22?})
github.com/mattermost/mattermost/server/v8/cmd/mattermost/commands/server.go:51 +0xae fp=0xc000055cd0 sp=0xc000055c58 pc=0x284376e
github.com/spf13/cobra.(*Command).execute(0x5325d00, {0xc000126040, 0x0, 0x0})
github.com/spf13/cobra@v1.8.1/command.go:985 +0xaca fp=0xc000055e58 sp=0xc000055cd0 pc=0x282efca
github.com/spf13/cobra.(*Command).ExecuteC(0x5325d00)
github.com/spf13/cobra@v1.8.1/command.go:1117 +0x3ff fp=0xc000055f30 sp=0xc000055e58 pc=0x282f89f
github.com/spf13/cobra.(*Command).Execute(...)
github.com/spf13/cobra@v1.8.1/command.go:1041
github.com/mattermost/mattermost/server/v8/cmd/mattermost/commands.Run(...)
github.com/mattermost/mattermost/server/v8/cmd/mattermost/commands/root.go:17
main.main()
github.com/mattermost/mattermost/server/v8/cmd/mattermost/main.go:20 +0x88 fp=0xc000055f50 sp=0xc000055f30 pc=0x2845f08
runtime.main()
runtime/proc.go:271 +0x29d fp=0xc000055fe0 sp=0xc000055f50 pc=0x4426bd
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc000055fe8 sp=0xc000055fe0 pc=0x47a881
goroutine 2 gp=0xc000006c40 m=nil [force gc (idle), 2 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006efa8 sp=0xc00006ef88 pc=0x442b0e
runtime.goparkunlock(...)
runtime/proc.go:408
runtime.forcegchelper()
runtime/proc.go:326 +0xb3 fp=0xc00006efe0 sp=0xc00006efa8 pc=0x442973
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006efe8 sp=0xc00006efe0 pc=0x47a881
created by runtime.init.6 in goroutine 1
runtime/proc.go:314 +0x1a
goroutine 3 gp=0xc000007180 m=nil [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006f780 sp=0xc00006f760 pc=0x442b0e
runtime.goparkunlock(...)
runtime/proc.go:408
runtime.bgsweep(0xc00007e000)
runtime/mgcsweep.go:318 +0xdf fp=0xc00006f7c8 sp=0xc00006f780 pc=0x42c65f
runtime.gcenable.gowrap1()
runtime/mgc.go:203 +0x25 fp=0xc00006f7e0 sp=0xc00006f7c8 pc=0x420f45
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006f7e8 sp=0xc00006f7e0 pc=0x47a881
created by runtime.gcenable in goroutine 1
runtime/mgc.go:203 +0x66
goroutine 4 gp=0xc000007340 m=nil [GC scavenge wait]:
runtime.gopark(0x10000?, 0x385c828?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006ff78 sp=0xc00006ff58 pc=0x442b0e
runtime.goparkunlock(...)
runtime/proc.go:408
runtime.(*scavengerState).park(0x5357820)
runtime/mgcscavenge.go:425 +0x49 fp=0xc00006ffa8 sp=0xc00006ff78 pc=0x429fe9
runtime.bgscavenge(0xc00007e000)
runtime/mgcscavenge.go:658 +0x59 fp=0xc00006ffc8 sp=0xc00006ffa8 pc=0x42a599
runtime.gcenable.gowrap2()
runtime/mgc.go:204 +0x25 fp=0xc00006ffe0 sp=0xc00006ffc8 pc=0x420ee5
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006ffe8 sp=0xc00006ffe0 pc=0x47a881
created by runtime.gcenable in goroutine 1
runtime/mgc.go:204 +0xa5
goroutine 18 gp=0xc000102700 m=nil [finalizer wait, 2 minutes]:
runtime.gopark(0xc00006e648?, 0x413d85?, 0xa8?, 0x1?, 0xc0000061c0?)
runtime/proc.go:402 +0xce fp=0xc00006e620 sp=0xc00006e600 pc=0x442b0e
runtime.runfinq()
runtime/mfinal.go:194 +0x107 fp=0xc00006e7e0 sp=0xc00006e620 pc=0x41ff87
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006e7e8 sp=0xc00006e7e0 pc=0x47a881
created by runtime.createfing in goroutine 1
runtime/mfinal.go:164 +0x3d
goroutine 19 gp=0xc0001028c0 m=nil [GC worker (idle)]:
runtime.gopark(0xee336ade96a9?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006a750 sp=0xc00006a730 pc=0x442b0e
runtime.gcBgMarkWorker()
runtime/mgc.go:1310 +0xe5 fp=0xc00006a7e0 sp=0xc00006a750 pc=0x423025
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006a7e8 sp=0xc00006a7e0 pc=0x47a881
created by runtime.gcBgMarkStartWorkers in goroutine 1
runtime/mgc.go:1234 +0x1c
goroutine 20 gp=0xc000102a80 m=nil [GC worker (idle), 2 minutes]:
runtime.gopark(0xee33674fb751?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006af50 sp=0xc00006af30 pc=0x442b0e
runtime.gcBgMarkWorker()
runtime/mgc.go:1310 +0xe5 fp=0xc00006afe0 sp=0xc00006af50 pc=0x423025
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006afe8 sp=0xc00006afe0 pc=0x47a881
created by runtime.gcBgMarkStartWorkers in goroutine 1
runtime/mgc.go:1234 +0x1c
goroutine 34 gp=0xc0004c8000 m=nil [GC worker (idle)]:
runtime.gopark(0xee336ade87a9?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc0004ce750 sp=0xc0004ce730 pc=0x442b0e
runtime.gcBgMarkWorker()
runtime/mgc.go:1310 +0xe5 fp=0xc0004ce7e0 sp=0xc0004ce750 pc=0x423025
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc0004ce7e8 sp=0xc0004ce7e0 pc=0x47a881
created by runtime.gcBgMarkStartWorkers in goroutine 1
runtime/mgc.go:1234 +0x1c
goroutine 21 gp=0xc000102c40 m=nil [GC worker (idle)]:
runtime.gopark(0xee336ade862d?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:402 +0xce fp=0xc00006b750 sp=0xc00006b730 pc=0x442b0e
runtime.gcBgMarkWorker()
runtime/mgc.go:1310 +0xe5 fp=0xc00006b7e0 sp=0xc00006b750 pc=0x423025
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006b7e8 sp=0xc00006b7e0 pc=0x47a881
created by runtime.gcBgMarkStartWorkers in goroutine 1
runtime/mgc.go:1234 +0x1c
goroutine 5 gp=0xc000103dc0 m=nil [select, 2 minutes]:
runtime.gopark(0xc000070f98?, 0x2?, 0x0?, 0x0?, 0xc000070f94?)
runtime/proc.go:402 +0xce fp=0xc000070e40 sp=0xc000070e20 pc=0x442b0e
runtime.selectgo(0xc000070f98, 0xc000070f90, 0x0?, 0x0, 0x0?, 0x1)
runtime/select.go:327 +0x725 fp=0xc000070f60 sp=0xc000070e40 pc=0x454b85
github.com/blevesearch/bleve_index_api.AnalysisWorker(...)
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:46
github.com/blevesearch/bleve_index_api.NewAnalysisQueue.gowrap1()
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0x8e fp=0xc000070fe0 sp=0xc000070f60 pc=0x1a7f34e
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc000070fe8 sp=0xc000070fe0 pc=0x47a881
created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0xa5
goroutine 6 gp=0xc0004c9a40 m=nil [select, 2 minutes]:
runtime.gopark(0xc000071798?, 0x2?, 0x0?, 0x0?, 0xc000071794?)
runtime/proc.go:402 +0xce fp=0xc000071640 sp=0xc000071620 pc=0x442b0e
runtime.selectgo(0xc000071798, 0xc000071790, 0x0?, 0x0, 0x0?, 0x1)
runtime/select.go:327 +0x725 fp=0xc000071760 sp=0xc000071640 pc=0x454b85
github.com/blevesearch/bleve_index_api.AnalysisWorker(...)
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:46
github.com/blevesearch/bleve_index_api.NewAnalysisQueue.gowrap1()
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0x8e fp=0xc0000717e0 sp=0xc000071760 pc=0x1a7f34e
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc0000717e8 sp=0xc0000717e0 pc=0x47a881
created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0xa5
goroutine 7 gp=0xc0004c9c00 m=nil [select, 2 minutes]:
runtime.gopark(0xc000071f98?, 0x2?, 0x0?, 0x0?, 0xc000071f94?)
runtime/proc.go:402 +0xce fp=0xc000071e40 sp=0xc000071e20 pc=0x442b0e
runtime.selectgo(0xc000071f98, 0xc000071f90, 0x0?, 0x0, 0x0?, 0x1)
runtime/select.go:327 +0x725 fp=0xc000071f60 sp=0xc000071e40 pc=0x454b85
github.com/blevesearch/bleve_index_api.AnalysisWorker(...)
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:46
github.com/blevesearch/bleve_index_api.NewAnalysisQueue.gowrap1()
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0x8e fp=0xc000071fe0 sp=0xc000071f60 pc=0x1a7f34e
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc000071fe8 sp=0xc000071fe0 pc=0x47a881
created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0xa5
goroutine 8 gp=0xc0004c9dc0 m=nil [select, 2 minutes]:
runtime.gopark(0xc00006bf98?, 0x2?, 0x0?, 0x0?, 0xc00006bf94?)
runtime/proc.go:402 +0xce fp=0xc00006be40 sp=0xc00006be20 pc=0x442b0e
runtime.selectgo(0xc00006bf98, 0xc00006bf90, 0x0?, 0x0, 0x0?, 0x1)
runtime/select.go:327 +0x725 fp=0xc00006bf60 sp=0xc00006be40 pc=0x454b85
github.com/blevesearch/bleve_index_api.AnalysisWorker(...)
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:46
github.com/blevesearch/bleve_index_api.NewAnalysisQueue.gowrap1()
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0x8e fp=0xc00006bfe0 sp=0xc00006bf60 pc=0x1a7f34e
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc00006bfe8 sp=0xc00006bfe0 pc=0x47a881
created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1
github.com/blevesearch/bleve_index_api@v1.1.9/analysis.go:38 +0xa5
goroutine 22 gp=0xc000e36e00 m=nil [select]:
runtime.gopark(0xc0004d0f88?, 0x2?, 0x60?, 0x0?, 0xc0004d0f84?)
runtime/proc.go:402 +0xce fp=0xc0004d0e30 sp=0xc0004d0e10 pc=0x442b0e
runtime.selectgo(0xc0004d0f88, 0xc0004d0f80, 0x0?, 0x0, 0x0?, 0x1)
runtime/select.go:327 +0x725 fp=0xc0004d0f50 sp=0xc0004d0e30 pc=0x454b85
database/sql.(*DB).connectionOpener(0xc00241b5f0, {0x3882ef8, 0xc001492190})
database/sql/sql.go:1246 +0x87 fp=0xc0004d0fb8 sp=0xc0004d0f50 pc=0x7dada7
database/sql.OpenDB.gowrap1()
database/sql/sql.go:824 +0x28 fp=0xc0004d0fe0 sp=0xc0004d0fb8 pc=0x7d90c8
runtime.goexit({})
runtime/asm_amd64.s:1695 +0x1 fp=0xc0004d0fe8 sp=0xc0004d0fe0 pc=0x47a881
created by database/sql.OpenDB in goroutine 1
database/sql/sql.go:824 +0x14c
rax 0xca
rbx 0x0
rcx 0x47c6a3
rdx 0x0
rdi 0x535a000
rsi 0x80
rbp 0x7ffe0e0ee630
rsp 0x7ffe0e0ee5e8
r8 0x0
r9 0x0
r10 0x0
r11 0x286
r12 0x0
r13 0x1
r14 0x5358880
r15 0x1
rip 0x47c6a1
rflags 0x286
cs 0x33
fs 0x0
gs 0x0
Mattermost CRD
apiVersion: installation.mattermost.com/v1beta1
kind: Mattermost
metadata:
annotations:
creationTimestamp: "2025-11-20T19:46:38Z"
generation: 2
name: pg-test
namespace: mattermost
resourceVersion: "491405737"
uid: faf84069-035e-4106-a047-3170ba5ae3de
spec:
database:
external:
secret: pg-connection
elasticSearch: {}
fileStore:
external:
bucket: mm-test
secret: s3-creds
url: s3.amazonaws.com
image: mattermost/mattermost-team-edition
imagePullPolicy: IfNotPresent
ingress:
enabled: false
ingressName: ""
mattermostEnv:
- name: MM_FILESETTINGS_AMAZONS3SSL
value: "true"
- name: MM_FILESETTINGS_AMAZONS3SSE
value: "true"
- name: MM_LOGSETTINGS_CONSOLELEVEL
value: INFO
- name: MM_SQLSETTINGS_DRIVERNAME
value: postgres
- name: MM_SQLSETTINGS_DATASOURCE
value: SUPER SECRET
- name: AWS_SESSION_TOKEN
value: ALSO SUPER SECRET
podExtensions: {}
probes:
livenessProbe:
failureThreshold: 100
initialDelaySeconds: 259200
periodSeconds: 259200
readinessProbe:
failureThreshold: 100
initialDelaySeconds: 259200
periodSeconds: 259200
replicas: 1
scheduling:
resources:
limits:
cpu: "4"
memory: 8Gi
requests:
cpu: 500m
memory: 500Mi
version: 10.5.9
volumeMounts:
- mountPath: /etc/ssl/certs/rds/
name: rds-ca
volumes:
- name: rds-ca
secret:
defaultMode: 420
items:
- key: rds-ca-file
path: rds-combined-ca-bundle.pem
secretName: rds-ca
status:
observedGeneration: 2
replicas: 1
state: reconciling