4
0

rollback-cloud.yml 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. name: "[Prod] Rollback"
  2. # Rolls production back to an earlier image, and optionally restores the
  3. # database from a dump taken by the deploy workflow.
  4. #
  5. # Rolling the image back is the normal case and loses nothing. Restoring the
  6. # database is a last resort for when the new schema is incompatible with the
  7. # old code: it discards EVERYTHING CUSTOMERS WROTE since the dump was taken.
  8. # That is why a database restore additionally requires typing
  9. # "restore-production" into the confirm field.
  10. #
  11. # Reuses the compose file and .env that "Deploy: app.torqvoice.com" wrote, so
  12. # prod must have been deployed at least once before this can run.
  13. on:
  14. workflow_dispatch:
  15. inputs:
  16. tag:
  17. description: "Version tag to roll back to (e.g. v1.2.38). 'latest' is refused."
  18. required: true
  19. restore_database:
  20. description: "DESTRUCTIVE. Also restore the database, discarding everything written since the dump. Leave off unless the old code cannot run against the new schema."
  21. type: boolean
  22. default: false
  23. dump:
  24. description: "Dump filename to restore from (e.g. torqvoice-20260815-120000-pre-v1.2.39.dump). Only used when restore_database is on."
  25. required: false
  26. default: ""
  27. confirm:
  28. description: "Type restore-production to allow a database restore. Ignored otherwise."
  29. required: false
  30. default: ""
  31. jobs:
  32. rollback:
  33. runs-on: [self-hosted, Linux, X64, hetzner]
  34. steps:
  35. # Only for .github/scripts; the app itself arrives as an image.
  36. - uses: actions/checkout@v4
  37. with:
  38. sparse-checkout: .github/scripts
  39. - name: Refuse floating tags
  40. run: |
  41. if [ "${{ inputs.tag }}" = "latest" ] || [ -z "${{ inputs.tag }}" ]; then
  42. echo "::error::Roll back to a version tag (e.g. v1.2.38), never 'latest'."
  43. exit 1
  44. fi
  45. - name: Show available dumps
  46. run: ls -lht "${{ secrets.DATA_PATH }}/db-backups/torqvoice" 2>/dev/null | head -20 || echo "No dumps yet."
  47. # Everything that can fail is checked before the app is stopped, so a bad
  48. # input leaves production untouched and still running.
  49. - name: Validate restore request
  50. if: inputs.restore_database
  51. env:
  52. DATABASE_URL: ${{ secrets.CLOUD_DATABASE_URL }}
  53. run: |
  54. set -euo pipefail
  55. if [ "${{ inputs.confirm }}" != "restore-production" ]; then
  56. echo "::error::Database restore requested without typing restore-production in the confirm field."
  57. exit 1
  58. fi
  59. # This job may only ever drop the production database. If the secret
  60. # is ever miswired, refuse rather than drop whatever it points at.
  61. DB_NAME=$(echo "$DATABASE_URL" | sed -E 's|.*/([^/?]+)(\?.*)?$|\1|')
  62. if [ "$DB_NAME" != "torqvoice" ]; then
  63. echo "::error::CLOUD_DATABASE_URL points at \"$DB_NAME\", expected \"torqvoice\". Refusing."
  64. exit 1
  65. fi
  66. if [ -z "${{ inputs.dump }}" ]; then
  67. echo "::error::restore_database is on but no dump filename was given."
  68. exit 1
  69. fi
  70. # The dump must belong to the production database: a staging dump
  71. # here would replace real customer data with test data.
  72. case "${{ inputs.dump }}" in
  73. "$DB_NAME"-[0-9]*) ;;
  74. *)
  75. echo "::error::Dump \"${{ inputs.dump }}\" is not a $DB_NAME dump. Refusing."
  76. exit 1
  77. ;;
  78. esac
  79. DUMP="${{ secrets.DATA_PATH }}/db-backups/torqvoice/${{ inputs.dump }}"
  80. if [ ! -f "$DUMP" ]; then
  81. echo "::error::Dump not found: $DUMP"
  82. exit 1
  83. fi
  84. docker exec -i torqvoice-db pg_restore --list < "$DUMP" > /dev/null
  85. echo "Dump is readable: $DUMP"
  86. - name: Stop prod app
  87. run: |
  88. cd $HOME/torqvoice-deploy/prod
  89. docker compose stop torqvoice-app
  90. - name: Restore database
  91. if: inputs.restore_database
  92. env:
  93. DATABASE_URL: ${{ secrets.CLOUD_DATABASE_URL }}
  94. run: |
  95. set -euo pipefail
  96. DUMP="${{ secrets.DATA_PATH }}/db-backups/torqvoice/${{ inputs.dump }}"
  97. DB_USER=$(echo "$DATABASE_URL" | sed -E 's|^[^:]+://([^:]+):.*|\1|')
  98. DB_NAME=$(echo "$DATABASE_URL" | sed -E 's|.*/([^/?]+)(\?.*)?$|\1|')
  99. echo "Restoring $DB_NAME from $(basename "$DUMP")"
  100. docker exec -i torqvoice-db psql -U "$DB_USER" -d postgres -c "DROP DATABASE \"$DB_NAME\" WITH (FORCE);"
  101. docker exec -i torqvoice-db psql -U "$DB_USER" -d postgres -c "CREATE DATABASE \"$DB_NAME\" OWNER \"$DB_USER\";"
  102. docker exec -i torqvoice-db pg_restore -U "$DB_USER" -d "$DB_NAME" --no-owner < "$DUMP"
  103. echo "Restore complete."
  104. - name: Deploy previous tag
  105. run: |
  106. cd $HOME/torqvoice-deploy/prod
  107. APP_TAG=${{ inputs.tag }} docker compose up -d --pull always
  108. - name: Health check
  109. run: |
  110. if bash "$GITHUB_WORKSPACE/.github/scripts/wait-healthy.sh" torqvoice-app; then
  111. echo "Rolled back to ${{ inputs.tag }}."
  112. exit 0
  113. fi
  114. echo "::error::Not healthy after rolling back to ${{ inputs.tag }}"
  115. docker logs torqvoice-app --tail 80
  116. exit 1