From c745c11c43ca2ebdebd4945d452a4a4a7a69d03b Mon Sep 17 00:00:00 2001
From: Ahmad Farhat <ahmad.af.farhat@gmail.com>
Date: Wed, 5 Jul 2023 15:29:48 -0400
Subject: [PATCH] Added google cloud support (#5308)
* Added google cloud support
* Rubocop
* Update development env
* Rubocop again
* Add to sample.env
---
Gemfile | 1 +
Gemfile.lock | 54 +++++++++++++++++++
config/environments/development.rb | 10 +++-
config/environments/production.rb | 2 +
config/storage.yml | 20 +++++--
..._to_active_storage_blobs.active_storage.rb | 24 +++++++++
..._storage_variant_records.active_storage.rb | 31 +++++++++++
...e_storage_blobs_checksum.active_storage.rb | 10 ++++
db/schema.rb | 2 +-
sample.env | 10 ++++
10 files changed, 157 insertions(+), 7 deletions(-)
create mode 100644 db/migrate/20230705183745_add_service_name_to_active_storage_blobs.active_storage.rb
create mode 100644 db/migrate/20230705183746_create_active_storage_variant_records.active_storage.rb
create mode 100644 db/migrate/20230705183747_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb
diff --git a/Gemfile b/Gemfile
index 6b17f812..495b4d6d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -14,6 +14,7 @@ gem 'bootsnap', require: false
gem 'cssbundling-rails'
gem 'data_migrate'
gem 'dotenv-rails'
+gem 'google-cloud-storage', '~> 1.44', require: false
gem 'hcaptcha'
gem 'hiredis', '~> 0.6.0'
gem 'i18n-language-mapping'
diff --git a/Gemfile.lock b/Gemfile.lock
index f86fc320..7348ebf8 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -136,7 +136,10 @@ GEM
debug (1.7.1)
irb (>= 1.5.0)
reline (>= 0.3.1)
+ declarative (0.0.20)
diff-lcs (1.5.0)
+ digest-crc (0.6.5)
+ rake (>= 12.0.0, < 14.0.0)
dotenv (2.8.1)
dotenv-rails (2.8.1)
dotenv (= 2.8.1)
@@ -158,6 +161,40 @@ GEM
ffi (1.15.5)
globalid (1.1.0)
activesupport (>= 5.0)
+ google-apis-core (0.11.0)
+ addressable (~> 2.5, >= 2.5.1)
+ googleauth (>= 0.16.2, < 2.a)
+ httpclient (>= 2.8.1, < 3.a)
+ mini_mime (~> 1.0)
+ representable (~> 3.0)
+ retriable (>= 2.0, < 4.a)
+ rexml
+ webrick
+ google-apis-iamcredentials_v1 (0.17.0)
+ google-apis-core (>= 0.11.0, < 2.a)
+ google-apis-storage_v1 (0.19.0)
+ google-apis-core (>= 0.9.0, < 2.a)
+ google-cloud-core (1.6.0)
+ google-cloud-env (~> 1.0)
+ google-cloud-errors (~> 1.0)
+ google-cloud-env (1.6.0)
+ faraday (>= 0.17.3, < 3.0)
+ google-cloud-errors (1.3.1)
+ google-cloud-storage (1.44.0)
+ addressable (~> 2.8)
+ digest-crc (~> 0.4)
+ google-apis-iamcredentials_v1 (~> 0.1)
+ google-apis-storage_v1 (~> 0.19.0)
+ google-cloud-core (~> 1.6)
+ googleauth (>= 0.16.2, < 2.a)
+ mini_mime (~> 1.0)
+ googleauth (1.6.0)
+ faraday (>= 0.17.3, < 3.a)
+ jwt (>= 1.4, < 3.0)
+ memoist (~> 0.16)
+ multi_json (~> 1.11)
+ os (>= 0.9, < 2.0)
+ signet (>= 0.16, < 2.a)
hashdiff (1.0.1)
hashie (5.0.0)
hcaptcha (7.1.0)
@@ -203,12 +240,14 @@ GEM
net-smtp
marcel (1.0.2)
matrix (0.4.2)
+ memoist (0.16.2)
method_source (1.0.0)
mini_magick (4.12.0)
mini_mime (1.1.2)
mini_portile2 (2.8.2)
minitest (5.18.0)
msgpack (1.6.0)
+ multi_json (1.15.0)
net-imap (0.3.4)
date
net-protocol
@@ -245,6 +284,7 @@ GEM
validate_email
validate_url
webfinger (~> 1.2)
+ os (1.1.4)
pagy (5.10.1)
activesupport
parallel (1.22.1)
@@ -301,8 +341,13 @@ GEM
io-console (~> 0.5)
remote_syslog_logger (1.0.4)
syslog_protocol
+ representable (3.2.0)
+ declarative (< 0.1.0)
+ trailblazer-option (>= 0.1.1, < 0.2.0)
+ uber (< 0.2.0)
request_store (1.5.1)
rack (>= 1.4)
+ retriable (3.1.2)
rexml (3.2.5)
rspec-core (3.12.1)
rspec-support (~> 3.12.0)
@@ -353,6 +398,11 @@ GEM
websocket (~> 1.0)
shoulda-matchers (5.3.0)
activesupport (>= 5.2.0)
+ signet (0.17.0)
+ addressable (~> 2.8)
+ faraday (>= 0.17.5, < 3.a)
+ jwt (>= 1.5, < 3.0)
+ multi_json (~> 1.10)
sprockets (4.2.0)
concurrent-ruby (~> 1.0)
rack (>= 2.2.4, < 4)
@@ -367,8 +417,10 @@ GEM
syslog_protocol (0.9.2)
thor (1.2.2)
timeout (0.3.2)
+ trailblazer-option (0.1.2)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
+ uber (0.1.0)
unicode-display_width (2.4.2)
validate_email (0.1.6)
activemodel (>= 3.0)
@@ -392,6 +444,7 @@ GEM
addressable (>= 2.8.0)
crack (>= 0.3.2)
hashdiff (>= 0.4.0, < 2.0.0)
+ webrick (1.8.1)
websocket (1.2.9)
websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
@@ -420,6 +473,7 @@ DEPENDENCIES
dotenv-rails
factory_bot_rails
faker
+ google-cloud-storage (~> 1.44)
hcaptcha
hiredis (~> 0.6.0)
i18n-language-mapping
diff --git a/config/environments/development.rb b/config/environments/development.rb
index c4f401ba..0f1121f3 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -52,7 +52,15 @@ Rails.application.configure do
end
# Store uploaded files on the local file system (see config/storage.yml for options).
- config.active_storage.service = :local
+ config.active_storage.service = if ENV['S3_ACCESS_KEY_ID'].present? && ENV['S3_ENDPOINT'].present?
+ :s3
+ elsif ENV['S3_ACCESS_KEY_ID'].present?
+ :amazon
+ elsif ENV['GCS_PROJECT'].present?
+ :google
+ else
+ :local
+ end
if ENV['SMTP_SERVER'].present?
config.action_mailer.perform_deliveries = true
diff --git a/config/environments/production.rb b/config/environments/production.rb
index f3f5a4a0..a7b23143 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -60,6 +60,8 @@ Rails.application.configure do
:s3
elsif ENV['S3_ACCESS_KEY_ID'].present?
:amazon
+ elsif ENV['GCS_PROJECT'].present?
+ :google
else
:local
end
diff --git a/config/storage.yml b/config/storage.yml
index 0c67dd21..4287c0f0 100644
--- a/config/storage.yml
+++ b/config/storage.yml
@@ -38,11 +38,21 @@ s3:
bucket: <%= ENV['S3_BUCKET'] %>
# Remember not to checkin your GCS keyfile to a repository
-# google:
-# service: GCS
-# project: your_project
-# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
-# bucket: your_own_bucket-<%= Rails.env %>
+google:
+ service: GCS
+ project: "<%= ENV['GCS_PROJECT'] %>"
+ bucket: "<%= ENV['GCS_BUCKET'] %>"
+ credentials:
+ type: 'service_account'
+ project_id: "<%= ENV['GCS_PROJECT_ID'] %>"
+ private_key_id: "<%= ENV['GCS_PRIVATE_KEY_ID'] %>"
+ private_key: "<%= ENV['GCS_PRIVATE_KEY']&.lines&.join("\\n") %>"
+ client_email: "<%= ENV['GCS_CLIENT_EMAIL'] %>"
+ client_id: "<%= ENV['GCS_CLIENT_ID'] %>"
+ auth_uri: 'https://accounts.google.com/o/oauth2/auth'
+ token_uri: 'https://accounts.google.com/o/oauth2/token'
+ auth_provider_x509_cert_url: 'https://www.googleapis.com/oauth2/v1/certs'
+ client_x509_cert_url: "<%= ENV['GCS_CLIENT_CERT'] %>"
# Use bin/rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
# microsoft:
diff --git a/db/migrate/20230705183745_add_service_name_to_active_storage_blobs.active_storage.rb b/db/migrate/20230705183745_add_service_name_to_active_storage_blobs.active_storage.rb
new file mode 100644
index 00000000..2002bd25
--- /dev/null
+++ b/db/migrate/20230705183745_add_service_name_to_active_storage_blobs.active_storage.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This migration comes from active_storage (originally 20190112182829)
+class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0]
+ def up
+ return unless table_exists?(:active_storage_blobs)
+
+ return if column_exists?(:active_storage_blobs, :service_name)
+
+ add_column :active_storage_blobs, :service_name, :string
+
+ if (configured_service = ActiveStorage::Blob.service.name)
+ ActiveStorage::Blob.unscoped.update_all(service_name: configured_service) # rubocop:disable Rails/SkipsModelValidations
+ end
+
+ change_column :active_storage_blobs, :service_name, :string, null: false
+ end
+
+ def down
+ return unless table_exists?(:active_storage_blobs)
+
+ remove_column :active_storage_blobs, :service_name
+ end
+end
diff --git a/db/migrate/20230705183746_create_active_storage_variant_records.active_storage.rb b/db/migrate/20230705183746_create_active_storage_variant_records.active_storage.rb
new file mode 100644
index 00000000..91e341db
--- /dev/null
+++ b/db/migrate/20230705183746_create_active_storage_variant_records.active_storage.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This migration comes from active_storage (originally 20191206030411)
+class CreateActiveStorageVariantRecords < ActiveRecord::Migration[6.0]
+ def change
+ return unless table_exists?(:active_storage_blobs)
+
+ # Use Active Record's configured type for primary key
+
+ create_table :active_storage_variant_records, id: primary_key_type, if_not_exists: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
+ t.belongs_to :blob, null: false, index: false, type: blobs_primary_key_type
+ t.string :variation_digest, null: false
+
+ t.index %i[blob_id variation_digest], name: 'index_active_storage_variant_records_uniqueness', unique: true
+ t.foreign_key :active_storage_blobs, column: :blob_id
+ end
+ end
+
+ private
+
+ def primary_key_type
+ config = Rails.configuration.generators
+ config.options[config.orm][:primary_key_type] || :primary_key
+ end
+
+ def blobs_primary_key_type
+ pkey_name = connection.primary_key(:active_storage_blobs)
+ pkey_column = connection.columns(:active_storage_blobs).find { |c| c.name == pkey_name }
+ pkey_column.bigint? ? :bigint : pkey_column.type
+ end
+end
diff --git a/db/migrate/20230705183747_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb b/db/migrate/20230705183747_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb
new file mode 100644
index 00000000..6f318221
--- /dev/null
+++ b/db/migrate/20230705183747_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# This migration comes from active_storage (originally 20211119233751)
+class RemoveNotNullOnActiveStorageBlobsChecksum < ActiveRecord::Migration[6.0]
+ def change
+ return unless table_exists?(:active_storage_blobs)
+
+ change_column_null(:active_storage_blobs, :checksum, true)
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f4579674..f0760fed 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.0].define(version: 2023_03_07_184209) do
+ActiveRecord::Schema[7.0].define(version: 2023_07_05_183747) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
diff --git a/sample.env b/sample.env
index 3f2a2694..25da1adb 100644
--- a/sample.env
+++ b/sample.env
@@ -57,6 +57,16 @@ REDIS_URL=
#S3_BUCKET=
#S3_ENDPOINT=
+# Set these environment variables if you are using Google Cloud Storage
+#GCS_PROJECT=
+#GCS_BUCKET=
+#GCS_PROJECT_ID=
+#GCS_PRIVATE_KEY_ID=
+#GCS_PRIVATE_KEY=
+#GCS_CLIENT_EMAIL=
+#GCS_CLIENT_ID=
+#GCS_CLIENT_CERT=
+
# Define the default locale language code (i.e. 'en' for English) from the following list:
# [en, ar, fr, es]
#DEFAULT_LOCALE=en
--
GitLab