Menu
I am frequently getting this error Memory quota exceeded for my ruby on rails application hosted on heroku:
2014-12-29 11:09:37.876 355 <45>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - source=worker.1 dyno=heroku.22144543.00c11b4d-8ec6-46d9-addf-f03163e10f0c sample#memory_total=2899.25MB sample#memory_rss=1023.73MB sample#memory_cache=0.00MB sample#memory_swap=1875.52MB sample#memory_pgpgin=2603236pages sample#memory_pgpgout=2341160pages
2014-12-29 11:09:37.876 132 <4
2014-12-29 11:09:37.876 132 <455>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - Process running mem=2899M(283.1%)>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - Error R14 (Memory quota exceeded)
I read the blog of which suggests the solution i.e. set WEB_CONCURRENCY config variable but it is not clear to me.
My current configuration on heroku are:
worker_processes Integer(ENV["WEB_CONCURRENCY"] || 3)
timeout 29
preload_app true
Edited:
Worker Code
class PhotoWorker
include Sidekiq::Worker
sidekiq_options queue: "high"
# sidekiq_options retry: false
def perform(params)
begin
puts "Start Woker for #{params.inspect}"
site = Site.find params['site_id']
if params["id"].present?
photo = Photo.find(params["id"])
if params['key'].present?
photo.key = params['key']
photo.remote_image_url = photo.image.direct_fog_url(:with_path => true)
photo.image_name = params['key'].split('/').last
photo.save(validate: false)
puts photo.inspect
end
else
#photo = site.photos.build
#photo.hotel_detail_id = site.hotel_detail.id
#photo.album_id = site.hotel_detail.album.id
#photo.save(validate: false)
photo.key = params['key']
photo.remote_image_url = photo.image.direct_fog_url(:with_path => true)
#photo.image = params['key']
#photo.remote_image_url = params['key']
photo.image_name = params['key'].split('/').last
puts photo.inspect
photo.save(validate: false)
end
if params["id"].present? && params["crop_x"].present?
photo.crop_x = params["crop_x"]
photo.crop_y = params["crop_y"]
photo.crop_w = params["crop_w"]
photo.crop_h = params["crop_h"]
photo.save(validate: false)
end
if params["id"].present?
if params['key'].present?
s3 = AWS::S3.new.buckets[ENV["FOG_DIRECTORY"]]
s3.objects[params['key']].delete
end
else
AmazonFile.new(params['key']).delete
end
puts "Deleted temp file: #{params['key']}" if params['key'].present?
puts "Photo(#{photo.id}) saved successfully. URL: #{params['key']}"
rescue Exception => exc
#puts "Photo not saved. URL: #{params['key']}"
puts "Error: #{exc.message}"
end
end
end
Apart from sidekiq i am also using carrierwave_backgrounder gem.
I am using Kraken for image compression
Photo Uploader
class PhotoUploader < CarrierWave::Uploader::Base
#include ::CarrierWave::Backgrounder::Delay
include CarrierWaveDirect::Uploader
include CarrierWave::MiniMagick
include CarrierWave::MimeTypes
process :set_content_type
process :crop
storage :fog
after :store, :kraken
version :admin do
process :resize_to_fit => [200,200]
end
#Leisure theme
version :leisure_358x243, :if => :leisure_theme? do
process :resize_to_fill => [358,243]
end
version :leisure_900x500, :if => :leisure_theme? do
process :resize_to_fill => [900,500]
end
version :leisure_350x147, :if => :leisure_theme? do
process :resize_to_fill => [350,147]
end
version :leisure_1100x344, :if => :leisure_theme? do
process :resize_to_fill => [1100,344]
end
#Business theme
version :business_360x160, :if => :business_theme? do
process :resize_to_fill => [360,160]
end
version :business_1100x315, :if => :business_theme? do
process :resize_to_fill => [1100,315]
end
version :business_1140x530, :if => :business_theme? do
process :resize_to_fill => [1140,530]
end
version :business_1100x355, :if => :business_theme? do
process :resize_to_fill => [1100,355]
end
#Commthree theme
version :commthree_550x300, :if => :commthree_theme? do
process :resize_to_fill => [550,300]
end
version :commthree_319x183, :if => :commthree_theme? do
process :resize_to_fill => [319,183]
end
version :commthree_1920x700, :if => :commthree_theme? do
process :resize_to_fill => [1920,700]
end
#All theme
version :all_360x188 do
process :resize_to_fill => [360,188]
end
version :all_1100x401 do
process :resize_to_fill => [1100,401]
end
version :all_140x88 do
process :resize_to_fill => [140,88]
end
def kraken(file)
if version_name.to_s == ""
storepath = store_dir + "/" + filename
else
fname = filename.split('.')
#originalfile = "http://s3.amazonaws.com/" + ENV["FOG_DIRECTORY"] + "/" + store_dir + "/" + fname[0] + "_" + version_name.to_s + "." + fname.last
storepath = store_dir + "/" + fname[0] + "_" + version_name.to_s + "." + fname.last
end
originalfile = "http://s3.amazonaws.com/" + ENV["FOG_DIRECTORY"] + "/" + store_dir + "/" + filename
kraken = Kraken::API.new(
:api_key => ENV['KRAKEN_API'],
:api_secret => ENV['KRAKEN_SECRET']
)
params = {}
Rails.logger.info('Kraken About to Process: ' + originalfile);
p "version #{version_name}"
case version_name.to_s
when "admin"
params = {
'lossy' => true,
'resize' => {
'width' => 200,
'height' => 200,
'strategy' => "auto",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "leisure_358x243"
params = {
'lossy' => true,
'resize' => {
'width' => 358,
'height' => 243,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "leisure_900x500"
params = {
'lossy' => true,
'resize' => {
'width' => 900,
'height' => 500,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "leisure_350x147"
params = {
'lossy' => true,
'resize' => {
'width' => 350,
'height' => 147,
'strategy' => "exact",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "leisure_1100x344"
params = {
'lossy' => true,
'resize' => {
'width' => 1100,
'height' => 344,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "business_360x160"
params = {
'lossy' => true,
'resize' => {
'width' => 360,
'height' => 160,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "business_1100x315"
params = {
'lossy' => true,
'resize' => {
'width' => 1100,
'height' => 315,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "business_1140x530"
params = {
'lossy' => true,
'resize' => {
'width' => 1140,
'height' => 530,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "business_1100x355"
params = {
'lossy' => true,
'resize' => {
'width' => 1100,
'height' => 355,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "commthree_550x300"
params = {
'lossy' => true,
'resize' => {
'width' => 550,
'height' => 300,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "commthree_319x183"
params = {
'lossy' => true,
'resize' => {
'width' => 319,
'height' => 183,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "commthree_1920x700"
params = {
'lossy' => true,
'resize' => {
'width' => 1920,
'height' => 700,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "all_360x188"
params = {
'lossy' => true,
'resize' => {
'width' => 360,
'height' => 188,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "all_1100x401"
params = {
'lossy' => true,
'resize' => {
'width' => 1100,
'height' => 401,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
when "all_140x88"
params = {
'lossy' => true,
'resize' => {
'width' => 140,
'height' => 88,
'strategy' => "fit",
},
's3_store' => {
'key' => ENV['AWS_ACCESS_KEY_ID'],
'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
'bucket' => ENV["FOG_DIRECTORY"],
'acl' => 'public_read',
'path' => storepath
},
}
end
#Store the file online
if !version_name.blank?
Rails.logger.info('UPLOADING TO: ' + store_path);
data = kraken.url(originalfile,params)
if data.success
Rails.logger.info('KRAKEN: Success! Optimized image URL: ' + data.kraked_url)
else
puts
Rails.logger.info('KRAKEN: Fail. Error message: ' + data.message)
end
end
end
# Add a white list of extensions which are allowed to be uploaded.
# For images you might use something like this:
def extension_white_list
%w(jpg jpeg gif png)
end
# def cache_dir
# "#{Rails.root}/tmp/uploads"
# end
def store_dir
if model.id.present?
"photos/#{model.id}"
else
"photos"
end
end
#file name is missing extension!!!
def filename
original_filename if original_filename.present?
end
def crop
puts "i am uploader"
if model.crop_x.present?
manipulate! do |img|
puts "i am cropping"
cx = model.crop_x.to_i
cy = model.crop_y.to_i
cw = model.crop_w.to_i
ch = model.crop_h.to_i
img.crop"#{cw}x#{ch}+#{cx}+#{cy}"
#img
end
end
end
def business_theme? (image)
p "model:#{model.id}"
(model.hotel_detail.site.theme.layout == "x1")if model.id.present?
end
def leisure_theme? (image)
p "model:#{model.id}"
(model.hotel_detail.site.theme.layout == "x2")if model.id.present?
end
def commthree_theme? (image)
p "model:#{model.id}"
(model.hotel_detail.site.theme.layout == "x3") if model.id.present?
end
protected
def secure_token(length=16)
var = :"@#{mounted_as}_secure_token"
model.instance_variable_get(var) or model.instance_variable_set(var, SecureRandom.hex(length/2))
end
end
Please suggest me how to calculate the correct value of WEB_CONCURRENCY.
542
It looks like you are using both minimagick and kraken to do image manipulation. If you fully migrate to Kraken and let them handle all of your image manipulation you shouldn't have a problem with memory on your workers.
Remove all of your versions and see if that improves performance.
version :leisure_358x243, :if => :leisure_theme? do
process :resize_to_fill => [358,243]
end
If you want to stick with processing images on your workers there are more efficient libraries than carrierwave-minimagick. Take a look at carrierwave-vips.
70
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
