Reschedule failed jobs. Send only submission id to queue and not whole submission.

hermanzdosilovic · hermanzdosilovic · commit f7265cf8d53e · 2020-10-02T00:26:28.000+02:00
diff --git a/Gemfile b/Gemfile
@@ -11,6 +11,7 @@ gem 'pry-rails', '~> 0.3.9'
 gem 'puma', '~> 4.3.5'
 gem 'rack-cors', '~> 1.1.1'
 gem 'resque', '~> 2.0.0'
+gem 'resque-scheduler', '~> 4.4'
 gem 'will_paginate', '~> 3.2.1'
 
 group :development do
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -63,7 +63,12 @@ GEM
       activesupport
       i18n
     erubi (1.9.0)
+    et-orbi (1.2.4)
+      tzinfo
     ffi (1.12.2)
+    fugit (1.3.9)
+      et-orbi (~> 1.1, >= 1.1.8)
+      raabro (~> 1.3)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     httparty (0.17.3)
@@ -109,6 +114,7 @@ GEM
       pry (>= 0.10.4)
     puma (4.3.5)
       nio4r (~> 2.0)
+    raabro (1.3.3)
     rack (2.2.3)
     rack-cors (1.1.1)
       rack (>= 2.0.0)
@@ -153,7 +159,14 @@ GEM
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
+    resque-scheduler (4.4.0)
+      mono_logger (~> 1.0)
+      redis (>= 3.3)
+      resque (>= 1.26)
+      rufus-scheduler (~> 3.2)
     ruby2_keywords (0.0.2)
+    rufus-scheduler (3.6.0)
+      fugit (~> 1.1, >= 1.1.6)
     sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
@@ -194,6 +207,7 @@ DEPENDENCIES
   rack-cors (~> 1.1.1)
   rails (~> 5.0)
   resque (~> 2.0.0)
+  resque-scheduler (~> 4.4)
   will_paginate (~> 3.2.1)
 
 BUNDLED WITH
diff --git a/Rakefile b/Rakefile
@@ -1,5 +1,6 @@
 require_relative 'config/application'
 require 'resque/tasks'
+require 'resque/scheduler/tasks'
 
 task 'resque:setup' => :environment
 
diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb
@@ -102,13 +102,13 @@ def create
     if submission.save
       if @wait
         begin
-          IsolateJob.perform_now(submission)
+          IsolateJob.perform_now(submission.id)
           render json: submission, status: :created, base64_encoded: @base64_encoded, fields: @requested_fields
         rescue Encoding::UndefinedConversionError => e
           render_conversion_error(:created, submission.token)
         end
       else
-        IsolateJob.perform_later(submission)
+        IsolateJob.perform_later(submission.id)
         render json: submission, status: :created, fields: [:token]
       end
     else
@@ -139,7 +139,7 @@ def batch_create
 
     submissions.each do |submission|
       if submission.save
-        IsolateJob.perform_later(submission)
+        IsolateJob.perform_later(submission.id)
         response << { token: submission.token }
         has_valid_submission = true
       else
diff --git a/app/jobs/isolate_job.rb b/app/jobs/isolate_job.rb
@@ -1,4 +1,6 @@
 class IsolateJob < ApplicationJob
+  retry_on RuntimeError, wait: 2.seconds, attempts: 100
+
   queue_as ENV["JUDGE0_VERSION"].to_sym
 
   STDIN_FILE_NAME = "stdin.txt"
@@ -12,8 +14,8 @@ class IsolateJob < ApplicationJob
               :source_file, :stdin_file, :stdout_file,
               :stderr_file, :metadata_file, :additional_files_archive_file
 
-  def perform(submission)
-    @submission = submission
+  def perform(submission_id)
+    @submission = Submission.find(submission_id)
 
     time = []
     memory = []
@@ -40,6 +42,7 @@ def perform(submission)
     submission.save
 
   rescue Exception => e
+    raise e.message unless submission
     submission.finished_at ||= DateTime.now
     submission.update(message: e.message, status: Status.boxerr)
     cleanup(raise_exception = false)
diff --git a/lib/tasks/run_submissions_in_queue.rake b/lib/tasks/run_submissions_in_queue.rake
@@ -5,9 +5,9 @@ namespace :judge0 do
     ARGV.each { |a| task a.to_sym do ; end }
     Submission.where(status_id: Status.queue).each do |s|
       if ARGV[1].to_s == "now"
-        IsolateJob.perform_now(s)
+        IsolateJob.perform_now(s.id)
       else
-        IsolateJob.perform_later(s)
+        IsolateJob.perform_later(s.id)
       end
     end
   end
diff --git a/scripts/workers b/scripts/workers
@@ -7,6 +7,7 @@ source ./scripts/load-config
 
 run_resque=1
 resque_pid=0
+scheduler_pid=0
 
 date_now() {
     echo -n $(date +"%Y-%m-%d-%H-%M-%S")
@@ -23,6 +24,12 @@ trap exit_gracefully SIGTERM SIGINT
 
 mkdir -p tmp/pids &> /dev/null
 while [[ $run_resque -eq 1 ]]; do
+    echo "[$(date_now)] Starting scheduler."
+    if ! ps -p $scheduler_pid &> /dev/null; then
+        rake resque:scheduler &
+        scheduler_pid=$!
+    fi
+
     rm -rf tmp/pids/resque.pid &> /dev/null
     echo "[$(date_now)] Starting workers."
     rails resque:workers &
