diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 99797a8c836ac72203b6963a4788ebdb0b2abcc7..733b3149528844c51c058cd5494c4445d3867d03 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -466,8 +466,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 		if (s_job->s_fence->parent &&
 		    dma_fence_remove_callback(s_job->s_fence->parent,
 					      &s_job->cb)) {
-			dma_fence_put(s_job->s_fence->parent);
-			s_job->s_fence->parent = NULL;
 			atomic_dec(&sched->hw_rq_count);
 		} else {
 			/*
@@ -591,6 +589,7 @@ void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
 		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
 			dma_fence_set_error(&s_fence->finished, -ECANCELED);
 
+      dma_fence_put(s_job->s_fence->parent);
 		fence = sched->ops->run_job(s_job);
 
 		if (IS_ERR_OR_NULL(fence)) {
@@ -599,11 +598,7 @@ void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
 
 			s_job->s_fence->parent = NULL;
 		} else {
-
-			s_job->s_fence->parent = dma_fence_get(fence);
-
-			/* Drop for orignal kref_init */
-			dma_fence_put(fence);
+			s_job->s_fence->parent = fence;
 		}
 	}
 }
@@ -1052,9 +1047,6 @@ static int drm_sched_main(void *param)
 		drm_sched_fence_scheduled(s_fence, fence);
 
 		if (!IS_ERR_OR_NULL(fence)) {
-			/* Drop for original kref_init of the fence */
-			dma_fence_put(fence);
-
 			r = dma_fence_add_callback(fence, &sched_job->cb,
 						   drm_sched_job_done_cb);
 			if (r == -ENOENT)
@@ -1062,6 +1054,7 @@ static int drm_sched_main(void *param)
 			else if (r)
 				DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n",
 					  r);
+            dma_fence_put(fence);
 		} else {
 			drm_sched_job_done(sched_job, IS_ERR(fence) ?
 					   PTR_ERR(fence) : 0);