send video to s3 applay async

apps/core/signals.py

@@ -2,7 +2,12 @@ from django.conf import settings
 from django.dispatch import receiver
 from django.db.models.signals import post_save
 
-from . import models
+from . import models, tasks
+
+
+@receiver(post_save, sender=models.Video)
+def post_save_video(sender, instance, created, **kwargs):
+    tasks.send_video_to_s3.apply_async((instance.id,))
 
 
 @receiver(post_save, sender=models.Playlist)

apps/core/tasks.py

@@ -4,9 +4,9 @@ import logging
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 
-from .models import Video, Playlist, TagToObject, READY, DYNAMIC
+from .models import Video, Playlist, TagToObject, READY, DYNAMIC, FAIL
 from .s3_uploader import upload_file
-from .utils import asset_upload, generate_thumbnail
+from .utils import asset_upload
 
 from cp_video.celery import app
 
@@ -17,14 +17,68 @@ playlist_content_type = ContentType.objects.get_for_model(Playlist)
 log = logging.getLogger('send_video_to_s3')
 
 
-@app.task(soft_time_limit=60)
-def send_video_to_s3():
-    local_videos = Video.objects.filter(
-        local_file__isnull=False
-    ).prefetch_related('tags', 'categories')
+# @app.task(soft_time_limit=60)
+# def send_video_to_s3():
+#     local_videos = Video.objects.filter(
+#         local_file__isnull=False
+#     ).prefetch_related('tags', 'categories')
+#
+#     for local_video in local_videos:
+#         log.info(f'Uploading video {local_video.id} to S3...')
+#         video_name = local_video.local_file.name.split('/')[-1]
+#         key = asset_upload(local_video, video_name)
+#         file_path = settings.DOMAIN_NAME + settings.MEDIA_URL + key
+#         log.info(f'File path: {file_path}')
+#
+#         response = requests.get(file_path)
+#         content = response.content
+#
+#         local_video.s3_file = upload_file(content, key, 'video/mp4')
+#         local_video.status = READY
+#
+#         # if not local_video.thumbnail and local_video.local_file:
+#         #     log.info(f'Uploading thumbnail for video {local_video.id} to S3...')
+#         #     thumbnail_name = generate_thumbnail(file_path, video_name)
+#         #     key = f'videos/{thumbnail_name}'
+#         #     thumbnail_path = settings.DOMAIN_NAME + settings.MEDIA_URL + thumbnail_name
+#         #     response = requests.get(thumbnail_path)
+#         #     content = response.content
+#         #     content_type = 'image/jpeg'
+#         #     local_video.thumbnail = upload_file(content, key, content_type)
+#         #     log.info(f'Thumbnail for video {local_video.id} uploaded to S3...')
+#
+#         local_video.save(update_fields=['s3_file', 'status', 'thumbnail'])
+#
+#         tag_ids = list(TagToObject.objects.filter(
+#             content_type=video_content_type,
+#             object_id=local_video.id,
+#         ).values_list('tag_id', flat=True))
+#
+#         if tag_ids:
+#             playlist_ids = list(TagToObject.objects.filter(
+#                 content_type=playlist_content_type,
+#                 tag_id__in=tag_ids,
+#             ).values_list('object_id', flat=True))
+#
+#             if playlist_ids:
+#                 playlists = Playlist.objects.filter(
+#                     id__in=playlist_ids,
+#                     type=DYNAMIC,
+#                 )
+#
+#                 for playlist in playlists:
+#                     playlist.videos.add(local_video)
+#
+#     local_videos.update(local_file=None)
+
+
+@app.task(soft_time_limit=600)
+def send_video_to_s3(video_id=None):
+    local_video = Video.objects.get(id=video_id)
+
+    try:
+        log.info(f'Uploading video id {local_video.id} to S3...')
 
-    for local_video in local_videos:
-        log.info(f'Uploading video {local_video.id} to S3...')
         video_name = local_video.local_file.name.split('/')[-1]
         key = asset_upload(local_video, video_name)
         file_path = settings.DOMAIN_NAME + settings.MEDIA_URL + key
@@ -35,23 +89,15 @@ def send_video_to_s3():
 
         local_video.s3_file = upload_file(content, key, 'video/mp4')
         local_video.status = READY
+        local_video.local_file = None
 
-        # if not local_video.thumbnail and local_video.local_file:
-        #     log.info(f'Uploading thumbnail for video {local_video.id} to S3...')
-        #     thumbnail_name = generate_thumbnail(file_path, video_name)
-        #     key = f'videos/{thumbnail_name}'
-        #     thumbnail_path = settings.DOMAIN_NAME + settings.MEDIA_URL + thumbnail_name
-        #     response = requests.get(thumbnail_path)
-        #     content = response.content
-        #     content_type = 'image/jpeg'
-        #     local_video.thumbnail = upload_file(content, key, content_type)
-        #     log.info(f'Thumbnail for video {local_video.id} uploaded to S3...')
+        local_video.save()
 
-        local_video.save(update_fields=['s3_file', 'status', 'thumbnail'])
+        log.info(f'Video id {local_video.id} uploaded to S3...')
 
         tag_ids = list(TagToObject.objects.filter(
             content_type=video_content_type,
-            object_id=local_video.id,
+            object_id=video_id,
         ).values_list('tag_id', flat=True))
 
         if tag_ids:
@@ -68,5 +114,8 @@ def send_video_to_s3():
 
                 for playlist in playlists:
                     playlist.videos.add(local_video)
+    except:
+        local_video.status = FAIL
+        local_video.save(update_fields=['status'])
+        log.info(f'Video {video_id} upload failed!')
 
-    local_videos.update(local_file=None)

cp_video/celery.py

@@ -21,8 +21,8 @@ app.autodiscover_tasks()
 
 
 app.conf.beat_schedule = {
-    'send_video_to_s3': {
-        'task': 'apps.core.tasks.send_video_to_s3',
-        'schedule': 60 * 5,
-    },
+    # 'send_video_to_s3': {
+    #     'task': 'apps.core.tasks.send_video_to_s3',
+    #     'schedule': 60 * 5,
+    # },
 }