Usage of pylib media for getting metadata from media files

admin.py

@@ -22,8 +22,7 @@ class ItemAdmin(admin.ModelAdmin):
         'num_other_c',
         'num_videos_c',
         'sil_c',
-        'camera_vendor_c',
+        'camera_c',
-        'camera_model_c',
         'width_c',
         'height_c',
         'exposure_program_c',
@@ -75,8 +74,7 @@ class ItemAdmin(admin.ModelAdmin):
                 ]
             if obj.type == TYPE_IMAGE:
                 rv += [
-                    'camera_vendor_c',
+                    'camera_c',
-                    'camera_model_c',
                     'width_c',
                     'height_c',
                     'lon_c',

help.py

@@ -74,8 +74,7 @@ more in the different type depending lists.
 * lat (NUMERIC):
 * height (NUMERIC):
 * iso (NUMERIC):
-* camera_vendor (TEXT):
+* camera (TEXT):
-* camera_model (TEXT):
 * orientation (NUMERIC):
 * width (NUMERIC):
 

migrations/0003_auto_20200130_2203.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.9 on 2020-01-30 22:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pygal', '0002_setting_suspend_puplic'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='item',
+            old_name='camera_model_c',
+            new_name='camera_c',
+        ),
+        migrations.RemoveField(
+            model_name='item',
+            name='camera_vendor_c',
+        ),
+    ]

models.py

@@ -6,8 +6,8 @@ from django.urls import reverse
 import fstools
 import json
 import logging
+import media
 import os
-from PIL import Image
 import pygal
 import subprocess
 import time
@@ -76,33 +76,6 @@ def get_item_by_rel_path(rel_path):
                 return i
 
 
-def ffprobe_lines(full_path):
-    def _ffprobe_command(full_path):
-        return ['ffprobe', '-v', 'quiet', '-show_format', '-show_streams', full_path]
-
-    def _avprobe_command(full_path):
-        return ['avprobe', '-v', 'quiet', '-show_format', '-show_streams', full_path]
-
-    def decode(string):
-        for i in ['utf-8', 'cp1252']:
-            try:
-                return string.decode(i)
-            except UnicodeEncodeError:
-                pass
-            except UnicodeDecodeError:
-                pass
-        return string
-
-    try:
-        try:
-            text_to_be_parsed = subprocess.check_output(_avprobe_command(full_path))
-        except OSError:
-            text_to_be_parsed = subprocess.check_output(_ffprobe_command(full_path))
-    except subprocess.CalledProcessError:
-        text_to_be_parsed = ''
-    return decode(text_to_be_parsed).splitlines()
-
-
 def is_valid_area(x1, y1, x2, y2):
     for p in [x1, y1, x2, y2]:
         if type(p) is not int:
@@ -125,8 +98,7 @@ class ItemData(dict):
         'lat',
         'height',
         'iso',
-        'camera_vendor',
+        'camera',
-        'camera_model',
         'orientation',
         'width',
         'duration',
@@ -154,8 +126,7 @@ class ItemData(dict):
         'f_number': 0,
         'focal_length': 0,
         'iso': 0,
-        'camera_vendor': '-',
+        'camera': '-',
-        'camera_model': '-',
         'orientation': 1,
         'duration': 3,
         'ratio': 1,
@@ -224,8 +195,7 @@ class Item(models.Model):
     lat_c = models.FloatField(null=True, blank=True)
     height_c = models.IntegerField(null=True, blank=True)
     iso_c = models.IntegerField(null=True, blank=True)
-    camera_vendor_c = models.CharField(max_length=100, null=True, blank=True)
+    camera_c = models.CharField(max_length=100, null=True, blank=True)
-    camera_model_c = models.CharField(max_length=100, null=True, blank=True)
     orientation_c = models.IntegerField(null=True, blank=True)
     width_c = models.IntegerField(null=True, blank=True)
     # video
@@ -247,6 +217,31 @@ class Item(models.Model):
     num_videos_c = models.IntegerField(null=True, blank=True)
     sil_c = models.TextField(null=True, blank=True)
 
+    MODEL_TO_MEDIA_DATA = {
+        media.metadata.KEY_SIZE: 'size_c',
+        media.metadata.KEY_TIME: 'datetime_c',
+        media.metadata.KEY_EXPOSURE_PROGRAM: 'exposure_program_c',
+        media.metadata.KEY_EXPOSURE_TIME: 'exposure_time_c',
+        media.metadata.KEY_FLASH: 'flash_c',
+        media.metadata.KEY_APERTURE: 'f_number_c',
+        media.metadata.KEY_FOCAL_LENGTH: 'focal_length_c',
+        media.metadata.KEY_GPS: {'lon': 'lon_c', 'lat': 'lat_c'},
+        media.metadata.KEY_HEIGHT: 'height_c',
+        media.metadata.KEY_ISO: 'iso_c',
+        media.metadata.KEY_CAMERA: 'camera_c',
+        media.metadata.KEY_ORIENTATION: 'orientation_c',
+        media.metadata.KEY_WIDTH: 'width_c',
+        media.metadata.KEY_DURATION: 'duration_c',
+        media.metadata.KEY_RATIO: 'ratio_c',
+        media.metadata.KEY_ALBUM: 'album_c',
+        media.metadata.KEY_ARTIST: 'artist_c',
+        media.metadata.KEY_BITRATE: 'bitrate_c',
+        media.metadata.KEY_GENRE: 'genre_c',
+        media.metadata.KEY_TITLE: 'title_c',
+        media.metadata.KEY_TRACK: 'track_c',
+        media.metadata.KEY_YEAR: 'year_c',
+    }
+
     def __init__(self, *args, **kwargs):
         self.__current_uid__ = None
         self.__current_settings__ = None
@@ -378,53 +373,15 @@ class Item(models.Model):
         self.uid_c = self.current_uid()
         self.settings_c = self.current_settings()
         self.data_version_c = self.DATA_VERSION_NUMBER
-        if self.type == TYPE_AUDIO:
+        if self.type == TYPE_FOLDER:
-            self.__update_audio_file_data__(full_path)
-        elif self.type == TYPE_FOLDER:
             self.__update_folder_file_data__(full_path)
-        elif self.type == TYPE_IMAGE:
-            self.__update_image_file_data__(full_path)
         elif self.type == TYPE_OTHER:
             self.__update_other_file_data__(full_path)
-        elif self.type == TYPE_VIDEO:
+        else:
-            self.__update_video_file_data__(full_path)
+            self.__update_media_file_data__(full_path)
         for key, value in self.cached_item_data.items():
             logger.debug('  - Adding %s=%s', key, repr(value))
 
-    def __update_audio_file_data__(self, full_path):
-        self.size_c = os.path.getsize(full_path)
-        #
-        tag_type_target_dict = {}
-        tag_type_target_dict['album'] = (str, 'album')
-        tag_type_target_dict['TAG:album'] = (str, 'album')
-        tag_type_target_dict['TAG:artist'] = (str, 'artist')
-        tag_type_target_dict['artist'] = (str, 'artist')
-        tag_type_target_dict['bit_rate'] = (self.__int_conv__, 'bitrate')
-        tag_type_target_dict['duration'] = (float, 'duration')
-        tag_type_target_dict['TAG:genre'] = (str, 'genre')
-        tag_type_target_dict['genre'] = (str, 'genre')
-        tag_type_target_dict['TAG:title'] = (str, 'title')
-        tag_type_target_dict['title'] = (str, 'title')
-        tag_type_target_dict['TAG:track'] = (self.__int_conv__, 'track')
-        tag_type_target_dict['track'] = (self.__int_conv__, 'track')
-        tag_type_target_dict['TAG:date'] = (self.__int_conv__, 'year')
-        tag_type_target_dict['date'] = (self.__int_conv__, 'year')
-        for line in ffprobe_lines(full_path):
-            try:
-                key, val = [snippet.strip() for snippet in line.split('=')]
-            except ValueError:
-                continue
-            else:
-                if key in tag_type_target_dict:
-                    tp, name = tag_type_target_dict[key]
-                    try:
-                        setattr(self, name + '_c', tp(val))
-                    except ValueError:
-                        logger.log(logging.WARNING if val else logger.INFO, 'Can\'t convert %s (%s) for %s', repr(val), name, self.name)
-        #
-        if self.year_c is not None and self.track_c is not None:
-            self.datetime_c = datetime.datetime(max(1971, self.year_c), 1, 1, 12, 0, (60 - self.track_c) % 60, tzinfo=timezone.utc)
-
     def __update_folder_file_data__(self, full_path):
         sil = []
         self.size_c = 0
@@ -462,151 +419,26 @@ class Item(models.Model):
         if len(sil) > 0:
             self.datetime_c = sil[0].datetime_c
 
-    def __update_image_file_data__(self, full_path):
+    def __update_media_file_data__(self, full_path):
-        self.size_c = os.path.getsize(full_path)
+        data = media.get_media_data(full_path) or {}
-        #
+        for key in self.MODEL_TO_MEDIA_DATA:
-        tag_type_target_dict = {}
+            value = data.get(key)
-        tag_type_target_dict[0x9003] = (self.__datetime_conv__, 'datetime')
+            if key == media.metadata.KEY_GPS:   # Split GPS data in lon and lat
-        tag_type_target_dict[0x8822] = (self.__exposure_program_conv__, 'exposure_program')
+                if value is not None:
-        tag_type_target_dict[0x829A] = (self.__num_denum_conv__, 'exposure_time')
+                    for k in self.MODEL_TO_MEDIA_DATA[key]:
-        tag_type_target_dict[0x9209] = (self.__flash_conv__, 'flash')
+                        setattr(self, self.MODEL_TO_MEDIA_DATA[key][k], value[k])
-        tag_type_target_dict[0x829D] = (self.__num_denum_conv__, 'f_number')
-        tag_type_target_dict[0x920A] = (self.__num_denum_conv__, 'focal_length')
-        tag_type_target_dict[0x8825] = (self.__gps_conv__, ('lon', 'lat'))
-        tag_type_target_dict[0xA003] = (self.__int_conv__, 'height')
-        tag_type_target_dict[0x8827] = (self.__int_conv__, 'iso')
-        tag_type_target_dict[0x010F] = (str, 'camera_vendor')
-        tag_type_target_dict[0x0110] = (str, 'camera_model')
-        tag_type_target_dict[0x0112] = (self.__int_conv__, 'orientation')
-        tag_type_target_dict[0xA002] = (self.__int_conv__, 'width')
-        im = Image.open(full_path)
-        try:
-            exif = dict(im._getexif().items())
-        except AttributeError:
-            logger.debug('%s does not have any exif information', full_path)
             else:
-            for key in tag_type_target_dict:
+                if key == media.metadata.KEY_TIME:  # convert time to datetime
-                if key in exif:
+                    if data.get(media.metadata.KEY_TIME_IS_SUBSTITUTION) and self.type == TYPE_IMAGE:   # don't use time substitution for images
-                    tp, name = tag_type_target_dict[key]
+                        break
-                    if type(name) is tuple:
+                    value = datetime.datetime.fromtimestamp(value, tz=datetime.timezone.utc)
-                        data = tp(exif[key]) or (None, None)
+                setattr(self, self.MODEL_TO_MEDIA_DATA[key], value)
-                        for name, val in zip(name, data):
-                            setattr(self, name + '_c', val)
-                    else:
-                        setattr(self, name + '_c', tp(exif[key]))
 
     def __update_other_file_data__(self, full_path):
         self.size_c = os.path.getsize(full_path)
         #
         self.datetime_c = datetime.datetime.fromtimestamp(os.path.getctime(full_path), tz=timezone.utc)
 
-    def __update_video_file_data__(self, full_path):
-        self.size_c = os.path.getsize(full_path)
-        #
-        tag_type_target_dict = {}
-        tag_type_target_dict['creation_time'] = (self.__vid_datetime_conv__, 'datetime')
-        tag_type_target_dict['TAG:creation_time'] = (self.__vid_datetime_conv__, 'datetime')
-        tag_type_target_dict['duration'] = (float, 'duration')
-        tag_type_target_dict['height'] = (self.__int_conv__, 'height')
-        tag_type_target_dict['width'] = (self.__int_conv__, 'width')
-        tag_type_target_dict['display_aspect_ratio'] = (self.__ratio_conv__, 'ratio')
-        for line in ffprobe_lines(full_path):
-            try:
-                key, val = [snippet.strip() for snippet in line.split('=')]
-            except ValueError:
-                continue
-            else:
-                if key in tag_type_target_dict:
-                    tp, name = tag_type_target_dict[key]
-                    try:
-                        setattr(self, name + '_c', tp(val))
-                    except ValueError:
-                        logger.log(logging.WARNING if val else logger.INFO, 'Can\'t convert %s (%s) for %s', repr(val), name, self.name)
-
-    def __datetime_conv__(self, dt):
-        format_string = "%Y:%m:%d %H:%M:%S%z"
-        return datetime.datetime.strptime(dt + '+0000', format_string)
-
-    def __exposure_program_conv__(self, n):
-        return {
-            0: 'Unidentified',
-            1: 'Manual',
-            2: 'Program Normal',
-            3: 'Aperture Priority',
-            4: 'Shutter Priority',
-            5: 'Program Creative',
-            6: 'Program Action',
-            7: 'Portrait Mode',
-            8: 'Landscape Mode'
-        }.get(n, '-')
-
-    def __flash_conv__(self, n):
-        return {
-            0: 'No',
-            1: 'Fired',
-            5: 'Fired (?)',  # no return sensed
-            7: 'Fired (!)',  # return sensed
-            9: 'Fill Fired',
-            13: 'Fill Fired (?)',
-            15: 'Fill Fired (!)',
-            16: 'Off',
-            24: 'Auto Off',
-            25: 'Auto Fired',
-            29: 'Auto Fired (?)',
-            31: 'Auto Fired (!)',
-            32: 'Not Available'
-        }.get(n, '-')
-
-    def __int_conv__(self, value):
-        try:
-            return int(value)
-        except ValueError:
-            for c in ['.', '/', '-']:
-                p = value.find(c)
-                if p >= 0:
-                    value = value[:p]
-        if value == '':
-            return 0
-        return int(value)
-
-    def __num_denum_conv__(self, data):
-        num, denum = data
-        return num / denum
-
-    def __gps_conv__(self, data):
-        def lat_lon_cal(lon_or_lat):
-            lon_lat = 0.
-            fac = 1.
-            for num, denum in lon_or_lat:
-                lon_lat += float(num) / float(denum) * fac
-                fac *= 1. / 60.
-            return lon_lat
-        try:
-            lon = lat_lon_cal(data[0x0004])
-            lat = lat_lon_cal(data[0x0002])
-            if lon != 0 or lat != 0:    # do not use lon and lat equal 0, caused by motorola gps weakness
-                return lon, lat
-        except KeyError:
-            logger.warning('GPS data extraction failed for %s: %s', self.name, repr(data))
-        return None
-
-    def __vid_datetime_conv__(self, dt):
-        try:
-            dt = dt[:dt.index('.')]
-        except ValueError:
-            pass  # time string seems to have no '.'
-        dt = dt.replace('T', ' ').replace('/', '').replace('\\', '')
-        if len(dt) == 16:
-            dt += ':00'
-        dt += '+0000'
-        format_string = '%Y-%m-%d %H:%M:%S%z'
-        return datetime.datetime.strptime(dt, format_string)
-
-    def __ratio_conv__(self, ratio):
-        ratio = ratio.replace('\\', '')
-        num, denum = ratio.split(':')
-        return float(num) / float(denum)
-
     def __str__(self):
         return 'Item: %s' % self.rel_path
 

search.py

@@ -31,8 +31,7 @@ SCHEMA = Schema(
     lat=NUMERIC,
     height=NUMERIC,
     iso=NUMERIC,
-    camera_vendor=TEXT,
+    camera=TEXT,
-    camera_model=TEXT,
     orientation=NUMERIC,
     width=NUMERIC,
     # Audio Cache

views/infoviews.py

@@ -75,7 +75,7 @@ class image_view(base_view):
     @property
     def item_information(self):
         rv = []
-        rv.append({'description': _('Camera'), 'data': '%s - %s' % (self.item.item_data.camera_vendor, self.item.item_data.camera_model), 'url': None})
+        rv.append({'description': _('Camera'), 'data': '%s' % (self.item.item_data.camera), 'url': None})
         rv.append({'description': _('Resolution'), 'data': '%d x %d' % (self.item.item_data.width, self.item.item_data.height), 'url': None})
         gps_data = self.item.item_data.gps
         if gps_data is not None: